diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/LICENSE.md b/extension/llm/custom_ops/spinquant/third-party/FFHT/LICENSE.md new file mode 100644 index 00000000000..52c4e01cd49 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/LICENSE.md @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Alexandr Andoni, Piotr Indyk, Thijs Laarhoven, +Ilya Razenshteyn, Ludwig Schmidt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/Makefile b/extension/llm/custom_ops/spinquant/third-party/FFHT/Makefile new file mode 100644 index 00000000000..cb68ad9f6f9 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/Makefile @@ -0,0 +1,21 @@ +CC = gcc +CFLAGS = -O3 -march=native -std=c99 -pedantic -Wall -Wextra -Wshadow -Wpointer-arith -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes + +all: test_float test_double fast_copy.o fht.o + +OBJ := fast_copy.o fht.o + +%.o: %.c + $(CC) $< -o $@ -c $(CFLAGS) + +test_%: test_%.c $(OBJ) + $(CC) $< $(OBJ) -o $@ $(CFLAGS) + +test_double_header_only: test_double_header_only.c + $(CC) $< -o $@ $(CFLAGS) + +test_float_header_only: test_double_header_only.c + $(CC) $< -o $@ $(CFLAGS) + +clean: + rm -f test_float test_double test_float_header_only test_double_header_only $(OBJ) diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/README.md b/extension/llm/custom_ops/spinquant/third-party/FFHT/README.md new file mode 100644 index 00000000000..7e00d0eca9a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/README.md @@ -0,0 +1,115 @@ +# Fast Fast Hadamard Transform + +FFHT (Fast Fast Hadamard Transform) is a library that provides a heavily +optimized C99 implementation of the Fast Hadamard Transform. FFHT also provides +a thin Python wrapper that allows to perform the Fast Hadamard Transform on +one-dimensional [NumPy](http://www.numpy.org/) arrays. + +The Hadamard Transform is a linear orthogonal map defined on real vectors whose +length is a _power of two_. For the precise definition, see the +[Wikipedia entry](https://en.wikipedia.org/wiki/Hadamard_transform). The +Hadamard Transform has been recently used a lot in various machine learning +and numerical algorithms. + +FFHT uses [AVX](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) +to speed up the computation. + +The header file `fht.h` exports two functions: `int fht_float(float *buf, int +log_n)` and `int fht_double(double *buf, int log_n)`. The +only difference between them is the type of vector entries. So, in what follows, +we describe how the version for floats `fht_float` works. + +The function `fht_float` takes two parameters: + +* `buf` is a pointer to the data on which one needs to perform the Fast +Hadamard Transform. +* `log_n` is the binary logarithm of the length of `buffer`. +That is, the length is equal to `2^log_n`. + +The return value is -1 if the input is invalid and is zero otherwise. + +A header-only version of the library is provided in `fht_header_only.h`. + +In addition to the Fast Hadamard Transform, we provide two auxiliary programs: +`test_float` and `test_double`, which are implemented in C99. The exhaustively +test and benchmark the library. + +FFHT has been tested on 64-bit versions of Linux, OS X and Windows (the latter +is via Cygwin). + +To install the Python package, run `python setup.py install`. The script +`example.py` shows how to use FFHT from Python. + +## Benchmarks + +Below are the times for the Fast Hadamard Transform for vectors of +various lengths. The benchmarks were run on a machine with Intel +Core i7-6700K and 2133 MHz DDR4 RAM. We compare FFHT, +[FFTW 3.3.6](http://fftw.org/), and +[fht](https://github.com/nbarbey/fht) by +[Nicolas Barbey](https://github.com/nbarbey). + +Let us stress that FFTW is a great versatile tool, and the authors of FFTW did +not try to optimize the performace of the Fast Hadamard Transform. On the other +hand, FFHT does one thing (the Fast Hadamard Transform), but does it extremely +well. + +Vector size | FFHT (float) | FFHT (double) | FFTW 3.3.6 (float) | FFTW 3.3.6 (double) | fht (float) | fht (double) +:---: | :---: | :---: | :---: | :---: | :---: | :---: +210 | 0.31 us | 0.49 us | 4.48 us | 7.72 us | 17.4 us | 19.3 us +220 | 0.68 ms | 1.39 ms | 8.81 ms | 17.07 ms | 29.8 ms | 35.0 ms +227 | 0.22 s | 0.50 s | 2.08 s | 3.57 s | 6.89 s | 7.49 s + +## Troubleshooting + +For some versions of OS X the native `clang` compiler (that mimicks `gcc`) may +not recognize the availability of AVX. A solution for this problem is to use a +genuine `gcc` (say from [Homebrew](http://brew.sh/)) or to use `-march=corei7-avx` +instead of `-march=native` for compiler flags. + +A symptom of the above happening is the undefined macros `__AVX__`. + +## Related Work + +FFHT has been created as a part of +[FALCONN](https://github.com/falconn-lib/falconn): a library for similarity +search over high-dimensional data. FALCONN's underlying algorithms are described +and analyzed in the following research paper: + +> Alexandr Andoni, Piotr Indyk, Thijs Laarhoven, Ilya Razenshteyn and Ludwig +> Schmidt, "Practical and Optimal LSH for Angular Distance", NIPS 2015, full +> version available at [arXiv:1509.02897](http://arxiv.org/abs/1509.02897) + +This is the right paper to cite, if you use FFHT for your research projects. + +## Acknowledgments + +We thank Ruslan Savchenko for useful discussions. + +Thanks to: + +* Clement Canonne +* Michal Forisek +* Rati Gelashvili +* Daniel Grier +* Dhiraj Holden +* Justin Holmgren +* Aleksandar Ivanovic +* Vladislav Isenbaev +* Jacob Kogler +* Ilya Kornakov +* Anton Lapshin +* Rio LaVigne +* Oleg Martynov +* Linar Mikeev +* Cameron Musco +* Sam Park +* Sunoo Park +* Amelia Perry +* Andrew Sabisch +* Abhishek Sarkar +* Ruslan Savchenko +* Vadim Semenov +* Arman Yessenamanov + +for helping us with testing FFHT. diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/_ffht_2.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/_ffht_2.c new file mode 100644 index 00000000000..2041e5eedec --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/_ffht_2.c @@ -0,0 +1,128 @@ +#include +#include +#include "fht.h" + +#define UNUSED(x) (void)(x) + +static char module_docstring[] = + "A C extension that computes the Fast Hadamard Transform"; +static char fht_docstring[] = + "Compute the Fast Hadamard Transform (FHT) for a given " + "one-dimensional NumPy array.\n\n" + "The Hadamard Transform is a linear orthogonal map defined on real vectors " + "whose length is a _power of two_. For the precise definition, see the " + "[Wikipedia entry](https://en.wikipedia.org/wiki/Hadamard_transform). The " + "Hadamard Transform has been recently used a lot in various machine " + "learning " + "and numerical algorithms.\n\n" + "The implementation uses " + "[AVX](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) " + "to speed up the computation. If AVX is not supported on your machine, " + "a simpler implementation without (explicit) vectorization is used.\n\n" + "The function takes two parameters:\n\n" + "* `buffer` is a NumPy array which is being transformed. It must be a " + "one-dimensional array with `dtype` equal to `float32` or `float64` (the " + "former is recommended unless you need high accuracy) and of size being a " + "power " + "of two. If your CPU supports AVX, then `buffer` must be aligned to 32 " + "bytes. " + "To allocate such an aligned buffer, use the function `created_aligned` " + "from this " + "module.\n" + "* `chunk` is a positive integer that controls when the implementation " + "switches " + "from recursive to iterative algorithm. The overall algorithm is " + "recursive, but as " + "soon as the vector becomes no longer than `chunk`, the iterative " + "algorithm is " + "invoked. For technical reasons, `chunk` must be at least 8. A good choice " + "is to " + "set `chunk` to 1024. But to fine-tune the performance one should use a " + "program " + "`best_chunk` supplied with the library.\n"; + +static PyObject *ffht_fht(PyObject *self, PyObject *args); + +static PyMethodDef module_methods[] = { + {"fht", ffht_fht, METH_VARARGS, fht_docstring}, {NULL, NULL, 0, NULL}}; + +PyMODINIT_FUNC initffht(void); + +PyMODINIT_FUNC initffht(void) { + PyObject *m = Py_InitModule3("ffht", module_methods, module_docstring); + if (!m) return; + + import_array(); +} + +static PyObject *ffht_fht(PyObject *self, PyObject *args) { + UNUSED(self); + + PyObject *buffer_obj; + + if (!PyArg_ParseTuple(args, "O", &buffer_obj)) { + return NULL; + } + + PyArray_Descr *dtype; + int ndim; + npy_intp dims[NPY_MAXDIMS]; + PyArrayObject *arr = NULL; + + if (PyArray_GetArrayParamsFromObject(buffer_obj, NULL, 1, &dtype, &ndim, dims, + &arr, NULL) < 0) { + return NULL; + } + + if (arr == NULL) { + PyErr_SetString(PyExc_TypeError, "not a numpy array"); + return NULL; + } + + dtype = PyArray_DESCR(arr); + + if (dtype->type_num != NPY_FLOAT && dtype->type_num != NPY_DOUBLE) { + PyErr_SetString(PyExc_TypeError, "array must consist of floats or doubles"); + Py_DECREF(arr); + return NULL; + } + + if (PyArray_NDIM(arr) != 1) { + PyErr_SetString(PyExc_TypeError, "array must be one-dimensional"); + Py_DECREF(arr); + return NULL; + } + + int n = PyArray_DIM(arr, 0); + + if (n == 0 || (n & (n - 1))) { + PyErr_SetString(PyExc_ValueError, "array's length must be a power of two"); + Py_DECREF(arr); + return NULL; + } + + int log_n = 0; + while ((1 << log_n) < n) { + ++log_n; + } + + void *raw_buffer = PyArray_DATA(arr); + int res; + if (dtype->type_num == NPY_FLOAT) { + float *buffer = (float *)raw_buffer; + res = fht_float(buffer, log_n); + } else { + double *buffer = (double *)raw_buffer; + res = fht_double(buffer, log_n); + } + + if (res) { + PyErr_SetString(PyExc_RuntimeError, "FHT did not work properly"); + Py_DECREF(arr); + return NULL; + } + + Py_DECREF(arr); + + return Py_BuildValue(""); +} \ No newline at end of file diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/_ffht_3.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/_ffht_3.c new file mode 100644 index 00000000000..1afe8013e46 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/_ffht_3.c @@ -0,0 +1,142 @@ +#include +#include +#include "fht.h" + +#define UNUSED(x) (void)(x) + +static char module_docstring[] = + "A C extension that computes the Fast Hadamard Transform"; +static char fht_docstring[] = + "Compute the Fast Hadamard Transform (FHT) for a given " + "one-dimensional NumPy array.\n\n" + "The Hadamard Transform is a linear orthogonal map defined on real vectors " + "whose length is a _power of two_. For the precise definition, see the " + "[Wikipedia entry](https://en.wikipedia.org/wiki/Hadamard_transform). The " + "Hadamard Transform has been recently used a lot in various machine " + "learning " + "and numerical algorithms.\n\n" + "The implementation uses " + "[AVX](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) " + "to speed up the computation. If AVX is not supported on your machine, " + "a simpler implementation without (explicit) vectorization is used.\n\n" + "The function takes two parameters:\n\n" + "* `buffer` is a NumPy array which is being transformed. It must be a " + "one-dimensional array with `dtype` equal to `float32` or `float64` (the " + "former is recommended unless you need high accuracy) and of size being a " + "power " + "of two. If your CPU supports AVX, then `buffer` must be aligned to 32 " + "bytes. " + "To allocate such an aligned buffer, use the function `created_aligned` " + "from this " + "module.\n" + "* `chunk` is a positive integer that controls when the implementation " + "switches " + "from recursive to iterative algorithm. The overall algorithm is " + "recursive, but as " + "soon as the vector becomes no longer than `chunk`, the iterative " + "algorithm is " + "invoked. For technical reasons, `chunk` must be at least 8. A good choice " + "is to " + "set `chunk` to 1024. But to fine-tune the performance one should use a " + "program " + "`best_chunk` supplied with the library.\n"; + +static PyObject *ffht_fht(PyObject *self, PyObject *args) { + UNUSED(self); + + PyObject *buffer_obj; + + if (!PyArg_ParseTuple(args, "O", &buffer_obj)) { + return NULL; + } + + PyArray_Descr *dtype; + int ndim; + npy_intp dims[NPY_MAXDIMS]; + PyArrayObject *arr = NULL; + + if (PyArray_GetArrayParamsFromObject(buffer_obj, NULL, 1, &dtype, &ndim, dims, + &arr, NULL) < 0) { + return NULL; + } + + if (arr == NULL) { + PyErr_SetString(PyExc_TypeError, "not a numpy array"); + return NULL; + } + + dtype = PyArray_DESCR(arr); + + if (dtype->type_num != NPY_FLOAT && dtype->type_num != NPY_DOUBLE) { + PyErr_SetString(PyExc_TypeError, "array must consist of floats or doubles"); + Py_DECREF(arr); + return NULL; + } + + if (PyArray_NDIM(arr) != 1) { + PyErr_SetString(PyExc_TypeError, "array must be one-dimensional"); + Py_DECREF(arr); + return NULL; + } + + int n = PyArray_DIM(arr, 0); + + if (n == 0 || (n & (n - 1))) { + PyErr_SetString(PyExc_ValueError, "array's length must be a power of two"); + Py_DECREF(arr); + return NULL; + } + + int log_n = 0; + while ((1 << log_n) < n) { + ++log_n; + } + + void *raw_buffer = PyArray_DATA(arr); + int res; + if (dtype->type_num == NPY_FLOAT) { + float *buffer = (float *)raw_buffer; + res = fht_float(buffer, log_n); + } else { + double *buffer = (double *)raw_buffer; + res = fht_double(buffer, log_n); + } + + if (res) { + PyErr_SetString(PyExc_RuntimeError, "FHT did not work properly"); + Py_DECREF(arr); + return NULL; + } + + Py_DECREF(arr); + + return Py_BuildValue(""); +} + +static PyMethodDef module_methods[] = { + {"fht", ffht_fht, METH_VARARGS, fht_docstring}, + {NULL, NULL, 0, NULL} +}; + + +static struct PyModuleDef ffhtmodule = { + PyModuleDef_HEAD_INIT, + "ffht", + module_docstring, + -1, + module_methods +}; + +PyMODINIT_FUNC PyInit_ffht(void) { + PyObject *module = PyModule_Create(&ffhtmodule); + + if (module == NULL) { + printf("NULL"); + return NULL; + } + + import_array(); + return module; +} + + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/example.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/example.py new file mode 100644 index 00000000000..576c89830da --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/example.py @@ -0,0 +1,20 @@ +import numpy as np +import ffht +import timeit +import sys + +reps = 1000 +n = 2**20 +chunk_size = 1024 + +a = np.random.randn(n).astype(np.float32) + +t1 = timeit.default_timer() +for i in range(reps): + ffht.fht(a) +t2 = timeit.default_timer() + +if sys.version_info[0] == 2: + print (t2 - t1 + 0.0) / (reps + 0.0) +if sys.version_info[0] == 3: + print('{}'.format((t2 - t1 + 0.0) / (reps + 0.0))) diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.clang-format b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.clang-format new file mode 100644 index 00000000000..4b3f13fa55e --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.clang-format @@ -0,0 +1,5 @@ +--- +Language: Cpp +BasedOnStyle: Google +... + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.gitignore b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.gitignore new file mode 100644 index 00000000000..3c1b4f2183e --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.gitignore @@ -0,0 +1,46 @@ +*.a +*.so +*.so.?* +*.dll +*.exe +*.dylib +*.cmake +!/cmake/*.cmake +*~ +*.pyc +__pycache__ + +# lcov +*.lcov +/lcov + +# cmake files. +/Testing +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake + +# makefiles. +Makefile + +# in-source build. +bin/ +lib/ +/test/*_test + +# exuberant ctags. +tags + +# YouCompleteMe configuration. +.ycm_extra_conf.pyc + +# ninja generated files. +.ninja_deps +.ninja_log +build.ninja +install_manifest.txt +rules.ninja + +# out-of-source build top-level folders. +build/ +_build/ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.travis-libcxx-setup.sh b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.travis-libcxx-setup.sh new file mode 100644 index 00000000000..a591743c6a6 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.travis-libcxx-setup.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Install a newer CMake version +curl -sSL https://cmake.org/files/v3.6/cmake-3.6.1-Linux-x86_64.sh -o install-cmake.sh +chmod +x install-cmake.sh +sudo ./install-cmake.sh --prefix=/usr/local --skip-license + +# Checkout LLVM sources +git clone --depth=1 https://github.com/llvm-mirror/llvm.git llvm-source +git clone --depth=1 https://github.com/llvm-mirror/libcxx.git llvm-source/projects/libcxx +git clone --depth=1 https://github.com/llvm-mirror/libcxxabi.git llvm-source/projects/libcxxabi + +# Setup libc++ options +if [ -z "$BUILD_32_BITS" ]; then + export BUILD_32_BITS=OFF && echo disabling 32 bit build +fi + +# Build and install libc++ (Use unstable ABI for better sanitizer coverage) +mkdir llvm-build && cd llvm-build +cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/usr \ + -DLIBCXX_ABI_UNSTABLE=ON \ + -DLLVM_USE_SANITIZER=${LIBCXX_SANITIZER} \ + -DLLVM_BUILD_32_BITS=${BUILD_32_BITS} \ + ../llvm-source +make cxx -j2 +sudo make install-cxxabi install-cxx +cd ../ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.travis.yml b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.travis.yml new file mode 100644 index 00000000000..36df088446c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/.travis.yml @@ -0,0 +1,157 @@ +sudo: required +dist: trusty +language: cpp + +env: + global: + - /usr/local/bin:$PATH + +matrix: + include: + - compiler: gcc + addons: + apt: + packages: + - lcov + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Coverage + - compiler: gcc + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug + - compiler: gcc + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release + - compiler: gcc + addons: + apt: + packages: + - g++-multilib + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Debug BUILD_32_BITS=ON + - compiler: gcc + addons: + apt: + packages: + - g++-multilib + env: COMPILER=g++ C_COMPILER=gcc BUILD_TYPE=Release BUILD_32_BITS=ON + - compiler: gcc + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-6 + env: + - COMPILER=g++-6 C_COMPILER=gcc-6 BUILD_TYPE=Debug + - EXTRA_FLAGS="-fno-omit-frame-pointer -g -O2 -fsanitize=undefined,address -fuse-ld=gold" + - compiler: clang + env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Debug + - compiler: clang + env: COMPILER=clang++ C_COMPILER=clang BUILD_TYPE=Release + # Clang w/ libc++ + - compiler: clang + addons: + apt: + packages: + clang-3.8 + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug + - LIBCXX_BUILD=1 + - EXTRA_FLAGS="-stdlib=libc++" + - compiler: clang + addons: + apt: + packages: + clang-3.8 + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release + - LIBCXX_BUILD=1 + - EXTRA_FLAGS="-stdlib=libc++" + # Clang w/ 32bit libc++ + - compiler: clang + addons: + apt: + packages: + - clang-3.8 + - g++-multilib + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug + - LIBCXX_BUILD=1 + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-stdlib=libc++ -m32" + # Clang w/ 32bit libc++ + - compiler: clang + addons: + apt: + packages: + - clang-3.8 + - g++-multilib + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Release + - LIBCXX_BUILD=1 + - BUILD_32_BITS=ON + - EXTRA_FLAGS="-stdlib=libc++ -m32" + # Clang w/ libc++, ASAN, UBSAN + - compiler: clang + addons: + apt: + packages: + clang-3.8 + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug + - LIBCXX_BUILD=1 LIBCXX_SANITIZER="Undefined;Address" + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=undefined,address -fno-sanitize-recover=all" + - UBSAN_OPTIONS=print_stacktrace=1 + # Clang w/ libc++ and MSAN + - compiler: clang + addons: + apt: + packages: + clang-3.8 + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=Debug + - LIBCXX_BUILD=1 LIBCXX_SANITIZER=MemoryWithOrigins + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=memory -fsanitize-memory-track-origins" + # Clang w/ libc++ and MSAN + - compiler: clang + addons: + apt: + packages: + clang-3.8 + env: + - COMPILER=clang++-3.8 C_COMPILER=clang-3.8 BUILD_TYPE=RelWithDebInfo + - LIBCXX_BUILD=1 LIBCXX_SANITIZER=Thread + - EXTRA_FLAGS="-stdlib=libc++ -g -O2 -fno-omit-frame-pointer -fsanitize=thread -fno-sanitize-recover=all" + + - os: osx + osx_image: xcode8.3 + compiler: clang + env: + - COMPILER=clang++ BUILD_TYPE=Debug + - os: osx + osx_image: xcode8.3 + compiler: clang + env: + - COMPILER=clang++ BUILD_TYPE=Release + +before_script: + - if [ -z "$BUILD_32_BITS" ]; then + export BUILD_32_BITS=OFF && echo disabling 32 bit build; + fi + - if [ -n "${LIBCXX_BUILD}" ]; then + source .travis-libcxx-setup.sh; + fi + - mkdir build && cd build + +install: + - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then + PATH=~/.local/bin:${PATH}; + pip install --user --upgrade pip; + pip install --user cpp-coveralls; + fi + +script: + - cmake -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_CXX_FLAGS="${EXTRA_FLAGS}" -DBENCHMARK_BUILD_32_BITS=${BUILD_32_BITS} .. + - make + - ctest -C ${BUILD_TYPE} --output-on-failure + +after_success: + - if [ "${BUILD_TYPE}" == "Coverage" -a "${TRAVIS_OS_NAME}" == "linux" ]; then + coveralls --include src --include include --gcov-options '\-lp' --root .. --build-root .; + fi diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/AUTHORS b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/AUTHORS new file mode 100644 index 00000000000..ae278df4046 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/AUTHORS @@ -0,0 +1,40 @@ +# This is the official list of benchmark authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Albert Pretorius +Arne Beer +Christopher Seymour +David Coeurjolly +Dominic Hamon +Eric Fiselier +Eugene Zhuk +Evgeny Safronov +Felix Homann +Google Inc. +International Business Machines Corporation +Ismael Jimenez Martinez +Jern-Kuan Leong +Joao Paulo Magalhaes +JianXiong Zhou +Jussi Knuuttila +Kaito Udagawa +Lei Xu +Matt Clarkson +Maxim Vafin +Nick Hutchinson +Oleksandr Sochka +Paul Redmond +Radoslav Yovchev +Shuo Chen +Yixuan Qiu +Yusuke Suzuki +Dirac Research +Zbigniew Skowron +Dominik Czarnota diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CMakeLists.txt b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CMakeLists.txt new file mode 100644 index 00000000000..f7f1566f569 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CMakeLists.txt @@ -0,0 +1,202 @@ +cmake_minimum_required (VERSION 2.8.12) + +project (benchmark) + +foreach(p + CMP0054 # CMake 3.1 + CMP0056 # export EXE_LINKER_FLAGS to try_run + ) + if(POLICY ${p}) + cmake_policy(SET ${p} NEW) + endif() +endforeach() + +option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON) +option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON) +option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark library." OFF) +option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library." OFF) +option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library" OFF) + +# Make sure we can import out CMake functions +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +# Read the git tags to determine the project version +include(GetGitVersion) +get_git_version(GIT_VERSION) + +# Tell the user what versions we are using +string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" VERSION ${GIT_VERSION}) +message("-- Version: ${VERSION}") + +# The version of the libraries +set(GENERIC_LIB_VERSION ${VERSION}) +string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION) + +# Import our CMake modules +include(CheckCXXCompilerFlag) +include(AddCXXCompilerFlag) +include(CXXFeatureCheck) + +if (BENCHMARK_BUILD_32_BITS) + add_required_cxx_compiler_flag(-m32) +endif() + +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + # Turn compiler warnings up to 11 + string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + + if (NOT BENCHMARK_ENABLE_EXCEPTIONS) + add_cxx_compiler_flag(-EHs-) + add_cxx_compiler_flag(-EHa-) + endif() + # Link time optimisation + if (BENCHMARK_ENABLE_LTO) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GL") + set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "${CMAKE_STATIC_LINKER_FLAGS_RELEASE} /LTCG") + set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG") + + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /GL") + string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO}") + set(CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO} /LTCG") + string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO}") + set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /LTCG") + string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}") + set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /LTCG") + + set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /GL") + set(CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL "${CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL} /LTCG") + set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL "${CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL} /LTCG") + set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG") + endif() +else() + # Try and enable C++11. Don't use C++14 because it doesn't work in some + # configurations. + add_cxx_compiler_flag(-std=c++11) + if (NOT HAVE_CXX_FLAG_STD_CXX11) + add_cxx_compiler_flag(-std=c++0x) + endif() + + # Turn compiler warnings up to 11 + add_cxx_compiler_flag(-Wall) + + add_cxx_compiler_flag(-Wextra) + add_cxx_compiler_flag(-Wshadow) + add_cxx_compiler_flag(-Werror RELEASE) + add_cxx_compiler_flag(-Werror RELWITHDEBINFO) + add_cxx_compiler_flag(-Werror MINSIZEREL) + add_cxx_compiler_flag(-pedantic) + add_cxx_compiler_flag(-pedantic-errors) + add_cxx_compiler_flag(-Wshorten-64-to-32) + add_cxx_compiler_flag(-Wfloat-equal) + add_cxx_compiler_flag(-fstrict-aliasing) + if (NOT BENCHMARK_ENABLE_EXCEPTIONS) + add_cxx_compiler_flag(-fno-exceptions) + endif() + if (NOT BENCHMARK_USE_LIBCXX) + add_cxx_compiler_flag(-Wzero-as-null-pointer-constant) + endif() + if (HAVE_CXX_FLAG_FSTRICT_ALIASING) + if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing + add_cxx_compiler_flag(-Wstrict-aliasing) + endif() + endif() + # ICC17u2: overloaded virtual function "benchmark::Fixture::SetUp" is only partially overridden + # (because of deprecated overload) + add_cxx_compiler_flag(-wd654) + add_cxx_compiler_flag(-Wthread-safety) + if (HAVE_CXX_FLAG_WTHREAD_SAFETY) + cxx_feature_check(THREAD_SAFETY_ATTRIBUTES) + endif() + + # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a + # predefined macro, which turns on all of the wonderful libc extensions. + # However g++ doesn't do this in Cygwin so we have to define it ourselfs + # since we depend on GNU/POSIX/BSD extensions. + if (CYGWIN) + add_definitions(-D_GNU_SOURCE=1) + endif() + + # Link time optimisation + if (BENCHMARK_ENABLE_LTO) + add_cxx_compiler_flag(-flto) + if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") + find_program(GCC_AR gcc-ar) + if (GCC_AR) + set(CMAKE_AR ${GCC_AR}) + endif() + find_program(GCC_RANLIB gcc-ranlib) + if (GCC_RANLIB) + set(CMAKE_RANLIB ${GCC_RANLIB}) + endif() + endif() + endif() + + # Coverage build type + set(CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING + "Flags used by the C++ compiler during coverage builds." + FORCE) + set(CMAKE_EXE_LINKER_FLAGS_COVERAGE + "${CMAKE_EXE_LINKER_FLAGS_DEBUG}" CACHE STRING + "Flags used for linking binaries during coverage builds." + FORCE) + set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE + "${CMAKE_SHARED_LINKER_FLAGS_DEBUG}" CACHE STRING + "Flags used by the shared libraries linker during coverage builds." + FORCE) + mark_as_advanced( + CMAKE_CXX_FLAGS_COVERAGE + CMAKE_EXE_LINKER_FLAGS_COVERAGE + CMAKE_SHARED_LINKER_FLAGS_COVERAGE) + set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE STRING + "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel Coverage." + FORCE) + add_cxx_compiler_flag(--coverage COVERAGE) +endif() + +if (BENCHMARK_USE_LIBCXX) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + add_cxx_compiler_flag(-stdlib=libc++) + elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR + "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + add_cxx_compiler_flag(-nostdinc++) + message("libc++ header path must be manually specified using CMAKE_CXX_FLAGS") + # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break + # configuration checks such as 'find_package(Threads)' + list(APPEND BENCHMARK_CXX_LINKER_FLAGS -nodefaultlibs) + # -lc++ cannot be added directly to CMAKE__LINKER_FLAGS because + # linker flags appear before all linker inputs and -lc++ must appear after. + list(APPEND BENCHMARK_CXX_LIBRARIES c++) + else() + message(FATAL "-DBENCHMARK_USE_LIBCXX:BOOL=ON is not supported for compiler") + endif() +endif(BENCHMARK_USE_LIBCXX) + +# C++ feature checks +# Determine the correct regular expression engine to use +cxx_feature_check(STD_REGEX) +cxx_feature_check(GNU_POSIX_REGEX) +cxx_feature_check(POSIX_REGEX) +if(NOT HAVE_STD_REGEX AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX) + message(FATAL_ERROR "Failed to determine the source files for the regular expression backend") +endif() +if (NOT BENCHMARK_ENABLE_EXCEPTIONS AND HAVE_STD_REGEX + AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX) + message(WARNING "Using std::regex with exceptions disabled is not fully supported") +endif() +cxx_feature_check(STEADY_CLOCK) +# Ensure we have pthreads +find_package(Threads REQUIRED) + +# Set up directories +include_directories(${PROJECT_SOURCE_DIR}/include) + +# Build the targets +add_subdirectory(src) + +if (BENCHMARK_ENABLE_TESTING) + enable_testing() + add_subdirectory(test) +endif() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CONTRIBUTING.md b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CONTRIBUTING.md new file mode 100644 index 00000000000..43de4c9d470 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# How to contribute # + +We'd love to accept your patches and contributions to this project. There are +a just a few small guidelines you need to follow. + + +## Contributor License Agreement ## + +Contributions to any Google project must be accompanied by a Contributor +License Agreement. This is not a copyright **assignment**, it simply gives +Google permission to use and redistribute your contributions as part of the +project. + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual + CLA][]. + + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA][]. + +You generally only need to submit a CLA once, so if you've already submitted +one (even if it was for a different project), you probably don't need to do it +again. + +[individual CLA]: https://developers.google.com/open-source/cla/individual +[corporate CLA]: https://developers.google.com/open-source/cla/corporate + +Once your CLA is submitted (or if you already submitted one for +another Google project), make a commit adding yourself to the +[AUTHORS][] and [CONTRIBUTORS][] files. This commit can be part +of your first [pull request][]. + +[AUTHORS]: AUTHORS +[CONTRIBUTORS]: CONTRIBUTORS + + +## Submitting a patch ## + + 1. It's generally best to start by opening a new issue describing the bug or + feature you're intending to fix. Even if you think it's relatively minor, + it's helpful to know what people are working on. Mention in the initial + issue that you are planning to work on that bug or feature so that it can + be assigned to you. + + 1. Follow the normal process of [forking][] the project, and setup a new + branch to work in. It's important that each group of changes be done in + separate branches in order to ensure that a pull request only includes the + commits related to that bug or feature. + + 1. Do your best to have [well-formed commit messages][] for each change. + This provides consistency throughout the project, and ensures that commit + messages are able to be formatted properly by various git tools. + + 1. Finally, push the commits to your fork and submit a [pull request][]. + +[forking]: https://help.github.com/articles/fork-a-repo +[well-formed commit messages]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html +[pull request]: https://help.github.com/articles/creating-a-pull-request diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CONTRIBUTORS b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CONTRIBUTORS new file mode 100644 index 00000000000..9abb60865eb --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/CONTRIBUTORS @@ -0,0 +1,59 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. +# +# Names should be added to this file as: +# Name +# +# Please keep the list sorted. + +Albert Pretorius +Arne Beer +Billy Robert O'Neal III +Chris Kennelly +Christopher Seymour +David Coeurjolly +Dominic Hamon +Eric Fiselier +Eugene Zhuk +Evgeny Safronov +Felix Homann +Ismael Jimenez Martinez +Jern-Kuan Leong +Joao Paulo Magalhaes +JianXiong Zhou +Jussi Knuuttila +Kaito Udagawa +Kai Wolf +Lei Xu +Matt Clarkson +Maxim Vafin +Nick Hutchinson +Oleksandr Sochka +Pascal Leroy +Paul Redmond +Pierre Phaneuf +Radoslav Yovchev +Ray Glover +Shuo Chen +Tom Madams +Yixuan Qiu +Yusuke Suzuki +Tobias Ulvgård +Zbigniew Skowron +Dominik Czarnota diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/LICENSE b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/README.md b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/README.md new file mode 100644 index 00000000000..2430d93bf9c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/README.md @@ -0,0 +1,726 @@ +# benchmark +[![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark) +[![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master) +[![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark) + +A library to support the benchmarking of functions, similar to unit-tests. + +Discussion group: https://groups.google.com/d/forum/benchmark-discuss + +IRC channel: https://freenode.net #googlebenchmark + +[Known issues and common problems](#known-issues) + +[Additional Tooling Documentation](docs/tools.md) + +## Example usage +### Basic usage +Define a function that executes the code to be measured. + +```c++ +static void BM_StringCreation(benchmark::State& state) { + while (state.KeepRunning()) + std::string empty_string; +} +// Register the function as a benchmark +BENCHMARK(BM_StringCreation); + +// Define another benchmark +static void BM_StringCopy(benchmark::State& state) { + std::string x = "hello"; + while (state.KeepRunning()) + std::string copy(x); +} +BENCHMARK(BM_StringCopy); + +BENCHMARK_MAIN(); +``` + +### Passing arguments +Sometimes a family of benchmarks can be implemented with just one routine that +takes an extra argument to specify which one of the family of benchmarks to +run. For example, the following code defines a family of benchmarks for +measuring the speed of `memcpy()` calls of different lengths: + +```c++ +static void BM_memcpy(benchmark::State& state) { + char* src = new char[state.range(0)]; + char* dst = new char[state.range(0)]; + memset(src, 'x', state.range(0)); + while (state.KeepRunning()) + memcpy(dst, src, state.range(0)); + state.SetBytesProcessed(int64_t(state.iterations()) * + int64_t(state.range(0))); + delete[] src; + delete[] dst; +} +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); +``` + +The preceding code is quite repetitive, and can be replaced with the following +short-hand. The following invocation will pick a few appropriate arguments in +the specified range and will generate a benchmark for each such argument. + +```c++ +BENCHMARK(BM_memcpy)->Range(8, 8<<10); +``` + +By default the arguments in the range are generated in multiples of eight and +the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the +range multiplier is changed to multiples of two. + +```c++ +BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10); +``` +Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ]. + +You might have a benchmark that depends on two or more inputs. For example, the +following code defines a family of benchmarks for measuring the speed of set +insertion. + +```c++ +static void BM_SetInsert(benchmark::State& state) { + while (state.KeepRunning()) { + state.PauseTiming(); + std::set data = ConstructRandomSet(state.range(0)); + state.ResumeTiming(); + for (int j = 0; j < state.range(1); ++j) + data.insert(RandomNumber()); + } +} +BENCHMARK(BM_SetInsert) + ->Args({1<<10, 1}) + ->Args({1<<10, 8}) + ->Args({1<<10, 64}) + ->Args({1<<10, 512}) + ->Args({8<<10, 1}) + ->Args({8<<10, 8}) + ->Args({8<<10, 64}) + ->Args({8<<10, 512}); +``` + +The preceding code is quite repetitive, and can be replaced with the following +short-hand. The following macro will pick a few appropriate arguments in the +product of the two specified ranges and will generate a benchmark for each such +pair. + +```c++ +BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {1, 512}}); +``` + +For more complex patterns of inputs, passing a custom function to `Apply` allows +programmatic specification of an arbitrary set of arguments on which to run the +benchmark. The following example enumerates a dense range on one parameter, +and a sparse range on the second. + +```c++ +static void CustomArguments(benchmark::internal::Benchmark* b) { + for (int i = 0; i <= 10; ++i) + for (int j = 32; j <= 1024*1024; j *= 8) + b->Args({i, j}); +} +BENCHMARK(BM_SetInsert)->Apply(CustomArguments); +``` + +### Calculate asymptotic complexity (Big O) +Asymptotic complexity might be calculated for a family of benchmarks. The +following code will calculate the coefficient for the high-order term in the +running time and the normalized root-mean square error of string comparison. + +```c++ +static void BM_StringCompare(benchmark::State& state) { + std::string s1(state.range(0), '-'); + std::string s2(state.range(0), '-'); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(s1.compare(s2)); + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_StringCompare) + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN); +``` + +As shown in the following invocation, asymptotic complexity might also be +calculated automatically. + +```c++ +BENCHMARK(BM_StringCompare) + ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(); +``` + +The following code will specify asymptotic complexity with a lambda function, +that might be used to customize high-order term calculation. + +```c++ +BENCHMARK(BM_StringCompare)->RangeMultiplier(2) + ->Range(1<<10, 1<<18)->Complexity([](int n)->double{return n; }); +``` + +### Templated benchmarks +Templated benchmarks work the same way: This example produces and consumes +messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the +absence of multiprogramming. + +```c++ +template int BM_Sequential(benchmark::State& state) { + Q q; + typename Q::value_type v; + while (state.KeepRunning()) { + for (int i = state.range(0); i--; ) + q.push(v); + for (int e = state.range(0); e--; ) + q.Wait(&v); + } + // actually messages, not bytes: + state.SetBytesProcessed( + static_cast(state.iterations())*state.range(0)); +} +BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); +``` + +Three macros are provided for adding benchmark templates. + +```c++ +#if __cplusplus >= 201103L // C++11 and greater. +#define BENCHMARK_TEMPLATE(func, ...) // Takes any number of parameters. +#else // C++ < C++11 +#define BENCHMARK_TEMPLATE(func, arg1) +#endif +#define BENCHMARK_TEMPLATE1(func, arg1) +#define BENCHMARK_TEMPLATE2(func, arg1, arg2) +``` + +## Passing arbitrary arguments to a benchmark +In C++11 it is possible to define a benchmark that takes an arbitrary number +of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)` +macro creates a benchmark that invokes `func` with the `benchmark::State` as +the first argument followed by the specified `args...`. +The `test_case_name` is appended to the name of the benchmark and +should describe the values passed. + +```c++ +template ` +void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { + [...] +} +// Registers a benchmark named "BM_takes_args/int_string_test` that passes +// the specified values to `extra_args`. +BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +``` +Note that elements of `...args` may refer to global variables. Users should +avoid modifying global state inside of a benchmark. + +## Using RegisterBenchmark(name, fn, args...) + +The `RegisterBenchmark(name, func, args...)` function provides an alternative +way to create and register benchmarks. +`RegisterBenchmark(name, func, args...)` creates, registers, and returns a +pointer to a new benchmark with the specified `name` that invokes +`func(st, args...)` where `st` is a `benchmark::State` object. + +Unlike the `BENCHMARK` registration macros, which can only be used at the global +scope, the `RegisterBenchmark` can be called anywhere. This allows for +benchmark tests to be registered programmatically. + +Additionally `RegisterBenchmark` allows any callable object to be registered +as a benchmark. Including capturing lambdas and function objects. This +allows the creation + +For Example: +```c++ +auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ }; + +int main(int argc, char** argv) { + for (auto& test_input : { /* ... */ }) + benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input); + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} +``` + +### Multithreaded benchmarks +In a multithreaded test (benchmark invoked by multiple threads simultaneously), +it is guaranteed that none of the threads will start until all have called +`KeepRunning`, and all will have finished before KeepRunning returns false. As +such, any global setup or teardown can be wrapped in a check against the thread +index: + +```c++ +static void BM_MultiThreaded(benchmark::State& state) { + if (state.thread_index == 0) { + // Setup code here. + } + while (state.KeepRunning()) { + // Run the test as normal. + } + if (state.thread_index == 0) { + // Teardown code here. + } +} +BENCHMARK(BM_MultiThreaded)->Threads(2); +``` + +If the benchmarked code itself uses threads and you want to compare it to +single-threaded code, you may want to use real-time ("wallclock") measurements +for latency comparisons: + +```c++ +BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime(); +``` + +Without `UseRealTime`, CPU time is used by default. + + +## Manual timing +For benchmarking something for which neither CPU time nor real-time are +correct or accurate enough, completely manual timing is supported using +the `UseManualTime` function. + +When `UseManualTime` is used, the benchmarked code must call +`SetIterationTime` once per iteration of the `KeepRunning` loop to +report the manually measured time. + +An example use case for this is benchmarking GPU execution (e.g. OpenCL +or CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot +be accurately measured using CPU time or real-time. Instead, they can be +measured accurately using a dedicated API, and these measurement results +can be reported back with `SetIterationTime`. + +```c++ +static void BM_ManualTiming(benchmark::State& state) { + int microseconds = state.range(0); + std::chrono::duration sleep_duration { + static_cast(microseconds) + }; + + while (state.KeepRunning()) { + auto start = std::chrono::high_resolution_clock::now(); + // Simulate some useful workload with a sleep + std::this_thread::sleep_for(sleep_duration); + auto end = std::chrono::high_resolution_clock::now(); + + auto elapsed_seconds = + std::chrono::duration_cast>( + end - start); + + state.SetIterationTime(elapsed_seconds.count()); + } +} +BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime(); +``` + +### Preventing optimisation +To prevent a value or expression from being optimized away by the compiler +the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()` +functions can be used. + +```c++ +static void BM_test(benchmark::State& state) { + while (state.KeepRunning()) { + int x = 0; + for (int i=0; i < 64; ++i) { + benchmark::DoNotOptimize(x += i); + } + } +} +``` + +`DoNotOptimize()` forces the *result* of `` to be stored in either +memory or a register. For GNU based compilers it acts as read/write barrier +for global memory. More specifically it forces the compiler to flush pending +writes to memory and reload any other values as necessary. + +Note that `DoNotOptimize()` does not prevent optimizations on `` +in any way. `` may even be removed entirely when the result is already +known. For example: + +```c++ + /* Example 1: `` is removed entirely. */ + int foo(int x) { return x + 42; } + while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42); + + /* Example 2: Result of '' is only reused */ + int bar(int) __attribute__((const)); + while (...) DoNotOptimize(bar(0)); // Optimized to: + // int __result__ = bar(0); + // while (...) DoNotOptimize(__result__); +``` + +The second tool for preventing optimizations is `ClobberMemory()`. In essence +`ClobberMemory()` forces the compiler to perform all pending writes to global +memory. Memory managed by block scope objects must be "escaped" using +`DoNotOptimize(...)` before it can be clobbered. In the below example +`ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized +away. + +```c++ +static void BM_vector_push_back(benchmark::State& state) { + while (state.KeepRunning()) { + std::vector v; + v.reserve(1); + benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered. + v.push_back(42); + benchmark::ClobberMemory(); // Force 42 to be written to memory. + } +} +``` + +Note that `ClobberMemory()` is only available for GNU or MSVC based compilers. + +### Set time unit manually +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: + +```c++ +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); +``` + +## Controlling number of iterations +In all cases, the number of iterations for which the benchmark is run is +governed by the amount of time the benchmark takes. Concretely, the number of +iterations is at least one, not more than 1e9, until CPU time is greater than +the minimum time, or the wallclock time is 5x minimum time. The minimum time is +set as a flag `--benchmark_min_time` or per benchmark by calling `MinTime` on +the registered benchmark object. + +## Reporting the mean and standard devation by repeated benchmarks +By default each benchmark is run once and that single result is reported. +However benchmarks are often noisy and a single result may not be representative +of the overall behavior. For this reason it's possible to repeatedly rerun the +benchmark. + +The number of runs of each benchmark is specified globally by the +`--benchmark_repetitions` flag or on a per benchmark basis by calling +`Repetitions` on the registered benchmark object. When a benchmark is run +more than once the mean and standard deviation of the runs will be reported. + +Additionally the `--benchmark_report_aggregates_only={true|false}` flag or +`ReportAggregatesOnly(bool)` function can be used to change how repeated tests +are reported. By default the result of each repeated run is reported. When this +option is 'true' only the mean and standard deviation of the runs is reported. +Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides +the value of the flag for that benchmark. + +## Fixtures +Fixture tests are created by +first defining a type that derives from ::benchmark::Fixture and then +creating/registering the tests using the following macros: + +* `BENCHMARK_F(ClassName, Method)` +* `BENCHMARK_DEFINE_F(ClassName, Method)` +* `BENCHMARK_REGISTER_F(ClassName, Method)` + +For Example: + +```c++ +class MyFixture : public benchmark::Fixture {}; + +BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) { + while (st.KeepRunning()) { + ... + } +} + +BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) { + while (st.KeepRunning()) { + ... + } +} +/* BarTest is NOT registered */ +BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2); +/* BarTest is now registered */ +``` + + +## User-defined counters + +You can add your own counters with user-defined names. The example below +will add columns "Foo", "Bar" and "Baz" in its output: + +```c++ +static void UserCountersExample1(benchmark::State& state) { + double numFoos = 0, numBars = 0, numBazs = 0; + while (state.KeepRunning()) { + // ... count Foo,Bar,Baz events + } + state.counters["Foo"] = numFoos; + state.counters["Bar"] = numBars; + state.counters["Baz"] = numBazs; +} +``` + +The `state.counters` object is a `std::map` with `std::string` keys +and `Counter` values. The latter is a `double`-like class, via an implicit +conversion to `double&`. Thus you can use all of the standard arithmetic +assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter. + +In multithreaded benchmarks, each counter is set on the calling thread only. +When the benchmark finishes, the counters from each thread will be summed; +the resulting sum is the value which will be shown for the benchmark. + +The `Counter` constructor accepts two parameters: the value as a `double` +and a bit flag which allows you to show counters as rates and/or as +per-thread averages: + +```c++ + // sets a simple counter + state.counters["Foo"] = numFoos; + + // Set the counter as a rate. It will be presented divided + // by the duration of the benchmark. + state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate); + + // Set the counter as a thread-average quantity. It will + // be presented divided by the number of threads. + state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads); + + // There's also a combined flag: + state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate); +``` + +When you're compiling in C++11 mode or later you can use `insert()` with +`std::initializer_list`: + +```c++ + // With C++11, this can be done: + state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}}); + // ... instead of: + state.counters["Foo"] = numFoos; + state.counters["Bar"] = numBars; + state.counters["Baz"] = numBazs; +``` + +### Counter reporting + +When using the console reporter, by default, user counters are are printed at +the end after the table, the same way as ``bytes_processed`` and +``items_processed``. This is best for cases in which there are few counters, +or where there are only a couple of lines per benchmark. Here's an example of +the default output: + +``` +------------------------------------------------------------------------------ +Benchmark Time CPU Iterations UserCounters... +------------------------------------------------------------------------------ +BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8 +BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m +BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2 +BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4 +BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8 +BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16 +BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32 +BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4 +BM_Factorial 26 ns 26 ns 26608979 40320 +BM_Factorial/real_time 26 ns 26 ns 26587936 40320 +BM_CalculatePiRange/1 16 ns 16 ns 45704255 0 +BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374 +BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746 +BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355 +``` + +If this doesn't suit you, you can print each counter as a table column by +passing the flag `--benchmark_counters_tabular=true` to the benchmark +application. This is best for cases in which there are a lot of counters, or +a lot of lines per individual benchmark. Note that this will trigger a +reprinting of the table header any time the counter set changes between +individual benchmarks. Here's an example of corresponding output when +`--benchmark_counters_tabular=true` is passed: + +``` +--------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations Bar Bat Baz Foo +--------------------------------------------------------------------------------------- +BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8 +BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1 +BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2 +BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4 +BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8 +BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16 +BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32 +BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4 +-------------------------------------------------------------- +Benchmark Time CPU Iterations +-------------------------------------------------------------- +BM_Factorial 26 ns 26 ns 26392245 40320 +BM_Factorial/real_time 26 ns 26 ns 26494107 40320 +BM_CalculatePiRange/1 15 ns 15 ns 45571597 0 +BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374 +BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746 +BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355 +BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184 +BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162 +BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416 +BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159 +BM_CalculatePi/threads:8 2255 ns 9943 ns 70936 +``` +Note above the additional header printed when the benchmark changes from +``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does +not have the same counter set as ``BM_UserCounter``. + +## Exiting Benchmarks in Error + +When errors caused by external influences, such as file I/O and network +communication, occur within a benchmark the +`State::SkipWithError(const char* msg)` function can be used to skip that run +of benchmark and report the error. Note that only future iterations of the +`KeepRunning()` are skipped. Users may explicitly return to exit the +benchmark immediately. + +The `SkipWithError(...)` function may be used at any point within the benchmark, +including before and after the `KeepRunning()` loop. + +For example: + +```c++ +static void BM_test(benchmark::State& state) { + auto resource = GetResource(); + if (!resource.good()) { + state.SkipWithError("Resource is not good!"); + // KeepRunning() loop will not be entered. + } + while (state.KeepRunning()) { + auto data = resource.read_data(); + if (!resource.good()) { + state.SkipWithError("Failed to read data!"); + break; // Needed to skip the rest of the iteration. + } + do_stuff(data); + } +} +``` + +## Running a subset of the benchmarks + +The `--benchmark_filter=` option can be used to only run the benchmarks +which match the specified ``. For example: + +```bash +$ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32 +Run on (1 X 2300 MHz CPU ) +2016-06-25 19:34:24 +Benchmark Time CPU Iterations +---------------------------------------------------- +BM_memcpy/32 11 ns 11 ns 79545455 +BM_memcpy/32k 2181 ns 2185 ns 324074 +BM_memcpy/32 12 ns 12 ns 54687500 +BM_memcpy/32k 1834 ns 1837 ns 357143 +``` + + +## Output Formats +The library supports multiple output formats. Use the +`--benchmark_format=` flag to set the format type. `console` +is the default format. + +The Console format is intended to be a human readable format. By default +the format generates color output. Context is output on stderr and the +tabular data on stdout. Example tabular output looks like: +``` +Benchmark Time(ns) CPU(ns) Iterations +---------------------------------------------------------------------- +BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s +BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s +BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s +``` + +The JSON format outputs human readable json split into two top level attributes. +The `context` attribute contains information about the run in general, including +information about the CPU and the date. +The `benchmarks` attribute contains a list of ever benchmark run. Example json +output looks like: +```json +{ + "context": { + "date": "2015/03/17-18:40:25", + "num_cpus": 40, + "mhz_per_cpu": 2801, + "cpu_scaling_enabled": false, + "build_type": "debug" + }, + "benchmarks": [ + { + "name": "BM_SetInsert/1024/1", + "iterations": 94877, + "real_time": 29275, + "cpu_time": 29836, + "bytes_per_second": 134066, + "items_per_second": 33516 + }, + { + "name": "BM_SetInsert/1024/8", + "iterations": 21609, + "real_time": 32317, + "cpu_time": 32429, + "bytes_per_second": 986770, + "items_per_second": 246693 + }, + { + "name": "BM_SetInsert/1024/10", + "iterations": 21393, + "real_time": 32724, + "cpu_time": 33355, + "bytes_per_second": 1199226, + "items_per_second": 299807 + } + ] +} +``` + +The CSV format outputs comma-separated values. The `context` is output on stderr +and the CSV itself on stdout. Example CSV output looks like: +``` +name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label +"BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942, +"BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115, +"BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06, +``` + +## Output Files +The library supports writing the output of the benchmark to a file specified +by `--benchmark_out=`. The format of the output can be specified +using `--benchmark_out_format={json|console|csv}`. Specifying +`--benchmark_out` does not suppress the console output. + +## Debug vs Release +By default, benchmark builds as a debug library. You will see a warning in the output when this is the case. To build it as a release library instead, use: + +``` +cmake -DCMAKE_BUILD_TYPE=Release +``` + +To enable link-time optimisation, use + +``` +cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true +``` + +## Linking against the library +When using gcc, it is necessary to link against pthread to avoid runtime exceptions. +This is due to how gcc implements std::thread. +See [issue #67](https://github.com/google/benchmark/issues/67) for more details. + +## Compiler Support + +Google Benchmark uses C++11 when building the library. As such we require +a modern C++ toolchain, both compiler and standard library. + +The following minimum versions are strongly recommended build the library: + +* GCC 4.8 +* Clang 3.4 +* Visual Studio 2013 +* Intel 2015 Update 1 + +Anything older *may* work. + +Note: Using the library and its headers in C++03 is supported. C++11 is only +required to build the library. + +# Known Issues + +### Windows + +* Users must manually link `shlwapi.lib`. Failure to do so may result +in unresolved symbols. + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/appveyor.yml b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/appveyor.yml new file mode 100644 index 00000000000..e084f386b77 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/appveyor.yml @@ -0,0 +1,56 @@ +version: '{build}' + +image: Visual Studio 2017 + +configuration: + - Debug + - Release + +environment: + matrix: + - compiler: msvc-15-seh + generator: "Visual Studio 15 2017" + + - compiler: msvc-15-seh + generator: "Visual Studio 15 2017 Win64" + + - compiler: msvc-14-seh + generator: "Visual Studio 14 2015" + + - compiler: msvc-14-seh + generator: "Visual Studio 14 2015 Win64" + + - compiler: msvc-12-seh + generator: "Visual Studio 12 2013" + + - compiler: msvc-12-seh + generator: "Visual Studio 12 2013 Win64" + + - compiler: gcc-5.3.0-posix + generator: "MinGW Makefiles" + cxx_path: 'C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin' + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 + +matrix: + fast_finish: true + +install: + # git bash conflicts with MinGW makefiles + - if "%generator%"=="MinGW Makefiles" (set "PATH=%PATH:C:\Program Files\Git\usr\bin;=%") + - if not "%cxx_path%"=="" (set "PATH=%PATH%;%cxx_path%") + +build_script: + - md _build -Force + - cd _build + - echo %configuration% + - cmake -G "%generator%" "-DCMAKE_BUILD_TYPE=%configuration%" .. + - cmake --build . --config %configuration% + +test_script: + - ctest -c %configuration% --timeout 300 --output-on-failure + +artifacts: + - path: '_build/CMakeFiles/*.log' + name: logs + - path: '_build/Testing/**/*.xml' + name: test_results diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/AddCXXCompilerFlag.cmake b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/AddCXXCompilerFlag.cmake new file mode 100644 index 00000000000..0b176ba27f1 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/AddCXXCompilerFlag.cmake @@ -0,0 +1,64 @@ +# - Adds a compiler flag if it is supported by the compiler +# +# This function checks that the supplied compiler flag is supported and then +# adds it to the corresponding compiler flags +# +# add_cxx_compiler_flag( []) +# +# - Example +# +# include(AddCXXCompilerFlag) +# add_cxx_compiler_flag(-Wall) +# add_cxx_compiler_flag(-no-strict-aliasing RELEASE) +# Requires CMake 2.6+ + +if(__add_cxx_compiler_flag) + return() +endif() +set(__add_cxx_compiler_flag INCLUDED) + +include(CheckCXXCompilerFlag) + +function(mangle_compiler_flag FLAG OUTPUT) + string(TOUPPER "HAVE_CXX_FLAG_${FLAG}" SANITIZED_FLAG) + string(REPLACE "+" "X" SANITIZED_FLAG ${SANITIZED_FLAG}) + string(REGEX REPLACE "[^A-Za-z_0-9]" "_" SANITIZED_FLAG ${SANITIZED_FLAG}) + string(REGEX REPLACE "_+" "_" SANITIZED_FLAG ${SANITIZED_FLAG}) + set(${OUTPUT} "${SANITIZED_FLAG}" PARENT_SCOPE) +endfunction(mangle_compiler_flag) + +function(add_cxx_compiler_flag FLAG) + mangle_compiler_flag("${FLAG}" MANGLED_FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") + if(${MANGLED_FLAG}) + set(VARIANT ${ARGV1}) + if(ARGV1) + string(TOUPPER "_${VARIANT}" VARIANT) + endif() + set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE) + endif() +endfunction() + +function(add_required_cxx_compiler_flag FLAG) + mangle_compiler_flag("${FLAG}" MANGLED_FLAG) + set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}") + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}") + check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG}) + set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}") + if(${MANGLED_FLAG}) + set(VARIANT ${ARGV1}) + if(ARGV1) + string(TOUPPER "_${VARIANT}" VARIANT) + endif() + set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE) + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}" PARENT_SCOPE) + else() + message(FATAL_ERROR "Required flag '${FLAG}' is not supported by the compiler") + endif() +endfunction() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/CXXFeatureCheck.cmake b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/CXXFeatureCheck.cmake new file mode 100644 index 00000000000..2c4460f0e30 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/CXXFeatureCheck.cmake @@ -0,0 +1,46 @@ +# - Compile and run code to check for C++ features +# +# This functions compiles a source file under the `cmake` folder +# and adds the corresponding `HAVE_[FILENAME]` flag to the CMake +# environment +# +# cxx_feature_check( []) +# +# - Example +# +# include(CXXFeatureCheck) +# cxx_feature_check(STD_REGEX) +# Requires CMake 2.8.12+ + +if(__cxx_feature_check) + return() +endif() +set(__cxx_feature_check INCLUDED) + +function(cxx_feature_check FILE) + string(TOLOWER ${FILE} FILE) + string(TOUPPER ${FILE} VAR) + string(TOUPPER "HAVE_${VAR}" FEATURE) + if (DEFINED HAVE_${VAR}) + set(HAVE_${VAR} 1 CACHE INTERNAL "Feature test for ${FILE}" PARENT_SCOPE) + add_definitions(-DHAVE_${VAR}) + return() + endif() + message("-- Performing Test ${FEATURE}") + try_run(RUN_${FEATURE} COMPILE_${FEATURE} + ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp + CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS} + LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES}) + if(RUN_${FEATURE} EQUAL 0) + message("-- Performing Test ${FEATURE} -- success") + set(HAVE_${VAR} 1 CACHE INTERNAL "Feature test for ${FILE}" PARENT_SCOPE) + add_definitions(-DHAVE_${VAR}) + else() + if(NOT COMPILE_${FEATURE}) + message("-- Performing Test ${FEATURE} -- failed to compile") + else() + message("-- Performing Test ${FEATURE} -- compiled but failed to run") + endif() + endif() +endfunction() + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/Config.cmake.in b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/Config.cmake.in new file mode 100644 index 00000000000..6e9256eea8a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/Config.cmake.in @@ -0,0 +1 @@ +include("${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake") diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/GetGitVersion.cmake b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/GetGitVersion.cmake new file mode 100644 index 00000000000..8dd94800459 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/GetGitVersion.cmake @@ -0,0 +1,51 @@ +# - Returns a version string from Git tags +# +# This function inspects the annotated git tags for the project and returns a string +# into a CMake variable +# +# get_git_version() +# +# - Example +# +# include(GetGitVersion) +# get_git_version(GIT_VERSION) +# +# Requires CMake 2.8.11+ +find_package(Git) + +if(__get_git_version) + return() +endif() +set(__get_git_version INCLUDED) + +function(get_git_version var) + if(GIT_EXECUTABLE) + execute_process(COMMAND ${GIT_EXECUTABLE} describe --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8 + RESULT_VARIABLE status + OUTPUT_VARIABLE GIT_VERSION + ERROR_QUIET) + if(${status}) + set(GIT_VERSION "v0.0.0") + else() + string(STRIP ${GIT_VERSION} GIT_VERSION) + string(REGEX REPLACE "-[0-9]+-g" "-" GIT_VERSION ${GIT_VERSION}) + endif() + + # Work out if the repository is dirty + execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh + OUTPUT_QUIET + ERROR_QUIET) + execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD -- + OUTPUT_VARIABLE GIT_DIFF_INDEX + ERROR_QUIET) + string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY) + if (${GIT_DIRTY}) + set(GIT_VERSION "${GIT_VERSION}-dirty") + endif() + else() + set(GIT_VERSION "v0.0.0") + endif() + + message("-- git Version: ${GIT_VERSION}") + set(${var} ${GIT_VERSION} PARENT_SCOPE) +endfunction() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/gnu_posix_regex.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/gnu_posix_regex.cpp new file mode 100644 index 00000000000..b5b91cdab7c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/gnu_posix_regex.cpp @@ -0,0 +1,12 @@ +#include +#include +int main() { + std::string str = "test0159"; + regex_t re; + int ec = regcomp(&re, "^[a-z]+[0-9]+$", REG_EXTENDED | REG_NOSUB); + if (ec != 0) { + return ec; + } + return regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0; +} + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/posix_regex.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/posix_regex.cpp new file mode 100644 index 00000000000..466dc62560a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/posix_regex.cpp @@ -0,0 +1,14 @@ +#include +#include +int main() { + std::string str = "test0159"; + regex_t re; + int ec = regcomp(&re, "^[a-z]+[0-9]+$", REG_EXTENDED | REG_NOSUB); + if (ec != 0) { + return ec; + } + int ret = regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0; + regfree(&re); + return ret; +} + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/std_regex.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/std_regex.cpp new file mode 100644 index 00000000000..696f2a26bce --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/std_regex.cpp @@ -0,0 +1,10 @@ +#include +#include +int main() { + const std::string str = "test0159"; + std::regex re; + re = std::regex("^[a-z]+[0-9]+$", + std::regex_constants::extended | std::regex_constants::nosubs); + return std::regex_search(str, re) ? 0 : -1; +} + diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/steady_clock.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/steady_clock.cpp new file mode 100644 index 00000000000..66d50d17e9e --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/steady_clock.cpp @@ -0,0 +1,7 @@ +#include + +int main() { + typedef std::chrono::steady_clock Clock; + Clock::time_point tp = Clock::now(); + ((void)tp); +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/thread_safety_attributes.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/thread_safety_attributes.cpp new file mode 100644 index 00000000000..46161babdb1 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/cmake/thread_safety_attributes.cpp @@ -0,0 +1,4 @@ +#define HAVE_THREAD_SAFETY_ATTRIBUTES +#include "../src/mutex.h" + +int main() {} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/docs/tools.md b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/docs/tools.md new file mode 100644 index 00000000000..f176f74a48f --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/docs/tools.md @@ -0,0 +1,59 @@ +# Benchmark Tools + +## compare_bench.py + +The `compare_bench.py` utility which can be used to compare the result of benchmarks. +The program is invoked like: + +``` bash +$ compare_bench.py [benchmark options]... +``` + +Where `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file. + +The sample output using the JSON test files under `Inputs/` gives: + +``` bash +$ ./compare_bench.py ./gbench/Inputs/test1_run1.json ./gbench/Inputs/test1_run2.json +Comparing ./gbench/Inputs/test1_run1.json to ./gbench/Inputs/test1_run2.json +Benchmark Time CPU +---------------------------------------------- +BM_SameTimes +0.00 +0.00 +BM_2xFaster -0.50 -0.50 +BM_2xSlower +1.00 +1.00 +BM_10PercentFaster -0.10 -0.10 +BM_10PercentSlower +0.10 +0.10 +``` + +When a benchmark executable is run, the raw output from the benchmark is printed in real time to stdout. The sample output using `benchmark/basic_test` for both arguments looks like: + +``` +./compare_bench.py test/basic_test test/basic_test --benchmark_filter=BM_empty.* +RUNNING: test/basic_test --benchmark_filter=BM_empty.* +Run on (4 X 4228.32 MHz CPU s) +2016-08-02 19:21:33 +Benchmark Time CPU Iterations +-------------------------------------------------------------------- +BM_empty 9 ns 9 ns 79545455 +BM_empty/threads:4 4 ns 9 ns 75268816 +BM_empty_stop_start 8 ns 8 ns 83333333 +BM_empty_stop_start/threads:4 3 ns 8 ns 83333332 +RUNNING: test/basic_test --benchmark_filter=BM_empty.* +Run on (4 X 4228.32 MHz CPU s) +2016-08-02 19:21:35 +Benchmark Time CPU Iterations +-------------------------------------------------------------------- +BM_empty 9 ns 9 ns 76086957 +BM_empty/threads:4 4 ns 9 ns 76086956 +BM_empty_stop_start 8 ns 8 ns 87500000 +BM_empty_stop_start/threads:4 3 ns 8 ns 88607596 +Comparing test/basic_test to test/basic_test +Benchmark Time CPU +--------------------------------------------------------- +BM_empty +0.00 +0.00 +BM_empty/threads:4 +0.00 +0.00 +BM_empty_stop_start +0.00 +0.00 +BM_empty_stop_start/threads:4 +0.00 +0.00 +``` + +Obviously this example doesn't give any useful output, but it's intended to show the output format when 'compare_bench.py' needs to run benchmarks. diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/benchmark.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/benchmark.h new file mode 100644 index 00000000000..bd3b0ffb4cb --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/benchmark.h @@ -0,0 +1,1210 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Support for registering benchmarks for functions. + +/* Example usage: +// Define a function that executes the code to be measured a +// specified number of times: +static void BM_StringCreation(benchmark::State& state) { + while (state.KeepRunning()) + std::string empty_string; +} + +// Register the function as a benchmark +BENCHMARK(BM_StringCreation); + +// Define another benchmark +static void BM_StringCopy(benchmark::State& state) { + std::string x = "hello"; + while (state.KeepRunning()) + std::string copy(x); +} +BENCHMARK(BM_StringCopy); + +// Augment the main() program to invoke benchmarks if specified +// via the --benchmarks command line flag. E.g., +// my_unittest --benchmark_filter=all +// my_unittest --benchmark_filter=BM_StringCreation +// my_unittest --benchmark_filter=String +// my_unittest --benchmark_filter='Copy|Creation' +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); + return 0; +} + +// Sometimes a family of microbenchmarks can be implemented with +// just one routine that takes an extra argument to specify which +// one of the family of benchmarks to run. For example, the following +// code defines a family of microbenchmarks for measuring the speed +// of memcpy() calls of different lengths: + +static void BM_memcpy(benchmark::State& state) { + char* src = new char[state.range(0)]; char* dst = new char[state.range(0)]; + memset(src, 'x', state.range(0)); + while (state.KeepRunning()) + memcpy(dst, src, state.range(0)); + state.SetBytesProcessed(int64_t(state.iterations()) * + int64_t(state.range(0))); + delete[] src; delete[] dst; +} +BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10); + +// The preceding code is quite repetitive, and can be replaced with the +// following short-hand. The following invocation will pick a few +// appropriate arguments in the specified range and will generate a +// microbenchmark for each such argument. +BENCHMARK(BM_memcpy)->Range(8, 8<<10); + +// You might have a microbenchmark that depends on two inputs. For +// example, the following code defines a family of microbenchmarks for +// measuring the speed of set insertion. +static void BM_SetInsert(benchmark::State& state) { + while (state.KeepRunning()) { + state.PauseTiming(); + set data = ConstructRandomSet(state.range(0)); + state.ResumeTiming(); + for (int j = 0; j < state.range(1); ++j) + data.insert(RandomNumber()); + } +} +BENCHMARK(BM_SetInsert) + ->Args({1<<10, 1}) + ->Args({1<<10, 8}) + ->Args({1<<10, 64}) + ->Args({1<<10, 512}) + ->Args({8<<10, 1}) + ->Args({8<<10, 8}) + ->Args({8<<10, 64}) + ->Args({8<<10, 512}); + +// The preceding code is quite repetitive, and can be replaced with +// the following short-hand. The following macro will pick a few +// appropriate arguments in the product of the two specified ranges +// and will generate a microbenchmark for each such pair. +BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {1, 512}}); + +// For more complex patterns of inputs, passing a custom function +// to Apply allows programmatic specification of an +// arbitrary set of arguments to run the microbenchmark on. +// The following example enumerates a dense range on +// one parameter, and a sparse range on the second. +static void CustomArguments(benchmark::internal::Benchmark* b) { + for (int i = 0; i <= 10; ++i) + for (int j = 32; j <= 1024*1024; j *= 8) + b->Args({i, j}); +} +BENCHMARK(BM_SetInsert)->Apply(CustomArguments); + +// Templated microbenchmarks work the same way: +// Produce then consume 'size' messages 'iters' times +// Measures throughput in the absence of multiprogramming. +template int BM_Sequential(benchmark::State& state) { + Q q; + typename Q::value_type v; + while (state.KeepRunning()) { + for (int i = state.range(0); i--; ) + q.push(v); + for (int e = state.range(0); e--; ) + q.Wait(&v); + } + // actually messages, not bytes: + state.SetBytesProcessed( + static_cast(state.iterations())*state.range(0)); +} +BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10); + +Use `Benchmark::MinTime(double t)` to set the minimum time used to run the +benchmark. This option overrides the `benchmark_min_time` flag. + +void BM_test(benchmark::State& state) { + ... body ... +} +BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds. + +In a multithreaded test, it is guaranteed that none of the threads will start +until all have called KeepRunning, and all will have finished before KeepRunning +returns false. As such, any global setup or teardown you want to do can be +wrapped in a check against the thread index: + +static void BM_MultiThreaded(benchmark::State& state) { + if (state.thread_index == 0) { + // Setup code here. + } + while (state.KeepRunning()) { + // Run the test as normal. + } + if (state.thread_index == 0) { + // Teardown code here. + } +} +BENCHMARK(BM_MultiThreaded)->Threads(4); + + +If a benchmark runs a few milliseconds it may be hard to visually compare the +measured times, since the output data is given in nanoseconds per default. In +order to manually set the time unit, you can specify it manually: + +BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); +*/ + +#ifndef BENCHMARK_BENCHMARK_H_ +#define BENCHMARK_BENCHMARK_H_ + + +#if __cplusplus >= 201103L +#define BENCHMARK_HAS_CXX11 +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include + +#if defined(BENCHMARK_HAS_CXX11) +#include +#include +#include +#endif + +#if defined(_MSC_VER) +#include // for _ReadWriteBarrier +#endif + +#ifndef BENCHMARK_HAS_CXX11 +#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + TypeName& operator=(const TypeName&) +#else +#define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&) = delete; \ + TypeName& operator=(const TypeName&) = delete +#endif + +#if defined(__GNUC__) +#define BENCHMARK_UNUSED __attribute__((unused)) +#define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline)) +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#elif defined(_MSC_VER) && !defined(__clang__) +#define BENCHMARK_UNUSED +#define BENCHMARK_ALWAYS_INLINE __forceinline +#if _MSC_VER >= 1900 +#define BENCHMARK_NOEXCEPT noexcept +#define BENCHMARK_NOEXCEPT_OP(x) noexcept(x) +#else +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif +#define __func__ __FUNCTION__ +#else +#define BENCHMARK_UNUSED +#define BENCHMARK_ALWAYS_INLINE +#define BENCHMARK_NOEXCEPT +#define BENCHMARK_NOEXCEPT_OP(x) +#endif + +#define BENCHMARK_INTERNAL_TOSTRING2(x) #x +#define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x) + +#if defined(__GNUC__) +#define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y) +#define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg))) +#else +#define BENCHMARK_BUILTIN_EXPECT(x, y) x +#define BENCHMARK_DEPRECATED_MSG(msg) +#define BENCHMARK_WARNING_MSG(msg) __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING(__LINE__) ") : warning note: " msg)) +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif + + +namespace benchmark { +class BenchmarkReporter; + +void Initialize(int* argc, char** argv); + +// Report to stdout all arguments in 'argv' as unrecognized except the first. +// Returns true there is at least on unrecognized argument (i.e. 'argc' > 1). +bool ReportUnrecognizedArguments(int argc, char** argv); + +// Generate a list of benchmarks matching the specified --benchmark_filter flag +// and if --benchmark_list_tests is specified return after printing the name +// of each matching benchmark. Otherwise run each matching benchmark and +// report the results. +// +// The second and third overload use the specified 'console_reporter' and +// 'file_reporter' respectively. 'file_reporter' will write to the file +// specified +// by '--benchmark_output'. If '--benchmark_output' is not given the +// 'file_reporter' is ignored. +// +// RETURNS: The number of matching benchmarks. +size_t RunSpecifiedBenchmarks(); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter); +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter); + +// If this routine is called, peak memory allocation past this point in the +// benchmark is reported at the end of the benchmark report line. (It is +// computed by running the benchmark once with a single iteration and a memory +// tracer.) +// TODO(dominic) +// void MemoryUsage(); + +namespace internal { +class Benchmark; +class BenchmarkImp; +class BenchmarkFamilies; + +void UseCharPointer(char const volatile*); + +// Take ownership of the pointer and register the benchmark. Return the +// registered benchmark. +Benchmark* RegisterBenchmarkInternal(Benchmark*); + +// Ensure that the standard streams are properly initialized in every TU. +int InitializeStreams(); +BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams(); + +} // namespace internal + + +#if !defined(__GNUC__) || defined(__pnacl__) || defined(EMSCRIPTN) +# define BENCHMARK_HAS_NO_INLINE_ASSEMBLY +#endif + +// The DoNotOptimize(...) function can be used to prevent a value or +// expression from being optimized away by the compiler. This function is +// intended to add little to no overhead. +// See: https://youtu.be/nXaxk27zwlk?t=2441 +#ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + // Clang doesn't like the 'X' constraint on `value` and certain GCC versions + // don't like the 'g' constraint. Attempt to placate them both. +#if defined(__clang__) + asm volatile("" : : "g"(value) : "memory"); +#else + asm volatile("" : : "i,r,m"(value) : "memory"); +#endif +} +// Force the compiler to flush pending writes to global memory. Acts as an +// effective read/write barrier +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { + asm volatile("" : : : "memory"); +} +#elif defined(_MSC_VER) +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast(value)); + _ReadWriteBarrier(); +} + +inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { + _ReadWriteBarrier(); +} +#else +template +inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast(value)); +} +// FIXME Add ClobberMemory() for non-gnu and non-msvc compilers +#endif + + + +// This class is used for user-defined counters. +class Counter { +public: + + enum Flags { + kDefaults = 0, + // Mark the counter as a rate. It will be presented divided + // by the duration of the benchmark. + kIsRate = 1, + // Mark the counter as a thread-average quantity. It will be + // presented divided by the number of threads. + kAvgThreads = 2, + // Mark the counter as a thread-average rate. See above. + kAvgThreadsRate = kIsRate|kAvgThreads + }; + + double value; + Flags flags; + + BENCHMARK_ALWAYS_INLINE + Counter(double v = 0., Flags f = kDefaults) : value(v), flags(f) {} + + BENCHMARK_ALWAYS_INLINE operator double const& () const { return value; } + BENCHMARK_ALWAYS_INLINE operator double & () { return value; } + +}; + +// This is the container for the user-defined counters. +typedef std::map UserCounters; + + +// TimeUnit is passed to a benchmark in order to specify the order of magnitude +// for the measured time. +enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond }; + +// BigO is passed to a benchmark in order to specify the asymptotic +// computational +// complexity for the benchmark. In case oAuto is selected, complexity will be +// calculated automatically to the best fit. +enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda }; + +// BigOFunc is passed to a benchmark in order to specify the asymptotic +// computational complexity for the benchmark. +typedef double(BigOFunc)(int); + +namespace internal { +class ThreadTimer; +class ThreadManager; + +enum ReportMode +#if defined(BENCHMARK_HAS_CXX11) + : unsigned +#else +#endif + { + RM_Unspecified, // The mode has not been manually specified + RM_Default, // The mode is user-specified as default. + RM_ReportAggregatesOnly +}; +} // namespace internal + +// State is passed to a running Benchmark and contains state for the +// benchmark to use. +class State { + public: + // Returns true if the benchmark should continue through another iteration. + // NOTE: A benchmark may not return from the test until KeepRunning() has + // returned false. + bool KeepRunning() { + if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) { + StartKeepRunning(); + } + bool const res = total_iterations_++ < max_iterations; + if (BENCHMARK_BUILTIN_EXPECT(!res, false)) { + FinishKeepRunning(); + } + return res; + } + + // REQUIRES: timer is running and 'SkipWithError(...)' has not been called + // by the current thread. + // Stop the benchmark timer. If not called, the timer will be + // automatically stopped after KeepRunning() returns false for the first time. + // + // For threaded benchmarks the PauseTiming() function only pauses the timing + // for the current thread. + // + // NOTE: The "real time" measurement is per-thread. If different threads + // report different measurements the largest one is reported. + // + // NOTE: PauseTiming()/ResumeTiming() are relatively + // heavyweight, and so their use should generally be avoided + // within each benchmark iteration, if possible. + void PauseTiming(); + + // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called + // by the current thread. + // Start the benchmark timer. The timer is NOT running on entrance to the + // benchmark function. It begins running after the first call to KeepRunning() + // + // NOTE: PauseTiming()/ResumeTiming() are relatively + // heavyweight, and so their use should generally be avoided + // within each benchmark iteration, if possible. + void ResumeTiming(); + + // REQUIRES: 'SkipWithError(...)' has not been called previously by the + // current thread. + // Skip any future iterations of the 'KeepRunning()' loop in the current + // thread and report an error with the specified 'msg'. After this call + // the user may explicitly 'return' from the benchmark. + // + // For threaded benchmarks only the current thread stops executing and future + // calls to `KeepRunning()` will block until all threads have completed + // the `KeepRunning()` loop. If multiple threads report an error only the + // first error message is used. + // + // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit + // the current scope immediately. If the function is called from within + // the 'KeepRunning()' loop the current iteration will finish. It is the users + // responsibility to exit the scope as needed. + void SkipWithError(const char* msg); + + // REQUIRES: called exactly once per iteration of the KeepRunning loop. + // Set the manually measured time for this benchmark iteration, which + // is used instead of automatically measured time if UseManualTime() was + // specified. + // + // For threaded benchmarks the final value will be set to the largest + // reported values. + void SetIterationTime(double seconds); + + // Set the number of bytes processed by the current benchmark + // execution. This routine is typically called once at the end of a + // throughput oriented benchmark. If this routine is called with a + // value > 0, the report is printed in MB/sec instead of nanoseconds + // per iteration. + // + // REQUIRES: a benchmark has exited its KeepRunning loop. + BENCHMARK_ALWAYS_INLINE + void SetBytesProcessed(size_t bytes) { bytes_processed_ = bytes; } + + BENCHMARK_ALWAYS_INLINE + size_t bytes_processed() const { return bytes_processed_; } + + // If this routine is called with complexity_n > 0 and complexity report is + // requested for the + // family benchmark, then current benchmark will be part of the computation + // and complexity_n will + // represent the length of N. + BENCHMARK_ALWAYS_INLINE + void SetComplexityN(int complexity_n) { complexity_n_ = complexity_n; } + + BENCHMARK_ALWAYS_INLINE + int complexity_length_n() { return complexity_n_; } + + // If this routine is called with items > 0, then an items/s + // label is printed on the benchmark report line for the currently + // executing benchmark. It is typically called at the end of a processing + // benchmark where a processing items/second output is desired. + // + // REQUIRES: a benchmark has exited its KeepRunning loop. + BENCHMARK_ALWAYS_INLINE + void SetItemsProcessed(size_t items) { items_processed_ = items; } + + BENCHMARK_ALWAYS_INLINE + size_t items_processed() const { return items_processed_; } + + // If this routine is called, the specified label is printed at the + // end of the benchmark report line for the currently executing + // benchmark. Example: + // static void BM_Compress(benchmark::State& state) { + // ... + // double compress = input_size / output_size; + // state.SetLabel(StringPrintf("compress:%.1f%%", 100.0*compression)); + // } + // Produces output that looks like: + // BM_Compress 50 50 14115038 compress:27.3% + // + // REQUIRES: a benchmark has exited its KeepRunning loop. + void SetLabel(const char* label); + + void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) { + this->SetLabel(str.c_str()); + } + + // Range arguments for this run. CHECKs if the argument has been set. + BENCHMARK_ALWAYS_INLINE + int range(std::size_t pos = 0) const { + assert(range_.size() > pos); + return range_[pos]; + } + + BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead") + int range_x() const { return range(0); } + + BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead") + int range_y() const { return range(1); } + + BENCHMARK_ALWAYS_INLINE + size_t iterations() const { return total_iterations_; } + + private: + bool started_; + bool finished_; + size_t total_iterations_; + + std::vector range_; + + size_t bytes_processed_; + size_t items_processed_; + + int complexity_n_; + + bool error_occurred_; + + public: + // Container for user-defined counters. + UserCounters counters; + // Index of the executing thread. Values from [0, threads). + const int thread_index; + // Number of threads concurrently executing the benchmark. + const int threads; + const size_t max_iterations; + + // TODO(EricWF) make me private + State(size_t max_iters, const std::vector& ranges, int thread_i, + int n_threads, internal::ThreadTimer* timer, + internal::ThreadManager* manager); + + private: + void StartKeepRunning(); + void FinishKeepRunning(); + internal::ThreadTimer* timer_; + internal::ThreadManager* manager_; + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(State); +}; + +namespace internal { + +typedef void(Function)(State&); + +// ------------------------------------------------------ +// Benchmark registration object. The BENCHMARK() macro expands +// into an internal::Benchmark* object. Various methods can +// be called on this object to change the properties of the benchmark. +// Each method returns "this" so that multiple method calls can +// chained into one expression. +class Benchmark { + public: + virtual ~Benchmark(); + + // Note: the following methods all return "this" so that multiple + // method calls can be chained together in one expression. + + // Run this benchmark once with "x" as the extra argument passed + // to the function. + // REQUIRES: The function passed to the constructor must accept an arg1. + Benchmark* Arg(int x); + + // Run this benchmark with the given time unit for the generated output report + Benchmark* Unit(TimeUnit unit); + + // Run this benchmark once for a number of values picked from the + // range [start..limit]. (start and limit are always picked.) + // REQUIRES: The function passed to the constructor must accept an arg1. + Benchmark* Range(int start, int limit); + + // Run this benchmark once for all values in the range [start..limit] with + // specific step + // REQUIRES: The function passed to the constructor must accept an arg1. + Benchmark* DenseRange(int start, int limit, int step = 1); + + // Run this benchmark once with "args" as the extra arguments passed + // to the function. + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* Args(const std::vector& args); + + // Equivalent to Args({x, y}) + // NOTE: This is a legacy C++03 interface provided for compatibility only. + // New code should use 'Args'. + Benchmark* ArgPair(int x, int y) { + std::vector args; + args.push_back(x); + args.push_back(y); + return Args(args); + } + + // Run this benchmark once for a number of values picked from the + // ranges [start..limit]. (starts and limits are always picked.) + // REQUIRES: The function passed to the constructor must accept arg1, arg2 ... + Benchmark* Ranges(const std::vector >& ranges); + + // Equivalent to ArgNames({name}) + Benchmark* ArgName(const std::string& name); + + // Set the argument names to display in the benchmark name. If not called, + // only argument values will be shown. + Benchmark* ArgNames(const std::vector& names); + + // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}). + // NOTE: This is a legacy C++03 interface provided for compatibility only. + // New code should use 'Ranges'. + Benchmark* RangePair(int lo1, int hi1, int lo2, int hi2) { + std::vector > ranges; + ranges.push_back(std::make_pair(lo1, hi1)); + ranges.push_back(std::make_pair(lo2, hi2)); + return Ranges(ranges); + } + + // Pass this benchmark object to *func, which can customize + // the benchmark by calling various methods like Arg, Args, + // Threads, etc. + Benchmark* Apply(void (*func)(Benchmark* benchmark)); + + // Set the range multiplier for non-dense range. If not called, the range + // multiplier kRangeMultiplier will be used. + Benchmark* RangeMultiplier(int multiplier); + + // Set the minimum amount of time to use when running this benchmark. This + // option overrides the `benchmark_min_time` flag. + // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark. + Benchmark* MinTime(double t); + + // Specify the amount of iterations that should be run by this benchmark. + // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark. + // + // NOTE: This function should only be used when *exact* iteration control is + // needed and never to control or limit how long a benchmark runs, where + // `--benchmark_min_time=N` or `MinTime(...)` should be used instead. + Benchmark* Iterations(size_t n); + + // Specify the amount of times to repeat this benchmark. This option overrides + // the `benchmark_repetitions` flag. + // REQUIRES: `n > 0` + Benchmark* Repetitions(int n); + + // Specify if each repetition of the benchmark should be reported separately + // or if only the final statistics should be reported. If the benchmark + // is not repeated then the single result is always reported. + Benchmark* ReportAggregatesOnly(bool value = true); + + // If a particular benchmark is I/O bound, runs multiple threads internally or + // if for some reason CPU timings are not representative, call this method. If + // called, the elapsed time will be used to control how many iterations are + // run, and in the printing of items/second or MB/seconds values. If not + // called, the cpu time used by the benchmark will be used. + Benchmark* UseRealTime(); + + // If a benchmark must measure time manually (e.g. if GPU execution time is + // being + // measured), call this method. If called, each benchmark iteration should + // call + // SetIterationTime(seconds) to report the measured time, which will be used + // to control how many iterations are run, and in the printing of items/second + // or MB/second values. + Benchmark* UseManualTime(); + + // Set the asymptotic computational complexity for the benchmark. If called + // the asymptotic computational complexity will be shown on the output. + Benchmark* Complexity(BigO complexity = benchmark::oAuto); + + // Set the asymptotic computational complexity for the benchmark. If called + // the asymptotic computational complexity will be shown on the output. + Benchmark* Complexity(BigOFunc* complexity); + + // Support for running multiple copies of the same benchmark concurrently + // in multiple threads. This may be useful when measuring the scaling + // of some piece of code. + + // Run one instance of this benchmark concurrently in t threads. + Benchmark* Threads(int t); + + // Pick a set of values T from [min_threads,max_threads]. + // min_threads and max_threads are always included in T. Run this + // benchmark once for each value in T. The benchmark run for a + // particular value t consists of t threads running the benchmark + // function concurrently. For example, consider: + // BENCHMARK(Foo)->ThreadRange(1,16); + // This will run the following benchmarks: + // Foo in 1 thread + // Foo in 2 threads + // Foo in 4 threads + // Foo in 8 threads + // Foo in 16 threads + Benchmark* ThreadRange(int min_threads, int max_threads); + + // For each value n in the range, run this benchmark once using n threads. + // min_threads and max_threads are always included in the range. + // stride specifies the increment. E.g. DenseThreadRange(1, 8, 3) starts + // a benchmark with 1, 4, 7 and 8 threads. + Benchmark* DenseThreadRange(int min_threads, int max_threads, int stride = 1); + + // Equivalent to ThreadRange(NumCPUs(), NumCPUs()) + Benchmark* ThreadPerCpu(); + + virtual void Run(State& state) = 0; + + // Used inside the benchmark implementation + struct Instance; + + protected: + explicit Benchmark(const char* name); + Benchmark(Benchmark const&); + void SetName(const char* name); + + int ArgsCnt() const; + + static void AddRange(std::vector* dst, int lo, int hi, int mult); + + private: + friend class BenchmarkFamilies; + + std::string name_; + ReportMode report_mode_; + std::vector arg_names_; // Args for all benchmark runs + std::vector > args_; // Args for all benchmark runs + TimeUnit time_unit_; + int range_multiplier_; + double min_time_; + size_t iterations_; + int repetitions_; + bool use_real_time_; + bool use_manual_time_; + BigO complexity_; + BigOFunc* complexity_lambda_; + std::vector thread_counts_; + + Benchmark& operator=(Benchmark const&); +}; + +} // namespace internal + +// Create and register a benchmark with the specified 'name' that invokes +// the specified functor 'fn'. +// +// RETURNS: A pointer to the registered benchmark. +internal::Benchmark* RegisterBenchmark(const char* name, + internal::Function* fn); + +#if defined(BENCHMARK_HAS_CXX11) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn); +#endif + +// Remove all registered benchmarks. All pointers to previously registered +// benchmarks are invalidated. +void ClearRegisteredBenchmarks(); + +namespace internal { +// The class used to hold all Benchmarks created from static function. +// (ie those created using the BENCHMARK(...) macros. +class FunctionBenchmark : public Benchmark { + public: + FunctionBenchmark(const char* name, Function* func) + : Benchmark(name), func_(func) {} + + virtual void Run(State& st); + + private: + Function* func_; +}; + +#ifdef BENCHMARK_HAS_CXX11 +template +class LambdaBenchmark : public Benchmark { + public: + virtual void Run(State& st) { lambda_(st); } + + private: + template + LambdaBenchmark(const char* name, OLambda&& lam) + : Benchmark(name), lambda_(std::forward(lam)) {} + + LambdaBenchmark(LambdaBenchmark const&) = delete; + + private: + template + friend Benchmark* ::benchmark::RegisterBenchmark(const char*, Lam&&); + + Lambda lambda_; +}; +#endif + +} // namespace internal + +inline internal::Benchmark* RegisterBenchmark(const char* name, + internal::Function* fn) { + return internal::RegisterBenchmarkInternal( + ::new internal::FunctionBenchmark(name, fn)); +} + +#ifdef BENCHMARK_HAS_CXX11 +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn) { + using BenchType = + internal::LambdaBenchmark::type>; + return internal::RegisterBenchmarkInternal( + ::new BenchType(name, std::forward(fn))); +} +#endif + +#if defined(BENCHMARK_HAS_CXX11) && \ + (!defined(BENCHMARK_GCC_VERSION) || BENCHMARK_GCC_VERSION >= 409) +template +internal::Benchmark* RegisterBenchmark(const char* name, Lambda&& fn, + Args&&... args) { + return benchmark::RegisterBenchmark( + name, [=](benchmark::State& st) { fn(st, args...); }); +} +#else +#define BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK +#endif + +// The base class for all fixture tests. +class Fixture : public internal::Benchmark { + public: + Fixture() : internal::Benchmark("") {} + + virtual void Run(State& st) { + this->SetUp(st); + this->BenchmarkCase(st); + this->TearDown(st); + } + + // These will be deprecated ... + virtual void SetUp(const State&) {} + virtual void TearDown(const State&) {} + // ... In favor of these. + virtual void SetUp(State& st) { SetUp(const_cast(st)); } + virtual void TearDown(State& st) { TearDown(const_cast(st)); } + + protected: + virtual void BenchmarkCase(State&) = 0; +}; + +} // namespace benchmark + +// ------------------------------------------------------ +// Macro to register benchmarks + +// Check that __COUNTER__ is defined and that __COUNTER__ increases by 1 +// every time it is expanded. X + 1 == X + 0 is used in case X is defined to be +// empty. If X is empty the expression becomes (+1 == +0). +#if defined(__COUNTER__) && (__COUNTER__ + 1 == __COUNTER__ + 0) +#define BENCHMARK_PRIVATE_UNIQUE_ID __COUNTER__ +#else +#define BENCHMARK_PRIVATE_UNIQUE_ID __LINE__ +#endif + +// Helpers for generating unique variable names +#define BENCHMARK_PRIVATE_NAME(n) \ + BENCHMARK_PRIVATE_CONCAT(_benchmark_, BENCHMARK_PRIVATE_UNIQUE_ID, n) +#define BENCHMARK_PRIVATE_CONCAT(a, b, c) BENCHMARK_PRIVATE_CONCAT2(a, b, c) +#define BENCHMARK_PRIVATE_CONCAT2(a, b, c) a##b##c + +#define BENCHMARK_PRIVATE_DECLARE(n) \ + static ::benchmark::internal::Benchmark* BENCHMARK_PRIVATE_NAME(n) \ + BENCHMARK_UNUSED + +#define BENCHMARK(n) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#n, n))) + +// Old-style macros +#define BENCHMARK_WITH_ARG(n, a) BENCHMARK(n)->Arg((a)) +#define BENCHMARK_WITH_ARG2(n, a1, a2) BENCHMARK(n)->Args({(a1), (a2)}) +#define BENCHMARK_WITH_UNIT(n, t) BENCHMARK(n)->Unit((t)) +#define BENCHMARK_RANGE(n, lo, hi) BENCHMARK(n)->Range((lo), (hi)) +#define BENCHMARK_RANGE2(n, l1, h1, l2, h2) \ + BENCHMARK(n)->RangePair({{(l1), (h1)}, {(l2), (h2)}}) + +#if __cplusplus >= 201103L + +// Register a benchmark which invokes the function specified by `func` +// with the additional arguments specified by `...`. +// +// For example: +// +// template ` +// void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) { +// [...] +//} +// /* Registers a benchmark named "BM_takes_args/int_string_test` */ +// BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc")); +#define BENCHMARK_CAPTURE(func, test_case_name, ...) \ + BENCHMARK_PRIVATE_DECLARE(func) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #func "/" #test_case_name, \ + [](::benchmark::State& st) { func(st, __VA_ARGS__); }))) + +#endif // __cplusplus >= 11 + +// This will register a benchmark for a templatized function. For example: +// +// template +// void BM_Foo(int iters); +// +// BENCHMARK_TEMPLATE(BM_Foo, 1); +// +// will register BM_Foo<1> as a benchmark. +#define BENCHMARK_TEMPLATE1(n, a) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#n "<" #a ">", n))) + +#define BENCHMARK_TEMPLATE2(n, a, b) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark(#n "<" #a "," #b ">", \ + n))) + +#if __cplusplus >= 201103L +#define BENCHMARK_TEMPLATE(n, ...) \ + BENCHMARK_PRIVATE_DECLARE(n) = \ + (::benchmark::internal::RegisterBenchmarkInternal( \ + new ::benchmark::internal::FunctionBenchmark( \ + #n "<" #__VA_ARGS__ ">", n<__VA_ARGS__>))) +#else +#define BENCHMARK_TEMPLATE(n, a) BENCHMARK_TEMPLATE1(n, a) +#endif + +#define BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + class BaseClass##_##Method##_Benchmark : public BaseClass { \ + public: \ + BaseClass##_##Method##_Benchmark() : BaseClass() { \ + this->SetName(#BaseClass "/" #Method); \ + } \ + \ + protected: \ + virtual void BenchmarkCase(::benchmark::State&); \ + }; + +#define BENCHMARK_DEFINE_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + void BaseClass##_##Method##_Benchmark::BenchmarkCase + +#define BENCHMARK_REGISTER_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_REGISTER_F(BaseClass##_##Method##_Benchmark) + +#define BENCHMARK_PRIVATE_REGISTER_F(TestName) \ + BENCHMARK_PRIVATE_DECLARE(TestName) = \ + (::benchmark::internal::RegisterBenchmarkInternal(new TestName())) + +// This macro will define and register a benchmark within a fixture class. +#define BENCHMARK_F(BaseClass, Method) \ + BENCHMARK_PRIVATE_DECLARE_F(BaseClass, Method) \ + BENCHMARK_REGISTER_F(BaseClass, Method); \ + void BaseClass##_##Method##_Benchmark::BenchmarkCase + +// Helper macro to create a main routine in a test that runs the benchmarks +#define BENCHMARK_MAIN() \ + int main(int argc, char** argv) { \ + ::benchmark::Initialize(&argc, argv); \ + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; \ + ::benchmark::RunSpecifiedBenchmarks(); \ + } + + +// ------------------------------------------------------ +// Benchmark Reporters + +namespace benchmark { + +// Interface for custom benchmark result printers. +// By default, benchmark reports are printed to stdout. However an application +// can control the destination of the reports by calling +// RunSpecifiedBenchmarks and passing it a custom reporter object. +// The reporter object must implement the following interface. +class BenchmarkReporter { + public: + struct Context { + int num_cpus; + double mhz_per_cpu; + bool cpu_scaling_enabled; + + // The number of chars in the longest benchmark name. + size_t name_field_width; + }; + + struct Run { + Run() + : error_occurred(false), + iterations(1), + time_unit(kNanosecond), + real_accumulated_time(0), + cpu_accumulated_time(0), + bytes_per_second(0), + items_per_second(0), + max_heapbytes_used(0), + complexity(oNone), + complexity_lambda(), + complexity_n(0), + report_big_o(false), + report_rms(false), + counters() {} + + std::string benchmark_name; + std::string report_label; // Empty if not set by benchmark. + bool error_occurred; + std::string error_message; + + int64_t iterations; + TimeUnit time_unit; + double real_accumulated_time; + double cpu_accumulated_time; + + // Return a value representing the real time per iteration in the unit + // specified by 'time_unit'. + // NOTE: If 'iterations' is zero the returned value represents the + // accumulated time. + double GetAdjustedRealTime() const; + + // Return a value representing the cpu time per iteration in the unit + // specified by 'time_unit'. + // NOTE: If 'iterations' is zero the returned value represents the + // accumulated time. + double GetAdjustedCPUTime() const; + + // Zero if not set by benchmark. + double bytes_per_second; + double items_per_second; + + // This is set to 0.0 if memory tracing is not enabled. + double max_heapbytes_used; + + // Keep track of arguments to compute asymptotic complexity + BigO complexity; + BigOFunc* complexity_lambda; + int complexity_n; + + // Inform print function whether the current run is a complexity report + bool report_big_o; + bool report_rms; + + UserCounters counters; + }; + + // Construct a BenchmarkReporter with the output stream set to 'std::cout' + // and the error stream set to 'std::cerr' + BenchmarkReporter(); + + // Called once for every suite of benchmarks run. + // The parameter "context" contains information that the + // reporter may wish to use when generating its report, for example the + // platform under which the benchmarks are running. The benchmark run is + // never started if this function returns false, allowing the reporter + // to skip runs based on the context information. + virtual bool ReportContext(const Context& context) = 0; + + // Called once for each group of benchmark runs, gives information about + // cpu-time and heap memory usage during the benchmark run. If the group + // of runs contained more than two entries then 'report' contains additional + // elements representing the mean and standard deviation of those runs. + // Additionally if this group of runs was the last in a family of benchmarks + // 'reports' contains additional entries representing the asymptotic + // complexity and RMS of that benchmark family. + virtual void ReportRuns(const std::vector& report) = 0; + + // Called once and only once after ever group of benchmarks is run and + // reported. + virtual void Finalize() {} + + // REQUIRES: The object referenced by 'out' is valid for the lifetime + // of the reporter. + void SetOutputStream(std::ostream* out) { + assert(out); + output_stream_ = out; + } + + // REQUIRES: The object referenced by 'err' is valid for the lifetime + // of the reporter. + void SetErrorStream(std::ostream* err) { + assert(err); + error_stream_ = err; + } + + std::ostream& GetOutputStream() const { return *output_stream_; } + + std::ostream& GetErrorStream() const { return *error_stream_; } + + virtual ~BenchmarkReporter(); + + // Write a human readable string to 'out' representing the specified + // 'context'. + // REQUIRES: 'out' is non-null. + static void PrintBasicContext(std::ostream* out, Context const& context); + + private: + std::ostream* output_stream_; + std::ostream* error_stream_; +}; + +// Simple reporter that outputs benchmark data to the console. This is the +// default reporter used by RunSpecifiedBenchmarks(). +class ConsoleReporter : public BenchmarkReporter { +public: + enum OutputOptions { + OO_None = 0, + OO_Color = 1, + OO_Tabular = 2, + OO_ColorTabular = OO_Color|OO_Tabular, + OO_Defaults = OO_ColorTabular + }; + explicit ConsoleReporter(OutputOptions opts_ = OO_Defaults) + : output_options_(opts_), name_field_width_(0), + prev_counters_(), printed_header_(false) {} + + virtual bool ReportContext(const Context& context); + virtual void ReportRuns(const std::vector& reports); + + protected: + virtual void PrintRunData(const Run& report); + virtual void PrintHeader(const Run& report); + + OutputOptions output_options_; + size_t name_field_width_; + UserCounters prev_counters_; + bool printed_header_; +}; + +class JSONReporter : public BenchmarkReporter { + public: + JSONReporter() : first_report_(true) {} + virtual bool ReportContext(const Context& context); + virtual void ReportRuns(const std::vector& reports); + virtual void Finalize(); + + private: + void PrintRunData(const Run& report); + + bool first_report_; +}; + +class CSVReporter : public BenchmarkReporter { + public: + CSVReporter() : printed_header_(false) {} + virtual bool ReportContext(const Context& context); + virtual void ReportRuns(const std::vector& reports); + + private: + void PrintRunData(const Run& report); + + bool printed_header_; + std::set< std::string > user_counter_names_; +}; + +inline const char* GetTimeUnitString(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return "ms"; + case kMicrosecond: + return "us"; + case kNanosecond: + default: + return "ns"; + } +} + +inline double GetTimeUnitMultiplier(TimeUnit unit) { + switch (unit) { + case kMillisecond: + return 1e3; + case kMicrosecond: + return 1e6; + case kNanosecond: + default: + return 1e9; + } +} + +} // namespace benchmark + +#endif // BENCHMARK_BENCHMARK_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/benchmark_api.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/benchmark_api.h new file mode 100644 index 00000000000..a9ae67147c5 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/benchmark_api.h @@ -0,0 +1,27 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef BENCHMARK_BENCHMARK_API_H_ +#define BENCHMARK_BENCHMARK_API_H_ + +#ifdef __DEPRECATED +# ifndef BENCHMARK_WARNING_MSG +# warning the benchmark_api.h header has been deprecated and will be removed, please include benchmark.h instead +# else + BENCHMARK_WARNING_MSG("the benchmark_api.h header has been deprecated and will be removed, please include benchmark.h instead") +# endif +#endif + +#include "benchmark.h" // For forward declaration of BenchmarkReporter + +#endif // BENCHMARK_BENCHMARK_API_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/reporter.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/reporter.h new file mode 100644 index 00000000000..5baca1a740a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/include/benchmark/reporter.h @@ -0,0 +1,27 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef BENCHMARK_REPORTER_H_ +#define BENCHMARK_REPORTER_H_ + +#ifdef __DEPRECATED +# ifndef BENCHMARK_WARNING_MSG +# warning the reporter.h header has been deprecated and will be removed, please include benchmark.h instead +# else + BENCHMARK_WARNING_MSG("the reporter.h header has been deprecated and will be removed, please include benchmark.h instead") +# endif +#endif + +#include "benchmark.h" // For forward declaration of BenchmarkReporter + +#endif // BENCHMARK_REPORTER_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/mingw.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/mingw.py new file mode 100644 index 00000000000..706ad559db9 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/mingw.py @@ -0,0 +1,320 @@ +#! /usr/bin/env python +# encoding: utf-8 + +import argparse +import errno +import logging +import os +import platform +import re +import sys +import subprocess +import tempfile + +try: + import winreg +except ImportError: + import _winreg as winreg +try: + import urllib.request as request +except ImportError: + import urllib as request +try: + import urllib.parse as parse +except ImportError: + import urlparse as parse + +class EmptyLogger(object): + ''' + Provides an implementation that performs no logging + ''' + def debug(self, *k, **kw): + pass + def info(self, *k, **kw): + pass + def warn(self, *k, **kw): + pass + def error(self, *k, **kw): + pass + def critical(self, *k, **kw): + pass + def setLevel(self, *k, **kw): + pass + +urls = ( + 'http://downloads.sourceforge.net/project/mingw-w64/Toolchains%20' + 'targetting%20Win32/Personal%20Builds/mingw-builds/installer/' + 'repository.txt', + 'http://downloads.sourceforge.net/project/mingwbuilds/host-windows/' + 'repository.txt' +) +''' +A list of mingw-build repositories +''' + +def repository(urls = urls, log = EmptyLogger()): + ''' + Downloads and parse mingw-build repository files and parses them + ''' + log.info('getting mingw-builds repository') + versions = {} + re_sourceforge = re.compile(r'http://sourceforge.net/projects/([^/]+)/files') + re_sub = r'http://downloads.sourceforge.net/project/\1' + for url in urls: + log.debug(' - requesting: %s', url) + socket = request.urlopen(url) + repo = socket.read() + if not isinstance(repo, str): + repo = repo.decode(); + socket.close() + for entry in repo.split('\n')[:-1]: + value = entry.split('|') + version = tuple([int(n) for n in value[0].strip().split('.')]) + version = versions.setdefault(version, {}) + arch = value[1].strip() + if arch == 'x32': + arch = 'i686' + elif arch == 'x64': + arch = 'x86_64' + arch = version.setdefault(arch, {}) + threading = arch.setdefault(value[2].strip(), {}) + exceptions = threading.setdefault(value[3].strip(), {}) + revision = exceptions.setdefault(int(value[4].strip()[3:]), + re_sourceforge.sub(re_sub, value[5].strip())) + return versions + +def find_in_path(file, path=None): + ''' + Attempts to find an executable in the path + ''' + if platform.system() == 'Windows': + file += '.exe' + if path is None: + path = os.environ.get('PATH', '') + if type(path) is type(''): + path = path.split(os.pathsep) + return list(filter(os.path.exists, + map(lambda dir, file=file: os.path.join(dir, file), path))) + +def find_7zip(log = EmptyLogger()): + ''' + Attempts to find 7zip for unpacking the mingw-build archives + ''' + log.info('finding 7zip') + path = find_in_path('7z') + if not path: + key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\7-Zip') + path, _ = winreg.QueryValueEx(key, 'Path') + path = [os.path.join(path, '7z.exe')] + log.debug('found \'%s\'', path[0]) + return path[0] + +find_7zip() + +def unpack(archive, location, log = EmptyLogger()): + ''' + Unpacks a mingw-builds archive + ''' + sevenzip = find_7zip(log) + log.info('unpacking %s', os.path.basename(archive)) + cmd = [sevenzip, 'x', archive, '-o' + location, '-y'] + log.debug(' - %r', cmd) + with open(os.devnull, 'w') as devnull: + subprocess.check_call(cmd, stdout = devnull) + +def download(url, location, log = EmptyLogger()): + ''' + Downloads and unpacks a mingw-builds archive + ''' + log.info('downloading MinGW') + log.debug(' - url: %s', url) + log.debug(' - location: %s', location) + + re_content = re.compile(r'attachment;[ \t]*filename=(")?([^"]*)(")?[\r\n]*') + + stream = request.urlopen(url) + try: + content = stream.getheader('Content-Disposition') or '' + except AttributeError: + content = stream.headers.getheader('Content-Disposition') or '' + matches = re_content.match(content) + if matches: + filename = matches.group(2) + else: + parsed = parse.urlparse(stream.geturl()) + filename = os.path.basename(parsed.path) + + try: + os.makedirs(location) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(location): + pass + else: + raise + + archive = os.path.join(location, filename) + with open(archive, 'wb') as out: + while True: + buf = stream.read(1024) + if not buf: + break + out.write(buf) + unpack(archive, location, log = log) + os.remove(archive) + + possible = os.path.join(location, 'mingw64') + if not os.path.exists(possible): + possible = os.path.join(location, 'mingw32') + if not os.path.exists(possible): + raise ValueError('Failed to find unpacked MinGW: ' + possible) + return possible + +def root(location = None, arch = None, version = None, threading = None, + exceptions = None, revision = None, log = EmptyLogger()): + ''' + Returns the root folder of a specific version of the mingw-builds variant + of gcc. Will download the compiler if needed + ''' + + # Get the repository if we don't have all the information + if not (arch and version and threading and exceptions and revision): + versions = repository(log = log) + + # Determine some defaults + version = version or max(versions.keys()) + if not arch: + arch = platform.machine().lower() + if arch == 'x86': + arch = 'i686' + elif arch == 'amd64': + arch = 'x86_64' + if not threading: + keys = versions[version][arch].keys() + if 'posix' in keys: + threading = 'posix' + elif 'win32' in keys: + threading = 'win32' + else: + threading = keys[0] + if not exceptions: + keys = versions[version][arch][threading].keys() + if 'seh' in keys: + exceptions = 'seh' + elif 'sjlj' in keys: + exceptions = 'sjlj' + else: + exceptions = keys[0] + if revision == None: + revision = max(versions[version][arch][threading][exceptions].keys()) + if not location: + location = os.path.join(tempfile.gettempdir(), 'mingw-builds') + + # Get the download url + url = versions[version][arch][threading][exceptions][revision] + + # Tell the user whatzzup + log.info('finding MinGW %s', '.'.join(str(v) for v in version)) + log.debug(' - arch: %s', arch) + log.debug(' - threading: %s', threading) + log.debug(' - exceptions: %s', exceptions) + log.debug(' - revision: %s', revision) + log.debug(' - url: %s', url) + + # Store each specific revision differently + slug = '{version}-{arch}-{threading}-{exceptions}-rev{revision}' + slug = slug.format( + version = '.'.join(str(v) for v in version), + arch = arch, + threading = threading, + exceptions = exceptions, + revision = revision + ) + if arch == 'x86_64': + root_dir = os.path.join(location, slug, 'mingw64') + elif arch == 'i686': + root_dir = os.path.join(location, slug, 'mingw32') + else: + raise ValueError('Unknown MinGW arch: ' + arch) + + # Download if needed + if not os.path.exists(root_dir): + downloaded = download(url, os.path.join(location, slug), log = log) + if downloaded != root_dir: + raise ValueError('The location of mingw did not match\n%s\n%s' + % (downloaded, root_dir)) + + return root_dir + +def str2ver(string): + ''' + Converts a version string into a tuple + ''' + try: + version = tuple(int(v) for v in string.split('.')) + if len(version) is not 3: + raise ValueError() + except ValueError: + raise argparse.ArgumentTypeError( + 'please provide a three digit version string') + return version + +def main(): + ''' + Invoked when the script is run directly by the python interpreter + ''' + parser = argparse.ArgumentParser( + description = 'Downloads a specific version of MinGW', + formatter_class = argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('--location', + help = 'the location to download the compiler to', + default = os.path.join(tempfile.gettempdir(), 'mingw-builds')) + parser.add_argument('--arch', required = True, choices = ['i686', 'x86_64'], + help = 'the target MinGW architecture string') + parser.add_argument('--version', type = str2ver, + help = 'the version of GCC to download') + parser.add_argument('--threading', choices = ['posix', 'win32'], + help = 'the threading type of the compiler') + parser.add_argument('--exceptions', choices = ['sjlj', 'seh', 'dwarf'], + help = 'the method to throw exceptions') + parser.add_argument('--revision', type=int, + help = 'the revision of the MinGW release') + group = parser.add_mutually_exclusive_group() + group.add_argument('-v', '--verbose', action='store_true', + help='increase the script output verbosity') + group.add_argument('-q', '--quiet', action='store_true', + help='only print errors and warning') + args = parser.parse_args() + + # Create the logger + logger = logging.getLogger('mingw') + handler = logging.StreamHandler() + formatter = logging.Formatter('%(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + if args.quiet: + logger.setLevel(logging.WARN) + if args.verbose: + logger.setLevel(logging.DEBUG) + + # Get MinGW + root_dir = root(location = args.location, arch = args.arch, + version = args.version, threading = args.threading, + exceptions = args.exceptions, revision = args.revision, + log = logger) + + sys.stdout.write('%s\n' % os.path.join(root_dir, 'bin')) + +if __name__ == '__main__': + try: + main() + except IOError as e: + sys.stderr.write('IO error: %s\n' % e) + sys.exit(1) + except OSError as e: + sys.stderr.write('OS error: %s\n' % e) + sys.exit(1) + except KeyboardInterrupt as e: + sys.stderr.write('Killed\n') + sys.exit(1) diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/CMakeLists.txt b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/CMakeLists.txt new file mode 100644 index 00000000000..244484b8b05 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/CMakeLists.txt @@ -0,0 +1,78 @@ +# Allow the source files to find headers in src/ +include_directories(${PROJECT_SOURCE_DIR}/src) + +if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) + list(APPEND CMAKE_SHARED_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) + list(APPEND CMAKE_MODULE_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) +endif() + +file(GLOB + SOURCE_FILES + *.cc + ${PROJECT_SOURCE_DIR}/include/benchmark/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.h) + +add_library(benchmark ${SOURCE_FILES}) +set_target_properties(benchmark PROPERTIES + OUTPUT_NAME "benchmark" + VERSION ${GENERIC_LIB_VERSION} + SOVERSION ${GENERIC_LIB_SOVERSION} +) +target_include_directories(benchmark PUBLIC + $ + ) + +# Link threads. +target_link_libraries(benchmark ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +find_library(LIBRT rt) +if(LIBRT) + target_link_libraries(benchmark ${LIBRT}) +endif() + +# We need extra libraries on Windows +if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + target_link_libraries(benchmark Shlwapi) +endif() + +set(include_install_dir "include") +set(lib_install_dir "lib/") +set(bin_install_dir "bin/") +set(config_install_dir "lib/cmake/${PROJECT_NAME}") + +set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") + +set(version_config "${generated_dir}/${PROJECT_NAME}ConfigVersion.cmake") +set(project_config "${generated_dir}/${PROJECT_NAME}Config.cmake") +set(targets_export_name "${PROJECT_NAME}Targets") + +set(namespace "${PROJECT_NAME}::") + +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + "${version_config}" VERSION ${GIT_VERSION} COMPATIBILITY SameMajorVersion +) + +configure_file("${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in" "${project_config}" @ONLY) + +# Install target (will install the library to specified CMAKE_INSTALL_PREFIX variable) +install( + TARGETS benchmark + EXPORT ${targets_export_name} + ARCHIVE DESTINATION ${lib_install_dir} + LIBRARY DESTINATION ${lib_install_dir} + RUNTIME DESTINATION ${bin_install_dir} + INCLUDES DESTINATION ${include_install_dir}) + +install( + DIRECTORY "${PROJECT_SOURCE_DIR}/include/benchmark" + DESTINATION ${include_install_dir} + FILES_MATCHING PATTERN "*.*h") + +install( + FILES "${project_config}" "${version_config}" + DESTINATION "${config_install_dir}") + +install( + EXPORT "${targets_export_name}" + NAMESPACE "${namespace}" + DESTINATION "${config_install_dir}") diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/arraysize.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/arraysize.h new file mode 100644 index 00000000000..51a50f2dff2 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/arraysize.h @@ -0,0 +1,33 @@ +#ifndef BENCHMARK_ARRAYSIZE_H_ +#define BENCHMARK_ARRAYSIZE_H_ + +#include "internal_macros.h" + +namespace benchmark { +namespace internal { +// The arraysize(arr) macro returns the # of elements in an array arr. +// The expression is a compile-time constant, and therefore can be +// used in defining new arrays, for example. If you use arraysize on +// a pointer by mistake, you will get a compile-time error. +// + +// This template function declaration is used in defining arraysize. +// Note that the function doesn't need an implementation, as we only +// use its type. +template +char (&ArraySizeHelper(T (&array)[N]))[N]; + +// That gcc wants both of these prototypes seems mysterious. VC, for +// its part, can't decide which to use (another mystery). Matching of +// template overloads: the final frontier. +#ifndef COMPILER_MSVC +template +char (&ArraySizeHelper(const T (&array)[N]))[N]; +#endif + +#define arraysize(array) (sizeof(::benchmark::internal::ArraySizeHelper(array))) + +} // end namespace internal +} // end namespace benchmark + +#endif // BENCHMARK_ARRAYSIZE_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark.cc new file mode 100644 index 00000000000..1ba0a50adf8 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark.cc @@ -0,0 +1,715 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "benchmark_api_internal.h" +#include "internal_macros.h" + +#ifndef BENCHMARK_OS_WINDOWS +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "check.h" +#include "colorprint.h" +#include "commandlineflags.h" +#include "complexity.h" +#include "counter.h" +#include "log.h" +#include "mutex.h" +#include "re.h" +#include "stat.h" +#include "string_util.h" +#include "sysinfo.h" +#include "timers.h" + +DEFINE_bool(benchmark_list_tests, false, + "Print a list of benchmarks. This option overrides all other " + "options."); + +DEFINE_string(benchmark_filter, ".", + "A regular expression that specifies the set of benchmarks " + "to execute. If this flag is empty, no benchmarks are run. " + "If this flag is the string \"all\", all benchmarks linked " + "into the process are run."); + +DEFINE_double(benchmark_min_time, 0.5, + "Minimum number of seconds we should run benchmark before " + "results are considered significant. For cpu-time based " + "tests, this is the lower bound on the total cpu time " + "used by all threads that make up the test. For real-time " + "based tests, this is the lower bound on the elapsed time " + "of the benchmark execution, regardless of number of " + "threads."); + +DEFINE_int32(benchmark_repetitions, 1, + "The number of runs of each benchmark. If greater than 1, the " + "mean and standard deviation of the runs will be reported."); + +DEFINE_bool(benchmark_report_aggregates_only, false, + "Report the result of each benchmark repetitions. When 'true' is " + "specified only the mean, standard deviation, and other statistics " + "are reported for repeated benchmarks."); + +DEFINE_string(benchmark_format, "console", + "The format to use for console output. Valid values are " + "'console', 'json', or 'csv'."); + +DEFINE_string(benchmark_out_format, "json", + "The format to use for file output. Valid values are " + "'console', 'json', or 'csv'."); + +DEFINE_string(benchmark_out, "", "The file to write additonal output to"); + +DEFINE_string(benchmark_color, "auto", + "Whether to use colors in the output. Valid values: " + "'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use " + "colors if the output is being sent to a terminal and the TERM " + "environment variable is set to a terminal type that supports " + "colors."); + +DEFINE_bool(benchmark_counters_tabular, false, + "Whether to use tabular format when printing user counters to " + "the console. Valid values: 'true'/'yes'/1, 'false'/'no'/0." + "Defaults to false."); + +DEFINE_int32(v, 0, "The level of verbose logging to output"); + +namespace benchmark { +namespace internal { + +void UseCharPointer(char const volatile*) {} + +} // end namespace internal + +namespace { + +static const size_t kMaxIterations = 1000000000; + +} // end namespace + +namespace internal { + +class ThreadManager { + public: + ThreadManager(int num_threads) + : alive_threads_(num_threads), start_stop_barrier_(num_threads) {} + + Mutex& GetBenchmarkMutex() const RETURN_CAPABILITY(benchmark_mutex_) { + return benchmark_mutex_; + } + + bool StartStopBarrier() EXCLUDES(end_cond_mutex_) { + return start_stop_barrier_.wait(); + } + + void NotifyThreadComplete() EXCLUDES(end_cond_mutex_) { + start_stop_barrier_.removeThread(); + if (--alive_threads_ == 0) { + MutexLock lock(end_cond_mutex_); + end_condition_.notify_all(); + } + } + + void WaitForAllThreads() EXCLUDES(end_cond_mutex_) { + MutexLock lock(end_cond_mutex_); + end_condition_.wait(lock.native_handle(), + [this]() { return alive_threads_ == 0; }); + } + + public: + struct Result { + double real_time_used = 0; + double cpu_time_used = 0; + double manual_time_used = 0; + int64_t bytes_processed = 0; + int64_t items_processed = 0; + int complexity_n = 0; + std::string report_label_; + std::string error_message_; + bool has_error_ = false; + UserCounters counters; + }; + GUARDED_BY(GetBenchmarkMutex()) Result results; + + private: + mutable Mutex benchmark_mutex_; + std::atomic alive_threads_; + Barrier start_stop_barrier_; + Mutex end_cond_mutex_; + Condition end_condition_; +}; + +// Timer management class +class ThreadTimer { + public: + ThreadTimer() = default; + + // Called by each thread + void StartTimer() { + running_ = true; + start_real_time_ = ChronoClockNow(); + start_cpu_time_ = ThreadCPUUsage(); + } + + // Called by each thread + void StopTimer() { + CHECK(running_); + running_ = false; + real_time_used_ += ChronoClockNow() - start_real_time_; + cpu_time_used_ += ThreadCPUUsage() - start_cpu_time_; + } + + // Called by each thread + void SetIterationTime(double seconds) { manual_time_used_ += seconds; } + + bool running() const { return running_; } + + // REQUIRES: timer is not running + double real_time_used() { + CHECK(!running_); + return real_time_used_; + } + + // REQUIRES: timer is not running + double cpu_time_used() { + CHECK(!running_); + return cpu_time_used_; + } + + // REQUIRES: timer is not running + double manual_time_used() { + CHECK(!running_); + return manual_time_used_; + } + + private: + bool running_ = false; // Is the timer running + double start_real_time_ = 0; // If running_ + double start_cpu_time_ = 0; // If running_ + + // Accumulated time so far (does not contain current slice if running_) + double real_time_used_ = 0; + double cpu_time_used_ = 0; + // Manually set iteration time. User sets this with SetIterationTime(seconds). + double manual_time_used_ = 0; +}; + +namespace { + +BenchmarkReporter::Run CreateRunReport( + const benchmark::internal::Benchmark::Instance& b, + const internal::ThreadManager::Result& results, size_t iters, + double seconds) { + // Create report about this benchmark run. + BenchmarkReporter::Run report; + + report.benchmark_name = b.name; + report.error_occurred = results.has_error_; + report.error_message = results.error_message_; + report.report_label = results.report_label_; + // Report the total iterations across all threads. + report.iterations = static_cast(iters) * b.threads; + report.time_unit = b.time_unit; + + if (!report.error_occurred) { + double bytes_per_second = 0; + if (results.bytes_processed > 0 && seconds > 0.0) { + bytes_per_second = (results.bytes_processed / seconds); + } + double items_per_second = 0; + if (results.items_processed > 0 && seconds > 0.0) { + items_per_second = (results.items_processed / seconds); + } + + if (b.use_manual_time) { + report.real_accumulated_time = results.manual_time_used; + } else { + report.real_accumulated_time = results.real_time_used; + } + report.cpu_accumulated_time = results.cpu_time_used; + report.bytes_per_second = bytes_per_second; + report.items_per_second = items_per_second; + report.complexity_n = results.complexity_n; + report.complexity = b.complexity; + report.complexity_lambda = b.complexity_lambda; + report.counters = results.counters; + internal::Finish(&report.counters, seconds, b.threads); + } + return report; +} + +// Execute one thread of benchmark b for the specified number of iterations. +// Adds the stats collected for the thread into *total. +void RunInThread(const benchmark::internal::Benchmark::Instance* b, + size_t iters, int thread_id, + internal::ThreadManager* manager) { + internal::ThreadTimer timer; + State st(iters, b->arg, thread_id, b->threads, &timer, manager); + b->benchmark->Run(st); + CHECK(st.iterations() == st.max_iterations) + << "Benchmark returned before State::KeepRunning() returned false!"; + { + MutexLock l(manager->GetBenchmarkMutex()); + internal::ThreadManager::Result& results = manager->results; + results.cpu_time_used += timer.cpu_time_used(); + results.real_time_used += timer.real_time_used(); + results.manual_time_used += timer.manual_time_used(); + results.bytes_processed += st.bytes_processed(); + results.items_processed += st.items_processed(); + results.complexity_n += st.complexity_length_n(); + internal::Increment(&results.counters, st.counters); + } + manager->NotifyThreadComplete(); +} + +std::vector RunBenchmark( + const benchmark::internal::Benchmark::Instance& b, + std::vector* complexity_reports) { + std::vector reports; // return value + + const bool has_explicit_iteration_count = b.iterations != 0; + size_t iters = has_explicit_iteration_count ? b.iterations : 1; + std::unique_ptr manager; + std::vector pool(b.threads - 1); + const int repeats = + b.repetitions != 0 ? b.repetitions : FLAGS_benchmark_repetitions; + const bool report_aggregates_only = + repeats != 1 && + (b.report_mode == internal::RM_Unspecified + ? FLAGS_benchmark_report_aggregates_only + : b.report_mode == internal::RM_ReportAggregatesOnly); + for (int repetition_num = 0; repetition_num < repeats; repetition_num++) { + for (;;) { + // Try benchmark + VLOG(2) << "Running " << b.name << " for " << iters << "\n"; + + manager.reset(new internal::ThreadManager(b.threads)); + for (std::size_t ti = 0; ti < pool.size(); ++ti) { + pool[ti] = std::thread(&RunInThread, &b, iters, + static_cast(ti + 1), manager.get()); + } + RunInThread(&b, iters, 0, manager.get()); + manager->WaitForAllThreads(); + for (std::thread& thread : pool) thread.join(); + internal::ThreadManager::Result results; + { + MutexLock l(manager->GetBenchmarkMutex()); + results = manager->results; + } + manager.reset(); + // Adjust real/manual time stats since they were reported per thread. + results.real_time_used /= b.threads; + results.manual_time_used /= b.threads; + + VLOG(2) << "Ran in " << results.cpu_time_used << "/" + << results.real_time_used << "\n"; + + // Base decisions off of real time if requested by this benchmark. + double seconds = results.cpu_time_used; + if (b.use_manual_time) { + seconds = results.manual_time_used; + } else if (b.use_real_time) { + seconds = results.real_time_used; + } + + const double min_time = + !IsZero(b.min_time) ? b.min_time : FLAGS_benchmark_min_time; + + // Determine if this run should be reported; Either it has + // run for a sufficient amount of time or because an error was reported. + const bool should_report = repetition_num > 0 + || has_explicit_iteration_count // An exact iteration count was requested + || results.has_error_ + || iters >= kMaxIterations + || seconds >= min_time // the elapsed time is large enough + // CPU time is specified but the elapsed real time greatly exceeds the + // minimum time. Note that user provided timers are except from this + // sanity check. + || ((results.real_time_used >= 5 * min_time) && !b.use_manual_time); + + if (should_report) { + BenchmarkReporter::Run report = + CreateRunReport(b, results, iters, seconds); + if (!report.error_occurred && b.complexity != oNone) + complexity_reports->push_back(report); + reports.push_back(report); + break; + } + + // See how much iterations should be increased by + // Note: Avoid division by zero with max(seconds, 1ns). + double multiplier = min_time * 1.4 / std::max(seconds, 1e-9); + // If our last run was at least 10% of FLAGS_benchmark_min_time then we + // use the multiplier directly. Otherwise we use at most 10 times + // expansion. + // NOTE: When the last run was at least 10% of the min time the max + // expansion should be 14x. + bool is_significant = (seconds / min_time) > 0.1; + multiplier = is_significant ? multiplier : std::min(10.0, multiplier); + if (multiplier <= 1.0) multiplier = 2.0; + double next_iters = std::max(multiplier * iters, iters + 1.0); + if (next_iters > kMaxIterations) { + next_iters = kMaxIterations; + } + VLOG(3) << "Next iters: " << next_iters << ", " << multiplier << "\n"; + iters = static_cast(next_iters + 0.5); + } + } + // Calculate additional statistics + auto stat_reports = ComputeStats(reports); + if ((b.complexity != oNone) && b.last_benchmark_instance) { + auto additional_run_stats = ComputeBigO(*complexity_reports); + stat_reports.insert(stat_reports.end(), additional_run_stats.begin(), + additional_run_stats.end()); + complexity_reports->clear(); + } + + if (report_aggregates_only) reports.clear(); + reports.insert(reports.end(), stat_reports.begin(), stat_reports.end()); + return reports; +} + +} // namespace +} // namespace internal + +State::State(size_t max_iters, const std::vector& ranges, int thread_i, + int n_threads, internal::ThreadTimer* timer, + internal::ThreadManager* manager) + : started_(false), + finished_(false), + total_iterations_(0), + range_(ranges), + bytes_processed_(0), + items_processed_(0), + complexity_n_(0), + error_occurred_(false), + counters(), + thread_index(thread_i), + threads(n_threads), + max_iterations(max_iters), + timer_(timer), + manager_(manager) { + CHECK(max_iterations != 0) << "At least one iteration must be run"; + CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; +} + +void State::PauseTiming() { + // Add in time accumulated so far + CHECK(started_ && !finished_ && !error_occurred_); + timer_->StopTimer(); +} + +void State::ResumeTiming() { + CHECK(started_ && !finished_ && !error_occurred_); + timer_->StartTimer(); +} + +void State::SkipWithError(const char* msg) { + CHECK(msg); + error_occurred_ = true; + { + MutexLock l(manager_->GetBenchmarkMutex()); + if (manager_->results.has_error_ == false) { + manager_->results.error_message_ = msg; + manager_->results.has_error_ = true; + } + } + total_iterations_ = max_iterations; + if (timer_->running()) timer_->StopTimer(); +} + +void State::SetIterationTime(double seconds) { + timer_->SetIterationTime(seconds); +} + +void State::SetLabel(const char* label) { + MutexLock l(manager_->GetBenchmarkMutex()); + manager_->results.report_label_ = label; +} + +void State::StartKeepRunning() { + CHECK(!started_ && !finished_); + started_ = true; + manager_->StartStopBarrier(); + if (!error_occurred_) ResumeTiming(); +} + +void State::FinishKeepRunning() { + CHECK(started_ && (!finished_ || error_occurred_)); + if (!error_occurred_) { + PauseTiming(); + } + // Total iterations now is one greater than max iterations. Fix this. + total_iterations_ = max_iterations; + finished_ = true; + manager_->StartStopBarrier(); +} + +namespace internal { +namespace { + +void RunBenchmarks(const std::vector& benchmarks, + BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter) { + // Note the file_reporter can be null. + CHECK(console_reporter != nullptr); + + // Determine the width of the name field using a minimum width of 10. + bool has_repetitions = FLAGS_benchmark_repetitions > 1; + size_t name_field_width = 10; + for (const Benchmark::Instance& benchmark : benchmarks) { + name_field_width = + std::max(name_field_width, benchmark.name.size()); + has_repetitions |= benchmark.repetitions > 1; + } + if (has_repetitions) name_field_width += std::strlen("_stddev"); + + // Print header here + BenchmarkReporter::Context context; + context.num_cpus = NumCPUs(); + context.mhz_per_cpu = CyclesPerSecond() / 1000000.0f; + + context.cpu_scaling_enabled = CpuScalingEnabled(); + context.name_field_width = name_field_width; + + // Keep track of runing times of all instances of current benchmark + std::vector complexity_reports; + + // We flush streams after invoking reporter methods that write to them. This + // ensures users get timely updates even when streams are not line-buffered. + auto flushStreams = [](BenchmarkReporter* reporter) { + if (!reporter) return; + std::flush(reporter->GetOutputStream()); + std::flush(reporter->GetErrorStream()); + }; + + if (console_reporter->ReportContext(context) && + (!file_reporter || file_reporter->ReportContext(context))) { + flushStreams(console_reporter); + flushStreams(file_reporter); + for (const auto& benchmark : benchmarks) { + std::vector reports = + RunBenchmark(benchmark, &complexity_reports); + console_reporter->ReportRuns(reports); + if (file_reporter) file_reporter->ReportRuns(reports); + flushStreams(console_reporter); + flushStreams(file_reporter); + } + } + console_reporter->Finalize(); + if (file_reporter) file_reporter->Finalize(); + flushStreams(console_reporter); + flushStreams(file_reporter); +} + +std::unique_ptr CreateReporter( + std::string const& name, ConsoleReporter::OutputOptions output_opts) { + typedef std::unique_ptr PtrType; + if (name == "console") { + return PtrType(new ConsoleReporter(output_opts)); + } else if (name == "json") { + return PtrType(new JSONReporter); + } else if (name == "csv") { + return PtrType(new CSVReporter); + } else { + std::cerr << "Unexpected format: '" << name << "'\n"; + std::exit(1); + } +} + +} // end namespace + +bool IsZero(double n) { + return std::abs(n) < std::numeric_limits::epsilon(); +} + +ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { + int output_opts = ConsoleReporter::OO_Defaults; + if ((FLAGS_benchmark_color == "auto" && IsColorTerminal()) || + IsTruthyFlagValue(FLAGS_benchmark_color)) { + output_opts |= ConsoleReporter::OO_Color; + } else { + output_opts &= ~ConsoleReporter::OO_Color; + } + if(force_no_color) { + output_opts &= ~ConsoleReporter::OO_Color; + } + if(FLAGS_benchmark_counters_tabular) { + output_opts |= ConsoleReporter::OO_Tabular; + } else { + output_opts &= ~ConsoleReporter::OO_Tabular; + } + return static_cast< ConsoleReporter::OutputOptions >(output_opts); +} + +} // end namespace internal + +size_t RunSpecifiedBenchmarks() { + return RunSpecifiedBenchmarks(nullptr, nullptr); +} + +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter) { + return RunSpecifiedBenchmarks(console_reporter, nullptr); +} + +size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter, + BenchmarkReporter* file_reporter) { + std::string spec = FLAGS_benchmark_filter; + if (spec.empty() || spec == "all") + spec = "."; // Regexp that matches all benchmarks + + // Setup the reporters + std::ofstream output_file; + std::unique_ptr default_console_reporter; + std::unique_ptr default_file_reporter; + if (!console_reporter) { + default_console_reporter = internal::CreateReporter( + FLAGS_benchmark_format, internal::GetOutputOptions()); + console_reporter = default_console_reporter.get(); + } + auto& Out = console_reporter->GetOutputStream(); + auto& Err = console_reporter->GetErrorStream(); + + std::string const& fname = FLAGS_benchmark_out; + if (fname.empty() && file_reporter) { + Err << "A custom file reporter was provided but " + "--benchmark_out= was not specified." + << std::endl; + std::exit(1); + } + if (!fname.empty()) { + output_file.open(fname); + if (!output_file.is_open()) { + Err << "invalid file name: '" << fname << std::endl; + std::exit(1); + } + if (!file_reporter) { + default_file_reporter = internal::CreateReporter( + FLAGS_benchmark_out_format, ConsoleReporter::OO_None); + file_reporter = default_file_reporter.get(); + } + file_reporter->SetOutputStream(&output_file); + file_reporter->SetErrorStream(&output_file); + } + + std::vector benchmarks; + if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; + + if (benchmarks.empty()) { + Err << "Failed to match any benchmarks against regex: " << spec << "\n"; + return 0; + } + + if (FLAGS_benchmark_list_tests) { + for (auto const& benchmark : benchmarks) Out << benchmark.name << "\n"; + } else { + internal::RunBenchmarks(benchmarks, console_reporter, file_reporter); + } + + return benchmarks.size(); +} + +namespace internal { + +void PrintUsageAndExit() { + fprintf(stdout, + "benchmark" + " [--benchmark_list_tests={true|false}]\n" + " [--benchmark_filter=]\n" + " [--benchmark_min_time=]\n" + " [--benchmark_repetitions=]\n" + " [--benchmark_report_aggregates_only={true|false}\n" + " [--benchmark_format=]\n" + " [--benchmark_out=]\n" + " [--benchmark_out_format=]\n" + " [--benchmark_color={auto|true|false}]\n" + " [--benchmark_counters_tabular={true|false}]\n" + " [--v=]\n"); + exit(0); +} + +void ParseCommandLineFlags(int* argc, char** argv) { + using namespace benchmark; + for (int i = 1; i < *argc; ++i) { + if (ParseBoolFlag(argv[i], "benchmark_list_tests", + &FLAGS_benchmark_list_tests) || + ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || + ParseDoubleFlag(argv[i], "benchmark_min_time", + &FLAGS_benchmark_min_time) || + ParseInt32Flag(argv[i], "benchmark_repetitions", + &FLAGS_benchmark_repetitions) || + ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", + &FLAGS_benchmark_report_aggregates_only) || + ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || + ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) || + ParseStringFlag(argv[i], "benchmark_out_format", + &FLAGS_benchmark_out_format) || + ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) || + // "color_print" is the deprecated name for "benchmark_color". + // TODO: Remove this. + ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) || + ParseBoolFlag(argv[i], "benchmark_counters_tabular", + &FLAGS_benchmark_counters_tabular) || + ParseInt32Flag(argv[i], "v", &FLAGS_v)) { + for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; + + --(*argc); + --i; + } else if (IsFlag(argv[i], "help")) { + PrintUsageAndExit(); + } + } + for (auto const* flag : + {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) + if (*flag != "console" && *flag != "json" && *flag != "csv") { + PrintUsageAndExit(); + } + if (FLAGS_benchmark_color.empty()) { + PrintUsageAndExit(); + } +} + +int InitializeStreams() { + static std::ios_base::Init init; + return 0; +} + +} // end namespace internal + +void Initialize(int* argc, char** argv) { + internal::ParseCommandLineFlags(argc, argv); + internal::LogLevel() = FLAGS_v; +} + +bool ReportUnrecognizedArguments(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], argv[i]); + } + return argc > 1; +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark_api_internal.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark_api_internal.h new file mode 100644 index 00000000000..36d23404717 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark_api_internal.h @@ -0,0 +1,46 @@ +#ifndef BENCHMARK_API_INTERNAL_H +#define BENCHMARK_API_INTERNAL_H + +#include "benchmark/benchmark.h" + +#include +#include +#include +#include +#include + +namespace benchmark { +namespace internal { + +// Information kept per benchmark we may want to run +struct Benchmark::Instance { + std::string name; + Benchmark* benchmark; + ReportMode report_mode; + std::vector arg; + TimeUnit time_unit; + int range_multiplier; + bool use_real_time; + bool use_manual_time; + BigO complexity; + BigOFunc* complexity_lambda; + UserCounters counters; + bool last_benchmark_instance; + int repetitions; + double min_time; + size_t iterations; + int threads; // Number of concurrent threads to us +}; + +bool FindBenchmarksInternal(const std::string& re, + std::vector* benchmarks, + std::ostream* Err); + +bool IsZero(double n); + +ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color = false); + +} // end namespace internal +} // end namespace benchmark + +#endif // BENCHMARK_API_INTERNAL_H diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark_register.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark_register.cc new file mode 100644 index 00000000000..ed70d820dee --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/benchmark_register.cc @@ -0,0 +1,467 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "benchmark_api_internal.h" +#include "internal_macros.h" + +#ifndef BENCHMARK_OS_WINDOWS +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "check.h" +#include "commandlineflags.h" +#include "complexity.h" +#include "log.h" +#include "mutex.h" +#include "re.h" +#include "stat.h" +#include "string_util.h" +#include "sysinfo.h" +#include "timers.h" + +namespace benchmark { + +namespace { +// For non-dense Range, intermediate values are powers of kRangeMultiplier. +static const int kRangeMultiplier = 8; +// The size of a benchmark family determines is the number of inputs to repeat +// the benchmark on. If this is "large" then warn the user during configuration. +static const size_t kMaxFamilySize = 100; +} // end namespace + +namespace internal { + +//=============================================================================// +// BenchmarkFamilies +//=============================================================================// + +// Class for managing registered benchmarks. Note that each registered +// benchmark identifies a family of related benchmarks to run. +class BenchmarkFamilies { + public: + static BenchmarkFamilies* GetInstance(); + + // Registers a benchmark family and returns the index assigned to it. + size_t AddBenchmark(std::unique_ptr family); + + // Clear all registered benchmark families. + void ClearBenchmarks(); + + // Extract the list of benchmark instances that match the specified + // regular expression. + bool FindBenchmarks(const std::string& re, + std::vector* benchmarks, + std::ostream* Err); + + private: + BenchmarkFamilies() {} + + std::vector> families_; + Mutex mutex_; +}; + +BenchmarkFamilies* BenchmarkFamilies::GetInstance() { + static BenchmarkFamilies instance; + return &instance; +} + +size_t BenchmarkFamilies::AddBenchmark(std::unique_ptr family) { + MutexLock l(mutex_); + size_t index = families_.size(); + families_.push_back(std::move(family)); + return index; +} + +void BenchmarkFamilies::ClearBenchmarks() { + MutexLock l(mutex_); + families_.clear(); + families_.shrink_to_fit(); +} + +bool BenchmarkFamilies::FindBenchmarks( + const std::string& spec, std::vector* benchmarks, + std::ostream* ErrStream) { + CHECK(ErrStream); + auto& Err = *ErrStream; + // Make regular expression out of command-line flag + std::string error_msg; + Regex re; + if (!re.Init(spec, &error_msg)) { + Err << "Could not compile benchmark re: " << error_msg << std::endl; + return false; + } + + // Special list of thread counts to use when none are specified + const std::vector one_thread = {1}; + + MutexLock l(mutex_); + for (std::unique_ptr& family : families_) { + // Family was deleted or benchmark doesn't match + if (!family) continue; + + if (family->ArgsCnt() == -1) { + family->Args({}); + } + const std::vector* thread_counts = + (family->thread_counts_.empty() + ? &one_thread + : &static_cast&>(family->thread_counts_)); + const size_t family_size = family->args_.size() * thread_counts->size(); + // The benchmark will be run at least 'family_size' different inputs. + // If 'family_size' is very large warn the user. + if (family_size > kMaxFamilySize) { + Err << "The number of inputs is very large. " << family->name_ + << " will be repeated at least " << family_size << " times.\n"; + } + // reserve in the special case the regex ".", since we know the final + // family size. + if (spec == ".") benchmarks->reserve(family_size); + + for (auto const& args : family->args_) { + for (int num_threads : *thread_counts) { + Benchmark::Instance instance; + instance.name = family->name_; + instance.benchmark = family.get(); + instance.report_mode = family->report_mode_; + instance.arg = args; + instance.time_unit = family->time_unit_; + instance.range_multiplier = family->range_multiplier_; + instance.min_time = family->min_time_; + instance.iterations = family->iterations_; + instance.repetitions = family->repetitions_; + instance.use_real_time = family->use_real_time_; + instance.use_manual_time = family->use_manual_time_; + instance.complexity = family->complexity_; + instance.complexity_lambda = family->complexity_lambda_; + instance.threads = num_threads; + + // Add arguments to instance name + size_t arg_i = 0; + for (auto const& arg : args) { + instance.name += "/"; + + if (arg_i < family->arg_names_.size()) { + const auto& arg_name = family->arg_names_[arg_i]; + if (!arg_name.empty()) { + instance.name += + StringPrintF("%s:", family->arg_names_[arg_i].c_str()); + } + } + + instance.name += StringPrintF("%d", arg); + ++arg_i; + } + + if (!IsZero(family->min_time_)) + instance.name += StringPrintF("/min_time:%0.3f", family->min_time_); + if (family->iterations_ != 0) + instance.name += StringPrintF("/iterations:%d", family->iterations_); + if (family->repetitions_ != 0) + instance.name += StringPrintF("/repeats:%d", family->repetitions_); + + if (family->use_manual_time_) { + instance.name += "/manual_time"; + } else if (family->use_real_time_) { + instance.name += "/real_time"; + } + + // Add the number of threads used to the name + if (!family->thread_counts_.empty()) { + instance.name += StringPrintF("/threads:%d", instance.threads); + } + + if (re.Match(instance.name)) { + instance.last_benchmark_instance = (&args == &family->args_.back()); + benchmarks->push_back(std::move(instance)); + } + } + } + } + return true; +} + +Benchmark* RegisterBenchmarkInternal(Benchmark* bench) { + std::unique_ptr bench_ptr(bench); + BenchmarkFamilies* families = BenchmarkFamilies::GetInstance(); + families->AddBenchmark(std::move(bench_ptr)); + return bench; +} + +// FIXME: This function is a hack so that benchmark.cc can access +// `BenchmarkFamilies` +bool FindBenchmarksInternal(const std::string& re, + std::vector* benchmarks, + std::ostream* Err) { + return BenchmarkFamilies::GetInstance()->FindBenchmarks(re, benchmarks, Err); +} + +//=============================================================================// +// Benchmark +//=============================================================================// + +Benchmark::Benchmark(const char* name) + : name_(name), + report_mode_(RM_Unspecified), + time_unit_(kNanosecond), + range_multiplier_(kRangeMultiplier), + min_time_(0), + iterations_(0), + repetitions_(0), + use_real_time_(false), + use_manual_time_(false), + complexity_(oNone), + complexity_lambda_(nullptr) {} + +Benchmark::~Benchmark() {} + +void Benchmark::AddRange(std::vector* dst, int lo, int hi, int mult) { + CHECK_GE(lo, 0); + CHECK_GE(hi, lo); + CHECK_GE(mult, 2); + + // Add "lo" + dst->push_back(lo); + + static const int kint32max = std::numeric_limits::max(); + + // Now space out the benchmarks in multiples of "mult" + for (int32_t i = 1; i < kint32max / mult; i *= mult) { + if (i >= hi) break; + if (i > lo) { + dst->push_back(i); + } + } + // Add "hi" (if different from "lo") + if (hi != lo) { + dst->push_back(hi); + } +} + +Benchmark* Benchmark::Arg(int x) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + args_.push_back({x}); + return this; +} + +Benchmark* Benchmark::Unit(TimeUnit unit) { + time_unit_ = unit; + return this; +} + +Benchmark* Benchmark::Range(int start, int limit) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + std::vector arglist; + AddRange(&arglist, start, limit, range_multiplier_); + + for (int i : arglist) { + args_.push_back({i}); + } + return this; +} + +Benchmark* Benchmark::Ranges(const std::vector>& ranges) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(ranges.size())); + std::vector> arglists(ranges.size()); + std::size_t total = 1; + for (std::size_t i = 0; i < ranges.size(); i++) { + AddRange(&arglists[i], ranges[i].first, ranges[i].second, + range_multiplier_); + total *= arglists[i].size(); + } + + std::vector ctr(arglists.size(), 0); + + for (std::size_t i = 0; i < total; i++) { + std::vector tmp; + tmp.reserve(arglists.size()); + + for (std::size_t j = 0; j < arglists.size(); j++) { + tmp.push_back(arglists[j].at(ctr[j])); + } + + args_.push_back(std::move(tmp)); + + for (std::size_t j = 0; j < arglists.size(); j++) { + if (ctr[j] + 1 < arglists[j].size()) { + ++ctr[j]; + break; + } + ctr[j] = 0; + } + } + return this; +} + +Benchmark* Benchmark::ArgName(const std::string& name) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + arg_names_ = {name}; + return this; +} + +Benchmark* Benchmark::ArgNames(const std::vector& names) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(names.size())); + arg_names_ = names; + return this; +} + +Benchmark* Benchmark::DenseRange(int start, int limit, int step) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == 1); + CHECK_GE(start, 0); + CHECK_LE(start, limit); + for (int arg = start; arg <= limit; arg += step) { + args_.push_back({arg}); + } + return this; +} + +Benchmark* Benchmark::Args(const std::vector& args) { + CHECK(ArgsCnt() == -1 || ArgsCnt() == static_cast(args.size())); + args_.push_back(args); + return this; +} + +Benchmark* Benchmark::Apply(void (*custom_arguments)(Benchmark* benchmark)) { + custom_arguments(this); + return this; +} + +Benchmark* Benchmark::RangeMultiplier(int multiplier) { + CHECK(multiplier > 1); + range_multiplier_ = multiplier; + return this; +} + + +Benchmark* Benchmark::MinTime(double t) { + CHECK(t > 0.0); + CHECK(iterations_ == 0); + min_time_ = t; + return this; +} + + +Benchmark* Benchmark::Iterations(size_t n) { + CHECK(n > 0); + CHECK(IsZero(min_time_)); + iterations_ = n; + return this; +} + +Benchmark* Benchmark::Repetitions(int n) { + CHECK(n > 0); + repetitions_ = n; + return this; +} + +Benchmark* Benchmark::ReportAggregatesOnly(bool value) { + report_mode_ = value ? RM_ReportAggregatesOnly : RM_Default; + return this; +} + +Benchmark* Benchmark::UseRealTime() { + CHECK(!use_manual_time_) + << "Cannot set UseRealTime and UseManualTime simultaneously."; + use_real_time_ = true; + return this; +} + +Benchmark* Benchmark::UseManualTime() { + CHECK(!use_real_time_) + << "Cannot set UseRealTime and UseManualTime simultaneously."; + use_manual_time_ = true; + return this; +} + +Benchmark* Benchmark::Complexity(BigO complexity) { + complexity_ = complexity; + return this; +} + +Benchmark* Benchmark::Complexity(BigOFunc* complexity) { + complexity_lambda_ = complexity; + complexity_ = oLambda; + return this; +} + +Benchmark* Benchmark::Threads(int t) { + CHECK_GT(t, 0); + thread_counts_.push_back(t); + return this; +} + +Benchmark* Benchmark::ThreadRange(int min_threads, int max_threads) { + CHECK_GT(min_threads, 0); + CHECK_GE(max_threads, min_threads); + + AddRange(&thread_counts_, min_threads, max_threads, 2); + return this; +} + +Benchmark* Benchmark::DenseThreadRange(int min_threads, int max_threads, + int stride) { + CHECK_GT(min_threads, 0); + CHECK_GE(max_threads, min_threads); + CHECK_GE(stride, 1); + + for (auto i = min_threads; i < max_threads; i += stride) { + thread_counts_.push_back(i); + } + thread_counts_.push_back(max_threads); + return this; +} + +Benchmark* Benchmark::ThreadPerCpu() { + static int num_cpus = NumCPUs(); + thread_counts_.push_back(num_cpus); + return this; +} + +void Benchmark::SetName(const char* name) { name_ = name; } + +int Benchmark::ArgsCnt() const { + if (args_.empty()) { + if (arg_names_.empty()) return -1; + return static_cast(arg_names_.size()); + } + return static_cast(args_.front().size()); +} + +//=============================================================================// +// FunctionBenchmark +//=============================================================================// + +void FunctionBenchmark::Run(State& st) { func_(st); } + +} // end namespace internal + +void ClearRegisteredBenchmarks() { + internal::BenchmarkFamilies::GetInstance()->ClearBenchmarks(); +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/check.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/check.h new file mode 100644 index 00000000000..73bead2fb55 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/check.h @@ -0,0 +1,79 @@ +#ifndef CHECK_H_ +#define CHECK_H_ + +#include +#include +#include + +#include "internal_macros.h" +#include "log.h" + +namespace benchmark { +namespace internal { + +typedef void(AbortHandlerT)(); + +inline AbortHandlerT*& GetAbortHandler() { + static AbortHandlerT* handler = &std::abort; + return handler; +} + +BENCHMARK_NORETURN inline void CallAbortHandler() { + GetAbortHandler()(); + std::abort(); // fallback to enforce noreturn +} + +// CheckHandler is the class constructed by failing CHECK macros. CheckHandler +// will log information about the failures and abort when it is destructed. +class CheckHandler { + public: + CheckHandler(const char* check, const char* file, const char* func, int line) + : log_(GetErrorLogInstance()) { + log_ << file << ":" << line << ": " << func << ": Check `" << check + << "' failed. "; + } + + LogType& GetLog() { return log_; } + + BENCHMARK_NORETURN ~CheckHandler() BENCHMARK_NOEXCEPT_OP(false) { + log_ << std::endl; + CallAbortHandler(); + } + + CheckHandler& operator=(const CheckHandler&) = delete; + CheckHandler(const CheckHandler&) = delete; + CheckHandler() = delete; + + private: + LogType& log_; +}; + +} // end namespace internal +} // end namespace benchmark + +// The CHECK macro returns a std::ostream object that can have extra information +// written to it. +#ifndef NDEBUG +#define CHECK(b) \ + (b ? ::benchmark::internal::GetNullLogInstance() \ + : ::benchmark::internal::CheckHandler(#b, __FILE__, __func__, __LINE__) \ + .GetLog()) +#else +#define CHECK(b) ::benchmark::internal::GetNullLogInstance() +#endif + +#define CHECK_EQ(a, b) CHECK((a) == (b)) +#define CHECK_NE(a, b) CHECK((a) != (b)) +#define CHECK_GE(a, b) CHECK((a) >= (b)) +#define CHECK_LE(a, b) CHECK((a) <= (b)) +#define CHECK_GT(a, b) CHECK((a) > (b)) +#define CHECK_LT(a, b) CHECK((a) < (b)) + +#define CHECK_FLOAT_EQ(a, b, eps) CHECK(std::fabs((a) - (b)) < (eps)) +#define CHECK_FLOAT_NE(a, b, eps) CHECK(std::fabs((a) - (b)) >= (eps)) +#define CHECK_FLOAT_GE(a, b, eps) CHECK((a) - (b) > -(eps)) +#define CHECK_FLOAT_LE(a, b, eps) CHECK((b) - (a) > -(eps)) +#define CHECK_FLOAT_GT(a, b, eps) CHECK((a) - (b) > (eps)) +#define CHECK_FLOAT_LT(a, b, eps) CHECK((b) - (a) > (eps)) + +#endif // CHECK_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/colorprint.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/colorprint.cc new file mode 100644 index 00000000000..2dec4a8b28b --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/colorprint.cc @@ -0,0 +1,188 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "colorprint.h" + +#include +#include +#include +#include +#include +#include + +#include "check.h" +#include "internal_macros.h" + +#ifdef BENCHMARK_OS_WINDOWS +#include +#include +#else +#include +#endif // BENCHMARK_OS_WINDOWS + +namespace benchmark { +namespace { +#ifdef BENCHMARK_OS_WINDOWS +typedef WORD PlatformColorCode; +#else +typedef const char* PlatformColorCode; +#endif + +PlatformColorCode GetPlatformColorCode(LogColor color) { +#ifdef BENCHMARK_OS_WINDOWS + switch (color) { + case COLOR_RED: + return FOREGROUND_RED; + case COLOR_GREEN: + return FOREGROUND_GREEN; + case COLOR_YELLOW: + return FOREGROUND_RED | FOREGROUND_GREEN; + case COLOR_BLUE: + return FOREGROUND_BLUE; + case COLOR_MAGENTA: + return FOREGROUND_BLUE | FOREGROUND_RED; + case COLOR_CYAN: + return FOREGROUND_BLUE | FOREGROUND_GREEN; + case COLOR_WHITE: // fall through to default + default: + return 0; + } +#else + switch (color) { + case COLOR_RED: + return "1"; + case COLOR_GREEN: + return "2"; + case COLOR_YELLOW: + return "3"; + case COLOR_BLUE: + return "4"; + case COLOR_MAGENTA: + return "5"; + case COLOR_CYAN: + return "6"; + case COLOR_WHITE: + return "7"; + default: + return nullptr; + }; +#endif +} + +} // end namespace + +std::string FormatString(const char* msg, va_list args) { + // we might need a second shot at this, so pre-emptivly make a copy + va_list args_cp; + va_copy(args_cp, args); + + std::size_t size = 256; + char local_buff[256]; + auto ret = vsnprintf(local_buff, size, msg, args_cp); + + va_end(args_cp); + + // currently there is no error handling for failure, so this is hack. + CHECK(ret >= 0); + + if (ret == 0) // handle empty expansion + return {}; + else if (static_cast(ret) < size) + return local_buff; + else { + // we did not provide a long enough buffer on our first attempt. + size = (size_t)ret + 1; // + 1 for the null byte + std::unique_ptr buff(new char[size]); + ret = vsnprintf(buff.get(), size, msg, args); + CHECK(ret > 0 && ((size_t)ret) < size); + return buff.get(); + } +} + +std::string FormatString(const char* msg, ...) { + va_list args; + va_start(args, msg); + auto tmp = FormatString(msg, args); + va_end(args); + return tmp; +} + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + ColorPrintf(out, color, fmt, args); + va_end(args); +} + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, + va_list args) { +#ifdef BENCHMARK_OS_WINDOWS + ((void)out); // suppress unused warning + + const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + + // Gets the current text color. + CONSOLE_SCREEN_BUFFER_INFO buffer_info; + GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); + const WORD old_color_attrs = buffer_info.wAttributes; + + // We need to flush the stream buffers into the console before each + // SetConsoleTextAttribute call lest it affect the text that is already + // printed but has not yet reached the console. + fflush(stdout); + SetConsoleTextAttribute(stdout_handle, + GetPlatformColorCode(color) | FOREGROUND_INTENSITY); + vprintf(fmt, args); + + fflush(stdout); + // Restores the text color. + SetConsoleTextAttribute(stdout_handle, old_color_attrs); +#else + const char* color_code = GetPlatformColorCode(color); + if (color_code) out << FormatString("\033[0;3%sm", color_code); + out << FormatString(fmt, args) << "\033[m"; +#endif +} + +bool IsColorTerminal() { +#if BENCHMARK_OS_WINDOWS + // On Windows the TERM variable is usually not set, but the + // console there does support colors. + return 0 != _isatty(_fileno(stdout)); +#else + // On non-Windows platforms, we rely on the TERM variable. This list of + // supported TERM values is copied from Google Test: + // . + const char* const SUPPORTED_TERM_VALUES[] = { + "xterm", "xterm-color", "xterm-256color", + "screen", "screen-256color", "tmux", + "tmux-256color", "rxvt-unicode", "rxvt-unicode-256color", + "linux", "cygwin", + }; + + const char* const term = getenv("TERM"); + + bool term_supports_color = false; + for (const char* candidate : SUPPORTED_TERM_VALUES) { + if (term && 0 == strcmp(term, candidate)) { + term_supports_color = true; + break; + } + } + + return 0 != isatty(fileno(stdout)) && term_supports_color; +#endif // BENCHMARK_OS_WINDOWS +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/colorprint.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/colorprint.h new file mode 100644 index 00000000000..9f6fab9b342 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/colorprint.h @@ -0,0 +1,33 @@ +#ifndef BENCHMARK_COLORPRINT_H_ +#define BENCHMARK_COLORPRINT_H_ + +#include +#include +#include + +namespace benchmark { +enum LogColor { + COLOR_DEFAULT, + COLOR_RED, + COLOR_GREEN, + COLOR_YELLOW, + COLOR_BLUE, + COLOR_MAGENTA, + COLOR_CYAN, + COLOR_WHITE +}; + +std::string FormatString(const char* msg, va_list args); +std::string FormatString(const char* msg, ...); + +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, + va_list args); +void ColorPrintf(std::ostream& out, LogColor color, const char* fmt, ...); + +// Returns true if stdout appears to be a terminal that supports colored +// output, false otherwise. +bool IsColorTerminal(); + +} // end namespace benchmark + +#endif // BENCHMARK_COLORPRINT_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/commandlineflags.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/commandlineflags.cc new file mode 100644 index 00000000000..2fc92517a32 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/commandlineflags.cc @@ -0,0 +1,218 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "commandlineflags.h" + +#include +#include +#include +#include +#include + +namespace benchmark { +// Parses 'str' for a 32-bit signed integer. If successful, writes +// the result to *value and returns true; otherwise leaves *value +// unchanged and returns false. +bool ParseInt32(const std::string& src_text, const char* str, int32_t* value) { + // Parses the environment variable as a decimal integer. + char* end = nullptr; + const long long_value = strtol(str, &end, 10); // NOLINT + + // Has strtol() consumed all characters in the string? + if (*end != '\0') { + // No - an invalid character was encountered. + std::cerr << src_text << " is expected to be a 32-bit integer, " + << "but actually has value \"" << str << "\".\n"; + return false; + } + + // Is the parsed value in the range of an Int32? + const int32_t result = static_cast(long_value); + if (long_value == std::numeric_limits::max() || + long_value == std::numeric_limits::min() || + // The parsed value overflows as a long. (strtol() returns + // LONG_MAX or LONG_MIN when the input overflows.) + result != long_value + // The parsed value overflows as an Int32. + ) { + std::cerr << src_text << " is expected to be a 32-bit integer, " + << "but actually has value \"" << str << "\", " + << "which overflows.\n"; + return false; + } + + *value = result; + return true; +} + +// Parses 'str' for a double. If successful, writes the result to *value and +// returns true; otherwise leaves *value unchanged and returns false. +bool ParseDouble(const std::string& src_text, const char* str, double* value) { + // Parses the environment variable as a decimal integer. + char* end = nullptr; + const double double_value = strtod(str, &end); // NOLINT + + // Has strtol() consumed all characters in the string? + if (*end != '\0') { + // No - an invalid character was encountered. + std::cerr << src_text << " is expected to be a double, " + << "but actually has value \"" << str << "\".\n"; + return false; + } + + *value = double_value; + return true; +} + +// Returns the name of the environment variable corresponding to the +// given flag. For example, FlagToEnvVar("foo") will return +// "BENCHMARK_FOO" in the open-source version. +static std::string FlagToEnvVar(const char* flag) { + const std::string flag_str(flag); + + std::string env_var; + for (size_t i = 0; i != flag_str.length(); ++i) + env_var += static_cast(::toupper(flag_str.c_str()[i])); + + return "BENCHMARK_" + env_var; +} + +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromEnv(const char* flag, bool default_value) { + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = getenv(env_var.c_str()); + return string_value == nullptr ? default_value + : strcmp(string_value, "0") != 0; +} + +// Reads and returns a 32-bit integer stored in the environment +// variable corresponding to the given flag; if it isn't set or +// doesn't represent a valid 32-bit integer, returns default_value. +int32_t Int32FromEnv(const char* flag, int32_t default_value) { + const std::string env_var = FlagToEnvVar(flag); + const char* const string_value = getenv(env_var.c_str()); + if (string_value == nullptr) { + // The environment variable is not set. + return default_value; + } + + int32_t result = default_value; + if (!ParseInt32(std::string("Environment variable ") + env_var, string_value, + &result)) { + std::cout << "The default value " << default_value << " is used.\n"; + return default_value; + } + + return result; +} + +// Reads and returns the string environment variable corresponding to +// the given flag; if it's not set, returns default_value. +const char* StringFromEnv(const char* flag, const char* default_value) { + const std::string env_var = FlagToEnvVar(flag); + const char* const value = getenv(env_var.c_str()); + return value == nullptr ? default_value : value; +} + +// Parses a string as a command line flag. The string should have +// the format "--flag=value". When def_optional is true, the "=value" +// part can be omitted. +// +// Returns the value of the flag, or nullptr if the parsing failed. +const char* ParseFlagValue(const char* str, const char* flag, + bool def_optional) { + // str and flag must not be nullptr. + if (str == nullptr || flag == nullptr) return nullptr; + + // The flag must start with "--". + const std::string flag_str = std::string("--") + std::string(flag); + const size_t flag_len = flag_str.length(); + if (strncmp(str, flag_str.c_str(), flag_len) != 0) return nullptr; + + // Skips the flag name. + const char* flag_end = str + flag_len; + + // When def_optional is true, it's OK to not have a "=value" part. + if (def_optional && (flag_end[0] == '\0')) return flag_end; + + // If def_optional is true and there are more characters after the + // flag name, or if def_optional is false, there must be a '=' after + // the flag name. + if (flag_end[0] != '=') return nullptr; + + // Returns the string after "=". + return flag_end + 1; +} + +bool ParseBoolFlag(const char* str, const char* flag, bool* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, true); + + // Aborts if the parsing failed. + if (value_str == nullptr) return false; + + // Converts the string value to a bool. + *value = IsTruthyFlagValue(value_str); + return true; +} + +bool ParseInt32Flag(const char* str, const char* flag, int32_t* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == nullptr) return false; + + // Sets *value to the value of the flag. + return ParseInt32(std::string("The value of flag --") + flag, value_str, + value); +} + +bool ParseDoubleFlag(const char* str, const char* flag, double* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == nullptr) return false; + + // Sets *value to the value of the flag. + return ParseDouble(std::string("The value of flag --") + flag, value_str, + value); +} + +bool ParseStringFlag(const char* str, const char* flag, std::string* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == nullptr) return false; + + *value = value_str; + return true; +} + +bool IsFlag(const char* str, const char* flag) { + return (ParseFlagValue(str, flag, true) != nullptr); +} + +bool IsTruthyFlagValue(const std::string& value) { + if (value.empty()) return true; + char ch = value[0]; + return isalnum(ch) && + !(ch == '0' || ch == 'f' || ch == 'F' || ch == 'n' || ch == 'N'); +} +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/commandlineflags.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/commandlineflags.h new file mode 100644 index 00000000000..945c9a9fc4a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/commandlineflags.h @@ -0,0 +1,79 @@ +#ifndef BENCHMARK_COMMANDLINEFLAGS_H_ +#define BENCHMARK_COMMANDLINEFLAGS_H_ + +#include +#include + +// Macro for referencing flags. +#define FLAG(name) FLAGS_##name + +// Macros for declaring flags. +#define DECLARE_bool(name) extern bool FLAG(name) +#define DECLARE_int32(name) extern int32_t FLAG(name) +#define DECLARE_int64(name) extern int64_t FLAG(name) +#define DECLARE_double(name) extern double FLAG(name) +#define DECLARE_string(name) extern std::string FLAG(name) + +// Macros for defining flags. +#define DEFINE_bool(name, default_val, doc) bool FLAG(name) = (default_val) +#define DEFINE_int32(name, default_val, doc) int32_t FLAG(name) = (default_val) +#define DEFINE_int64(name, default_val, doc) int64_t FLAG(name) = (default_val) +#define DEFINE_double(name, default_val, doc) double FLAG(name) = (default_val) +#define DEFINE_string(name, default_val, doc) \ + std::string FLAG(name) = (default_val) + +namespace benchmark { +// Parses 'str' for a 32-bit signed integer. If successful, writes the result +// to *value and returns true; otherwise leaves *value unchanged and returns +// false. +bool ParseInt32(const std::string& src_text, const char* str, int32_t* value); + +// Parses a bool/Int32/string from the environment variable +// corresponding to the given Google Test flag. +bool BoolFromEnv(const char* flag, bool default_val); +int32_t Int32FromEnv(const char* flag, int32_t default_val); +double DoubleFromEnv(const char* flag, double default_val); +const char* StringFromEnv(const char* flag, const char* default_val); + +// Parses a string for a bool flag, in the form of either +// "--flag=value" or "--flag". +// +// In the former case, the value is taken as true if it passes IsTruthyValue(). +// +// In the latter case, the value is taken as true. +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseBoolFlag(const char* str, const char* flag, bool* value); + +// Parses a string for an Int32 flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseInt32Flag(const char* str, const char* flag, int32_t* value); + +// Parses a string for a Double flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseDoubleFlag(const char* str, const char* flag, double* value); + +// Parses a string for a string flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseStringFlag(const char* str, const char* flag, std::string* value); + +// Returns true if the string matches the flag. +bool IsFlag(const char* str, const char* flag); + +// Returns true unless value starts with one of: '0', 'f', 'F', 'n' or 'N', or +// some non-alphanumeric character. As a special case, also returns true if +// value is the empty string. +bool IsTruthyFlagValue(const std::string& value); +} // end namespace benchmark + +#endif // BENCHMARK_COMMANDLINEFLAGS_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/complexity.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/complexity.cc new file mode 100644 index 00000000000..33975be55ec --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/complexity.cc @@ -0,0 +1,324 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Adapted to be used with google benchmark + +#include "benchmark/benchmark.h" + +#include +#include +#include "check.h" +#include "complexity.h" +#include "stat.h" + +namespace benchmark { + +// Internal function to calculate the different scalability forms +BigOFunc* FittingCurve(BigO complexity) { + switch (complexity) { + case oN: + return [](int n) -> double { return n; }; + case oNSquared: + return [](int n) -> double { return std::pow(n, 2); }; + case oNCubed: + return [](int n) -> double { return std::pow(n, 3); }; + case oLogN: + return [](int n) { return log2(n); }; + case oNLogN: + return [](int n) { return n * log2(n); }; + case o1: + default: + return [](int) { return 1.0; }; + } +} + +// Function to return an string for the calculated complexity +std::string GetBigOString(BigO complexity) { + switch (complexity) { + case oN: + return "N"; + case oNSquared: + return "N^2"; + case oNCubed: + return "N^3"; + case oLogN: + return "lgN"; + case oNLogN: + return "NlgN"; + case o1: + return "(1)"; + default: + return "f(N)"; + } +} + +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error, for the fitting curve +// given by the lambda expresion. +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). + +// For a deeper explanation on the algorithm logic, look the README file at +// http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit + +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, + BigOFunc* fitting_curve) { + double sigma_gn = 0.0; + double sigma_gn_squared = 0.0; + double sigma_time = 0.0; + double sigma_time_gn = 0.0; + + // Calculate least square fitting parameter + for (size_t i = 0; i < n.size(); ++i) { + double gn_i = fitting_curve(n[i]); + sigma_gn += gn_i; + sigma_gn_squared += gn_i * gn_i; + sigma_time += time[i]; + sigma_time_gn += time[i] * gn_i; + } + + LeastSq result; + result.complexity = oLambda; + + // Calculate complexity. + result.coef = sigma_time_gn / sigma_gn_squared; + + // Calculate RMS + double rms = 0.0; + for (size_t i = 0; i < n.size(); ++i) { + double fit = result.coef * fitting_curve(n[i]); + rms += pow((time[i] - fit), 2); + } + + // Normalized RMS by the mean of the observed values + double mean = sigma_time / n.size(); + result.rms = sqrt(rms / n.size()) / mean; + + return result; +} + +// Find the coefficient for the high-order term in the running time, by +// minimizing the sum of squares of relative error. +// - n : Vector containing the size of the benchmark tests. +// - time : Vector containing the times for the benchmark tests. +// - complexity : If different than oAuto, the fitting curve will stick to +// this one. If it is oAuto, it will be calculated the best +// fitting curve. +LeastSq MinimalLeastSq(const std::vector& n, + const std::vector& time, const BigO complexity) { + CHECK_EQ(n.size(), time.size()); + CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two + // benchmark runs are given + CHECK_NE(complexity, oNone); + + LeastSq best_fit; + + if (complexity == oAuto) { + std::vector fit_curves = {oLogN, oN, oNLogN, oNSquared, oNCubed}; + + // Take o1 as default best fitting curve + best_fit = MinimalLeastSq(n, time, FittingCurve(o1)); + best_fit.complexity = o1; + + // Compute all possible fitting curves and stick to the best one + for (const auto& fit : fit_curves) { + LeastSq current_fit = MinimalLeastSq(n, time, FittingCurve(fit)); + if (current_fit.rms < best_fit.rms) { + best_fit = current_fit; + best_fit.complexity = fit; + } + } + } else { + best_fit = MinimalLeastSq(n, time, FittingCurve(complexity)); + best_fit.complexity = complexity; + } + + return best_fit; +} + +std::vector ComputeStats( + const std::vector& reports) { + typedef BenchmarkReporter::Run Run; + std::vector results; + + auto error_count = + std::count_if(reports.begin(), reports.end(), + [](Run const& run) { return run.error_occurred; }); + + if (reports.size() - error_count < 2) { + // We don't report aggregated data if there was a single run. + return results; + } + // Accumulators. + Stat1_d real_accumulated_time_stat; + Stat1_d cpu_accumulated_time_stat; + Stat1_d bytes_per_second_stat; + Stat1_d items_per_second_stat; + // All repetitions should be run with the same number of iterations so we + // can take this information from the first benchmark. + int64_t const run_iterations = reports.front().iterations; + // create stats for user counters + struct CounterStat { + Counter c; + Stat1_d s; + }; + std::map< std::string, CounterStat > counter_stats; + for(Run const& r : reports) { + for(auto const& cnt : r.counters) { + auto it = counter_stats.find(cnt.first); + if(it == counter_stats.end()) { + counter_stats.insert({cnt.first, {cnt.second, Stat1_d{}}}); + } else { + CHECK_EQ(counter_stats[cnt.first].c.flags, cnt.second.flags); + } + } + } + + // Populate the accumulators. + for (Run const& run : reports) { + CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); + CHECK_EQ(run_iterations, run.iterations); + if (run.error_occurred) continue; + real_accumulated_time_stat += + Stat1_d(run.real_accumulated_time / run.iterations); + cpu_accumulated_time_stat += + Stat1_d(run.cpu_accumulated_time / run.iterations); + items_per_second_stat += Stat1_d(run.items_per_second); + bytes_per_second_stat += Stat1_d(run.bytes_per_second); + // user counters + for(auto const& cnt : run.counters) { + auto it = counter_stats.find(cnt.first); + CHECK_NE(it, counter_stats.end()); + it->second.s += Stat1_d(cnt.second); + } + } + + // Get the data from the accumulator to BenchmarkReporter::Run's. + Run mean_data; + mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; + mean_data.iterations = run_iterations; + mean_data.real_accumulated_time = + real_accumulated_time_stat.Mean() * run_iterations; + mean_data.cpu_accumulated_time = + cpu_accumulated_time_stat.Mean() * run_iterations; + mean_data.bytes_per_second = bytes_per_second_stat.Mean(); + mean_data.items_per_second = items_per_second_stat.Mean(); + mean_data.time_unit = reports[0].time_unit; + // user counters + for(auto const& kv : counter_stats) { + auto c = Counter(kv.second.s.Mean(), counter_stats[kv.first].c.flags); + mean_data.counters[kv.first] = c; + } + + // Only add label to mean/stddev if it is same for all runs + mean_data.report_label = reports[0].report_label; + for (std::size_t i = 1; i < reports.size(); i++) { + if (reports[i].report_label != reports[0].report_label) { + mean_data.report_label = ""; + break; + } + } + + Run stddev_data; + stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; + stddev_data.report_label = mean_data.report_label; + stddev_data.iterations = 0; + stddev_data.real_accumulated_time = real_accumulated_time_stat.StdDev(); + stddev_data.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); + stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); + stddev_data.items_per_second = items_per_second_stat.StdDev(); + stddev_data.time_unit = reports[0].time_unit; + // user counters + for(auto const& kv : counter_stats) { + auto c = Counter(kv.second.s.StdDev(), counter_stats[kv.first].c.flags); + stddev_data.counters[kv.first] = c; + } + + results.push_back(mean_data); + results.push_back(stddev_data); + return results; +} + +std::vector ComputeBigO( + const std::vector& reports) { + typedef BenchmarkReporter::Run Run; + std::vector results; + + if (reports.size() < 2) return results; + + // Accumulators. + std::vector n; + std::vector real_time; + std::vector cpu_time; + + // Populate the accumulators. + for (const Run& run : reports) { + CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; + n.push_back(run.complexity_n); + real_time.push_back(run.real_accumulated_time / run.iterations); + cpu_time.push_back(run.cpu_accumulated_time / run.iterations); + } + + LeastSq result_cpu; + LeastSq result_real; + + if (reports[0].complexity == oLambda) { + result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); + result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); + } else { + result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); + result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); + } + std::string benchmark_name = + reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); + + // Get the data from the accumulator to BenchmarkReporter::Run's. + Run big_o; + big_o.benchmark_name = benchmark_name + "_BigO"; + big_o.iterations = 0; + big_o.real_accumulated_time = result_real.coef; + big_o.cpu_accumulated_time = result_cpu.coef; + big_o.report_big_o = true; + big_o.complexity = result_cpu.complexity; + + // All the time results are reported after being multiplied by the + // time unit multiplier. But since RMS is a relative quantity it + // should not be multiplied at all. So, here, we _divide_ it by the + // multiplier so that when it is multiplied later the result is the + // correct one. + double multiplier = GetTimeUnitMultiplier(reports[0].time_unit); + + // Only add label to mean/stddev if it is same for all runs + Run rms; + big_o.report_label = reports[0].report_label; + rms.benchmark_name = benchmark_name + "_RMS"; + rms.report_label = big_o.report_label; + rms.iterations = 0; + rms.real_accumulated_time = result_real.rms / multiplier; + rms.cpu_accumulated_time = result_cpu.rms / multiplier; + rms.report_rms = true; + rms.complexity = result_cpu.complexity; + // don't forget to keep the time unit, or we won't be able to + // recover the correct value. + rms.time_unit = reports[0].time_unit; + + results.push_back(big_o); + results.push_back(rms); + return results; +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/complexity.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/complexity.h new file mode 100644 index 00000000000..c0ca60e6bec --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/complexity.h @@ -0,0 +1,60 @@ +// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Source project : https://github.com/ismaelJimenez/cpp.leastsq +// Adapted to be used with google benchmark + +#ifndef COMPLEXITY_H_ +#define COMPLEXITY_H_ + +#include +#include + +#include "benchmark/benchmark.h" + +namespace benchmark { + +// Return a vector containing the mean and standard devation information for +// the specified list of reports. If 'reports' contains less than two +// non-errored runs an empty vector is returned +std::vector ComputeStats( + const std::vector& reports); + +// Return a vector containing the bigO and RMS information for the specified +// list of reports. If 'reports.size() < 2' an empty vector is returned. +std::vector ComputeBigO( + const std::vector& reports); + +// This data structure will contain the result returned by MinimalLeastSq +// - coef : Estimated coeficient for the high-order term as +// interpolated from data. +// - rms : Normalized Root Mean Squared Error. +// - complexity : Scalability form (e.g. oN, oNLogN). In case a scalability +// form has been provided to MinimalLeastSq this will return +// the same value. In case BigO::oAuto has been selected, this +// parameter will return the best fitting curve detected. + +struct LeastSq { + LeastSq() : coef(0.0), rms(0.0), complexity(oNone) {} + + double coef; + double rms; + BigO complexity; +}; + +// Function to return an string for the calculated complexity +std::string GetBigOString(BigO complexity); + +} // end namespace benchmark +#endif // COMPLEXITY_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/console_reporter.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/console_reporter.cc new file mode 100644 index 00000000000..4bb6f71271c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/console_reporter.cc @@ -0,0 +1,180 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "complexity.h" +#include "counter.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "check.h" +#include "colorprint.h" +#include "commandlineflags.h" +#include "internal_macros.h" +#include "string_util.h" +#include "timers.h" + +namespace benchmark { + +bool ConsoleReporter::ReportContext(const Context& context) { + name_field_width_ = context.name_field_width; + printed_header_ = false; + prev_counters_.clear(); + + PrintBasicContext(&GetErrorStream(), context); + +#ifdef BENCHMARK_OS_WINDOWS + if ((output_options_ & OO_Color) && &std::cout != &GetOutputStream()) { + GetErrorStream() + << "Color printing is only supported for stdout on windows." + " Disabling color printing\n"; + output_options_ = static_cast< OutputOptions >(output_options_ & ~OO_Color); + } +#endif + + return true; +} + +void ConsoleReporter::PrintHeader(const Run& run) { + std::string str = FormatString("%-*s %13s %13s %10s", static_cast(name_field_width_), + "Benchmark", "Time", "CPU", "Iterations"); + if(!run.counters.empty()) { + if(output_options_ & OO_Tabular) { + for(auto const& c : run.counters) { + str += FormatString(" %10s", c.first.c_str()); + } + } else { + str += " UserCounters..."; + } + } + str += "\n"; + std::string line = std::string(str.length(), '-'); + GetOutputStream() << line << "\n" << str << line << "\n"; +} + +void ConsoleReporter::ReportRuns(const std::vector& reports) { + for (const auto& run : reports) { + // print the header: + // --- if none was printed yet + bool print_header = !printed_header_; + // --- or if the format is tabular and this run + // has different fields from the prev header + print_header |= (output_options_ & OO_Tabular) && + (!internal::SameNames(run.counters, prev_counters_)); + if (print_header) { + printed_header_ = true; + prev_counters_ = run.counters; + PrintHeader(run); + } + // As an alternative to printing the headers like this, we could sort + // the benchmarks by header and then print. But this would require + // waiting for the full results before printing, or printing twice. + PrintRunData(run); + } +} + +static void IgnoreColorPrint(std::ostream& out, LogColor, const char* fmt, + ...) { + va_list args; + va_start(args, fmt); + out << FormatString(fmt, args); + va_end(args); +} + +void ConsoleReporter::PrintRunData(const Run& result) { + typedef void(PrinterFn)(std::ostream&, LogColor, const char*, ...); + auto& Out = GetOutputStream(); + PrinterFn* printer = (output_options_ & OO_Color) ? + (PrinterFn*)ColorPrintf : IgnoreColorPrint; + auto name_color = + (result.report_big_o || result.report_rms) ? COLOR_BLUE : COLOR_GREEN; + printer(Out, name_color, "%-*s ", name_field_width_, + result.benchmark_name.c_str()); + + if (result.error_occurred) { + printer(Out, COLOR_RED, "ERROR OCCURRED: \'%s\'", + result.error_message.c_str()); + printer(Out, COLOR_DEFAULT, "\n"); + return; + } + // Format bytes per second + std::string rate; + if (result.bytes_per_second > 0) { + rate = StrCat(" ", HumanReadableNumber(result.bytes_per_second), "B/s"); + } + + // Format items per second + std::string items; + if (result.items_per_second > 0) { + items = + StrCat(" ", HumanReadableNumber(result.items_per_second), " items/s"); + } + + const double real_time = result.GetAdjustedRealTime(); + const double cpu_time = result.GetAdjustedCPUTime(); + + if (result.report_big_o) { + std::string big_o = GetBigOString(result.complexity); + printer(Out, COLOR_YELLOW, "%10.2f %s %10.2f %s ", real_time, big_o.c_str(), + cpu_time, big_o.c_str()); + } else if (result.report_rms) { + printer(Out, COLOR_YELLOW, "%10.0f %% %10.0f %% ", real_time * 100, + cpu_time * 100); + } else { + const char* timeLabel = GetTimeUnitString(result.time_unit); + printer(Out, COLOR_YELLOW, "%10.0f %s %10.0f %s ", real_time, timeLabel, + cpu_time, timeLabel); + } + + if (!result.report_big_o && !result.report_rms) { + printer(Out, COLOR_CYAN, "%10lld", result.iterations); + } + + for (auto& c : result.counters) { + auto const& s = HumanReadableNumber(c.second.value); + if (output_options_ & OO_Tabular) { + if (c.second.flags & Counter::kIsRate) { + printer(Out, COLOR_DEFAULT, " %8s/s", s.c_str()); + } else { + printer(Out, COLOR_DEFAULT, " %10s", s.c_str()); + } + } else { + const char* unit = (c.second.flags & Counter::kIsRate) ? "/s" : ""; + printer(Out, COLOR_DEFAULT, " %s=%s%s", c.first.c_str(), s.c_str(), + unit); + } + } + + if (!rate.empty()) { + printer(Out, COLOR_DEFAULT, " %*s", 13, rate.c_str()); + } + + if (!items.empty()) { + printer(Out, COLOR_DEFAULT, " %*s", 18, items.c_str()); + } + + if (!result.report_label.empty()) { + printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); + } + + printer(Out, COLOR_DEFAULT, "\n"); +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/counter.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/counter.cc new file mode 100644 index 00000000000..ed1aa044ee7 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/counter.cc @@ -0,0 +1,68 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "counter.h" + +namespace benchmark { +namespace internal { + +double Finish(Counter const& c, double cpu_time, double num_threads) { + double v = c.value; + if (c.flags & Counter::kIsRate) { + v /= cpu_time; + } + if (c.flags & Counter::kAvgThreads) { + v /= num_threads; + } + return v; +} + +void Finish(UserCounters *l, double cpu_time, double num_threads) { + for (auto &c : *l) { + c.second.value = Finish(c.second, cpu_time, num_threads); + } +} + +void Increment(UserCounters *l, UserCounters const& r) { + // add counters present in both or just in *l + for (auto &c : *l) { + auto it = r.find(c.first); + if (it != r.end()) { + c.second.value = c.second + it->second; + } + } + // add counters present in r, but not in *l + for (auto const &tc : r) { + auto it = l->find(tc.first); + if (it == l->end()) { + (*l)[tc.first] = tc.second; + } + } +} + +bool SameNames(UserCounters const& l, UserCounters const& r) { + if (&l == &r) return true; + if (l.size() != r.size()) { + return false; + } + for (auto const& c : l) { + if (r.find(c.first) == r.end()) { + return false; + } + } + return true; +} + +} // end namespace internal +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/counter.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/counter.h new file mode 100644 index 00000000000..dd6865a31d7 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/counter.h @@ -0,0 +1,26 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" + +namespace benchmark { + +// these counter-related functions are hidden to reduce API surface. +namespace internal { +void Finish(UserCounters *l, double time, double num_threads); +void Increment(UserCounters *l, UserCounters const& r); +bool SameNames(UserCounters const& l, UserCounters const& r); +} // end namespace internal + +} //end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/csv_reporter.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/csv_reporter.cc new file mode 100644 index 00000000000..35510645b08 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/csv_reporter.cc @@ -0,0 +1,149 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "complexity.h" + +#include +#include +#include +#include +#include +#include + +#include "string_util.h" +#include "timers.h" +#include "check.h" + +// File format reference: http://edoceo.com/utilitas/csv-file-format. + +namespace benchmark { + +namespace { +std::vector elements = { + "name", "iterations", "real_time", "cpu_time", + "time_unit", "bytes_per_second", "items_per_second", "label", + "error_occurred", "error_message"}; +} // namespace + +bool CSVReporter::ReportContext(const Context& context) { + PrintBasicContext(&GetErrorStream(), context); + return true; +} + +void CSVReporter::ReportRuns(const std::vector & reports) { + std::ostream& Out = GetOutputStream(); + + if (!printed_header_) { + // save the names of all the user counters + for (const auto& run : reports) { + for (const auto& cnt : run.counters) { + user_counter_names_.insert(cnt.first); + } + } + + // print the header + for (auto B = elements.begin(); B != elements.end();) { + Out << *B++; + if (B != elements.end()) Out << ","; + } + for (auto B = user_counter_names_.begin(); B != user_counter_names_.end();) { + Out << ",\"" << *B++ << "\""; + } + Out << "\n"; + + printed_header_ = true; + } else { + // check that all the current counters are saved in the name set + for (const auto& run : reports) { + for (const auto& cnt : run.counters) { + CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) + << "All counters must be present in each run. " + << "Counter named \"" << cnt.first + << "\" was not in a run after being added to the header"; + } + } + } + + // print results for each run + for (const auto& run : reports) { + PrintRunData(run); + } + +} + +void CSVReporter::PrintRunData(const Run & run) { + std::ostream& Out = GetOutputStream(); + + // Field with embedded double-quote characters must be doubled and the field + // delimited with double-quotes. + std::string name = run.benchmark_name; + ReplaceAll(&name, "\"", "\"\""); + Out << '"' << name << "\","; + if (run.error_occurred) { + Out << std::string(elements.size() - 3, ','); + Out << "true,"; + std::string msg = run.error_message; + ReplaceAll(&msg, "\"", "\"\""); + Out << '"' << msg << "\"\n"; + return; + } + + // Do not print iteration on bigO and RMS report + if (!run.report_big_o && !run.report_rms) { + Out << run.iterations; + } + Out << ","; + + Out << run.GetAdjustedRealTime() << ","; + Out << run.GetAdjustedCPUTime() << ","; + + // Do not print timeLabel on bigO and RMS report + if (run.report_big_o) { + Out << GetBigOString(run.complexity); + } else if (!run.report_rms) { + Out << GetTimeUnitString(run.time_unit); + } + Out << ","; + + if (run.bytes_per_second > 0.0) { + Out << run.bytes_per_second; + } + Out << ","; + if (run.items_per_second > 0.0) { + Out << run.items_per_second; + } + Out << ","; + if (!run.report_label.empty()) { + // Field with embedded double-quote characters must be doubled and the field + // delimited with double-quotes. + std::string label = run.report_label; + ReplaceAll(&label, "\"", "\"\""); + Out << "\"" << label << "\""; + } + Out << ",,"; // for error_occurred and error_message + + // Print user counters + for (const auto &ucn : user_counter_names_) { + auto it = run.counters.find(ucn); + if(it == run.counters.end()) { + Out << ","; + } else { + Out << "," << it->second; + } + } + Out << '\n'; +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/cycleclock.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/cycleclock.h new file mode 100644 index 00000000000..4251fe4c32a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/cycleclock.h @@ -0,0 +1,172 @@ +// ---------------------------------------------------------------------- +// CycleClock +// A CycleClock tells you the current time in Cycles. The "time" +// is actually time since power-on. This is like time() but doesn't +// involve a system call and is much more precise. +// +// NOTE: Not all cpu/platform/kernel combinations guarantee that this +// clock increments at a constant rate or is synchronized across all logical +// cpus in a system. +// +// If you need the above guarantees, please consider using a different +// API. There are efforts to provide an interface which provides a millisecond +// granularity and implemented as a memory read. A memory read is generally +// cheaper than the CycleClock for many architectures. +// +// Also, in some out of order CPU implementations, the CycleClock is not +// serializing. So if you're trying to count at cycles granularity, your +// data might be inaccurate due to out of order instruction execution. +// ---------------------------------------------------------------------- + +#ifndef BENCHMARK_CYCLECLOCK_H_ +#define BENCHMARK_CYCLECLOCK_H_ + +#include + +#include "benchmark/benchmark.h" +#include "internal_macros.h" + +#if defined(BENCHMARK_OS_MACOSX) +#include +#endif +// For MSVC, we want to use '_asm rdtsc' when possible (since it works +// with even ancient MSVC compilers), and when not possible the +// __rdtsc intrinsic, declared in . Unfortunately, in some +// environments, and have conflicting +// declarations of some other intrinsics, breaking compilation. +// Therefore, we simply declare __rdtsc ourselves. See also +// http://connect.microsoft.com/VisualStudio/feedback/details/262047 +#if defined(COMPILER_MSVC) && !defined(_M_IX86) +extern "C" uint64_t __rdtsc(); +#pragma intrinsic(__rdtsc) +#endif + +#ifndef BENCHMARK_OS_WINDOWS +#include +#include +#endif + +#ifdef BENCHMARK_OS_EMSCRIPTEN +#include +#endif + +namespace benchmark { +// NOTE: only i386 and x86_64 have been well tested. +// PPC, sparc, alpha, and ia64 are based on +// http://peter.kuscsik.com/wordpress/?p=14 +// with modifications by m3b. See also +// https://setisvn.ssl.berkeley.edu/svn/lib/fftw-3.0.1/kernel/cycle.h +namespace cycleclock { +// This should return the number of cycles since power-on. Thread-safe. +inline BENCHMARK_ALWAYS_INLINE int64_t Now() { +#if defined(BENCHMARK_OS_MACOSX) + // this goes at the top because we need ALL Macs, regardless of + // architecture, to return the number of "mach time units" that + // have passed since startup. See sysinfo.cc where + // InitializeSystemInfo() sets the supposed cpu clock frequency of + // macs to the number of mach time units per second, not actual + // CPU clock frequency (which can change in the face of CPU + // frequency scaling). Also note that when the Mac sleeps, this + // counter pauses; it does not continue counting, nor does it + // reset to zero. + return mach_absolute_time(); +#elif defined(BENCHMARK_OS_EMSCRIPTEN) + // this goes above x86-specific code because old versions of Emscripten + // define __x86_64__, although they have nothing to do with it. + return static_cast(emscripten_get_now() * 1e+6); +#elif defined(__i386__) + int64_t ret; + __asm__ volatile("rdtsc" : "=A"(ret)); + return ret; +#elif defined(__x86_64__) || defined(__amd64__) + uint64_t low, high; + __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); + return (high << 32) | low; +#elif defined(__powerpc__) || defined(__ppc__) + // This returns a time-base, which is not always precisely a cycle-count. + int64_t tbl, tbu0, tbu1; + asm("mftbu %0" : "=r"(tbu0)); + asm("mftb %0" : "=r"(tbl)); + asm("mftbu %0" : "=r"(tbu1)); + tbl &= -static_cast(tbu0 == tbu1); + // high 32 bits in tbu1; low 32 bits in tbl (tbu0 is garbage) + return (tbu1 << 32) | tbl; +#elif defined(__sparc__) + int64_t tick; + asm(".byte 0x83, 0x41, 0x00, 0x00"); + asm("mov %%g1, %0" : "=r"(tick)); + return tick; +#elif defined(__ia64__) + int64_t itc; + asm("mov %0 = ar.itc" : "=r"(itc)); + return itc; +#elif defined(COMPILER_MSVC) && defined(_M_IX86) + // Older MSVC compilers (like 7.x) don't seem to support the + // __rdtsc intrinsic properly, so I prefer to use _asm instead + // when I know it will work. Otherwise, I'll use __rdtsc and hope + // the code is being compiled with a non-ancient compiler. + _asm rdtsc +#elif defined(COMPILER_MSVC) + return __rdtsc(); +#elif defined(BENCHMARK_OS_NACL) + // Native Client validator on x86/x86-64 allows RDTSC instructions, + // and this case is handled above. Native Client validator on ARM + // rejects MRC instructions (used in the ARM-specific sequence below), + // so we handle it here. Portable Native Client compiles to + // architecture-agnostic bytecode, which doesn't provide any + // cycle counter access mnemonics. + + // Native Client does not provide any API to access cycle counter. + // Use clock_gettime(CLOCK_MONOTONIC, ...) instead of gettimeofday + // because is provides nanosecond resolution (which is noticable at + // least for PNaCl modules running on x86 Mac & Linux). + // Initialize to always return 0 if clock_gettime fails. + struct timespec ts = { 0, 0 }; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; +#elif defined(__aarch64__) + // System timer of ARMv8 runs at a different frequency than the CPU's. + // The frequency is fixed, typically in the range 1-50MHz. It can be + // read at CNTFRQ special register. We assume the OS has set up + // the virtual timer properly. + int64_t virtual_timer_value; + asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +#elif defined(__ARM_ARCH) + // V6 is the earliest arch that has a standard cyclecount + // Native Client validator doesn't allow MRC instructions. +#if (__ARM_ARCH >= 6) + uint32_t pmccntr; + uint32_t pmuseren; + uint32_t pmcntenset; + // Read the user mode perf monitor counter access permissions. + asm volatile("mrc p15, 0, %0, c9, c14, 0" : "=r"(pmuseren)); + if (pmuseren & 1) { // Allows reading perfmon counters for user mode code. + asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r"(pmcntenset)); + if (pmcntenset & 0x80000000ul) { // Is it counting? + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(pmccntr)); + // The counter is set up to count every 64th cycle + return static_cast(pmccntr) * 64; // Should optimize to << 6 + } + } +#endif + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#elif defined(__mips__) + // mips apparently only allows rdtsc for superusers, so we fall + // back to gettimeofday. It's possible clock_gettime would be better. + struct timeval tv; + gettimeofday(&tv, nullptr); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +#else +// The soft failover to a generic implementation is automatic only for ARM. +// For other platforms the developer is expected to make an attempt to create +// a fast implementation and use generic version if nothing better is available. +#error You need to define CycleTimer for your OS and CPU +#endif +} +} // end namespace cycleclock +} // end namespace benchmark + +#endif // BENCHMARK_CYCLECLOCK_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/internal_macros.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/internal_macros.h new file mode 100644 index 00000000000..942887457f1 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/internal_macros.h @@ -0,0 +1,57 @@ +#ifndef BENCHMARK_INTERNAL_MACROS_H_ +#define BENCHMARK_INTERNAL_MACROS_H_ + +#include "benchmark/benchmark.h" + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if defined(__clang__) +#define COMPILER_CLANG +#elif defined(_MSC_VER) +#define COMPILER_MSVC +#elif defined(__GNUC__) +#define COMPILER_GCC +#endif + +#if __has_feature(cxx_attributes) +#define BENCHMARK_NORETURN [[noreturn]] +#elif defined(__GNUC__) +#define BENCHMARK_NORETURN __attribute__((noreturn)) +#elif defined(COMPILER_MSVC) +#define BENCHMARK_NORETURN __declspec(noreturn) +#else +#define BENCHMARK_NORETURN +#endif + +#if defined(__CYGWIN__) +#define BENCHMARK_OS_CYGWIN 1 +#elif defined(_WIN32) +#define BENCHMARK_OS_WINDOWS 1 +#elif defined(__APPLE__) +#include "TargetConditionals.h" + #if defined(TARGET_OS_MAC) + #define BENCHMARK_OS_MACOSX 1 + #if defined(TARGET_OS_IPHONE) + #define BENCHMARK_OS_IOS 1 + #endif + #endif +#elif defined(__FreeBSD__) +#define BENCHMARK_OS_FREEBSD 1 +#elif defined(__linux__) +#define BENCHMARK_OS_LINUX 1 +#elif defined(__native_client__) +#define BENCHMARK_OS_NACL 1 +#elif defined(EMSCRIPTEN) +#define BENCHMARK_OS_EMSCRIPTEN 1 +#elif defined(__rtems__) +#define BENCHMARK_OS_RTEMS 1 +#endif + +#if !__has_feature(cxx_exceptions) && !defined(__cpp_exceptions) \ + && !defined(__EXCEPTIONS) +#define BENCHMARK_HAS_NO_EXCEPTIONS +#endif + +#endif // BENCHMARK_INTERNAL_MACROS_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/json_reporter.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/json_reporter.cc new file mode 100644 index 00000000000..edf6ecc836c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/json_reporter.cc @@ -0,0 +1,168 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "complexity.h" + +#include +#include +#include +#include +#include +#include + +#include "string_util.h" +#include "timers.h" + +namespace benchmark { + +namespace { + +std::string FormatKV(std::string const& key, std::string const& value) { + return StringPrintF("\"%s\": \"%s\"", key.c_str(), value.c_str()); +} + +std::string FormatKV(std::string const& key, const char* value) { + return StringPrintF("\"%s\": \"%s\"", key.c_str(), value); +} + +std::string FormatKV(std::string const& key, bool value) { + return StringPrintF("\"%s\": %s", key.c_str(), value ? "true" : "false"); +} + +std::string FormatKV(std::string const& key, int64_t value) { + std::stringstream ss; + ss << '"' << key << "\": " << value; + return ss.str(); +} + +std::string FormatKV(std::string const& key, double value) { + return StringPrintF("\"%s\": %.2f", key.c_str(), value); +} + +int64_t RoundDouble(double v) { return static_cast(v + 0.5); } + +} // end namespace + +bool JSONReporter::ReportContext(const Context& context) { + std::ostream& out = GetOutputStream(); + + out << "{\n"; + std::string inner_indent(2, ' '); + + // Open context block and print context information. + out << inner_indent << "\"context\": {\n"; + std::string indent(4, ' '); + + std::string walltime_value = LocalDateTimeString(); + out << indent << FormatKV("date", walltime_value) << ",\n"; + + out << indent << FormatKV("num_cpus", static_cast(context.num_cpus)) + << ",\n"; + out << indent << FormatKV("mhz_per_cpu", RoundDouble(context.mhz_per_cpu)) + << ",\n"; + out << indent << FormatKV("cpu_scaling_enabled", context.cpu_scaling_enabled) + << ",\n"; + +#if defined(NDEBUG) + const char build_type[] = "release"; +#else + const char build_type[] = "debug"; +#endif + out << indent << FormatKV("library_build_type", build_type) << "\n"; + // Close context block and open the list of benchmarks. + out << inner_indent << "},\n"; + out << inner_indent << "\"benchmarks\": [\n"; + return true; +} + +void JSONReporter::ReportRuns(std::vector const& reports) { + if (reports.empty()) { + return; + } + std::string indent(4, ' '); + std::ostream& out = GetOutputStream(); + if (!first_report_) { + out << ",\n"; + } + first_report_ = false; + + for (auto it = reports.begin(); it != reports.end(); ++it) { + out << indent << "{\n"; + PrintRunData(*it); + out << indent << '}'; + auto it_cp = it; + if (++it_cp != reports.end()) { + out << ",\n"; + } + } +} + +void JSONReporter::Finalize() { + // Close the list of benchmarks and the top level object. + GetOutputStream() << "\n ]\n}\n"; +} + +void JSONReporter::PrintRunData(Run const& run) { + std::string indent(6, ' '); + std::ostream& out = GetOutputStream(); + out << indent << FormatKV("name", run.benchmark_name) << ",\n"; + if (run.error_occurred) { + out << indent << FormatKV("error_occurred", run.error_occurred) << ",\n"; + out << indent << FormatKV("error_message", run.error_message) << ",\n"; + } + if (!run.report_big_o && !run.report_rms) { + out << indent << FormatKV("iterations", run.iterations) << ",\n"; + out << indent + << FormatKV("real_time", RoundDouble(run.GetAdjustedRealTime())) + << ",\n"; + out << indent + << FormatKV("cpu_time", RoundDouble(run.GetAdjustedCPUTime())); + out << ",\n" + << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); + } else if (run.report_big_o) { + out << indent + << FormatKV("cpu_coefficient", RoundDouble(run.GetAdjustedCPUTime())) + << ",\n"; + out << indent + << FormatKV("real_coefficient", RoundDouble(run.GetAdjustedRealTime())) + << ",\n"; + out << indent << FormatKV("big_o", GetBigOString(run.complexity)) << ",\n"; + out << indent << FormatKV("time_unit", GetTimeUnitString(run.time_unit)); + } else if (run.report_rms) { + out << indent + << FormatKV("rms", run.GetAdjustedCPUTime()); + } + if (run.bytes_per_second > 0.0) { + out << ",\n" + << indent + << FormatKV("bytes_per_second", RoundDouble(run.bytes_per_second)); + } + if (run.items_per_second > 0.0) { + out << ",\n" + << indent + << FormatKV("items_per_second", RoundDouble(run.items_per_second)); + } + for(auto &c : run.counters) { + out << ",\n" + << indent + << FormatKV(c.first, RoundDouble(c.second)); + } + if (!run.report_label.empty()) { + out << ",\n" << indent << FormatKV("label", run.report_label); + } + out << '\n'; +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/log.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/log.h new file mode 100644 index 00000000000..d06e1031db1 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/log.h @@ -0,0 +1,73 @@ +#ifndef BENCHMARK_LOG_H_ +#define BENCHMARK_LOG_H_ + +#include +#include + +#include "benchmark/benchmark.h" + +namespace benchmark { +namespace internal { + +typedef std::basic_ostream&(EndLType)(std::basic_ostream&); + +class LogType { + friend LogType& GetNullLogInstance(); + friend LogType& GetErrorLogInstance(); + + // FIXME: Add locking to output. + template + friend LogType& operator<<(LogType&, Tp const&); + friend LogType& operator<<(LogType&, EndLType*); + + private: + LogType(std::ostream* out) : out_(out) {} + std::ostream* out_; + BENCHMARK_DISALLOW_COPY_AND_ASSIGN(LogType); +}; + +template +LogType& operator<<(LogType& log, Tp const& value) { + if (log.out_) { + *log.out_ << value; + } + return log; +} + +inline LogType& operator<<(LogType& log, EndLType* m) { + if (log.out_) { + *log.out_ << m; + } + return log; +} + +inline int& LogLevel() { + static int log_level = 0; + return log_level; +} + +inline LogType& GetNullLogInstance() { + static LogType log(nullptr); + return log; +} + +inline LogType& GetErrorLogInstance() { + static LogType log(&std::clog); + return log; +} + +inline LogType& GetLogInstanceForLevel(int level) { + if (level <= LogLevel()) { + return GetErrorLogInstance(); + } + return GetNullLogInstance(); +} + +} // end namespace internal +} // end namespace benchmark + +#define VLOG(x) \ + (::benchmark::internal::GetLogInstanceForLevel(x) << "-- LOG(" << x << "):" \ + " ") + +#endif diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/mutex.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/mutex.h new file mode 100644 index 00000000000..5f461d05a0c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/mutex.h @@ -0,0 +1,155 @@ +#ifndef BENCHMARK_MUTEX_H_ +#define BENCHMARK_MUTEX_H_ + +#include +#include + +#include "check.h" + +// Enable thread safety attributes only with clang. +// The attributes can be safely erased when compiling with other compilers. +#if defined(HAVE_THREAD_SAFETY_ATTRIBUTES) +#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else +#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + +#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(capability(x)) + +#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) + +#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) + +#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) + +#define ACQUIRED_BEFORE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) + +#define ACQUIRED_AFTER(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) + +#define REQUIRES(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__)) + +#define REQUIRES_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__)) + +#define ACQUIRE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__)) + +#define ACQUIRE_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__)) + +#define RELEASE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__)) + +#define RELEASE_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__)) + +#define TRY_ACQUIRE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__)) + +#define TRY_ACQUIRE_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__)) + +#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) + +#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x)) + +#define ASSERT_SHARED_CAPABILITY(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x)) + +#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) + +#define NO_THREAD_SAFETY_ANALYSIS \ + THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) + +namespace benchmark { + +typedef std::condition_variable Condition; + +// NOTE: Wrappers for std::mutex and std::unique_lock are provided so that +// we can annotate them with thread safety attributes and use the +// -Wthread-safety warning with clang. The standard library types cannot be +// used directly because they do not provided the required annotations. +class CAPABILITY("mutex") Mutex { + public: + Mutex() {} + + void lock() ACQUIRE() { mut_.lock(); } + void unlock() RELEASE() { mut_.unlock(); } + std::mutex& native_handle() { return mut_; } + + private: + std::mutex mut_; +}; + +class SCOPED_CAPABILITY MutexLock { + typedef std::unique_lock MutexLockImp; + + public: + MutexLock(Mutex& m) ACQUIRE(m) : ml_(m.native_handle()) {} + ~MutexLock() RELEASE() {} + MutexLockImp& native_handle() { return ml_; } + + private: + MutexLockImp ml_; +}; + +class Barrier { + public: + Barrier(int num_threads) : running_threads_(num_threads) {} + + // Called by each thread + bool wait() EXCLUDES(lock_) { + bool last_thread = false; + { + MutexLock ml(lock_); + last_thread = createBarrier(ml); + } + if (last_thread) phase_condition_.notify_all(); + return last_thread; + } + + void removeThread() EXCLUDES(lock_) { + MutexLock ml(lock_); + --running_threads_; + if (entered_ != 0) phase_condition_.notify_all(); + } + + private: + Mutex lock_; + Condition phase_condition_; + int running_threads_; + + // State for barrier management + int phase_number_ = 0; + int entered_ = 0; // Number of threads that have entered this barrier + + // Enter the barrier and wait until all other threads have also + // entered the barrier. Returns iff this is the last thread to + // enter the barrier. + bool createBarrier(MutexLock& ml) REQUIRES(lock_) { + CHECK_LT(entered_, running_threads_); + entered_++; + if (entered_ < running_threads_) { + // Wait for all threads to enter + int phase_number_cp = phase_number_; + auto cb = [this, phase_number_cp]() { + return this->phase_number_ > phase_number_cp || + entered_ == running_threads_; // A thread has aborted in error + }; + phase_condition_.wait(ml.native_handle(), cb); + if (phase_number_ > phase_number_cp) return false; + // else (running_threads_ == entered_) and we are the last thread. + } + // Last thread has reached the barrier + phase_number_++; + entered_ = 0; + return true; + } +}; + +} // end namespace benchmark + +#endif // BENCHMARK_MUTEX_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/re.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/re.h new file mode 100644 index 00000000000..01e9736505e --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/re.h @@ -0,0 +1,140 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_RE_H_ +#define BENCHMARK_RE_H_ + +#include "internal_macros.h" + +// Prefer C regex libraries when compiling w/o exceptions so that we can +// correctly report errors. +#if defined(BENCHMARK_HAS_NO_EXCEPTIONS) && defined(HAVE_STD_REGEX) && \ + (defined(HAVE_GNU_POSIX_REGEX) || defined(HAVE_POSIX_REGEX)) +#undef HAVE_STD_REGEX +#endif + +#if defined(HAVE_STD_REGEX) +#include +#elif defined(HAVE_GNU_POSIX_REGEX) +#include +#elif defined(HAVE_POSIX_REGEX) +#include +#else +#error No regular expression backend was found! +#endif +#include + +#include "check.h" + +namespace benchmark { + +// A wrapper around the POSIX regular expression API that provides automatic +// cleanup +class Regex { + public: + Regex() : init_(false) {} + + ~Regex(); + + // Compile a regular expression matcher from spec. Returns true on success. + // + // On failure (and if error is not nullptr), error is populated with a human + // readable error message if an error occurs. + bool Init(const std::string& spec, std::string* error); + + // Returns whether str matches the compiled regular expression. + bool Match(const std::string& str); + + private: + bool init_; +// Underlying regular expression object +#if defined(HAVE_STD_REGEX) + std::regex re_; +#elif defined(HAVE_POSIX_REGEX) || defined(HAVE_GNU_POSIX_REGEX) + regex_t re_; +#else +#error No regular expression backend implementation available +#endif +}; + +#if defined(HAVE_STD_REGEX) + +inline bool Regex::Init(const std::string& spec, std::string* error) { +#ifdef BENCHMARK_HAS_NO_EXCEPTIONS + ((void)error); // suppress unused warning +#else + try { +#endif + re_ = std::regex(spec, std::regex_constants::extended); + init_ = true; +#ifndef BENCHMARK_HAS_NO_EXCEPTIONS + } catch (const std::regex_error& e) { + if (error) { + *error = e.what(); + } + } +#endif + return init_; +} + +inline Regex::~Regex() {} + +inline bool Regex::Match(const std::string& str) { + if (!init_) { + return false; + } + return std::regex_search(str, re_); +} + +#else +inline bool Regex::Init(const std::string& spec, std::string* error) { + int ec = regcomp(&re_, spec.c_str(), REG_EXTENDED | REG_NOSUB); + if (ec != 0) { + if (error) { + size_t needed = regerror(ec, &re_, nullptr, 0); + char* errbuf = new char[needed]; + regerror(ec, &re_, errbuf, needed); + + // regerror returns the number of bytes necessary to null terminate + // the string, so we move that when assigning to error. + CHECK_NE(needed, 0); + error->assign(errbuf, needed - 1); + + delete[] errbuf; + } + + return false; + } + + init_ = true; + return true; +} + +inline Regex::~Regex() { + if (init_) { + regfree(&re_); + } +} + +inline bool Regex::Match(const std::string& str) { + if (!init_) { + return false; + } + return regexec(&re_, str.c_str(), 0, nullptr, 0) == 0; +} +#endif + +} // end namespace benchmark + +#endif // BENCHMARK_RE_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/reporter.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/reporter.cc new file mode 100644 index 00000000000..aacd453179d --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/reporter.cc @@ -0,0 +1,68 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "timers.h" + +#include + +#include +#include +#include + +#include "check.h" +#include "stat.h" + +namespace benchmark { + +BenchmarkReporter::BenchmarkReporter() + : output_stream_(&std::cout), error_stream_(&std::cerr) {} + +BenchmarkReporter::~BenchmarkReporter() {} + +void BenchmarkReporter::PrintBasicContext(std::ostream *out, + Context const &context) { + CHECK(out) << "cannot be null"; + auto &Out = *out; + + Out << "Run on (" << context.num_cpus << " X " << context.mhz_per_cpu + << " MHz CPU " << ((context.num_cpus > 1) ? "s" : "") << ")\n"; + + Out << LocalDateTimeString() << "\n"; + + if (context.cpu_scaling_enabled) { + Out << "***WARNING*** CPU scaling is enabled, the benchmark " + "real time measurements may be noisy and will incur extra " + "overhead.\n"; + } + +#ifndef NDEBUG + Out << "***WARNING*** Library was built as DEBUG. Timings may be " + "affected.\n"; +#endif +} + +double BenchmarkReporter::Run::GetAdjustedRealTime() const { + double new_time = real_accumulated_time * GetTimeUnitMultiplier(time_unit); + if (iterations != 0) new_time /= static_cast(iterations); + return new_time; +} + +double BenchmarkReporter::Run::GetAdjustedCPUTime() const { + double new_time = cpu_accumulated_time * GetTimeUnitMultiplier(time_unit); + if (iterations != 0) new_time /= static_cast(iterations); + return new_time; +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sleep.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sleep.cc new file mode 100644 index 00000000000..54aa04a4224 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sleep.cc @@ -0,0 +1,51 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sleep.h" + +#include +#include +#include + +#include "internal_macros.h" + +#ifdef BENCHMARK_OS_WINDOWS +#include +#endif + +namespace benchmark { +#ifdef BENCHMARK_OS_WINDOWS +// Window's Sleep takes milliseconds argument. +void SleepForMilliseconds(int milliseconds) { Sleep(milliseconds); } +void SleepForSeconds(double seconds) { + SleepForMilliseconds(static_cast(kNumMillisPerSecond * seconds)); +} +#else // BENCHMARK_OS_WINDOWS +void SleepForMicroseconds(int microseconds) { + struct timespec sleep_time; + sleep_time.tv_sec = microseconds / kNumMicrosPerSecond; + sleep_time.tv_nsec = (microseconds % kNumMicrosPerSecond) * kNumNanosPerMicro; + while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) + ; // Ignore signals and wait for the full interval to elapse. +} + +void SleepForMilliseconds(int milliseconds) { + SleepForMicroseconds(milliseconds * kNumMicrosPerMilli); +} + +void SleepForSeconds(double seconds) { + SleepForMicroseconds(static_cast(seconds * kNumMicrosPerSecond)); +} +#endif // BENCHMARK_OS_WINDOWS +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sleep.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sleep.h new file mode 100644 index 00000000000..f98551afe28 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sleep.h @@ -0,0 +1,15 @@ +#ifndef BENCHMARK_SLEEP_H_ +#define BENCHMARK_SLEEP_H_ + +namespace benchmark { +const int kNumMillisPerSecond = 1000; +const int kNumMicrosPerMilli = 1000; +const int kNumMicrosPerSecond = kNumMillisPerSecond * 1000; +const int kNumNanosPerMicro = 1000; +const int kNumNanosPerSecond = kNumNanosPerMicro * kNumMicrosPerSecond; + +void SleepForMilliseconds(int milliseconds); +void SleepForSeconds(double seconds); +} // end namespace benchmark + +#endif // BENCHMARK_SLEEP_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/stat.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/stat.h new file mode 100644 index 00000000000..d356875b632 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/stat.h @@ -0,0 +1,310 @@ +#ifndef BENCHMARK_STAT_H_ +#define BENCHMARK_STAT_H_ + +#include +#include +#include +#include + +namespace benchmark { + +template +class Stat1; + +template +class Stat1MinMax; + +typedef Stat1 Stat1_f; +typedef Stat1 Stat1_d; +typedef Stat1MinMax Stat1MinMax_f; +typedef Stat1MinMax Stat1MinMax_d; + +template +class Vector2; +template +class Vector3; +template +class Vector4; + +template +class Stat1 { + public: + typedef Stat1 Self; + + Stat1() { Clear(); } + // Create a sample of value dat and weight 1 + explicit Stat1(const VType &dat) { + sum_ = dat; + sum_squares_ = Sqr(dat); + numsamples_ = 1; + } + // Create statistics for all the samples between begin (included) + // and end(excluded) + explicit Stat1(const VType *begin, const VType *end) { + Clear(); + for (const VType *item = begin; item < end; ++item) { + (*this) += Stat1(*item); + } + } + // Create a sample of value dat and weight w + Stat1(const VType &dat, const NumType &w) { + sum_ = w * dat; + sum_squares_ = w * Sqr(dat); + numsamples_ = w; + } + // Copy operator + Stat1(const Self &stat) { + sum_ = stat.sum_; + sum_squares_ = stat.sum_squares_; + numsamples_ = stat.numsamples_; + } + + void Clear() { + numsamples_ = NumType(); + sum_squares_ = sum_ = VType(); + } + + Self &operator=(const Self &stat) { + sum_ = stat.sum_; + sum_squares_ = stat.sum_squares_; + numsamples_ = stat.numsamples_; + return (*this); + } + // Merge statistics from two sample sets. + Self &operator+=(const Self &stat) { + sum_ += stat.sum_; + sum_squares_ += stat.sum_squares_; + numsamples_ += stat.numsamples_; + return (*this); + } + // The operation opposite to += + Self &operator-=(const Self &stat) { + sum_ -= stat.sum_; + sum_squares_ -= stat.sum_squares_; + numsamples_ -= stat.numsamples_; + return (*this); + } + // Multiply the weight of the set of samples by a factor k + Self &operator*=(const VType &k) { + sum_ *= k; + sum_squares_ *= k; + numsamples_ *= k; + return (*this); + } + + // Merge statistics from two sample sets. + Self operator+(const Self &stat) const { return Self(*this) += stat; } + + // The operation opposite to + + Self operator-(const Self &stat) const { return Self(*this) -= stat; } + + // Multiply the weight of the set of samples by a factor k + Self operator*(const VType &k) const { return Self(*this) *= k; } + + // Return the total weight of this sample set + NumType numSamples() const { return numsamples_; } + + // Return the sum of this sample set + VType Sum() const { return sum_; } + + // Return the mean of this sample set + VType Mean() const { + if (numsamples_ == 0) return VType(); + return sum_ * (1.0 / numsamples_); + } + + // Return the mean of this sample set and compute the standard deviation at + // the same time. + VType Mean(VType *stddev) const { + if (numsamples_ == 0) return VType(); + VType mean = sum_ * (1.0 / numsamples_); + if (stddev) { + // Sample standard deviation is undefined for n = 1 + if (numsamples_ == 1) { + *stddev = VType(); + } else { + VType avg_squares = sum_squares_ * (1.0 / numsamples_); + *stddev = Sqrt(numsamples_ / (numsamples_ - 1.0) * (avg_squares - Sqr(mean))); + } + } + return mean; + } + + // Return the standard deviation of the sample set + VType StdDev() const { + VType stddev = VType(); + Mean(&stddev); + return stddev; + } + + private: + static_assert(std::is_integral::value && + !std::is_same::value, + "NumType must be an integral type that is not bool."); + // Let i be the index of the samples provided (using +=) + // and weight[i],value[i] be the data of sample #i + // then the variables have the following meaning: + NumType numsamples_; // sum of weight[i]; + VType sum_; // sum of weight[i]*value[i]; + VType sum_squares_; // sum of weight[i]*value[i]^2; + + // Template function used to square a number. + // For a vector we square all components + template + static inline SType Sqr(const SType &dat) { + return dat * dat; + } + + template + static inline Vector2 Sqr(const Vector2 &dat) { + return dat.MulComponents(dat); + } + + template + static inline Vector3 Sqr(const Vector3 &dat) { + return dat.MulComponents(dat); + } + + template + static inline Vector4 Sqr(const Vector4 &dat) { + return dat.MulComponents(dat); + } + + // Template function used to take the square root of a number. + // For a vector we square all components + template + static inline SType Sqrt(const SType &dat) { + // Avoid NaN due to imprecision in the calculations + if (dat < 0) return 0; + return sqrt(dat); + } + + template + static inline Vector2 Sqrt(const Vector2 &dat) { + // Avoid NaN due to imprecision in the calculations + return Max(dat, Vector2()).Sqrt(); + } + + template + static inline Vector3 Sqrt(const Vector3 &dat) { + // Avoid NaN due to imprecision in the calculations + return Max(dat, Vector3()).Sqrt(); + } + + template + static inline Vector4 Sqrt(const Vector4 &dat) { + // Avoid NaN due to imprecision in the calculations + return Max(dat, Vector4()).Sqrt(); + } +}; + +// Useful printing function +template +std::ostream &operator<<(std::ostream &out, const Stat1 &s) { + out << "{ avg = " << s.Mean() << " std = " << s.StdDev() + << " nsamples = " << s.NumSamples() << "}"; + return out; +} + +// Stat1MinMax: same as Stat1, but it also +// keeps the Min and Max values; the "-" +// operator is disabled because it cannot be implemented +// efficiently +template +class Stat1MinMax : public Stat1 { + public: + typedef Stat1MinMax Self; + + Stat1MinMax() { Clear(); } + // Create a sample of value dat and weight 1 + explicit Stat1MinMax(const VType &dat) : Stat1(dat) { + max_ = dat; + min_ = dat; + } + // Create statistics for all the samples between begin (included) + // and end(excluded) + explicit Stat1MinMax(const VType *begin, const VType *end) { + Clear(); + for (const VType *item = begin; item < end; ++item) { + (*this) += Stat1MinMax(*item); + } + } + // Create a sample of value dat and weight w + Stat1MinMax(const VType &dat, const NumType &w) + : Stat1(dat, w) { + max_ = dat; + min_ = dat; + } + // Copy operator + Stat1MinMax(const Self &stat) : Stat1(stat) { + max_ = stat.max_; + min_ = stat.min_; + } + + void Clear() { + Stat1::Clear(); + if (std::numeric_limits::has_infinity) { + min_ = std::numeric_limits::infinity(); + max_ = -std::numeric_limits::infinity(); + } else { + min_ = std::numeric_limits::max(); + max_ = std::numeric_limits::min(); + } + } + + Self &operator=(const Self &stat) { + this->Stat1::operator=(stat); + max_ = stat.max_; + min_ = stat.min_; + return (*this); + } + // Merge statistics from two sample sets. + Self &operator+=(const Self &stat) { + this->Stat1::operator+=(stat); + if (stat.max_ > max_) max_ = stat.max_; + if (stat.min_ < min_) min_ = stat.min_; + return (*this); + } + // Multiply the weight of the set of samples by a factor k + Self &operator*=(const VType &stat) { + this->Stat1::operator*=(stat); + return (*this); + } + // Merge statistics from two sample sets. + Self operator+(const Self &stat) const { return Self(*this) += stat; } + // Multiply the weight of the set of samples by a factor k + Self operator*(const VType &k) const { return Self(*this) *= k; } + + // Return the maximal value in this sample set + VType Max() const { return max_; } + // Return the minimal value in this sample set + VType Min() const { return min_; } + + private: + // The - operation makes no sense with Min/Max + // unless we keep the full list of values (but we don't) + // make it private, and let it undefined so nobody can call it + Self &operator-=(const Self &stat); // senseless. let it undefined. + + // The operation opposite to - + Self operator-(const Self &stat) const; // senseless. let it undefined. + + // Let i be the index of the samples provided (using +=) + // and weight[i],value[i] be the data of sample #i + // then the variables have the following meaning: + VType max_; // max of value[i] + VType min_; // min of value[i] +}; + +// Useful printing function +template +std::ostream &operator<<(std::ostream &out, + const Stat1MinMax &s) { + out << "{ avg = " << s.Mean() << " std = " << s.StdDev() + << " nsamples = " << s.NumSamples() << " min = " << s.Min() + << " max = " << s.Max() << "}"; + return out; +} +} // end namespace benchmark + +#endif // BENCHMARK_STAT_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/string_util.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/string_util.cc new file mode 100644 index 00000000000..cd4e7cfde57 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/string_util.cc @@ -0,0 +1,172 @@ +#include "string_util.h" + +#include +#include +#include +#include +#include +#include + +#include "arraysize.h" + +namespace benchmark { +namespace { + +// kilo, Mega, Giga, Tera, Peta, Exa, Zetta, Yotta. +const char kBigSIUnits[] = "kMGTPEZY"; +// Kibi, Mebi, Gibi, Tebi, Pebi, Exbi, Zebi, Yobi. +const char kBigIECUnits[] = "KMGTPEZY"; +// milli, micro, nano, pico, femto, atto, zepto, yocto. +const char kSmallSIUnits[] = "munpfazy"; + +// We require that all three arrays have the same size. +static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), + "SI and IEC unit arrays must be the same size"); +static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits), + "Small SI and Big SI unit arrays must be the same size"); + +static const int64_t kUnitsSize = arraysize(kBigSIUnits); + +} // end anonymous namespace + +void ToExponentAndMantissa(double val, double thresh, int precision, + double one_k, std::string* mantissa, + int64_t* exponent) { + std::stringstream mantissa_stream; + + if (val < 0) { + mantissa_stream << "-"; + val = -val; + } + + // Adjust threshold so that it never excludes things which can't be rendered + // in 'precision' digits. + const double adjusted_threshold = + std::max(thresh, 1.0 / std::pow(10.0, precision)); + const double big_threshold = adjusted_threshold * one_k; + const double small_threshold = adjusted_threshold; + // Values in ]simple_threshold,small_threshold[ will be printed as-is + const double simple_threshold = 0.01; + + if (val > big_threshold) { + // Positive powers + double scaled = val; + for (size_t i = 0; i < arraysize(kBigSIUnits); ++i) { + scaled /= one_k; + if (scaled <= big_threshold) { + mantissa_stream << scaled; + *exponent = i + 1; + *mantissa = mantissa_stream.str(); + return; + } + } + mantissa_stream << val; + *exponent = 0; + } else if (val < small_threshold) { + // Negative powers + if (val < simple_threshold) { + double scaled = val; + for (size_t i = 0; i < arraysize(kSmallSIUnits); ++i) { + scaled *= one_k; + if (scaled >= small_threshold) { + mantissa_stream << scaled; + *exponent = -static_cast(i + 1); + *mantissa = mantissa_stream.str(); + return; + } + } + } + mantissa_stream << val; + *exponent = 0; + } else { + mantissa_stream << val; + *exponent = 0; + } + *mantissa = mantissa_stream.str(); +} + +std::string ExponentToPrefix(int64_t exponent, bool iec) { + if (exponent == 0) return ""; + + const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1); + if (index >= kUnitsSize) return ""; + + const char* array = + (exponent > 0 ? (iec ? kBigIECUnits : kBigSIUnits) : kSmallSIUnits); + if (iec) + return array[index] + std::string("i"); + else + return std::string(1, array[index]); +} + +std::string ToBinaryStringFullySpecified(double value, double threshold, + int precision) { + std::string mantissa; + int64_t exponent; + ToExponentAndMantissa(value, threshold, precision, 1024.0, &mantissa, + &exponent); + return mantissa + ExponentToPrefix(exponent, false); +} + +void AppendHumanReadable(int n, std::string* str) { + std::stringstream ss; + // Round down to the nearest SI prefix. + ss << ToBinaryStringFullySpecified(n, 1.0, 0); + *str += ss.str(); +} + +std::string HumanReadableNumber(double n) { + // 1.1 means that figures up to 1.1k should be shown with the next unit down; + // this softens edge effects. + // 1 means that we should show one decimal place of precision. + return ToBinaryStringFullySpecified(n, 1.1, 1); +} + +std::string StringPrintFImp(const char* msg, va_list args) { + // we might need a second shot at this, so pre-emptivly make a copy + va_list args_cp; + va_copy(args_cp, args); + + // TODO(ericwf): use std::array for first attempt to avoid one memory + // allocation guess what the size might be + std::array local_buff; + std::size_t size = local_buff.size(); + // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation + // in the android-ndk + auto ret = vsnprintf(local_buff.data(), size, msg, args_cp); + + va_end(args_cp); + + // handle empty expansion + if (ret == 0) return std::string{}; + if (static_cast(ret) < size) + return std::string(local_buff.data()); + + // we did not provide a long enough buffer on our first attempt. + // add 1 to size to account for null-byte in size cast to prevent overflow + size = static_cast(ret) + 1; + auto buff_ptr = std::unique_ptr(new char[size]); + // 2015-10-08: vsnprintf is used instead of snd::vsnprintf due to a limitation + // in the android-ndk + ret = vsnprintf(buff_ptr.get(), size, msg, args); + return std::string(buff_ptr.get()); +} + +std::string StringPrintF(const char* format, ...) { + va_list args; + va_start(args, format); + std::string tmp = StringPrintFImp(format, args); + va_end(args); + return tmp; +} + +void ReplaceAll(std::string* str, const std::string& from, + const std::string& to) { + std::size_t start = 0; + while ((start = str->find(from, start)) != std::string::npos) { + str->replace(start, from.length(), to); + start += to.length(); + } +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/string_util.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/string_util.h new file mode 100644 index 00000000000..0b190b91a16 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/string_util.h @@ -0,0 +1,40 @@ +#ifndef BENCHMARK_STRING_UTIL_H_ +#define BENCHMARK_STRING_UTIL_H_ + +#include +#include +#include +#include "internal_macros.h" + +namespace benchmark { + +void AppendHumanReadable(int n, std::string* str); + +std::string HumanReadableNumber(double n); + +std::string StringPrintF(const char* format, ...); + +inline std::ostream& StringCatImp(std::ostream& out) BENCHMARK_NOEXCEPT { + return out; +} + +template +inline std::ostream& StringCatImp(std::ostream& out, First&& f, + Rest&&... rest) { + out << std::forward(f); + return StringCatImp(out, std::forward(rest)...); +} + +template +inline std::string StrCat(Args&&... args) { + std::ostringstream ss; + StringCatImp(ss, std::forward(args)...); + return ss.str(); +} + +void ReplaceAll(std::string* str, const std::string& from, + const std::string& to); + +} // end namespace benchmark + +#endif // BENCHMARK_STRING_UTIL_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sysinfo.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sysinfo.cc new file mode 100644 index 00000000000..7feb79e65f2 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sysinfo.cc @@ -0,0 +1,355 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "sysinfo.h" +#include "internal_macros.h" + +#ifdef BENCHMARK_OS_WINDOWS +#include +#include +#include +#else +#include +#include +#include +#include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD +#include +#if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_MACOSX +#include +#endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arraysize.h" +#include "check.h" +#include "cycleclock.h" +#include "internal_macros.h" +#include "log.h" +#include "sleep.h" +#include "string_util.h" + +namespace benchmark { +namespace { +std::once_flag cpuinfo_init; +double cpuinfo_cycles_per_second = 1.0; +int cpuinfo_num_cpus = 1; // Conservative guess + +#if !defined BENCHMARK_OS_MACOSX +const int64_t estimate_time_ms = 1000; + +// Helper function estimates cycles/sec by observing cycles elapsed during +// sleep(). Using small sleep time decreases accuracy significantly. +int64_t EstimateCyclesPerSecond() { + const int64_t start_ticks = cycleclock::Now(); + SleepForMilliseconds(estimate_time_ms); + return cycleclock::Now() - start_ticks; +} +#endif + +#if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN +// Helper function for reading an int from a file. Returns true if successful +// and the memory location pointed to by value is set to the value read. +bool ReadIntFromFile(const char* file, long* value) { + bool ret = false; + int fd = open(file, O_RDONLY); + if (fd != -1) { + char line[1024]; + char* err; + memset(line, '\0', sizeof(line)); + ssize_t read_err = read(fd, line, sizeof(line) - 1); + ((void)read_err); // prevent unused warning + CHECK(read_err >= 0); + const long temp_value = strtol(line, &err, 10); + if (line[0] != '\0' && (*err == '\n' || *err == '\0')) { + *value = temp_value; + ret = true; + } + close(fd); + } + return ret; +} +#endif + +#if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN +static std::string convertToLowerCase(std::string s) { + for (auto& ch : s) + ch = std::tolower(ch); + return s; +} +static bool startsWithKey(std::string Value, std::string Key, + bool IgnoreCase = true) { + if (IgnoreCase) { + Key = convertToLowerCase(std::move(Key)); + Value = convertToLowerCase(std::move(Value)); + } + return Value.compare(0, Key.size(), Key) == 0; +} +#endif + +void InitializeSystemInfo() { +#if defined BENCHMARK_OS_LINUX || defined BENCHMARK_OS_CYGWIN + char line[1024]; + char* err; + long freq; + + bool saw_mhz = false; + + // If the kernel is exporting the tsc frequency use that. There are issues + // where cpuinfo_max_freq cannot be relied on because the BIOS may be + // exporintg an invalid p-state (on x86) or p-states may be used to put the + // processor in a new mode (turbo mode). Essentially, those frequencies + // cannot always be relied upon. The same reasons apply to /proc/cpuinfo as + // well. + if (!saw_mhz && + ReadIntFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", &freq)) { + // The value is in kHz (as the file name suggests). For example, on a + // 2GHz warpstation, the file contains the value "2000000". + cpuinfo_cycles_per_second = freq * 1000.0; + saw_mhz = true; + } + + // If CPU scaling is in effect, we want to use the *maximum* frequency, + // not whatever CPU speed some random processor happens to be using now. + if (!saw_mhz && + ReadIntFromFile("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", + &freq)) { + // The value is in kHz. For example, on a 2GHz warpstation, the file + // contains the value "2000000". + cpuinfo_cycles_per_second = freq * 1000.0; + saw_mhz = true; + } + + // Read /proc/cpuinfo for other values, and if there is no cpuinfo_max_freq. + const char* pname = "/proc/cpuinfo"; + int fd = open(pname, O_RDONLY); + if (fd == -1) { + perror(pname); + if (!saw_mhz) { + cpuinfo_cycles_per_second = + static_cast(EstimateCyclesPerSecond()); + } + return; + } + + double bogo_clock = 1.0; + bool saw_bogo = false; + long max_cpu_id = 0; + int num_cpus = 0; + line[0] = line[1] = '\0'; + size_t chars_read = 0; + do { // we'll exit when the last read didn't read anything + // Move the next line to the beginning of the buffer + const size_t oldlinelen = strlen(line); + if (sizeof(line) == oldlinelen + 1) // oldlinelen took up entire line + line[0] = '\0'; + else // still other lines left to save + memmove(line, line + oldlinelen + 1, sizeof(line) - (oldlinelen + 1)); + // Terminate the new line, reading more if we can't find the newline + char* newline = strchr(line, '\n'); + if (newline == nullptr) { + const size_t linelen = strlen(line); + const size_t bytes_to_read = sizeof(line) - 1 - linelen; + CHECK(bytes_to_read > 0); // because the memmove recovered >=1 bytes + chars_read = read(fd, line + linelen, bytes_to_read); + line[linelen + chars_read] = '\0'; + newline = strchr(line, '\n'); + } + if (newline != nullptr) *newline = '\0'; + + // When parsing the "cpu MHz" and "bogomips" (fallback) entries, we only + // accept postive values. Some environments (virtual machines) report zero, + // which would cause infinite looping in WallTime_Init. + if (!saw_mhz && startsWithKey(line, "cpu MHz")) { + const char* freqstr = strchr(line, ':'); + if (freqstr) { + cpuinfo_cycles_per_second = strtod(freqstr + 1, &err) * 1000000.0; + if (freqstr[1] != '\0' && *err == '\0' && cpuinfo_cycles_per_second > 0) + saw_mhz = true; + } + } else if (startsWithKey(line, "bogomips")) { + const char* freqstr = strchr(line, ':'); + if (freqstr) { + bogo_clock = strtod(freqstr + 1, &err) * 1000000.0; + if (freqstr[1] != '\0' && *err == '\0' && bogo_clock > 0) + saw_bogo = true; + } + } else if (startsWithKey(line, "processor", /*IgnoreCase*/false)) { + // The above comparison is case-sensitive because ARM kernels often + // include a "Processor" line that tells you about the CPU, distinct + // from the usual "processor" lines that give you CPU ids. No current + // Linux architecture is using "Processor" for CPU ids. + num_cpus++; // count up every time we see an "processor :" entry + const char* id_str = strchr(line, ':'); + if (id_str) { + const long cpu_id = strtol(id_str + 1, &err, 10); + if (id_str[1] != '\0' && *err == '\0' && max_cpu_id < cpu_id) + max_cpu_id = cpu_id; + } + } + } while (chars_read > 0); + close(fd); + + if (!saw_mhz) { + if (saw_bogo) { + // If we didn't find anything better, we'll use bogomips, but + // we're not happy about it. + cpuinfo_cycles_per_second = bogo_clock; + } else { + // If we don't even have bogomips, we'll use the slow estimation. + cpuinfo_cycles_per_second = + static_cast(EstimateCyclesPerSecond()); + } + } + if (num_cpus == 0) { + fprintf(stderr, "Failed to read num. CPUs correctly from /proc/cpuinfo\n"); + } else { + if ((max_cpu_id + 1) != num_cpus) { + fprintf(stderr, + "CPU ID assignments in /proc/cpuinfo seem messed up." + " This is usually caused by a bad BIOS.\n"); + } + cpuinfo_num_cpus = num_cpus; + } + +#elif defined BENCHMARK_OS_FREEBSD +// For this sysctl to work, the machine must be configured without +// SMP, APIC, or APM support. hz should be 64-bit in freebsd 7.0 +// and later. Before that, it's a 32-bit quantity (and gives the +// wrong answer on machines faster than 2^32 Hz). See +// http://lists.freebsd.org/pipermail/freebsd-i386/2004-November/001846.html +// But also compare FreeBSD 7.0: +// http://fxr.watson.org/fxr/source/i386/i386/tsc.c?v=RELENG70#L223 +// 231 error = sysctl_handle_quad(oidp, &freq, 0, req); +// To FreeBSD 6.3 (it's the same in 6-STABLE): +// http://fxr.watson.org/fxr/source/i386/i386/tsc.c?v=RELENG6#L131 +// 139 error = sysctl_handle_int(oidp, &freq, sizeof(freq), req); +#if __FreeBSD__ >= 7 + uint64_t hz = 0; +#else + unsigned int hz = 0; +#endif + size_t sz = sizeof(hz); + const char* sysctl_path = "machdep.tsc_freq"; + if (sysctlbyname(sysctl_path, &hz, &sz, nullptr, 0) != 0) { + fprintf(stderr, "Unable to determine clock rate from sysctl: %s: %s\n", + sysctl_path, strerror(errno)); + cpuinfo_cycles_per_second = static_cast(EstimateCyclesPerSecond()); + } else { + cpuinfo_cycles_per_second = hz; + } +// TODO: also figure out cpuinfo_num_cpus + +#elif defined BENCHMARK_OS_WINDOWS + // In NT, read MHz from the registry. If we fail to do so or we're in win9x + // then make a crude estimate. + DWORD data, data_size = sizeof(data); + if (IsWindowsXPOrGreater() && + SUCCEEDED( + SHGetValueA(HKEY_LOCAL_MACHINE, + "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", + "~MHz", nullptr, &data, &data_size))) + cpuinfo_cycles_per_second = + static_cast((int64_t)data * (int64_t)(1000 * 1000)); // was mhz + else + cpuinfo_cycles_per_second = static_cast(EstimateCyclesPerSecond()); + + SYSTEM_INFO sysinfo; + // Use memset as opposed to = {} to avoid GCC missing initializer false + // positives. + std::memset(&sysinfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&sysinfo); + cpuinfo_num_cpus = sysinfo.dwNumberOfProcessors; // number of logical + // processors in the current + // group + +#elif defined BENCHMARK_OS_MACOSX + int32_t num_cpus = 0; + size_t size = sizeof(num_cpus); + if (::sysctlbyname("hw.ncpu", &num_cpus, &size, nullptr, 0) == 0 && + (size == sizeof(num_cpus))) { + cpuinfo_num_cpus = num_cpus; + } else { + fprintf(stderr, "%s\n", strerror(errno)); + std::exit(EXIT_FAILURE); + } + int64_t cpu_freq = 0; + size = sizeof(cpu_freq); + if (::sysctlbyname("hw.cpufrequency", &cpu_freq, &size, nullptr, 0) == 0 && + (size == sizeof(cpu_freq))) { + cpuinfo_cycles_per_second = cpu_freq; + } else { + #if defined BENCHMARK_OS_IOS + fprintf(stderr, "CPU frequency cannot be detected. \n"); + cpuinfo_cycles_per_second = 0; + #else + fprintf(stderr, "%s\n", strerror(errno)); + std::exit(EXIT_FAILURE); + #endif + } +#else + // Generic cycles per second counter + cpuinfo_cycles_per_second = static_cast(EstimateCyclesPerSecond()); +#endif +} + +} // end namespace + +double CyclesPerSecond(void) { + std::call_once(cpuinfo_init, InitializeSystemInfo); + return cpuinfo_cycles_per_second; +} + +int NumCPUs(void) { + std::call_once(cpuinfo_init, InitializeSystemInfo); + return cpuinfo_num_cpus; +} + +// The ""'s catch people who don't pass in a literal for "str" +#define strliterallen(str) (sizeof("" str "") - 1) + +// Must use a string literal for prefix. +#define memprefix(str, len, prefix) \ + ((((len) >= strliterallen(prefix)) && \ + std::memcmp(str, prefix, strliterallen(prefix)) == 0) \ + ? str + strliterallen(prefix) \ + : nullptr) + +bool CpuScalingEnabled() { +#ifndef BENCHMARK_OS_WINDOWS + // On Linux, the CPUfreq subsystem exposes CPU information as files on the + // local file system. If reading the exported files fails, then we may not be + // running on Linux, so we silently ignore all the read errors. + for (int cpu = 0, num_cpus = NumCPUs(); cpu < num_cpus; ++cpu) { + std::string governor_file = + StrCat("/sys/devices/system/cpu/cpu", cpu, "/cpufreq/scaling_governor"); + FILE* file = fopen(governor_file.c_str(), "r"); + if (!file) break; + char buff[16]; + size_t bytes_read = fread(buff, 1, sizeof(buff), file); + fclose(file); + if (memprefix(buff, bytes_read, "performance") == nullptr) return true; + } +#endif + return false; +} + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sysinfo.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sysinfo.h new file mode 100644 index 00000000000..c5d9916d2dd --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/sysinfo.h @@ -0,0 +1,10 @@ +#ifndef BENCHMARK_SYSINFO_H_ +#define BENCHMARK_SYSINFO_H_ + +namespace benchmark { +int NumCPUs(); +double CyclesPerSecond(); +bool CpuScalingEnabled(); +} // end namespace benchmark + +#endif // BENCHMARK_SYSINFO_H_ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/timers.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/timers.cc new file mode 100644 index 00000000000..817272d00bc --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/timers.cc @@ -0,0 +1,212 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "timers.h" +#include "internal_macros.h" + +#ifdef BENCHMARK_OS_WINDOWS +#include +#include +#include +#else +#include +#include +#include +#include // this header must be included before 'sys/sysctl.h' to avoid compilation error on FreeBSD +#include +#if defined BENCHMARK_OS_FREEBSD || defined BENCHMARK_OS_MACOSX +#include +#endif +#if defined(BENCHMARK_OS_MACOSX) +#include +#include +#include +#endif +#endif + +#ifdef BENCHMARK_OS_EMSCRIPTEN +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "check.h" +#include "log.h" +#include "sleep.h" +#include "string_util.h" + +namespace benchmark { + +// Suppress unused warnings on helper functions. +#if defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wunused-function" +#endif + +namespace { +#if defined(BENCHMARK_OS_WINDOWS) +double MakeTime(FILETIME const& kernel_time, FILETIME const& user_time) { + ULARGE_INTEGER kernel; + ULARGE_INTEGER user; + kernel.HighPart = kernel_time.dwHighDateTime; + kernel.LowPart = kernel_time.dwLowDateTime; + user.HighPart = user_time.dwHighDateTime; + user.LowPart = user_time.dwLowDateTime; + return (static_cast(kernel.QuadPart) + + static_cast(user.QuadPart)) * + 1e-7; +} +#else +double MakeTime(struct rusage const& ru) { + return (static_cast(ru.ru_utime.tv_sec) + + static_cast(ru.ru_utime.tv_usec) * 1e-6 + + static_cast(ru.ru_stime.tv_sec) + + static_cast(ru.ru_stime.tv_usec) * 1e-6); +} +#endif +#if defined(BENCHMARK_OS_MACOSX) +double MakeTime(thread_basic_info_data_t const& info) { + return (static_cast(info.user_time.seconds) + + static_cast(info.user_time.microseconds) * 1e-6 + + static_cast(info.system_time.seconds) + + static_cast(info.system_time.microseconds) * 1e-6); +} +#endif +#if defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_THREAD_CPUTIME_ID) +double MakeTime(struct timespec const& ts) { + return ts.tv_sec + (static_cast(ts.tv_nsec) * 1e-9); +} +#endif + +BENCHMARK_NORETURN static void DiagnoseAndExit(const char* msg) { + std::cerr << "ERROR: " << msg << std::endl; + std::exit(EXIT_FAILURE); +} + +} // end namespace + +double ProcessCPUUsage() { +#if defined(BENCHMARK_OS_WINDOWS) + HANDLE proc = GetCurrentProcess(); + FILETIME creation_time; + FILETIME exit_time; + FILETIME kernel_time; + FILETIME user_time; + if (GetProcessTimes(proc, &creation_time, &exit_time, &kernel_time, + &user_time)) + return MakeTime(kernel_time, user_time); + DiagnoseAndExit("GetProccessTimes() failed"); +#elif defined(BENCHMARK_OS_EMSCRIPTEN) + // clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) returns 0 on Emscripten. + // Use Emscripten-specific API. Reported CPU time would be exactly the + // same as total time, but this is ok because there aren't long-latency + // syncronous system calls in Emscripten. + return emscripten_get_now() * 1e-3; +#elif defined(CLOCK_PROCESS_CPUTIME_ID) && !defined(BENCHMARK_OS_MACOSX) + // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. See + // https://github.com/google/benchmark/pull/292 + struct timespec spec; + if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &spec) == 0) + return MakeTime(spec); + DiagnoseAndExit("clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ...) failed"); +#else + struct rusage ru; + if (getrusage(RUSAGE_SELF, &ru) == 0) return MakeTime(ru); + DiagnoseAndExit("getrusage(RUSAGE_SELF, ...) failed"); +#endif +} + +double ThreadCPUUsage() { +#if defined(BENCHMARK_OS_WINDOWS) + HANDLE this_thread = GetCurrentThread(); + FILETIME creation_time; + FILETIME exit_time; + FILETIME kernel_time; + FILETIME user_time; + GetThreadTimes(this_thread, &creation_time, &exit_time, &kernel_time, + &user_time); + return MakeTime(kernel_time, user_time); +#elif defined(BENCHMARK_OS_MACOSX) + // FIXME We want to use clock_gettime, but its not available in MacOS 10.11. See + // https://github.com/google/benchmark/pull/292 + mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; + thread_basic_info_data_t info; + mach_port_t thread = pthread_mach_thread_np(pthread_self()); + if (thread_info(thread, THREAD_BASIC_INFO, (thread_info_t)&info, &count) == + KERN_SUCCESS) { + return MakeTime(info); + } + DiagnoseAndExit("ThreadCPUUsage() failed when evaluating thread_info"); +#elif defined(BENCHMARK_OS_EMSCRIPTEN) + // Emscripten doesn't support traditional threads + return ProcessCPUUsage(); +#elif defined(BENCHMARK_OS_RTEMS) + // RTEMS doesn't support CLOCK_THREAD_CPUTIME_ID. See + // https://github.com/RTEMS/rtems/blob/master/cpukit/posix/src/clockgettime.c + return ProcessCPUUsage(); +#elif defined(CLOCK_THREAD_CPUTIME_ID) + struct timespec ts; + if (clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts) == 0) return MakeTime(ts); + DiagnoseAndExit("clock_gettime(CLOCK_THREAD_CPUTIME_ID, ...) failed"); +#else +#error Per-thread timing is not available on your system. +#endif +} + +namespace { + +std::string DateTimeString(bool local) { + typedef std::chrono::system_clock Clock; + std::time_t now = Clock::to_time_t(Clock::now()); + const std::size_t kStorageSize = 128; + char storage[kStorageSize]; + std::size_t written; + + if (local) { +#if defined(BENCHMARK_OS_WINDOWS) + written = + std::strftime(storage, sizeof(storage), "%x %X", ::localtime(&now)); +#else + std::tm timeinfo; + std::memset(&timeinfo, 0, sizeof(std::tm)); + ::localtime_r(&now, &timeinfo); + written = std::strftime(storage, sizeof(storage), "%F %T", &timeinfo); +#endif + } else { +#if defined(BENCHMARK_OS_WINDOWS) + written = std::strftime(storage, sizeof(storage), "%x %X", ::gmtime(&now)); +#else + std::tm timeinfo; + std::memset(&timeinfo, 0, sizeof(std::tm)); + ::gmtime_r(&now, &timeinfo); + written = std::strftime(storage, sizeof(storage), "%F %T", &timeinfo); +#endif + } + CHECK(written < kStorageSize); + ((void)written); // prevent unused variable in optimized mode. + return std::string(storage); +} + +} // end namespace + +std::string LocalDateTimeString() { return DateTimeString(true); } + +} // end namespace benchmark diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/timers.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/timers.h new file mode 100644 index 00000000000..65606ccd93d --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/src/timers.h @@ -0,0 +1,48 @@ +#ifndef BENCHMARK_TIMERS_H +#define BENCHMARK_TIMERS_H + +#include +#include + +namespace benchmark { + +// Return the CPU usage of the current process +double ProcessCPUUsage(); + +// Return the CPU usage of the children of the current process +double ChildrenCPUUsage(); + +// Return the CPU usage of the current thread +double ThreadCPUUsage(); + +#if defined(HAVE_STEADY_CLOCK) +template +struct ChooseSteadyClock { + typedef std::chrono::high_resolution_clock type; +}; + +template <> +struct ChooseSteadyClock { + typedef std::chrono::steady_clock type; +}; +#endif + +struct ChooseClockType { +#if defined(HAVE_STEADY_CLOCK) + typedef ChooseSteadyClock<>::type type; +#else + typedef std::chrono::high_resolution_clock type; +#endif +}; + +inline double ChronoClockNow() { + typedef ChooseClockType::type ClockType; + using FpSeconds = std::chrono::duration; + return FpSeconds(ClockType::now().time_since_epoch()).count(); +} + +std::string LocalDateTimeString(); + +} // end namespace benchmark + +#endif // BENCHMARK_TIMERS_H diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/CMakeLists.txt b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/CMakeLists.txt new file mode 100644 index 00000000000..b55612b4655 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/CMakeLists.txt @@ -0,0 +1,170 @@ +# Enable the tests + +find_package(Threads REQUIRED) +include(CheckCXXCompilerFlag) + +# NOTE: Some tests use `` to perform the test. Therefore we must +# strip -DNDEBUG from the default CMake flags in DEBUG mode. +string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) +if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" ) + add_definitions( -UNDEBUG ) + add_definitions(-DTEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS) + # Also remove /D NDEBUG to avoid MSVC warnings about conflicting defines. + foreach (flags_var_to_scrub + CMAKE_CXX_FLAGS_RELEASE + CMAKE_CXX_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS_MINSIZEREL + CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_C_FLAGS_MINSIZEREL) + string (REGEX REPLACE "(^| )[/-]D *NDEBUG($| )" " " + "${flags_var_to_scrub}" "${${flags_var_to_scrub}}") + endforeach() +endif() + +# NOTE: These flags must be added after find_package(Threads REQUIRED) otherwise +# they will break the configuration check. +if (DEFINED BENCHMARK_CXX_LINKER_FLAGS) + list(APPEND CMAKE_EXE_LINKER_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}) +endif() + +add_library(output_test_helper STATIC output_test_helper.cc output_test.h) + +macro(compile_benchmark_test name) + add_executable(${name} "${name}.cc") + target_link_libraries(${name} benchmark ${CMAKE_THREAD_LIBS_INIT}) +endmacro(compile_benchmark_test) + + +macro(compile_output_test name) + add_executable(${name} "${name}.cc" output_test.h) + target_link_libraries(${name} output_test_helper benchmark + ${BENCHMARK_CXX_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) +endmacro(compile_output_test) + + +# Demonstration executable +compile_benchmark_test(benchmark_test) +add_test(benchmark benchmark_test --benchmark_min_time=0.01) + +compile_benchmark_test(filter_test) +macro(add_filter_test name filter expect) + add_test(${name} filter_test --benchmark_min_time=0.01 --benchmark_filter=${filter} ${expect}) + add_test(${name}_list_only filter_test --benchmark_list_tests --benchmark_filter=${filter} ${expect}) +endmacro(add_filter_test) + +add_filter_test(filter_simple "Foo" 3) +add_filter_test(filter_suffix "BM_.*" 4) +add_filter_test(filter_regex_all ".*" 5) +add_filter_test(filter_regex_blank "" 5) +add_filter_test(filter_regex_none "monkey" 0) +add_filter_test(filter_regex_wildcard ".*Foo.*" 3) +add_filter_test(filter_regex_begin "^BM_.*" 4) +add_filter_test(filter_regex_begin2 "^N" 1) +add_filter_test(filter_regex_end ".*Ba$" 1) + +compile_benchmark_test(options_test) +add_test(options_benchmarks options_test --benchmark_min_time=0.01) + +compile_benchmark_test(basic_test) +add_test(basic_benchmark basic_test --benchmark_min_time=0.01) + +compile_benchmark_test(diagnostics_test) +add_test(diagnostics_test diagnostics_test --benchmark_min_time=0.01) + +compile_benchmark_test(skip_with_error_test) +add_test(skip_with_error_test skip_with_error_test --benchmark_min_time=0.01) + +compile_benchmark_test(donotoptimize_test) +# Some of the issues with DoNotOptimize only occur when optimization is enabled +check_cxx_compiler_flag(-O3 BENCHMARK_HAS_O3_FLAG) +if (BENCHMARK_HAS_O3_FLAG) + set_target_properties(donotoptimize_test PROPERTIES COMPILE_FLAGS "-O3") +endif() +add_test(donotoptimize_test donotoptimize_test --benchmark_min_time=0.01) + +compile_benchmark_test(fixture_test) +add_test(fixture_test fixture_test --benchmark_min_time=0.01) + +compile_benchmark_test(register_benchmark_test) +add_test(register_benchmark_test register_benchmark_test --benchmark_min_time=0.01) + +compile_benchmark_test(map_test) +add_test(map_test map_test --benchmark_min_time=0.01) + +compile_benchmark_test(multiple_ranges_test) +add_test(multiple_ranges_test multiple_ranges_test --benchmark_min_time=0.01) + +compile_output_test(reporter_output_test) +add_test(reporter_output_test reporter_output_test --benchmark_min_time=0.01) + +compile_output_test(user_counters_test) +add_test(user_counters_test user_counters_test --benchmark_min_time=0.01) + +compile_output_test(user_counters_tabular_test) +add_test(user_counters_tabular_test user_counters_tabular_test --benchmark_counters_tabular=true --benchmark_min_time=0.01) + +check_cxx_compiler_flag(-std=c++03 BENCHMARK_HAS_CXX03_FLAG) +if (BENCHMARK_HAS_CXX03_FLAG) + set(CXX03_FLAGS "${CMAKE_CXX_FLAGS}") + string(REPLACE "-std=c++11" "-std=c++03" CXX03_FLAGS "${CXX03_FLAGS}") + string(REPLACE "-std=c++0x" "-std=c++03" CXX03_FLAGS "${CXX03_FLAGS}") + + compile_benchmark_test(cxx03_test) + set_target_properties(cxx03_test + PROPERTIES COMPILE_FLAGS "${CXX03_FLAGS}") + add_test(cxx03 cxx03_test --benchmark_min_time=0.01) +endif() + +# Attempt to work around flaky test failures when running on Appveyor servers. +if (DEFINED ENV{APPVEYOR}) + set(COMPLEXITY_MIN_TIME "0.5") +else() + set(COMPLEXITY_MIN_TIME "0.01") +endif() +compile_output_test(complexity_test) +add_test(complexity_benchmark complexity_test --benchmark_min_time=${COMPLEXITY_MIN_TIME}) + +# Add the coverage command(s) +if(CMAKE_BUILD_TYPE) + string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) +endif() +if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage") + find_program(GCOV gcov) + find_program(LCOV lcov) + find_program(GENHTML genhtml) + find_program(CTEST ctest) + if (GCOV AND LCOV AND GENHTML AND CTEST AND HAVE_CXX_FLAG_COVERAGE) + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/lcov/index.html + COMMAND ${LCOV} -q -z -d . + COMMAND ${LCOV} -q --no-external -c -b "${CMAKE_SOURCE_DIR}" -d . -o before.lcov -i + COMMAND ${CTEST} --force-new-ctest-process + COMMAND ${LCOV} -q --no-external -c -b "${CMAKE_SOURCE_DIR}" -d . -o after.lcov + COMMAND ${LCOV} -q -a before.lcov -a after.lcov --output-file final.lcov + COMMAND ${LCOV} -q -r final.lcov "'${CMAKE_SOURCE_DIR}/test/*'" -o final.lcov + COMMAND ${GENHTML} final.lcov -o lcov --demangle-cpp --sort -p "${CMAKE_BINARY_DIR}" -t benchmark + DEPENDS filter_test benchmark_test options_test basic_test fixture_test cxx03_test complexity_test + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Running LCOV" + ) + add_custom_target(coverage + DEPENDS ${CMAKE_BINARY_DIR}/lcov/index.html + COMMENT "LCOV report at lcov/index.html" + ) + message(STATUS "Coverage command added") + else() + if (HAVE_CXX_FLAG_COVERAGE) + set(CXX_FLAG_COVERAGE_MESSAGE supported) + else() + set(CXX_FLAG_COVERAGE_MESSAGE unavailable) + endif() + message(WARNING + "Coverage not available:\n" + " gcov: ${GCOV}\n" + " lcov: ${LCOV}\n" + " genhtml: ${GENHTML}\n" + " ctest: ${CTEST}\n" + " --coverage flag: ${CXX_FLAG_COVERAGE_MESSAGE}") + endif() +endif() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/basic_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/basic_test.cc new file mode 100644 index 00000000000..bc1f96d9315 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/basic_test.cc @@ -0,0 +1,99 @@ + +#include "benchmark/benchmark.h" + +#define BASIC_BENCHMARK_TEST(x) BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192) + +void BM_empty(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } +} +BENCHMARK(BM_empty); +BENCHMARK(BM_empty)->ThreadPerCpu(); + +void BM_spin_empty(benchmark::State& state) { + while (state.KeepRunning()) { + for (int x = 0; x < state.range(0); ++x) { + benchmark::DoNotOptimize(x); + } + } +} +BASIC_BENCHMARK_TEST(BM_spin_empty); +BASIC_BENCHMARK_TEST(BM_spin_empty)->ThreadPerCpu(); + +void BM_spin_pause_before(benchmark::State& state) { + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + while (state.KeepRunning()) { + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + } +} +BASIC_BENCHMARK_TEST(BM_spin_pause_before); +BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu(); + +void BM_spin_pause_during(benchmark::State& state) { + while (state.KeepRunning()) { + state.PauseTiming(); + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + state.ResumeTiming(); + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + } +} +BASIC_BENCHMARK_TEST(BM_spin_pause_during); +BASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu(); + +void BM_pause_during(benchmark::State& state) { + while (state.KeepRunning()) { + state.PauseTiming(); + state.ResumeTiming(); + } +} +BENCHMARK(BM_pause_during); +BENCHMARK(BM_pause_during)->ThreadPerCpu(); +BENCHMARK(BM_pause_during)->UseRealTime(); +BENCHMARK(BM_pause_during)->UseRealTime()->ThreadPerCpu(); + +void BM_spin_pause_after(benchmark::State& state) { + while (state.KeepRunning()) { + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + } + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } +} +BASIC_BENCHMARK_TEST(BM_spin_pause_after); +BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu(); + +void BM_spin_pause_before_and_after(benchmark::State& state) { + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + while (state.KeepRunning()) { + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } + } + for (int i = 0; i < state.range(0); ++i) { + benchmark::DoNotOptimize(i); + } +} +BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after); +BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu(); + +void BM_empty_stop_start(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_empty_stop_start); +BENCHMARK(BM_empty_stop_start)->ThreadPerCpu(); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/benchmark_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/benchmark_test.cc new file mode 100644 index 00000000000..7a16466e208 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/benchmark_test.cc @@ -0,0 +1,240 @@ +#include "benchmark/benchmark.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GNUC__) +#define BENCHMARK_NOINLINE __attribute__((noinline)) +#else +#define BENCHMARK_NOINLINE +#endif + +namespace { + +int BENCHMARK_NOINLINE Factorial(uint32_t n) { + return (n == 1) ? 1 : n * Factorial(n - 1); +} + +double CalculatePi(int depth) { + double pi = 0.0; + for (int i = 0; i < depth; ++i) { + double numerator = static_cast(((i % 2) * 2) - 1); + double denominator = static_cast((2 * i) - 1); + pi += numerator / denominator; + } + return (pi - 1.0) * 4; +} + +std::set ConstructRandomSet(int size) { + std::set s; + for (int i = 0; i < size; ++i) s.insert(i); + return s; +} + +std::mutex test_vector_mu; +std::vector* test_vector = nullptr; + +} // end namespace + +static void BM_Factorial(benchmark::State& state) { + int fac_42 = 0; + while (state.KeepRunning()) fac_42 = Factorial(8); + // Prevent compiler optimizations + std::stringstream ss; + ss << fac_42; + state.SetLabel(ss.str()); +} +BENCHMARK(BM_Factorial); +BENCHMARK(BM_Factorial)->UseRealTime(); + +static void BM_CalculatePiRange(benchmark::State& state) { + double pi = 0.0; + while (state.KeepRunning()) pi = CalculatePi(state.range(0)); + std::stringstream ss; + ss << pi; + state.SetLabel(ss.str()); +} +BENCHMARK_RANGE(BM_CalculatePiRange, 1, 1024 * 1024); + +static void BM_CalculatePi(benchmark::State& state) { + static const int depth = 1024; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(CalculatePi(depth)); + } +} +BENCHMARK(BM_CalculatePi)->Threads(8); +BENCHMARK(BM_CalculatePi)->ThreadRange(1, 32); +BENCHMARK(BM_CalculatePi)->ThreadPerCpu(); + +static void BM_SetInsert(benchmark::State& state) { + while (state.KeepRunning()) { + state.PauseTiming(); + std::set data = ConstructRandomSet(state.range(0)); + state.ResumeTiming(); + for (int j = 0; j < state.range(1); ++j) data.insert(rand()); + } + state.SetItemsProcessed(state.iterations() * state.range(1)); + state.SetBytesProcessed(state.iterations() * state.range(1) * sizeof(int)); +} +BENCHMARK(BM_SetInsert)->Ranges({{1 << 10, 8 << 10}, {1, 10}}); + +template +static void BM_Sequential(benchmark::State& state) { + ValueType v = 42; + while (state.KeepRunning()) { + Container c; + for (int i = state.range(0); --i;) c.push_back(v); + } + const size_t items_processed = state.iterations() * state.range(0); + state.SetItemsProcessed(items_processed); + state.SetBytesProcessed(items_processed * sizeof(v)); +} +BENCHMARK_TEMPLATE2(BM_Sequential, std::vector, int) + ->Range(1 << 0, 1 << 10); +BENCHMARK_TEMPLATE(BM_Sequential, std::list)->Range(1 << 0, 1 << 10); +// Test the variadic version of BENCHMARK_TEMPLATE in C++11 and beyond. +#if __cplusplus >= 201103L +BENCHMARK_TEMPLATE(BM_Sequential, std::vector, int)->Arg(512); +#endif + +static void BM_StringCompare(benchmark::State& state) { + std::string s1(state.range(0), '-'); + std::string s2(state.range(0), '-'); + while (state.KeepRunning()) benchmark::DoNotOptimize(s1.compare(s2)); +} +BENCHMARK(BM_StringCompare)->Range(1, 1 << 20); + +static void BM_SetupTeardown(benchmark::State& state) { + if (state.thread_index == 0) { + // No need to lock test_vector_mu here as this is running single-threaded. + test_vector = new std::vector(); + } + int i = 0; + while (state.KeepRunning()) { + std::lock_guard l(test_vector_mu); + if (i % 2 == 0) + test_vector->push_back(i); + else + test_vector->pop_back(); + ++i; + } + if (state.thread_index == 0) { + delete test_vector; + } +} +BENCHMARK(BM_SetupTeardown)->ThreadPerCpu(); + +static void BM_LongTest(benchmark::State& state) { + double tracker = 0.0; + while (state.KeepRunning()) { + for (int i = 0; i < state.range(0); ++i) + benchmark::DoNotOptimize(tracker += i); + } +} +BENCHMARK(BM_LongTest)->Range(1 << 16, 1 << 28); + +static void BM_ParallelMemset(benchmark::State& state) { + int size = state.range(0) / static_cast(sizeof(int)); + int thread_size = size / state.threads; + int from = thread_size * state.thread_index; + int to = from + thread_size; + + if (state.thread_index == 0) { + test_vector = new std::vector(size); + } + + while (state.KeepRunning()) { + for (int i = from; i < to; i++) { + // No need to lock test_vector_mu as ranges + // do not overlap between threads. + benchmark::DoNotOptimize(test_vector->at(i) = 1); + } + } + + if (state.thread_index == 0) { + delete test_vector; + } +} +BENCHMARK(BM_ParallelMemset)->Arg(10 << 20)->ThreadRange(1, 4); + +static void BM_ManualTiming(benchmark::State& state) { + size_t slept_for = 0; + int microseconds = state.range(0); + std::chrono::duration sleep_duration{ + static_cast(microseconds)}; + + while (state.KeepRunning()) { + auto start = std::chrono::high_resolution_clock::now(); + // Simulate some useful workload with a sleep + std::this_thread::sleep_for( + std::chrono::duration_cast(sleep_duration)); + auto end = std::chrono::high_resolution_clock::now(); + + auto elapsed = + std::chrono::duration_cast>(end - start); + + state.SetIterationTime(elapsed.count()); + slept_for += microseconds; + } + state.SetItemsProcessed(slept_for); +} +BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime(); +BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime(); + +#if __cplusplus >= 201103L + +template +void BM_with_args(benchmark::State& state, Args&&...) { + while (state.KeepRunning()) { + } +} +BENCHMARK_CAPTURE(BM_with_args, int_test, 42, 43, 44); +BENCHMARK_CAPTURE(BM_with_args, string_and_pair_test, std::string("abc"), + std::pair(42, 3.8)); + +void BM_non_template_args(benchmark::State& state, int, double) { + while(state.KeepRunning()) {} +} +BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); + +#endif // __cplusplus >= 201103L + +static void BM_DenseThreadRanges(benchmark::State& st) { + switch (st.range(0)) { + case 1: + assert(st.threads == 1 || st.threads == 2 || st.threads == 3); + break; + case 2: + assert(st.threads == 1 || st.threads == 3 || st.threads == 4); + break; + case 3: + assert(st.threads == 5 || st.threads == 8 || st.threads == 11 || + st.threads == 14); + break; + default: + assert(false && "Invalid test case number"); + } + while (st.KeepRunning()) { + } +} +BENCHMARK(BM_DenseThreadRanges)->Arg(1)->DenseThreadRange(1, 3); +BENCHMARK(BM_DenseThreadRanges)->Arg(2)->DenseThreadRange(1, 4, 2); +BENCHMARK(BM_DenseThreadRanges)->Arg(3)->DenseThreadRange(5, 14, 3); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/complexity_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/complexity_test.cc new file mode 100644 index 00000000000..62d1154df0e --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/complexity_test.cc @@ -0,0 +1,167 @@ +#undef NDEBUG +#include +#include +#include +#include +#include +#include "benchmark/benchmark.h" +#include "output_test.h" + +namespace { + +#define ADD_COMPLEXITY_CASES(...) \ + int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) + +int AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, + std::string big_o) { + SetSubstitutions({{"%bigo_name", big_o_test_name}, + {"%rms_name", rms_test_name}, + {"%bigo_str", "[ ]* %float " + big_o}, + {"%bigo", big_o}, + {"%rms", "[ ]*[0-9]+ %"}}); + AddCases( + TC_ConsoleOut, + {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, + {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. + {"^%rms_name %rms %rms[ ]*$", MR_Next}}); + AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, + {"\"cpu_coefficient\": [0-9]+,$", MR_Next}, + {"\"real_coefficient\": [0-9]{1,5},$", MR_Next}, + {"\"big_o\": \"%bigo\",$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}, + {"\"name\": \"%rms_name\",$"}, + {"\"rms\": %float$", MR_Next}, + {"}", MR_Next}}); + AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, + {"^\"%bigo_name\"", MR_Not}, + {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); + return 0; +} + +} // end namespace + +// ========================================================================= // +// --------------------------- Testing BigO O(1) --------------------------- // +// ========================================================================= // + +void BM_Complexity_O1(benchmark::State& state) { + while (state.KeepRunning()) { + for (int i = 0; i < 1024; ++i) { + benchmark::DoNotOptimize(&i); + } + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int) { + return 1.0; +}); + +const char *big_o_1_test_name = "BM_Complexity_O1_BigO"; +const char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; +const char *enum_big_o_1 = "\\([0-9]+\\)"; +// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto +// deduced. +// See https://github.com/google/benchmark/issues/272 +const char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; +const char *lambda_big_o_1 = "f\\(N\\)"; + +// Add enum tests +ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1); + +// Add auto enum tests +ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1); + +// Add lambda tests +ADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); + +// ========================================================================= // +// --------------------------- Testing BigO O(N) --------------------------- // +// ========================================================================= // + +std::vector ConstructRandomVector(int size) { + std::vector v; + v.reserve(size); + for (int i = 0; i < size; ++i) { + v.push_back(std::rand() % size); + } + return v; +} + +void BM_Complexity_O_N(benchmark::State& state) { + auto v = ConstructRandomVector(state.range(0)); + const int item_not_in_vector = + state.range(0) * 2; // Test worst case scenario (item not in vector) + while (state.KeepRunning()) { + benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity(benchmark::oN); +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity([](int n) -> double { return n; }); +BENCHMARK(BM_Complexity_O_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity(); + +const char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; +const char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; +const char *enum_auto_big_o_n = "N"; +const char *lambda_big_o_n = "f\\(N\\)"; + +// Add enum tests +ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); + +// Add lambda tests +ADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); + +// ========================================================================= // +// ------------------------- Testing BigO O(N*lgN) ------------------------- // +// ========================================================================= // + +static void BM_Complexity_O_N_log_N(benchmark::State& state) { + auto v = ConstructRandomVector(state.range(0)); + while (state.KeepRunning()) { + std::sort(v.begin(), v.end()); + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_Complexity_O_N_log_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity(benchmark::oNLogN); +BENCHMARK(BM_Complexity_O_N_log_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity([](int n) { return n * log2(n); }); +BENCHMARK(BM_Complexity_O_N_log_N) + ->RangeMultiplier(2) + ->Range(1 << 10, 1 << 16) + ->Complexity(); + +const char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; +const char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; +const char *enum_auto_big_o_n_lg_n = "NlgN"; +const char *lambda_big_o_n_lg_n = "f\\(N\\)"; + +// Add enum tests +ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, + enum_auto_big_o_n_lg_n); + +// Add lambda tests +ADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, + lambda_big_o_n_lg_n); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char *argv[]) { RunOutputTests(argc, argv); } diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/cxx03_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/cxx03_test.cc new file mode 100644 index 00000000000..a79d964e17b --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/cxx03_test.cc @@ -0,0 +1,48 @@ +#undef NDEBUG +#include +#include + +#include "benchmark/benchmark.h" + +#if __cplusplus >= 201103L +#error C++11 or greater detected. Should be C++03. +#endif + +void BM_empty(benchmark::State& state) { + while (state.KeepRunning()) { + volatile std::size_t x = state.iterations(); + ((void)x); + } +} +BENCHMARK(BM_empty); + +// The new C++11 interface for args/ranges requires initializer list support. +// Therefore we provide the old interface to support C++03. +void BM_old_arg_range_interface(benchmark::State& state) { + assert((state.range(0) == 1 && state.range(1) == 2) || + (state.range(0) == 5 && state.range(1) == 6)); + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_old_arg_range_interface)->ArgPair(1, 2)->RangePair(5, 5, 6, 6); + +template +void BM_template2(benchmark::State& state) { + BM_empty(state); +} +BENCHMARK_TEMPLATE2(BM_template2, int, long); + +template +void BM_template1(benchmark::State& state) { + BM_empty(state); +} +BENCHMARK_TEMPLATE(BM_template1, long); +BENCHMARK_TEMPLATE1(BM_template1, int); + +void BM_counters(benchmark::State& state) { + BM_empty(state); + state.counters["Foo"] = 2; +} +BENCHMARK(BM_counters); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/diagnostics_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/diagnostics_test.cc new file mode 100644 index 00000000000..7aac8069e59 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/diagnostics_test.cc @@ -0,0 +1,64 @@ +// Testing: +// State::PauseTiming() +// State::ResumeTiming() +// Test that CHECK's within these function diagnose when they are called +// outside of the KeepRunning() loop. +// +// NOTE: Users should NOT include or use src/check.h. This is only done in +// order to test library internals. + +#include +#include + +#include "../src/check.h" +#include "benchmark/benchmark.h" + +#if defined(__GNUC__) && !defined(__EXCEPTIONS) +#define TEST_HAS_NO_EXCEPTIONS +#endif + +void TestHandler() { +#ifndef TEST_HAS_NO_EXCEPTIONS + throw std::logic_error(""); +#else + std::abort(); +#endif +} + +void try_invalid_pause_resume(benchmark::State& state) { +#if !defined(TEST_BENCHMARK_LIBRARY_HAS_NO_ASSERTIONS) && !defined(TEST_HAS_NO_EXCEPTIONS) + try { + state.PauseTiming(); + std::abort(); + } catch (std::logic_error const&) { + } + try { + state.ResumeTiming(); + std::abort(); + } catch (std::logic_error const&) { + } +#else + (void)state; // avoid unused warning +#endif +} + +void BM_diagnostic_test(benchmark::State& state) { + static bool called_once = false; + + if (called_once == false) try_invalid_pause_resume(state); + + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + + if (called_once == false) try_invalid_pause_resume(state); + + called_once = true; +} +BENCHMARK(BM_diagnostic_test); + +int main(int argc, char* argv[]) { + benchmark::internal::GetAbortHandler() = &TestHandler; + benchmark::Initialize(&argc, argv); + benchmark::RunSpecifiedBenchmarks(); +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/donotoptimize_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/donotoptimize_test.cc new file mode 100644 index 00000000000..a705654a269 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/donotoptimize_test.cc @@ -0,0 +1,52 @@ +#include "benchmark/benchmark.h" + +#include + +namespace { +#if defined(__GNUC__) +std::uint64_t double_up(const std::uint64_t x) __attribute__((const)); +#endif +std::uint64_t double_up(const std::uint64_t x) { return x * 2; } +} + +// Using DoNotOptimize on types like BitRef seem to cause a lot of problems +// with the inline assembly on both GCC and Clang. +struct BitRef { + int index; + unsigned char &byte; + +public: + static BitRef Make() { + static unsigned char arr[2] = {}; + BitRef b(1, arr[0]); + return b; + } +private: + BitRef(int i, unsigned char& b) : index(i), byte(b) {} +}; + +int main(int, char*[]) { + // this test verifies compilation of DoNotOptimize() for some types + + char buffer8[8]; + benchmark::DoNotOptimize(buffer8); + + char buffer20[20]; + benchmark::DoNotOptimize(buffer20); + + char buffer1024[1024]; + benchmark::DoNotOptimize(buffer1024); + benchmark::DoNotOptimize(&buffer1024[0]); + + int x = 123; + benchmark::DoNotOptimize(x); + benchmark::DoNotOptimize(&x); + benchmark::DoNotOptimize(x += 42); + + benchmark::DoNotOptimize(double_up(x)); + + // These tests are to e + benchmark::DoNotOptimize(BitRef::Make()); + BitRef lval = BitRef::Make(); + benchmark::DoNotOptimize(lval); +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/filter_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/filter_test.cc new file mode 100644 index 00000000000..3a205295f09 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/filter_test.cc @@ -0,0 +1,104 @@ +#include "benchmark/benchmark.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) { + ++count_; + ConsoleReporter::ReportRuns(report); + }; + + TestReporter() : count_(0) {} + + virtual ~TestReporter() {} + + size_t GetCount() const { return count_; } + + private: + mutable size_t count_; +}; + +} // end namespace + +static void NoPrefix(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(NoPrefix); + +static void BM_Foo(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_Foo); + +static void BM_Bar(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_Bar); + +static void BM_FooBar(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_FooBar); + +static void BM_FooBa(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_FooBa); + +int main(int argc, char **argv) { + bool list_only = false; + for (int i = 0; i < argc; ++i) + list_only |= std::string(argv[i]).find("--benchmark_list_tests") != + std::string::npos; + + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + const size_t returned_count = + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + if (argc == 2) { + // Make sure we ran all of the tests + std::stringstream ss(argv[1]); + size_t expected_return; + ss >> expected_return; + + if (returned_count != expected_return) { + std::cerr << "ERROR: Expected " << expected_return + << " tests to match the filter but returned_count = " + << returned_count << std::endl; + return -1; + } + + const size_t expected_reports = list_only ? 0 : expected_return; + const size_t reports_count = test_reporter.GetCount(); + if (reports_count != expected_reports) { + std::cerr << "ERROR: Expected " << expected_reports + << " tests to be run but reported_count = " << reports_count + << std::endl; + return -1; + } + } + + return 0; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/fixture_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/fixture_test.cc new file mode 100644 index 00000000000..bbc2f957902 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/fixture_test.cc @@ -0,0 +1,49 @@ + +#include "benchmark/benchmark.h" + +#include +#include + +class MyFixture : public ::benchmark::Fixture { + public: + void SetUp(const ::benchmark::State& state) { + if (state.thread_index == 0) { + assert(data.get() == nullptr); + data.reset(new int(42)); + } + } + + void TearDown(const ::benchmark::State& state) { + if (state.thread_index == 0) { + assert(data.get() != nullptr); + data.reset(); + } + } + + ~MyFixture() { assert(data == nullptr); } + + std::unique_ptr data; +}; + +BENCHMARK_F(MyFixture, Foo)(benchmark::State &st) { + assert(data.get() != nullptr); + assert(*data == 42); + while (st.KeepRunning()) { + } +} + +BENCHMARK_DEFINE_F(MyFixture, Bar)(benchmark::State& st) { + if (st.thread_index == 0) { + assert(data.get() != nullptr); + assert(*data == 42); + } + while (st.KeepRunning()) { + assert(data.get() != nullptr); + assert(*data == 42); + } + st.SetItemsProcessed(st.range(0)); +} +BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42); +BENCHMARK_REGISTER_F(MyFixture, Bar)->Arg(42)->ThreadPerCpu(); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/map_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/map_test.cc new file mode 100644 index 00000000000..83457c9981c --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/map_test.cc @@ -0,0 +1,56 @@ +#include "benchmark/benchmark.h" + +#include +#include + +namespace { + +std::map ConstructRandomMap(int size) { + std::map m; + for (int i = 0; i < size; ++i) { + m.insert(std::make_pair(rand() % size, rand() % size)); + } + return m; +} + +} // namespace + +// Basic version. +static void BM_MapLookup(benchmark::State& state) { + const int size = state.range(0); + while (state.KeepRunning()) { + state.PauseTiming(); + std::map m = ConstructRandomMap(size); + state.ResumeTiming(); + for (int i = 0; i < size; ++i) { + benchmark::DoNotOptimize(m.find(rand() % size)); + } + } + state.SetItemsProcessed(state.iterations() * size); +} +BENCHMARK(BM_MapLookup)->Range(1 << 3, 1 << 12); + +// Using fixtures. +class MapFixture : public ::benchmark::Fixture { + public: + void SetUp(const ::benchmark::State& st) { + m = ConstructRandomMap(st.range(0)); + } + + void TearDown(const ::benchmark::State&) { m.clear(); } + + std::map m; +}; + +BENCHMARK_DEFINE_F(MapFixture, Lookup)(benchmark::State& state) { + const int size = state.range(0); + while (state.KeepRunning()) { + for (int i = 0; i < size; ++i) { + benchmark::DoNotOptimize(m.find(rand() % size)); + } + } + state.SetItemsProcessed(state.iterations() * size); +} +BENCHMARK_REGISTER_F(MapFixture, Lookup)->Range(1 << 3, 1 << 12); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/multiple_ranges_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/multiple_ranges_test.cc new file mode 100644 index 00000000000..8e67b3b2a99 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/multiple_ranges_test.cc @@ -0,0 +1,74 @@ +#include "benchmark/benchmark.h" + +#include +#include + +class MultipleRangesFixture : public ::benchmark::Fixture { + public: + MultipleRangesFixture() + : expectedValues({{1, 3, 5}, + {1, 3, 8}, + {1, 3, 15}, + {2, 3, 5}, + {2, 3, 8}, + {2, 3, 15}, + {1, 4, 5}, + {1, 4, 8}, + {1, 4, 15}, + {2, 4, 5}, + {2, 4, 8}, + {2, 4, 15}, + {1, 7, 5}, + {1, 7, 8}, + {1, 7, 15}, + {2, 7, 5}, + {2, 7, 8}, + {2, 7, 15}, + {7, 6, 3}}) {} + + void SetUp(const ::benchmark::State& state) { + std::vector ranges = {state.range(0), state.range(1), state.range(2)}; + + assert(expectedValues.find(ranges) != expectedValues.end()); + + actualValues.insert(ranges); + } + + virtual ~MultipleRangesFixture() { + assert(actualValues.size() == expectedValues.size()); + } + + std::set> expectedValues; + std::set> actualValues; +}; + +BENCHMARK_DEFINE_F(MultipleRangesFixture, Empty)(benchmark::State& state) { + while (state.KeepRunning()) { + int product = state.range(0) * state.range(1) * state.range(2); + for (int x = 0; x < product; x++) { + benchmark::DoNotOptimize(x); + } + } +} + +BENCHMARK_REGISTER_F(MultipleRangesFixture, Empty) + ->RangeMultiplier(2) + ->Ranges({{1, 2}, {3, 7}, {5, 15}}) + ->Args({7, 6, 3}); + +void BM_CheckDefaultArgument(benchmark::State& state) { + // Test that the 'range()' without an argument is the same as 'range(0)'. + assert(state.range() == state.range(0)); + assert(state.range() != state.range(1)); + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_CheckDefaultArgument)->Ranges({{1, 5}, {6, 10}}); + +static void BM_MultipleRanges(benchmark::State& st) { + while (st.KeepRunning()) { + } +} +BENCHMARK(BM_MultipleRanges)->Ranges({{5, 5}, {6, 6}}); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/options_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/options_test.cc new file mode 100644 index 00000000000..8eac068b977 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/options_test.cc @@ -0,0 +1,65 @@ +#include "benchmark/benchmark.h" +#include +#include + +#if defined(NDEBUG) +#undef NDEBUG +#endif +#include + +void BM_basic(benchmark::State& state) { + while (state.KeepRunning()) { + } +} + +void BM_basic_slow(benchmark::State& state) { + std::chrono::milliseconds sleep_duration(state.range(0)); + while (state.KeepRunning()) { + std::this_thread::sleep_for( + std::chrono::duration_cast(sleep_duration)); + } +} + +BENCHMARK(BM_basic); +BENCHMARK(BM_basic)->Arg(42); +BENCHMARK(BM_basic_slow)->Arg(10)->Unit(benchmark::kNanosecond); +BENCHMARK(BM_basic_slow)->Arg(100)->Unit(benchmark::kMicrosecond); +BENCHMARK(BM_basic_slow)->Arg(1000)->Unit(benchmark::kMillisecond); +BENCHMARK(BM_basic)->Range(1, 8); +BENCHMARK(BM_basic)->RangeMultiplier(2)->Range(1, 8); +BENCHMARK(BM_basic)->DenseRange(10, 15); +BENCHMARK(BM_basic)->Args({42, 42}); +BENCHMARK(BM_basic)->Ranges({{64, 512}, {64, 512}}); +BENCHMARK(BM_basic)->MinTime(0.7); +BENCHMARK(BM_basic)->UseRealTime(); +BENCHMARK(BM_basic)->ThreadRange(2, 4); +BENCHMARK(BM_basic)->ThreadPerCpu(); +BENCHMARK(BM_basic)->Repetitions(3); + +void CustomArgs(benchmark::internal::Benchmark* b) { + for (int i = 0; i < 10; ++i) { + b->Arg(i); + } +} + +BENCHMARK(BM_basic)->Apply(CustomArgs); + +void BM_explicit_iteration_count(benchmark::State& st) { + // Test that benchmarks specified with an explicit iteration count are + // only run once. + static bool invoked_before = false; + assert(!invoked_before); + invoked_before = true; + + // Test that the requested iteration count is respected. + assert(st.max_iterations == 42); + size_t actual_iterations = 0; + while (st.KeepRunning()) + ++actual_iterations; + assert(st.iterations() == st.max_iterations); + assert(st.iterations() == 42); + +} +BENCHMARK(BM_explicit_iteration_count)->Iterations(42); + +BENCHMARK_MAIN() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/output_test.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/output_test.h new file mode 100644 index 00000000000..897a13866ba --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/output_test.h @@ -0,0 +1,201 @@ +#ifndef TEST_OUTPUT_TEST_H +#define TEST_OUTPUT_TEST_H + +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include + +#include "../src/re.h" +#include "benchmark/benchmark.h" + +#define CONCAT2(x, y) x##y +#define CONCAT(x, y) CONCAT2(x, y) + +#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__) + +#define SET_SUBSTITUTIONS(...) \ + int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) + +enum MatchRules { + MR_Default, // Skip non-matching lines until a match is found. + MR_Next, // Match must occur on the next line. + MR_Not // No line between the current position and the next match matches + // the regex +}; + +struct TestCase { + TestCase(std::string re, int rule = MR_Default); + + std::string regex_str; + int match_rule; + std::string substituted_regex; + std::shared_ptr regex; +}; + +enum TestCaseID { + TC_ConsoleOut, + TC_ConsoleErr, + TC_JSONOut, + TC_JSONErr, + TC_CSVOut, + TC_CSVErr, + + TC_NumID // PRIVATE +}; + +// Add a list of test cases to be run against the output specified by +// 'ID' +int AddCases(TestCaseID ID, std::initializer_list il); + +// Add or set a list of substitutions to be performed on constructed regex's +// See 'output_test_helper.cc' for a list of default substitutions. +int SetSubstitutions( + std::initializer_list> il); + +// Run all output tests. +void RunOutputTests(int argc, char* argv[]); + +// ========================================================================= // +// ------------------------- Results checking ------------------------------ // +// ========================================================================= // + +// Call this macro to register a benchmark for checking its results. This +// should be all that's needed. It subscribes a function to check the (CSV) +// results of a benchmark. This is done only after verifying that the output +// strings are really as expected. +// bm_name_pattern: a name or a regex pattern which will be matched against +// all the benchmark names. Matching benchmarks +// will be the subject of a call to checker_function +// checker_function: should be of type ResultsCheckFn (see below) +#define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ + size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) + +struct Results; +typedef std::function< void(Results const&) > ResultsCheckFn; + +size_t AddChecker(const char* bm_name_pattern, ResultsCheckFn fn); + +// Class holding the results of a benchmark. +// It is passed in calls to checker functions. +struct Results { + + // the benchmark name + std::string name; + // the benchmark fields + std::map< std::string, std::string > values; + + Results(const std::string& n) : name(n) {} + + int NumThreads() const; + + typedef enum { kCpuTime, kRealTime } BenchmarkTime; + + // get cpu_time or real_time in seconds + double GetTime(BenchmarkTime which) const; + + // get the real_time duration of the benchmark in seconds. + // it is better to use fuzzy float checks for this, as the float + // ASCII formatting is lossy. + double DurationRealTime() const { + return GetAs< double >("iterations") * GetTime(kRealTime); + } + // get the cpu_time duration of the benchmark in seconds + double DurationCPUTime() const { + return GetAs< double >("iterations") * GetTime(kCpuTime); + } + + // get the string for a result by name, or nullptr if the name + // is not found + const std::string* Get(const char* entry_name) const { + auto it = values.find(entry_name); + if(it == values.end()) return nullptr; + return &it->second; + } + + // get a result by name, parsed as a specific type. + // NOTE: for counters, use GetCounterAs instead. + template + T GetAs(const char* entry_name) const; + + // counters are written as doubles, so they have to be read first + // as a double, and only then converted to the asked type. + template + T GetCounterAs(const char* entry_name) const { + double dval = GetAs< double >(entry_name); + T tval = static_cast< T >(dval); + return tval; + } +}; + +template +T Results::GetAs(const char* entry_name) const { + auto *sv = Get(entry_name); + CHECK(sv != nullptr && !sv->empty()); + std::stringstream ss; + ss << *sv; + T out; + ss >> out; + CHECK(!ss.fail()); + return out; +} + +//---------------------------------- +// Macros to help in result checking. Do not use them with arguments causing +// side-effects. + +#define _CHECK_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value) \ + CONCAT(CHECK_, relationship) \ + (entry.getfn< var_type >(var_name), (value)) << "\n" \ + << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ + << __FILE__ << ":" << __LINE__ << ": " \ + << "expected (" << #var_type << ")" << (var_name) \ + << "=" << (entry).getfn< var_type >(var_name) \ + << " to be " #relationship " to " << (value) << "\n" + +// check with tolerance. eps_factor is the tolerance window, which is +// interpreted relative to value (eg, 0.1 means 10% of value). +#define _CHECK_FLOAT_RESULT_VALUE(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ + CONCAT(CHECK_FLOAT_, relationship) \ + (entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ + << __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ + << __FILE__ << ":" << __LINE__ << ": " \ + << "expected (" << #var_type << ")" << (var_name) \ + << "=" << (entry).getfn< var_type >(var_name) \ + << " to be " #relationship " to " << (value) << "\n" \ + << __FILE__ << ":" << __LINE__ << ": " \ + << "with tolerance of " << (eps_factor) * (value) \ + << " (" << (eps_factor)*100. << "%), " \ + << "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \ + << " (" << (((entry).getfn< var_type >(var_name) - (value)) \ + / \ + ((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \ + << "%)" + +#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ + _CHECK_RESULT_VALUE(entry, GetAs, var_type, var_name, relationship, value) + +#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ + _CHECK_RESULT_VALUE(entry, GetCounterAs, var_type, var_name, relationship, value) + +#define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \ + _CHECK_FLOAT_RESULT_VALUE(entry, GetAs, double, var_name, relationship, value, eps_factor) + +#define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \ + _CHECK_FLOAT_RESULT_VALUE(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) + +// ========================================================================= // +// --------------------------- Misc Utilities ------------------------------ // +// ========================================================================= // + +namespace { + +const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; + +} // end namespace + +#endif // TEST_OUTPUT_TEST_H diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/output_test_helper.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/output_test_helper.cc new file mode 100644 index 00000000000..24746f6d27f --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/output_test_helper.cc @@ -0,0 +1,423 @@ +#include +#include +#include +#include +#include + +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "../src/re.h" // NOTE: re.h is for internal use only +#include "output_test.h" +#include "../src/benchmark_api_internal.h" + +// ========================================================================= // +// ------------------------------ Internals -------------------------------- // +// ========================================================================= // +namespace internal { +namespace { + +using TestCaseList = std::vector; + +// Use a vector because the order elements are added matters during iteration. +// std::map/unordered_map don't guarantee that. +// For example: +// SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); +// Substitute("%HelloWorld") // Always expands to Hello. +using SubMap = std::vector>; + +TestCaseList& GetTestCaseList(TestCaseID ID) { + // Uses function-local statics to ensure initialization occurs + // before first use. + static TestCaseList lists[TC_NumID]; + return lists[ID]; +} + +SubMap& GetSubstitutions() { + // Don't use 'dec_re' from header because it may not yet be initialized. + static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; + static SubMap map = { + {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, + // human-readable float + {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, + {"%int", "[ ]*[0-9]+"}, + {" %s ", "[ ]+"}, + {"%time", "[ ]*[0-9]{1,5} ns"}, + {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"}, + {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"}, + {"%csv_header", + "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," + "items_per_second,label,error_occurred,error_message"}, + {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, + {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, + {"%csv_bytes_report", + "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, + {"%csv_items_report", + "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, + {"%csv_bytes_items_report", + "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + + "," + safe_dec_re + ",,,"}, + {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, + {"%csv_label_report_end", ",,"}}; + return map; +} + +std::string PerformSubstitutions(std::string source) { + SubMap const& subs = GetSubstitutions(); + using SizeT = std::string::size_type; + for (auto const& KV : subs) { + SizeT pos; + SizeT next_start = 0; + while ((pos = source.find(KV.first, next_start)) != std::string::npos) { + next_start = pos + KV.second.size(); + source.replace(pos, KV.first.size(), KV.second); + } + } + return source; +} + +void CheckCase(std::stringstream& remaining_output, TestCase const& TC, + TestCaseList const& not_checks) { + std::string first_line; + bool on_first = true; + std::string line; + while (remaining_output.eof() == false) { + CHECK(remaining_output.good()); + std::getline(remaining_output, line); + if (on_first) { + first_line = line; + on_first = false; + } + for (const auto& NC : not_checks) { + CHECK(!NC.regex->Match(line)) + << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" + << NC.regex_str << "\"" + << "\n actual regex string \"" << TC.substituted_regex << "\"" + << "\n started matching near: " << first_line; + } + if (TC.regex->Match(line)) return; + CHECK(TC.match_rule != MR_Next) + << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str + << "\"" + << "\n actual regex string \"" << TC.substituted_regex << "\"" + << "\n started matching near: " << first_line; + } + CHECK(remaining_output.eof() == false) + << "End of output reached before match for regex \"" << TC.regex_str + << "\" was found" + << "\n actual regex string \"" << TC.substituted_regex << "\"" + << "\n started matching near: " << first_line; +} + +void CheckCases(TestCaseList const& checks, std::stringstream& output) { + std::vector not_checks; + for (size_t i = 0; i < checks.size(); ++i) { + const auto& TC = checks[i]; + if (TC.match_rule == MR_Not) { + not_checks.push_back(TC); + continue; + } + CheckCase(output, TC, not_checks); + not_checks.clear(); + } +} + +class TestReporter : public benchmark::BenchmarkReporter { + public: + TestReporter(std::vector reps) + : reporters_(reps) {} + + virtual bool ReportContext(const Context& context) { + bool last_ret = false; + bool first = true; + for (auto rep : reporters_) { + bool new_ret = rep->ReportContext(context); + CHECK(first || new_ret == last_ret) + << "Reports return different values for ReportContext"; + first = false; + last_ret = new_ret; + } + (void)first; + return last_ret; + } + + void ReportRuns(const std::vector& report) { + for (auto rep : reporters_) rep->ReportRuns(report); + } + void Finalize() { + for (auto rep : reporters_) rep->Finalize(); + } + + private: + std::vector reporters_; +}; +} + +} // end namespace internal + +// ========================================================================= // +// -------------------------- Results checking ----------------------------- // +// ========================================================================= // + +namespace internal { + +// Utility class to manage subscribers for checking benchmark results. +// It works by parsing the CSV output to read the results. +class ResultsChecker { + public: + + struct PatternAndFn : public TestCase { // reusing TestCase for its regexes + PatternAndFn(const std::string& rx, ResultsCheckFn fn_) + : TestCase(rx), fn(fn_) {} + ResultsCheckFn fn; + }; + + std::vector< PatternAndFn > check_patterns; + std::vector< Results > results; + std::vector< std::string > field_names; + + void Add(const std::string& entry_pattern, ResultsCheckFn fn); + + void CheckResults(std::stringstream& output); + + private: + + void SetHeader_(const std::string& csv_header); + void SetValues_(const std::string& entry_csv_line); + + std::vector< std::string > SplitCsv_(const std::string& line); + +}; + +// store the static ResultsChecker in a function to prevent initialization +// order problems +ResultsChecker& GetResultsChecker() { + static ResultsChecker rc; + return rc; +} + +// add a results checker for a benchmark +void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { + check_patterns.emplace_back(entry_pattern, fn); +} + +// check the results of all subscribed benchmarks +void ResultsChecker::CheckResults(std::stringstream& output) { + // first reset the stream to the start + { + auto start = std::ios::streampos(0); + // clear before calling tellg() + output.clear(); + // seek to zero only when needed + if(output.tellg() > start) output.seekg(start); + // and just in case + output.clear(); + } + // now go over every line and publish it to the ResultsChecker + std::string line; + bool on_first = true; + while (output.eof() == false) { + CHECK(output.good()); + std::getline(output, line); + if (on_first) { + SetHeader_(line); // this is important + on_first = false; + continue; + } + SetValues_(line); + } + // finally we can call the subscribed check functions + for(const auto& p : check_patterns) { + VLOG(2) << "--------------------------------\n"; + VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; + for(const auto& r : results) { + if(!p.regex->Match(r.name)) { + VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; + continue; + } else { + VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; + } + VLOG(1) << "Checking results of " << r.name << ": ... \n"; + p.fn(r); + VLOG(1) << "Checking results of " << r.name << ": OK.\n"; + } + } +} + +// prepare for the names in this header +void ResultsChecker::SetHeader_(const std::string& csv_header) { + field_names = SplitCsv_(csv_header); +} + +// set the values for a benchmark +void ResultsChecker::SetValues_(const std::string& entry_csv_line) { + if(entry_csv_line.empty()) return; // some lines are empty + CHECK(!field_names.empty()); + auto vals = SplitCsv_(entry_csv_line); + CHECK_EQ(vals.size(), field_names.size()); + results.emplace_back(vals[0]); // vals[0] is the benchmark name + auto &entry = results.back(); + for (size_t i = 1, e = vals.size(); i < e; ++i) { + entry.values[field_names[i]] = vals[i]; + } +} + +// a quick'n'dirty csv splitter (eliminating quotes) +std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) { + std::vector< std::string > out; + if(line.empty()) return out; + if(!field_names.empty()) out.reserve(field_names.size()); + size_t prev = 0, pos = line.find_first_of(','), curr = pos; + while(pos != line.npos) { + CHECK(curr > 0); + if(line[prev] == '"') ++prev; + if(line[curr-1] == '"') --curr; + out.push_back(line.substr(prev, curr-prev)); + prev = pos + 1; + pos = line.find_first_of(',', pos + 1); + curr = pos; + } + curr = line.size(); + if(line[prev] == '"') ++prev; + if(line[curr-1] == '"') --curr; + out.push_back(line.substr(prev, curr-prev)); + return out; +} + +} // end namespace internal + +size_t AddChecker(const char* bm_name, ResultsCheckFn fn) +{ + auto &rc = internal::GetResultsChecker(); + rc.Add(bm_name, fn); + return rc.results.size(); +} + +int Results::NumThreads() const { + auto pos = name.find("/threads:"); + if(pos == name.npos) return 1; + auto end = name.find('/', pos + 9); + std::stringstream ss; + ss << name.substr(pos + 9, end); + int num = 1; + ss >> num; + CHECK(!ss.fail()); + return num; +} + +double Results::GetTime(BenchmarkTime which) const { + CHECK(which == kCpuTime || which == kRealTime); + const char *which_str = which == kCpuTime ? "cpu_time" : "real_time"; + double val = GetAs< double >(which_str); + auto unit = Get("time_unit"); + CHECK(unit); + if(*unit == "ns") { + return val * 1.e-9; + } else if(*unit == "us") { + return val * 1.e-6; + } else if(*unit == "ms") { + return val * 1.e-3; + } else if(*unit == "s") { + return val; + } else { + CHECK(1 == 0) << "unknown time unit: " << *unit; + return 0; + } +} + +// ========================================================================= // +// -------------------------- Public API Definitions------------------------ // +// ========================================================================= // + +TestCase::TestCase(std::string re, int rule) + : regex_str(std::move(re)), + match_rule(rule), + substituted_regex(internal::PerformSubstitutions(regex_str)), + regex(std::make_shared()) { + std::string err_str; + regex->Init(substituted_regex,& err_str); + CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex + << "\"" + << "\n originally \"" << regex_str << "\"" + << "\n got error: " << err_str; +} + +int AddCases(TestCaseID ID, std::initializer_list il) { + auto& L = internal::GetTestCaseList(ID); + L.insert(L.end(), il); + return 0; +} + +int SetSubstitutions( + std::initializer_list> il) { + auto& subs = internal::GetSubstitutions(); + for (auto KV : il) { + bool exists = false; + KV.second = internal::PerformSubstitutions(KV.second); + for (auto& EKV : subs) { + if (EKV.first == KV.first) { + EKV.second = std::move(KV.second); + exists = true; + break; + } + } + if (!exists) subs.push_back(std::move(KV)); + } + return 0; +} + +void RunOutputTests(int argc, char* argv[]) { + using internal::GetTestCaseList; + benchmark::Initialize(&argc, argv); + auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true); + benchmark::ConsoleReporter CR(options); + benchmark::JSONReporter JR; + benchmark::CSVReporter CSVR; + struct ReporterTest { + const char* name; + std::vector& output_cases; + std::vector& error_cases; + benchmark::BenchmarkReporter& reporter; + std::stringstream out_stream; + std::stringstream err_stream; + + ReporterTest(const char* n, std::vector& out_tc, + std::vector& err_tc, + benchmark::BenchmarkReporter& br) + : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { + reporter.SetOutputStream(&out_stream); + reporter.SetErrorStream(&err_stream); + } + } TestCases[] = { + {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), + GetTestCaseList(TC_ConsoleErr), CR}, + {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), + JR}, + {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), + CSVR}, + }; + + // Create the test reporter and run the benchmarks. + std::cout << "Running benchmarks...\n"; + internal::TestReporter test_rep({&CR, &JR, &CSVR}); + benchmark::RunSpecifiedBenchmarks(&test_rep); + + for (auto& rep_test : TestCases) { + std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; + std::string banner(msg.size() - 1, '-'); + std::cout << banner << msg << banner << "\n"; + + std::cerr << rep_test.err_stream.str(); + std::cout << rep_test.out_stream.str(); + + internal::CheckCases(rep_test.error_cases, rep_test.err_stream); + internal::CheckCases(rep_test.output_cases, rep_test.out_stream); + + std::cout << "\n"; + } + + // now that we know the output is as expected, we can dispatch + // the checks to subscribees. + auto &csv = TestCases[2]; + // would use == but gcc spits a warning + CHECK(std::strcmp(csv.name, "CSVReporter") == 0); + internal::GetResultsChecker().CheckResults(csv.out_stream); +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/register_benchmark_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/register_benchmark_test.cc new file mode 100644 index 00000000000..2769b7a6b63 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/register_benchmark_test.cc @@ -0,0 +1,182 @@ + +#undef NDEBUG +#include +#include + +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "benchmark/benchmark.h" + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual void ReportRuns(const std::vector& report) { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + std::vector all_runs_; +}; + +struct TestCase { + std::string name; + const char* label; + // Note: not explicit as we rely on it being converted through ADD_CASES. + TestCase(const char* xname) : TestCase(xname, nullptr) {} + TestCase(const char* xname, const char* xlabel) + : name(xname), label(xlabel) {} + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + CHECK(name == run.benchmark_name) << "expected " << name << " got " + << run.benchmark_name; + if (label) { + CHECK(run.report_label == label) << "expected " << label << " got " + << run.report_label; + } else { + CHECK(run.report_label == ""); + } + } +}; + +std::vector ExpectedResults; + +int AddCases(std::initializer_list const& v) { + for (auto N : v) { + ExpectedResults.push_back(N); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases({__VA_ARGS__}) + +} // end namespace + +typedef benchmark::internal::Benchmark* ReturnVal; + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with no additional arguments +//----------------------------------------------------------------------------// +void BM_function(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_function); +ReturnVal dummy = benchmark::RegisterBenchmark( + "BM_function_manual_registration", BM_function); +ADD_CASES({"BM_function"}, {"BM_function_manual_registration"}); + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with additional arguments +// Note: GCC <= 4.8 do not support this form of RegisterBenchmark because they +// reject the variadic pack expansion of lambda captures. +//----------------------------------------------------------------------------// +#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + +void BM_extra_args(benchmark::State& st, const char* label) { + while (st.KeepRunning()) { + } + st.SetLabel(label); +} +int RegisterFromFunction() { + std::pair cases[] = { + {"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}}; + for (auto const& c : cases) + benchmark::RegisterBenchmark(c.first, &BM_extra_args, c.second); + return 0; +} +int dummy2 = RegisterFromFunction(); +ADD_CASES({"test1", "One"}, {"test2", "Two"}, {"test3", "Three"}); + +#endif // BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + +//----------------------------------------------------------------------------// +// Test RegisterBenchmark with different callable types +//----------------------------------------------------------------------------// + +struct CustomFixture { + void operator()(benchmark::State& st) { + while (st.KeepRunning()) { + } + } +}; + +void TestRegistrationAtRuntime() { +#ifdef BENCHMARK_HAS_CXX11 + { + CustomFixture fx; + benchmark::RegisterBenchmark("custom_fixture", fx); + AddCases({"custom_fixture"}); + } +#endif +#ifndef BENCHMARK_HAS_NO_VARIADIC_REGISTER_BENCHMARK + { + const char* x = "42"; + auto capturing_lam = [=](benchmark::State& st) { + while (st.KeepRunning()) { + } + st.SetLabel(x); + }; + benchmark::RegisterBenchmark("lambda_benchmark", capturing_lam); + AddCases({{"lambda_benchmark", x}}); + } +#endif +} + +// Test that all benchmarks, registered at either during static init or runtime, +// are run and the results are passed to the reported. +void RunTestOne() { + TestRegistrationAtRuntime(); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); +} + +// Test that ClearRegisteredBenchmarks() clears all previously registered +// benchmarks. +// Also test that new benchmarks can be registered and ran afterwards. +void RunTestTwo() { + assert(ExpectedResults.size() != 0 && + "must have at least one registered benchmark"); + ExpectedResults.clear(); + benchmark::ClearRegisteredBenchmarks(); + + TestReporter test_reporter; + size_t num_ran = benchmark::RunSpecifiedBenchmarks(&test_reporter); + assert(num_ran == 0); + assert(test_reporter.all_runs_.begin() == test_reporter.all_runs_.end()); + + TestRegistrationAtRuntime(); + num_ran = benchmark::RunSpecifiedBenchmarks(&test_reporter); + assert(num_ran == ExpectedResults.size()); + + typedef benchmark::BenchmarkReporter::Run Run; + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); +} + +int main(int argc, char* argv[]) { + benchmark::Initialize(&argc, argv); + + RunTestOne(); + RunTestTwo(); +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/reporter_output_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/reporter_output_test.cc new file mode 100644 index 00000000000..4a481433485 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/reporter_output_test.cc @@ -0,0 +1,256 @@ + +#undef NDEBUG +#include + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// ========================================================================= // +// ---------------------- Testing Prologue Output -------------------------- // +// ========================================================================= // + +ADD_CASES(TC_ConsoleOut, + {{"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, + {"^[-]+$", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"%csv_header"}}); + +// ========================================================================= // +// ------------------------ Testing Basic Output --------------------------- // +// ========================================================================= // + +void BM_basic(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_basic); + +ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); + +// ========================================================================= // +// ------------------------ Testing Bytes per Second Output ---------------- // +// ========================================================================= // + +void BM_bytes_per_second(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.SetBytesProcessed(1); +} +BENCHMARK(BM_bytes_per_second); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_bytes_per_second %console_report +%floatB/s$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bytes_per_second\": %int$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); + +// ========================================================================= // +// ------------------------ Testing Items per Second Output ---------------- // +// ========================================================================= // + +void BM_items_per_second(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.SetItemsProcessed(1); +} +BENCHMARK(BM_items_per_second); + +ADD_CASES(TC_ConsoleOut, + {{"^BM_items_per_second %console_report +%float items/s$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"items_per_second\": %int$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}}); + +// ========================================================================= // +// ------------------------ Testing Label Output --------------------------- // +// ========================================================================= // + +void BM_label(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.SetLabel("some label"); +} +BENCHMARK(BM_label); + +ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"label\": \"some label\"$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " + "label\"%csv_label_report_end$"}}); + +// ========================================================================= // +// ------------------------ Testing Error Output --------------------------- // +// ========================================================================= // + +void BM_error(benchmark::State& state) { + state.SkipWithError("message"); + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_error); +ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, + {"\"error_occurred\": true,$", MR_Next}, + {"\"error_message\": \"message\",$", MR_Next}}); + +ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}}); + +// ========================================================================= // +// ------------------------ Testing No Arg Name Output ----------------------- +// // +// ========================================================================= // + +void BM_no_arg_name(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_no_arg_name)->Arg(3); +ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); + +// ========================================================================= // +// ------------------------ Testing Arg Name Output ----------------------- // +// ========================================================================= // + +void BM_arg_name(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); +ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); + +// ========================================================================= // +// ------------------------ Testing Arg Names Output ----------------------- // +// ========================================================================= // + +void BM_arg_names(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); +ADD_CASES(TC_ConsoleOut, + {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); + +// ========================================================================= // +// ----------------------- Testing Complexity Output ----------------------- // +// ========================================================================= // + +void BM_Complexity_O1(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.SetComplexityN(state.range(0)); +} +BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); +SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"}, + {"%RMS", "[ ]*[0-9]+ %"}}); +ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"}, + {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}}); + +// ========================================================================= // +// ----------------------- Testing Aggregate Output ------------------------ // +// ========================================================================= // + +// Test that non-aggregate data is printed by default +void BM_Repeat(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_Repeat)->Repetitions(3); +ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3 %console_report$"}, + {"^BM_Repeat/repeats:3_mean %console_report$"}, + {"^BM_Repeat/repeats:3_stddev %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"name\": \"BM_Repeat/repeats:3\",$"}, + {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, + {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, + {"^\"BM_Repeat/repeats:3\",%csv_report$"}, + {"^\"BM_Repeat/repeats:3\",%csv_report$"}, + {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, + {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); + +// Test that a non-repeated test still prints non-aggregate results even when +// only-aggregate reports have been requested +void BM_RepeatOnce(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); +ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); + +// Test that non-aggregate data is not reported +void BM_SummaryRepeat(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); +ADD_CASES(TC_ConsoleOut, + {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, + {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); +ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, + {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); +ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, + {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, + {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); + +void BM_RepeatTimeUnit(benchmark::State& state) { + while (state.KeepRunning()) { + } +} +BENCHMARK(BM_RepeatTimeUnit) + ->Repetitions(3) + ->ReportAggregatesOnly() + ->Unit(benchmark::kMicrosecond); +ADD_CASES(TC_ConsoleOut, + {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, + {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, + {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); +ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, + {"\"time_unit\": \"us\",?$"}, + {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, + {"\"time_unit\": \"us\",?$"}}); +ADD_CASES(TC_CSVOut, + {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, + {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, + {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/skip_with_error_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/skip_with_error_test.cc new file mode 100644 index 00000000000..b74d33c5899 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/skip_with_error_test.cc @@ -0,0 +1,150 @@ + +#undef NDEBUG +#include +#include + +#include "../src/check.h" // NOTE: check.h is for internal use only! +#include "benchmark/benchmark.h" + +namespace { + +class TestReporter : public benchmark::ConsoleReporter { + public: + virtual bool ReportContext(const Context& context) { + return ConsoleReporter::ReportContext(context); + }; + + virtual void ReportRuns(const std::vector& report) { + all_runs_.insert(all_runs_.end(), begin(report), end(report)); + ConsoleReporter::ReportRuns(report); + } + + TestReporter() {} + virtual ~TestReporter() {} + + mutable std::vector all_runs_; +}; + +struct TestCase { + std::string name; + bool error_occurred; + std::string error_message; + + typedef benchmark::BenchmarkReporter::Run Run; + + void CheckRun(Run const& run) const { + CHECK(name == run.benchmark_name) << "expected " << name << " got " + << run.benchmark_name; + CHECK(error_occurred == run.error_occurred); + CHECK(error_message == run.error_message); + if (error_occurred) { + // CHECK(run.iterations == 0); + } else { + CHECK(run.iterations != 0); + } + } +}; + +std::vector ExpectedResults; + +int AddCases(const char* base_name, std::initializer_list const& v) { + for (auto TC : v) { + TC.name = base_name + TC.name; + ExpectedResults.push_back(std::move(TC)); + } + return 0; +} + +#define CONCAT(x, y) CONCAT2(x, y) +#define CONCAT2(x, y) x##y +#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) + +} // end namespace + +void BM_error_before_running(benchmark::State& state) { + state.SkipWithError("error message"); + while (state.KeepRunning()) { + assert(false); + } +} +BENCHMARK(BM_error_before_running); +ADD_CASES("BM_error_before_running", {{"", true, "error message"}}); + +void BM_error_during_running(benchmark::State& state) { + int first_iter = true; + while (state.KeepRunning()) { + if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { + assert(first_iter); + first_iter = false; + state.SkipWithError("error message"); + } else { + state.PauseTiming(); + state.ResumeTiming(); + } + } +} +BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8); +ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"}, + {"/1/threads:2", true, "error message"}, + {"/1/threads:4", true, "error message"}, + {"/1/threads:8", true, "error message"}, + {"/2/threads:1", false, ""}, + {"/2/threads:2", false, ""}, + {"/2/threads:4", false, ""}, + {"/2/threads:8", false, ""}}); + +void BM_error_after_running(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + if (state.thread_index <= (state.threads / 2)) + state.SkipWithError("error message"); +} +BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); +ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"}, + {"/threads:2", true, "error message"}, + {"/threads:4", true, "error message"}, + {"/threads:8", true, "error message"}}); + +void BM_error_while_paused(benchmark::State& state) { + bool first_iter = true; + while (state.KeepRunning()) { + if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) { + assert(first_iter); + first_iter = false; + state.PauseTiming(); + state.SkipWithError("error message"); + } else { + state.PauseTiming(); + state.ResumeTiming(); + } + } +} +BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8); +ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"}, + {"/1/threads:2", true, "error message"}, + {"/1/threads:4", true, "error message"}, + {"/1/threads:8", true, "error message"}, + {"/2/threads:1", false, ""}, + {"/2/threads:2", false, ""}, + {"/2/threads:4", false, ""}, + {"/2/threads:8", false, ""}}); + +int main(int argc, char* argv[]) { + benchmark::Initialize(&argc, argv); + + TestReporter test_reporter; + benchmark::RunSpecifiedBenchmarks(&test_reporter); + + typedef benchmark::BenchmarkReporter::Run Run; + auto EB = ExpectedResults.begin(); + + for (Run const& run : test_reporter.all_runs_) { + assert(EB != ExpectedResults.end()); + EB->CheckRun(run); + ++EB; + } + assert(EB == ExpectedResults.end()); + + return 0; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/user_counters_tabular_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/user_counters_tabular_test.cc new file mode 100644 index 00000000000..5fc5b4d9b88 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/user_counters_tabular_test.cc @@ -0,0 +1,250 @@ + +#undef NDEBUG + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// @todo: this checks the full output at once; the rule for +// CounterSet1 was failing because it was not matching "^[-]+$". +// @todo: check that the counters are vertically aligned. +ADD_CASES(TC_ConsoleOut, { +// keeping these lines long improves readability, so: +// clang-format off + {"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next}, + {"^[-]+$", MR_Next}, + {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, + {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, + {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, + {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, + {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next}, + {"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next}, + {"^[-]+$", MR_Next}, + {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next}, + {"^[-]+$", MR_Next}, + {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next}, + {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"}, +// clang-format on +}); +ADD_CASES(TC_CSVOut, {{"%csv_header," + "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}}); + +// ========================================================================= // +// ------------------------- Tabular Counters Output ----------------------- // +// ========================================================================= // + +void BM_Counters_Tabular(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters.insert({ + {"Foo", { 1, bm::Counter::kAvgThreads}}, + {"Bar", { 2, bm::Counter::kAvgThreads}}, + {"Baz", { 4, bm::Counter::kAvgThreads}}, + {"Bat", { 8, bm::Counter::kAvgThreads}}, + {"Frob", {16, bm::Counter::kAvgThreads}}, + {"Lob", {32, bm::Counter::kAvgThreads}}, + }); +} +BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckTabular(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1); + CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2); + CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4); + CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8); + CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16); + CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular); + +// ========================================================================= // +// -------------------- Tabular+Rate Counters Output ----------------------- // +// ========================================================================= // + +void BM_CounterRates_Tabular(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters.insert({ + {"Foo", { 1, bm::Counter::kAvgThreadsRate}}, + {"Bar", { 2, bm::Counter::kAvgThreadsRate}}, + {"Baz", { 4, bm::Counter::kAvgThreadsRate}}, + {"Bat", { 8, bm::Counter::kAvgThreadsRate}}, + {"Frob", {16, bm::Counter::kAvgThreadsRate}}, + {"Lob", {32, bm::Counter::kAvgThreadsRate}}, + }); +} +BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float,$", MR_Next}, + {"\"Frob\": %float,$", MR_Next}, + {"\"Lob\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report," + "%float,%float,%float,%float,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckTabularRate(Results const& e) { + double t = e.DurationCPUTime(); + CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int", + &CheckTabularRate); + +// ========================================================================= // +// ------------------------- Tabular Counters Output ----------------------- // +// ========================================================================= // + +// set only some of the counters +void BM_CounterSet0_Tabular(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters.insert({ + {"Foo", {10, bm::Counter::kAvgThreads}}, + {"Bar", {20, bm::Counter::kAvgThreads}}, + {"Baz", {40, bm::Counter::kAvgThreads}}, + }); +} +BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report," + "%float,,%float,%float,,"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckSet0(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); + CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20); + CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); +} +CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0); + +// again. +void BM_CounterSet1_Tabular(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters.insert({ + {"Foo", {15, bm::Counter::kAvgThreads}}, + {"Bar", {25, bm::Counter::kAvgThreads}}, + {"Baz", {45, bm::Counter::kAvgThreads}}, + }); +} +BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bar\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report," + "%float,,%float,%float,,"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckSet1(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15); + CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25); + CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45); +} +CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1); + +// ========================================================================= // +// ------------------------- Tabular Counters Output ----------------------- // +// ========================================================================= // + +// set only some of the counters, different set now. +void BM_CounterSet2_Tabular(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters.insert({ + {"Foo", {10, bm::Counter::kAvgThreads}}, + {"Bat", {30, bm::Counter::kAvgThreads}}, + {"Baz", {40, bm::Counter::kAvgThreads}}, + }); +} +BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"Bat\": %float,$", MR_Next}, + {"\"Baz\": %float,$", MR_Next}, + {"\"Foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report," + ",%float,%float,%float,,"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckSet2(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10); + CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30); + CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40); +} +CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/user_counters_test.cc b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/user_counters_test.cc new file mode 100644 index 00000000000..66df48b31f8 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/test/user_counters_test.cc @@ -0,0 +1,217 @@ + +#undef NDEBUG + +#include "benchmark/benchmark.h" +#include "output_test.h" + +// ========================================================================= // +// ---------------------- Testing Prologue Output -------------------------- // +// ========================================================================= // + +ADD_CASES(TC_ConsoleOut, + {{"^[-]+$", MR_Next}, + {"^Benchmark %s Time %s CPU %s Iterations UserCounters...$", MR_Next}, + {"^[-]+$", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"%csv_header,\"bar\",\"foo\""}}); + +// ========================================================================= // +// ------------------------- Simple Counters Output ------------------------ // +// ========================================================================= // + +void BM_Counters_Simple(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.counters["foo"] = 1; + state.counters["bar"] = 2 * (double)state.iterations(); +} +BENCHMARK(BM_Counters_Simple); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Simple %console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Simple\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Simple\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckSimple(Results const& e) { + double its = e.GetAs< double >("iterations"); + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); + // check that the value of bar is within 0.1% of the expected value + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2.*its, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Simple", &CheckSimple); + +// ========================================================================= // +// --------------------- Counters+Items+Bytes/s Output --------------------- // +// ========================================================================= // + +namespace { int num_calls1 = 0; } +void BM_Counters_WithBytesAndItemsPSec(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.counters["foo"] = 1; + state.counters["bar"] = ++num_calls1; + state.SetBytesProcessed(364); + state.SetItemsProcessed(150); +} +BENCHMARK(BM_Counters_WithBytesAndItemsPSec); +ADD_CASES(TC_ConsoleOut, + {{"^BM_Counters_WithBytesAndItemsPSec %console_report " + "bar=%hrfloat foo=%hrfloat +%hrfloatB/s +%hrfloat items/s$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_WithBytesAndItemsPSec\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bytes_per_second\": %int,$", MR_Next}, + {"\"items_per_second\": %int,$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_WithBytesAndItemsPSec\"," + "%csv_bytes_items_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckBytesAndItemsPSec(Results const& e) { + double t = e.DurationCPUTime(); // this (and not real time) is the time used + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, num_calls1); + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_RESULT_VALUE(e, "bytes_per_second", EQ, 364./t, 0.001); + CHECK_FLOAT_RESULT_VALUE(e, "items_per_second", EQ, 150./t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_WithBytesAndItemsPSec", + &CheckBytesAndItemsPSec); + +// ========================================================================= // +// ------------------------- Rate Counters Output -------------------------- // +// ========================================================================= // + +void BM_Counters_Rate(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kIsRate}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kIsRate}; +} +BENCHMARK(BM_Counters_Rate); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Rate %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Rate\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Rate\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckRate(Results const& e) { + double t = e.DurationCPUTime(); // this (and not real time) is the time used + // check that the values are within 0.1% of the expected values + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1./t, 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2./t, 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Rate", &CheckRate); + +// ========================================================================= // +// ------------------------- Thread Counters Output ------------------------ // +// ========================================================================= // + +void BM_Counters_Threads(benchmark::State& state) { + while (state.KeepRunning()) { + } + state.counters["foo"] = 1; + state.counters["bar"] = 2; +} +BENCHMARK(BM_Counters_Threads)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_Threads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Threads/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Threads/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckThreads(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "foo", EQ, e.NumThreads()); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2 * e.NumThreads()); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_Threads/threads:%int", &CheckThreads); + +// ========================================================================= // +// ---------------------- ThreadAvg Counters Output ------------------------ // +// ========================================================================= // + +void BM_Counters_AvgThreads(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreads}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreads}; +} +BENCHMARK(BM_Counters_AvgThreads)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreads/threads:%int %console_report bar=%hrfloat foo=%hrfloat$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreads/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreads/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgThreads(Results const& e) { + CHECK_COUNTER_VALUE(e, int, "foo", EQ, 1); + CHECK_COUNTER_VALUE(e, int, "bar", EQ, 2); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreads/threads:%int", + &CheckAvgThreads); + +// ========================================================================= // +// ---------------------- ThreadAvg Counters Output ------------------------ // +// ========================================================================= // + +void BM_Counters_AvgThreadsRate(benchmark::State& state) { + while (state.KeepRunning()) { + } + namespace bm = benchmark; + state.counters["foo"] = bm::Counter{1, bm::Counter::kAvgThreadsRate}; + state.counters["bar"] = bm::Counter{2, bm::Counter::kAvgThreadsRate}; +} +BENCHMARK(BM_Counters_AvgThreadsRate)->ThreadRange(1, 8); +ADD_CASES(TC_ConsoleOut, {{"^BM_Counters_AvgThreadsRate/threads:%int %console_report bar=%hrfloat/s foo=%hrfloat/s$"}}); +ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_AvgThreadsRate/threads:%int\",$"}, + {"\"iterations\": %int,$", MR_Next}, + {"\"real_time\": %int,$", MR_Next}, + {"\"cpu_time\": %int,$", MR_Next}, + {"\"time_unit\": \"ns\",$", MR_Next}, + {"\"bar\": %float,$", MR_Next}, + {"\"foo\": %float$", MR_Next}, + {"}", MR_Next}}); +ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_AvgThreadsRate/threads:%int\",%csv_report,%float,%float$"}}); +// VS2013 does not allow this function to be passed as a lambda argument +// to CHECK_BENCHMARK_RESULTS() +void CheckAvgThreadsRate(Results const& e) { + CHECK_FLOAT_COUNTER_VALUE(e, "foo", EQ, 1./e.DurationCPUTime(), 0.001); + CHECK_FLOAT_COUNTER_VALUE(e, "bar", EQ, 2./e.DurationCPUTime(), 0.001); +} +CHECK_BENCHMARK_RESULTS("BM_Counters_AvgThreadsRate/threads:%int", + &CheckAvgThreadsRate); + +// ========================================================================= // +// --------------------------- TEST CASES END ------------------------------ // +// ========================================================================= // + +int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/compare_bench.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/compare_bench.py new file mode 100755 index 00000000000..d54baaa0e8f --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/compare_bench.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +""" +compare_bench.py - Compare two benchmarks or their results and report the + difference. +""" +import argparse +from argparse import ArgumentParser +import sys +import gbench +from gbench import util, report +from gbench.util import * + +def check_inputs(in1, in2, flags): + """ + Perform checking on the user provided inputs and diagnose any abnormalities + """ + in1_kind, in1_err = classify_input_file(in1) + in2_kind, in2_err = classify_input_file(in2) + output_file = find_benchmark_flag('--benchmark_out=', flags) + output_type = find_benchmark_flag('--benchmark_out_format=', flags) + if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file: + print(("WARNING: '--benchmark_out=%s' will be passed to both " + "benchmarks causing it to be overwritten") % output_file) + if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0: + print("WARNING: passing --benchmark flags has no effect since both " + "inputs are JSON") + if output_type is not None and output_type != 'json': + print(("ERROR: passing '--benchmark_out_format=%s' to 'compare_bench.py`" + " is not supported.") % output_type) + sys.exit(1) + + +def main(): + parser = ArgumentParser( + description='compare the results of two benchmarks') + parser.add_argument( + 'test1', metavar='test1', type=str, nargs=1, + help='A benchmark executable or JSON output file') + parser.add_argument( + 'test2', metavar='test2', type=str, nargs=1, + help='A benchmark executable or JSON output file') + # FIXME this is a dummy argument which will never actually match + # any --benchmark flags but it helps generate a better usage message + parser.add_argument( + 'benchmark_options', metavar='benchmark_option', nargs='*', + help='Arguments to pass when running benchmark executables' + ) + args, unknown_args = parser.parse_known_args() + # Parse the command line flags + test1 = args.test1[0] + test2 = args.test2[0] + if args.benchmark_options: + print("Unrecognized positional argument arguments: '%s'" + % args.benchmark_options) + exit(1) + benchmark_options = unknown_args + check_inputs(test1, test2, benchmark_options) + # Run the benchmarks and report the results + json1 = gbench.util.run_or_load_benchmark(test1, benchmark_options) + json2 = gbench.util.run_or_load_benchmark(test2, benchmark_options) + output_lines = gbench.report.generate_difference_report(json1, json2) + print('Comparing %s to %s' % (test1, test2)) + for ln in output_lines: + print(ln) + + +if __name__ == '__main__': + main() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/Inputs/test1_run1.json b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/Inputs/test1_run1.json new file mode 100644 index 00000000000..37faed46d13 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/Inputs/test1_run1.json @@ -0,0 +1,60 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_SameTimes", + "iterations": 1000, + "real_time": 10, + "cpu_time": 10, + "time_unit": "ns" + }, + { + "name": "BM_2xFaster", + "iterations": 1000, + "real_time": 50, + "cpu_time": 50, + "time_unit": "ns" + }, + { + "name": "BM_2xSlower", + "iterations": 1000, + "real_time": 50, + "cpu_time": 50, + "time_unit": "ns" + }, + { + "name": "BM_10PercentFaster", + "iterations": 1000, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_10PercentSlower", + "iterations": 1000, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_100xSlower", + "iterations": 1000, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_100xFaster", + "iterations": 1000, + "real_time": 10000, + "cpu_time": 10000, + "time_unit": "ns" + } + ] +} \ No newline at end of file diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/Inputs/test1_run2.json b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/Inputs/test1_run2.json new file mode 100644 index 00000000000..aed5151d392 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/Inputs/test1_run2.json @@ -0,0 +1,60 @@ +{ + "context": { + "date": "2016-08-02 17:44:46", + "num_cpus": 4, + "mhz_per_cpu": 4228, + "cpu_scaling_enabled": false, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_SameTimes", + "iterations": 1000, + "real_time": 10, + "cpu_time": 10, + "time_unit": "ns" + }, + { + "name": "BM_2xFaster", + "iterations": 1000, + "real_time": 25, + "cpu_time": 25, + "time_unit": "ns" + }, + { + "name": "BM_2xSlower", + "iterations": 20833333, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + }, + { + "name": "BM_10PercentFaster", + "iterations": 1000, + "real_time": 90, + "cpu_time": 90, + "time_unit": "ns" + }, + { + "name": "BM_10PercentSlower", + "iterations": 1000, + "real_time": 110, + "cpu_time": 110, + "time_unit": "ns" + }, + { + "name": "BM_100xSlower", + "iterations": 1000, + "real_time": 10000, + "cpu_time": 10000, + "time_unit": "ns" + }, + { + "name": "BM_100xFaster", + "iterations": 1000, + "real_time": 100, + "cpu_time": 100, + "time_unit": "ns" + } + ] +} \ No newline at end of file diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/__init__.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/__init__.py new file mode 100644 index 00000000000..fce1a1acfbb --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/__init__.py @@ -0,0 +1,8 @@ +"""Google Benchmark tooling""" + +__author__ = 'Eric Fiselier' +__email__ = 'eric@efcs.ca' +__versioninfo__ = (0, 5, 0) +__version__ = '.'.join(str(v) for v in __versioninfo__) + 'dev' + +__all__ = [] diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/report.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/report.py new file mode 100644 index 00000000000..015d33d9e49 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/report.py @@ -0,0 +1,146 @@ +"""report.py - Utilities for reporting statistics about benchmark results +""" +import os + +class BenchmarkColor(object): + def __init__(self, name, code): + self.name = name + self.code = code + + def __repr__(self): + return '%s%r' % (self.__class__.__name__, + (self.name, self.code)) + + def __format__(self, format): + return self.code + +# Benchmark Colors Enumeration +BC_NONE = BenchmarkColor('NONE', '') +BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m') +BC_CYAN = BenchmarkColor('CYAN', '\033[96m') +BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m') +BC_HEADER = BenchmarkColor('HEADER', '\033[92m') +BC_WARNING = BenchmarkColor('WARNING', '\033[93m') +BC_WHITE = BenchmarkColor('WHITE', '\033[97m') +BC_FAIL = BenchmarkColor('FAIL', '\033[91m') +BC_ENDC = BenchmarkColor('ENDC', '\033[0m') +BC_BOLD = BenchmarkColor('BOLD', '\033[1m') +BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m') + +def color_format(use_color, fmt_str, *args, **kwargs): + """ + Return the result of 'fmt_str.format(*args, **kwargs)' after transforming + 'args' and 'kwargs' according to the value of 'use_color'. If 'use_color' + is False then all color codes in 'args' and 'kwargs' are replaced with + the empty string. + """ + assert use_color is True or use_color is False + if not use_color: + args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for arg in args] + kwargs = {key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE + for key, arg in kwargs.items()} + return fmt_str.format(*args, **kwargs) + + +def find_longest_name(benchmark_list): + """ + Return the length of the longest benchmark name in a given list of + benchmark JSON objects + """ + longest_name = 1 + for bc in benchmark_list: + if len(bc['name']) > longest_name: + longest_name = len(bc['name']) + return longest_name + + +def calculate_change(old_val, new_val): + """ + Return a float representing the decimal change between old_val and new_val. + """ + if old_val == 0 and new_val == 0: + return 0.0 + if old_val == 0: + return float(new_val - old_val) / (float(old_val + new_val) / 2) + return float(new_val - old_val) / abs(old_val) + + +def generate_difference_report(json1, json2, use_color=True): + """ + Calculate and report the difference between each test of two benchmarks + runs specified as 'json1' and 'json2'. + """ + first_col_width = find_longest_name(json1['benchmarks']) + 5 + def find_test(name): + for b in json2['benchmarks']: + if b['name'] == name: + return b + return None + first_line = "{:<{}s} Time CPU Old New".format( + 'Benchmark', first_col_width) + output_strs = [first_line, '-' * len(first_line)] + + gen = (bn for bn in json1['benchmarks'] if 'real_time' in bn and 'cpu_time' in bn) + for bn in gen: + other_bench = find_test(bn['name']) + if not other_bench: + continue + + def get_color(res): + if res > 0.05: + return BC_FAIL + elif res > -0.07: + return BC_WHITE + else: + return BC_CYAN + fmt_str = "{}{:<{}s}{endc}{}{:+9.2f}{endc}{}{:+14.2f}{endc}{:14d}{:14d}" + tres = calculate_change(bn['real_time'], other_bench['real_time']) + cpures = calculate_change(bn['cpu_time'], other_bench['cpu_time']) + output_strs += [color_format(use_color, fmt_str, + BC_HEADER, bn['name'], first_col_width, + get_color(tres), tres, get_color(cpures), cpures, + bn['cpu_time'], other_bench['cpu_time'], + endc=BC_ENDC)] + return output_strs + +############################################################################### +# Unit tests + +import unittest + +class TestReportDifference(unittest.TestCase): + def load_results(self): + import json + testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Inputs') + testOutput1 = os.path.join(testInputs, 'test1_run1.json') + testOutput2 = os.path.join(testInputs, 'test1_run2.json') + with open(testOutput1, 'r') as f: + json1 = json.load(f) + with open(testOutput2, 'r') as f: + json2 = json.load(f) + return json1, json2 + + def test_basic(self): + expect_lines = [ + ['BM_SameTimes', '+0.00', '+0.00', '10', '10'], + ['BM_2xFaster', '-0.50', '-0.50', '50', '25'], + ['BM_2xSlower', '+1.00', '+1.00', '50', '100'], + ['BM_10PercentFaster', '-0.10', '-0.10', '100', '90'], + ['BM_10PercentSlower', '+0.10', '+0.10', '100', '110'], + ['BM_100xSlower', '+99.00', '+99.00', '100', '10000'], + ['BM_100xFaster', '-0.99', '-0.99', '10000', '100'], + ] + json1, json2 = self.load_results() + output_lines_with_header = generate_difference_report(json1, json2, use_color=False) + output_lines = output_lines_with_header[2:] + print("\n".join(output_lines_with_header)) + self.assertEqual(len(output_lines), len(expect_lines)) + for i in xrange(0, len(output_lines)): + parts = [x for x in output_lines[i].split(' ') if x] + self.assertEqual(len(parts), 5) + self.assertEqual(parts, expect_lines[i]) + + +if __name__ == '__main__': + unittest.main() diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/util.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/util.py new file mode 100644 index 00000000000..07c23772754 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/external/benchmark/tools/gbench/util.py @@ -0,0 +1,159 @@ +"""util.py - General utilities for running, loading, and processing benchmarks +""" +import json +import os +import tempfile +import subprocess +import sys + +# Input file type enumeration +IT_Invalid = 0 +IT_JSON = 1 +IT_Executable = 2 + +_num_magic_bytes = 2 if sys.platform.startswith('win') else 4 +def is_executable_file(filename): + """ + Return 'True' if 'filename' names a valid file which is likely + an executable. A file is considered an executable if it starts with the + magic bytes for a EXE, Mach O, or ELF file. + """ + if not os.path.isfile(filename): + return False + with open(filename, mode='rb') as f: + magic_bytes = f.read(_num_magic_bytes) + if sys.platform == 'darwin': + return magic_bytes in [ + b'\xfe\xed\xfa\xce', # MH_MAGIC + b'\xce\xfa\xed\xfe', # MH_CIGAM + b'\xfe\xed\xfa\xcf', # MH_MAGIC_64 + b'\xcf\xfa\xed\xfe', # MH_CIGAM_64 + b'\xca\xfe\xba\xbe', # FAT_MAGIC + b'\xbe\xba\xfe\xca' # FAT_CIGAM + ] + elif sys.platform.startswith('win'): + return magic_bytes == b'MZ' + else: + return magic_bytes == b'\x7FELF' + + +def is_json_file(filename): + """ + Returns 'True' if 'filename' names a valid JSON output file. + 'False' otherwise. + """ + try: + with open(filename, 'r') as f: + json.load(f) + return True + except: + pass + return False + + +def classify_input_file(filename): + """ + Return a tuple (type, msg) where 'type' specifies the classified type + of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable + string represeting the error. + """ + ftype = IT_Invalid + err_msg = None + if not os.path.exists(filename): + err_msg = "'%s' does not exist" % filename + elif not os.path.isfile(filename): + err_msg = "'%s' does not name a file" % filename + elif is_executable_file(filename): + ftype = IT_Executable + elif is_json_file(filename): + ftype = IT_JSON + else: + err_msg = "'%s' does not name a valid benchmark executable or JSON file" % filename + return ftype, err_msg + + +def check_input_file(filename): + """ + Classify the file named by 'filename' and return the classification. + If the file is classified as 'IT_Invalid' print an error message and exit + the program. + """ + ftype, msg = classify_input_file(filename) + if ftype == IT_Invalid: + print("Invalid input file: %s" % msg) + sys.exit(1) + return ftype + +def find_benchmark_flag(prefix, benchmark_flags): + """ + Search the specified list of flags for a flag matching `` and + if it is found return the arg it specifies. If specified more than once the + last value is returned. If the flag is not found None is returned. + """ + assert prefix.startswith('--') and prefix.endswith('=') + result = None + for f in benchmark_flags: + if f.startswith(prefix): + result = f[len(prefix):] + return result + +def remove_benchmark_flags(prefix, benchmark_flags): + """ + Return a new list containing the specified benchmark_flags except those + with the specified prefix. + """ + assert prefix.startswith('--') and prefix.endswith('=') + return [f for f in benchmark_flags if not f.startswith(prefix)] + +def load_benchmark_results(fname): + """ + Read benchmark output from a file and return the JSON object. + REQUIRES: 'fname' names a file containing JSON benchmark output. + """ + with open(fname, 'r') as f: + return json.load(f) + + +def run_benchmark(exe_name, benchmark_flags): + """ + Run a benchmark specified by 'exe_name' with the specified + 'benchmark_flags'. The benchmark is run directly as a subprocess to preserve + real time console output. + RETURNS: A JSON object representing the benchmark output + """ + output_name = find_benchmark_flag('--benchmark_out=', + benchmark_flags) + is_temp_output = False + if output_name is None: + is_temp_output = True + thandle, output_name = tempfile.mkstemp() + os.close(thandle) + benchmark_flags = list(benchmark_flags) + \ + ['--benchmark_out=%s' % output_name] + + cmd = [exe_name] + benchmark_flags + print("RUNNING: %s" % ' '.join(cmd)) + exitCode = subprocess.call(cmd) + if exitCode != 0: + print('TEST FAILED...') + sys.exit(exitCode) + json_res = load_benchmark_results(output_name) + if is_temp_output: + os.unlink(output_name) + return json_res + + +def run_or_load_benchmark(filename, benchmark_flags): + """ + Get the results for a specified benchmark. If 'filename' specifies + an executable benchmark then the results are generated by running the + benchmark. Otherwise 'filename' must name a valid JSON output file, + which is loaded and the result returned. + """ + ftype = check_input_file(filename) + if ftype == IT_JSON: + return load_benchmark_results(filename) + elif ftype == IT_Executable: + return run_benchmark(filename, benchmark_flags) + else: + assert False # This branch is unreachable \ No newline at end of file diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fast_copy.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/fast_copy.c new file mode 100644 index 00000000000..bf3cbd1986d --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fast_copy.c @@ -0,0 +1,60 @@ +#include "fast_copy.h" +#include +#include +#if (defined(__x86_64__) || defined(__i386__)) +# include +#endif + +#ifdef FHT_HEADER_ONLY +# define _STORAGE_ static inline +#else +# define _STORAGE_ +#endif + +// These functions all assume that the size of memory being copied is a power of 2. + +#if _FEATURE_AVX512F +// If n is less than 64, defaults to memcpy. Otherwise, being a power of 2, we can just use unaligned stores and loads. +_STORAGE_ void *fast_copy(void *out, void *in, size_t n) { + if(n >= FAST_COPY_MEMCPY_THRESHOLD) { + return memcpy(out, in, n); + } + n >>= 6; + for(__m512 *ov = (__m512 *)out, *iv = (__m512 *)in; n--;) { + _mm512_storeu_ps((float *)(ov++), _mm512_loadu_ps((float *)(iv++))); + } + return out; +} +#elif __AVX2__ +// If n is less than 32, defaults to memcpy. Otherwise, being a power of 2, we can just use unaligned stores and loads. +_STORAGE_ void *fast_copy(void *out, void *in, size_t n) { + if(n >= FAST_COPY_MEMCPY_THRESHOLD) { + return memcpy(out, in, n); + } + n >>= 5; + for(__m256 *ov = (__m256 *)out, *iv = (__m256 *)in; n--;) { + _mm256_storeu_ps((float *)(ov++), _mm256_loadu_ps((float *)(iv++))); + } + return out; +} +#elif __SSE2__ +// If n is less than 16, defaults to memcpy. Otherwise, being a power of 2, we can just use unaligned stores and loads. +_STORAGE_ void *fast_copy(void *out, void *in, size_t n) { + if(n >= FAST_COPY_MEMCPY_THRESHOLD) { + return memcpy(out, in, n); + } + n >>= 4; + for(__m128 *ov = (__m128 *)out, *iv = (__m128 *)in; n--;) { + _mm_storeu_ps((float *)(ov++), _mm_loadu_ps((float *)(iv++))); + } + return out; +} +#else +_STORAGE_ void *fast_copy(void *out, void *in, size_t n) { + return memcpy(out, in, n); +} +#endif + +#ifdef FHT_HEADER_ONLY +# undef _STORAGE_ +#endif diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fast_copy.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/fast_copy.h new file mode 100644 index 00000000000..f4d4fabc01a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fast_copy.h @@ -0,0 +1,21 @@ +#ifndef _FAST_COPY_H__ +#define _FAST_COPY_H__ +#include + +#ifndef FAST_COPY_MEMCPY_THRESHOLD +# define FAST_COPY_MEMCPY_THRESHOLD ((size_t)1ull << 20) +#endif + +#ifdef __cplusplus +extern "C" { +#endif +#ifdef FHT_HEADER_ONLY +#include "fast_copy.c" +#else +void *fast_copy(void *out, void *in, size_t m); +#endif +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // _FAST_COPY_H__ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fht.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht.c new file mode 100644 index 00000000000..c374ff618f6 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht.c @@ -0,0 +1,3 @@ +#include "fht.h" + +#include "fht_impl.h" diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fht.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht.h new file mode 100644 index 00000000000..52b1543f3f9 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht.h @@ -0,0 +1,38 @@ +#ifndef _FHT_H_ +#define _FHT_H_ +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +int fht_float(float *buf, int log_n); +int fht_double(double *buf, int log_n); +int fht_float_oop(float *in, float *out, int log_n); +int fht_double_oop(double *in, double *out, int log_n); + + +#ifdef __cplusplus + +} // extern "C" + +static inline int fht(float *buf, int log_n) { + return fht_float(buf, log_n); +} + +static inline int fht(double *buf, int log_n) { + return fht_double(buf, log_n); +} + +static inline int fht(float *buf, float *out, int log_n) { + return fht_float_oop(buf, out, log_n); +} + +static inline int fht(double *buf, double *out, int log_n) { + return fht_double_oop(buf, out, log_n); +} + +#endif + +#endif diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_avx.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_avx.c new file mode 100644 index 00000000000..721130dc9f6 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_avx.c @@ -0,0 +1,19671 @@ +#include "fht.h" +static inline void helper_float_1(float *buf); +static inline void helper_float_1(float *buf) { + for (int j = 0; j < 2; j += 2) { + for (int k = 0; k < 1; ++k) { + float u = buf[j + k]; + float v = buf[j + k + 1]; + buf[j + k] = u + v; + buf[j + k + 1] = u - v; + } + } +} +static inline void helper_float_2(float *buf); +static inline void helper_float_2(float *buf) { + for (int j = 0; j < 4; j += 2) { + for (int k = 0; k < 1; ++k) { + float u = buf[j + k]; + float v = buf[j + k + 1]; + buf[j + k] = u + v; + buf[j + k + 1] = u - v; + } + } + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; ++k) { + float u = buf[j + k]; + float v = buf[j + k + 2]; + buf[j + k] = u + v; + buf[j + k + 2] = u - v; + } + } +} +static inline void helper_float_3(float *buf); +static inline void helper_float_3(float *buf) { + for (int j = 0; j < 8; j += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vmovups %%ymm0, (%0)\n" + :: "r"(buf + j) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } +} +static inline void helper_float_4(float *buf); +static inline void helper_float_4(float *buf) { + for (int j = 0; j < 16; j += 16) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_float_5(float *buf); +static inline void helper_float_5(float *buf) { + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_float_6(float *buf); +static inline void helper_float_6(float *buf) { + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +void helper_float_7_recursive(float *buf, int depth); +void helper_float_7_recursive(float *buf, int depth) { + if (depth == 7) { + for (int j = 0; j < 128; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_7(float *buf); +void helper_float_7(float *buf) { + helper_float_7_recursive(buf, 7); +} +void helper_float_8_recursive(float *buf, int depth); +void helper_float_8_recursive(float *buf, int depth) { + if (depth == 6) { + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_float_8_recursive(buf + 0, 6); + helper_float_8_recursive(buf + 64, 6); + helper_float_8_recursive(buf + 128, 6); + helper_float_8_recursive(buf + 192, 6); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_8(float *buf); +void helper_float_8(float *buf) { + helper_float_8_recursive(buf, 8); +} +static inline void helper_float_9(float *buf); +static inline void helper_float_9(float *buf) { + for (int j = 0; j < 512; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +void helper_float_10_recursive(float *buf, int depth); +void helper_float_10_recursive(float *buf, int depth) { + if (depth == 10) { + for (int j = 0; j < 1024; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_10(float *buf); +void helper_float_10(float *buf) { + helper_float_10_recursive(buf, 10); +} +void helper_float_11_recursive(float *buf, int depth); +void helper_float_11_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_11(float *buf); +void helper_float_11(float *buf) { + helper_float_11_recursive(buf, 11); +} +static inline void helper_float_12(float *buf); +static inline void helper_float_12(float *buf) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +void helper_float_13_recursive(float *buf, int depth); +void helper_float_13_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_float_13_recursive(buf + 0, 11); + helper_float_13_recursive(buf + 2048, 11); + helper_float_13_recursive(buf + 4096, 11); + helper_float_13_recursive(buf + 6144, 11); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_13(float *buf); +void helper_float_13(float *buf) { + helper_float_13_recursive(buf, 13); +} +void helper_float_14_recursive(float *buf, int depth); +void helper_float_14_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_14_recursive(buf + 0, 12); + helper_float_14_recursive(buf + 4096, 12); + helper_float_14_recursive(buf + 8192, 12); + helper_float_14_recursive(buf + 12288, 12); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_14(float *buf); +void helper_float_14(float *buf) { + helper_float_14_recursive(buf, 14); +} +void helper_float_15_recursive(float *buf, int depth); +void helper_float_15_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_15_recursive(buf + 0, 13); + helper_float_15_recursive(buf + 8192, 13); + helper_float_15_recursive(buf + 16384, 13); + helper_float_15_recursive(buf + 24576, 13); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 8192; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_15(float *buf); +void helper_float_15(float *buf) { + helper_float_15_recursive(buf, 15); +} +void helper_float_16_recursive(float *buf, int depth); +void helper_float_16_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_16_recursive(buf + 0, 13); + helper_float_16_recursive(buf + 8192, 13); + helper_float_16_recursive(buf + 16384, 13); + helper_float_16_recursive(buf + 24576, 13); + helper_float_16_recursive(buf + 32768, 13); + helper_float_16_recursive(buf + 40960, 13); + helper_float_16_recursive(buf + 49152, 13); + helper_float_16_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_16(float *buf); +void helper_float_16(float *buf) { + helper_float_16_recursive(buf, 16); +} +void helper_float_17_recursive(float *buf, int depth); +void helper_float_17_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_17_recursive(buf + 0, 12); + helper_float_17_recursive(buf + 4096, 12); + helper_float_17_recursive(buf + 8192, 12); + helper_float_17_recursive(buf + 12288, 12); + helper_float_17_recursive(buf + 16384, 12); + helper_float_17_recursive(buf + 20480, 12); + helper_float_17_recursive(buf + 24576, 12); + helper_float_17_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_17_recursive(buf + 0, 15); + helper_float_17_recursive(buf + 32768, 15); + helper_float_17_recursive(buf + 65536, 15); + helper_float_17_recursive(buf + 98304, 15); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_17(float *buf); +void helper_float_17(float *buf) { + helper_float_17_recursive(buf, 17); +} +void helper_float_18_recursive(float *buf, int depth); +void helper_float_18_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_18_recursive(buf + 0, 12); + helper_float_18_recursive(buf + 4096, 12); + helper_float_18_recursive(buf + 8192, 12); + helper_float_18_recursive(buf + 12288, 12); + helper_float_18_recursive(buf + 16384, 12); + helper_float_18_recursive(buf + 20480, 12); + helper_float_18_recursive(buf + 24576, 12); + helper_float_18_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_18_recursive(buf + 0, 15); + helper_float_18_recursive(buf + 32768, 15); + helper_float_18_recursive(buf + 65536, 15); + helper_float_18_recursive(buf + 98304, 15); + helper_float_18_recursive(buf + 131072, 15); + helper_float_18_recursive(buf + 163840, 15); + helper_float_18_recursive(buf + 196608, 15); + helper_float_18_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_18(float *buf); +void helper_float_18(float *buf) { + helper_float_18_recursive(buf, 18); +} +void helper_float_19_recursive(float *buf, int depth); +void helper_float_19_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_19_recursive(buf + 0, 13); + helper_float_19_recursive(buf + 8192, 13); + helper_float_19_recursive(buf + 16384, 13); + helper_float_19_recursive(buf + 24576, 13); + helper_float_19_recursive(buf + 32768, 13); + helper_float_19_recursive(buf + 40960, 13); + helper_float_19_recursive(buf + 49152, 13); + helper_float_19_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_float_19_recursive(buf + 0, 16); + helper_float_19_recursive(buf + 65536, 16); + helper_float_19_recursive(buf + 131072, 16); + helper_float_19_recursive(buf + 196608, 16); + helper_float_19_recursive(buf + 262144, 16); + helper_float_19_recursive(buf + 327680, 16); + helper_float_19_recursive(buf + 393216, 16); + helper_float_19_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_19(float *buf); +void helper_float_19(float *buf) { + helper_float_19_recursive(buf, 19); +} +void helper_float_20_recursive(float *buf, int depth); +void helper_float_20_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_20_recursive(buf + 0, 12); + helper_float_20_recursive(buf + 4096, 12); + helper_float_20_recursive(buf + 8192, 12); + helper_float_20_recursive(buf + 12288, 12); + helper_float_20_recursive(buf + 16384, 12); + helper_float_20_recursive(buf + 20480, 12); + helper_float_20_recursive(buf + 24576, 12); + helper_float_20_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_20_recursive(buf + 0, 15); + helper_float_20_recursive(buf + 32768, 15); + helper_float_20_recursive(buf + 65536, 15); + helper_float_20_recursive(buf + 98304, 15); + helper_float_20_recursive(buf + 131072, 15); + helper_float_20_recursive(buf + 163840, 15); + helper_float_20_recursive(buf + 196608, 15); + helper_float_20_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_float_20_recursive(buf + 0, 18); + helper_float_20_recursive(buf + 262144, 18); + helper_float_20_recursive(buf + 524288, 18); + helper_float_20_recursive(buf + 786432, 18); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_20(float *buf); +void helper_float_20(float *buf) { + helper_float_20_recursive(buf, 20); +} +void helper_float_21_recursive(float *buf, int depth); +void helper_float_21_recursive(float *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_float_21_recursive(buf + 0, 9); + helper_float_21_recursive(buf + 512, 9); + helper_float_21_recursive(buf + 1024, 9); + helper_float_21_recursive(buf + 1536, 9); + helper_float_21_recursive(buf + 2048, 9); + helper_float_21_recursive(buf + 2560, 9); + helper_float_21_recursive(buf + 3072, 9); + helper_float_21_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_21_recursive(buf + 0, 12); + helper_float_21_recursive(buf + 4096, 12); + helper_float_21_recursive(buf + 8192, 12); + helper_float_21_recursive(buf + 12288, 12); + helper_float_21_recursive(buf + 16384, 12); + helper_float_21_recursive(buf + 20480, 12); + helper_float_21_recursive(buf + 24576, 12); + helper_float_21_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_21_recursive(buf + 0, 15); + helper_float_21_recursive(buf + 32768, 15); + helper_float_21_recursive(buf + 65536, 15); + helper_float_21_recursive(buf + 98304, 15); + helper_float_21_recursive(buf + 131072, 15); + helper_float_21_recursive(buf + 163840, 15); + helper_float_21_recursive(buf + 196608, 15); + helper_float_21_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_21_recursive(buf + 0, 18); + helper_float_21_recursive(buf + 262144, 18); + helper_float_21_recursive(buf + 524288, 18); + helper_float_21_recursive(buf + 786432, 18); + helper_float_21_recursive(buf + 1048576, 18); + helper_float_21_recursive(buf + 1310720, 18); + helper_float_21_recursive(buf + 1572864, 18); + helper_float_21_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_21(float *buf); +void helper_float_21(float *buf) { + helper_float_21_recursive(buf, 21); +} +void helper_float_22_recursive(float *buf, int depth); +void helper_float_22_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_22_recursive(buf + 0, 11); + helper_float_22_recursive(buf + 2048, 11); + helper_float_22_recursive(buf + 4096, 11); + helper_float_22_recursive(buf + 6144, 11); + helper_float_22_recursive(buf + 8192, 11); + helper_float_22_recursive(buf + 10240, 11); + helper_float_22_recursive(buf + 12288, 11); + helper_float_22_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_22_recursive(buf + 0, 14); + helper_float_22_recursive(buf + 16384, 14); + helper_float_22_recursive(buf + 32768, 14); + helper_float_22_recursive(buf + 49152, 14); + helper_float_22_recursive(buf + 65536, 14); + helper_float_22_recursive(buf + 81920, 14); + helper_float_22_recursive(buf + 98304, 14); + helper_float_22_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_float_22_recursive(buf + 0, 17); + helper_float_22_recursive(buf + 131072, 17); + helper_float_22_recursive(buf + 262144, 17); + helper_float_22_recursive(buf + 393216, 17); + helper_float_22_recursive(buf + 524288, 17); + helper_float_22_recursive(buf + 655360, 17); + helper_float_22_recursive(buf + 786432, 17); + helper_float_22_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_float_22_recursive(buf + 0, 20); + helper_float_22_recursive(buf + 1048576, 20); + helper_float_22_recursive(buf + 2097152, 20); + helper_float_22_recursive(buf + 3145728, 20); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 1048576; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_22(float *buf); +void helper_float_22(float *buf) { + helper_float_22_recursive(buf, 22); +} +void helper_float_23_recursive(float *buf, int depth); +void helper_float_23_recursive(float *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_float_23_recursive(buf + 0, 9); + helper_float_23_recursive(buf + 512, 9); + helper_float_23_recursive(buf + 1024, 9); + helper_float_23_recursive(buf + 1536, 9); + helper_float_23_recursive(buf + 2048, 9); + helper_float_23_recursive(buf + 2560, 9); + helper_float_23_recursive(buf + 3072, 9); + helper_float_23_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_23_recursive(buf + 0, 12); + helper_float_23_recursive(buf + 4096, 12); + helper_float_23_recursive(buf + 8192, 12); + helper_float_23_recursive(buf + 12288, 12); + helper_float_23_recursive(buf + 16384, 12); + helper_float_23_recursive(buf + 20480, 12); + helper_float_23_recursive(buf + 24576, 12); + helper_float_23_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_23_recursive(buf + 0, 15); + helper_float_23_recursive(buf + 32768, 15); + helper_float_23_recursive(buf + 65536, 15); + helper_float_23_recursive(buf + 98304, 15); + helper_float_23_recursive(buf + 131072, 15); + helper_float_23_recursive(buf + 163840, 15); + helper_float_23_recursive(buf + 196608, 15); + helper_float_23_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_23_recursive(buf + 0, 18); + helper_float_23_recursive(buf + 262144, 18); + helper_float_23_recursive(buf + 524288, 18); + helper_float_23_recursive(buf + 786432, 18); + helper_float_23_recursive(buf + 1048576, 18); + helper_float_23_recursive(buf + 1310720, 18); + helper_float_23_recursive(buf + 1572864, 18); + helper_float_23_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_float_23_recursive(buf + 0, 21); + helper_float_23_recursive(buf + 2097152, 21); + helper_float_23_recursive(buf + 4194304, 21); + helper_float_23_recursive(buf + 6291456, 21); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 2097152; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_23(float *buf); +void helper_float_23(float *buf) { + helper_float_23_recursive(buf, 23); +} +void helper_float_24_recursive(float *buf, int depth); +void helper_float_24_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_24_recursive(buf + 0, 12); + helper_float_24_recursive(buf + 4096, 12); + helper_float_24_recursive(buf + 8192, 12); + helper_float_24_recursive(buf + 12288, 12); + helper_float_24_recursive(buf + 16384, 12); + helper_float_24_recursive(buf + 20480, 12); + helper_float_24_recursive(buf + 24576, 12); + helper_float_24_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_24_recursive(buf + 0, 15); + helper_float_24_recursive(buf + 32768, 15); + helper_float_24_recursive(buf + 65536, 15); + helper_float_24_recursive(buf + 98304, 15); + helper_float_24_recursive(buf + 131072, 15); + helper_float_24_recursive(buf + 163840, 15); + helper_float_24_recursive(buf + 196608, 15); + helper_float_24_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_24_recursive(buf + 0, 18); + helper_float_24_recursive(buf + 262144, 18); + helper_float_24_recursive(buf + 524288, 18); + helper_float_24_recursive(buf + 786432, 18); + helper_float_24_recursive(buf + 1048576, 18); + helper_float_24_recursive(buf + 1310720, 18); + helper_float_24_recursive(buf + 1572864, 18); + helper_float_24_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_24_recursive(buf + 0, 21); + helper_float_24_recursive(buf + 2097152, 21); + helper_float_24_recursive(buf + 4194304, 21); + helper_float_24_recursive(buf + 6291456, 21); + helper_float_24_recursive(buf + 8388608, 21); + helper_float_24_recursive(buf + 10485760, 21); + helper_float_24_recursive(buf + 12582912, 21); + helper_float_24_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_24(float *buf); +void helper_float_24(float *buf) { + helper_float_24_recursive(buf, 24); +} +void helper_float_25_recursive(float *buf, int depth); +void helper_float_25_recursive(float *buf, int depth) { + if (depth == 7) { + for (int j = 0; j < 128; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 10) { + helper_float_25_recursive(buf + 0, 7); + helper_float_25_recursive(buf + 128, 7); + helper_float_25_recursive(buf + 256, 7); + helper_float_25_recursive(buf + 384, 7); + helper_float_25_recursive(buf + 512, 7); + helper_float_25_recursive(buf + 640, 7); + helper_float_25_recursive(buf + 768, 7); + helper_float_25_recursive(buf + 896, 7); + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_float_25_recursive(buf + 0, 10); + helper_float_25_recursive(buf + 1024, 10); + helper_float_25_recursive(buf + 2048, 10); + helper_float_25_recursive(buf + 3072, 10); + helper_float_25_recursive(buf + 4096, 10); + helper_float_25_recursive(buf + 5120, 10); + helper_float_25_recursive(buf + 6144, 10); + helper_float_25_recursive(buf + 7168, 10); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_25_recursive(buf + 0, 13); + helper_float_25_recursive(buf + 8192, 13); + helper_float_25_recursive(buf + 16384, 13); + helper_float_25_recursive(buf + 24576, 13); + helper_float_25_recursive(buf + 32768, 13); + helper_float_25_recursive(buf + 40960, 13); + helper_float_25_recursive(buf + 49152, 13); + helper_float_25_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_float_25_recursive(buf + 0, 16); + helper_float_25_recursive(buf + 65536, 16); + helper_float_25_recursive(buf + 131072, 16); + helper_float_25_recursive(buf + 196608, 16); + helper_float_25_recursive(buf + 262144, 16); + helper_float_25_recursive(buf + 327680, 16); + helper_float_25_recursive(buf + 393216, 16); + helper_float_25_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_float_25_recursive(buf + 0, 19); + helper_float_25_recursive(buf + 524288, 19); + helper_float_25_recursive(buf + 1048576, 19); + helper_float_25_recursive(buf + 1572864, 19); + helper_float_25_recursive(buf + 2097152, 19); + helper_float_25_recursive(buf + 2621440, 19); + helper_float_25_recursive(buf + 3145728, 19); + helper_float_25_recursive(buf + 3670016, 19); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 524288; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864), "r"(buf + j + k + 2097152), "r"(buf + j + k + 2621440), "r"(buf + j + k + 3145728), "r"(buf + j + k + 3670016) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 25) { + helper_float_25_recursive(buf + 0, 22); + helper_float_25_recursive(buf + 4194304, 22); + helper_float_25_recursive(buf + 8388608, 22); + helper_float_25_recursive(buf + 12582912, 22); + helper_float_25_recursive(buf + 16777216, 22); + helper_float_25_recursive(buf + 20971520, 22); + helper_float_25_recursive(buf + 25165824, 22); + helper_float_25_recursive(buf + 29360128, 22); + for (int j = 0; j < 33554432; j += 33554432) { + for (int k = 0; k < 4194304; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4194304), "r"(buf + j + k + 8388608), "r"(buf + j + k + 12582912), "r"(buf + j + k + 16777216), "r"(buf + j + k + 20971520), "r"(buf + j + k + 25165824), "r"(buf + j + k + 29360128) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_25(float *buf); +void helper_float_25(float *buf) { + helper_float_25_recursive(buf, 25); +} +void helper_float_26_recursive(float *buf, int depth); +void helper_float_26_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_26_recursive(buf + 0, 12); + helper_float_26_recursive(buf + 4096, 12); + helper_float_26_recursive(buf + 8192, 12); + helper_float_26_recursive(buf + 12288, 12); + helper_float_26_recursive(buf + 16384, 12); + helper_float_26_recursive(buf + 20480, 12); + helper_float_26_recursive(buf + 24576, 12); + helper_float_26_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_26_recursive(buf + 0, 15); + helper_float_26_recursive(buf + 32768, 15); + helper_float_26_recursive(buf + 65536, 15); + helper_float_26_recursive(buf + 98304, 15); + helper_float_26_recursive(buf + 131072, 15); + helper_float_26_recursive(buf + 163840, 15); + helper_float_26_recursive(buf + 196608, 15); + helper_float_26_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_26_recursive(buf + 0, 18); + helper_float_26_recursive(buf + 262144, 18); + helper_float_26_recursive(buf + 524288, 18); + helper_float_26_recursive(buf + 786432, 18); + helper_float_26_recursive(buf + 1048576, 18); + helper_float_26_recursive(buf + 1310720, 18); + helper_float_26_recursive(buf + 1572864, 18); + helper_float_26_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_26_recursive(buf + 0, 21); + helper_float_26_recursive(buf + 2097152, 21); + helper_float_26_recursive(buf + 4194304, 21); + helper_float_26_recursive(buf + 6291456, 21); + helper_float_26_recursive(buf + 8388608, 21); + helper_float_26_recursive(buf + 10485760, 21); + helper_float_26_recursive(buf + 12582912, 21); + helper_float_26_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_float_26_recursive(buf + 0, 24); + helper_float_26_recursive(buf + 16777216, 24); + helper_float_26_recursive(buf + 33554432, 24); + helper_float_26_recursive(buf + 50331648, 24); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 16777216; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_26(float *buf); +void helper_float_26(float *buf) { + helper_float_26_recursive(buf, 26); +} +void helper_float_27_recursive(float *buf, int depth); +void helper_float_27_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_27_recursive(buf + 0, 12); + helper_float_27_recursive(buf + 4096, 12); + helper_float_27_recursive(buf + 8192, 12); + helper_float_27_recursive(buf + 12288, 12); + helper_float_27_recursive(buf + 16384, 12); + helper_float_27_recursive(buf + 20480, 12); + helper_float_27_recursive(buf + 24576, 12); + helper_float_27_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_27_recursive(buf + 0, 15); + helper_float_27_recursive(buf + 32768, 15); + helper_float_27_recursive(buf + 65536, 15); + helper_float_27_recursive(buf + 98304, 15); + helper_float_27_recursive(buf + 131072, 15); + helper_float_27_recursive(buf + 163840, 15); + helper_float_27_recursive(buf + 196608, 15); + helper_float_27_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_27_recursive(buf + 0, 18); + helper_float_27_recursive(buf + 262144, 18); + helper_float_27_recursive(buf + 524288, 18); + helper_float_27_recursive(buf + 786432, 18); + helper_float_27_recursive(buf + 1048576, 18); + helper_float_27_recursive(buf + 1310720, 18); + helper_float_27_recursive(buf + 1572864, 18); + helper_float_27_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_27_recursive(buf + 0, 21); + helper_float_27_recursive(buf + 2097152, 21); + helper_float_27_recursive(buf + 4194304, 21); + helper_float_27_recursive(buf + 6291456, 21); + helper_float_27_recursive(buf + 8388608, 21); + helper_float_27_recursive(buf + 10485760, 21); + helper_float_27_recursive(buf + 12582912, 21); + helper_float_27_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_float_27_recursive(buf + 0, 24); + helper_float_27_recursive(buf + 16777216, 24); + helper_float_27_recursive(buf + 33554432, 24); + helper_float_27_recursive(buf + 50331648, 24); + helper_float_27_recursive(buf + 67108864, 24); + helper_float_27_recursive(buf + 83886080, 24); + helper_float_27_recursive(buf + 100663296, 24); + helper_float_27_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_27(float *buf); +void helper_float_27(float *buf) { + helper_float_27_recursive(buf, 27); +} +void helper_float_28_recursive(float *buf, int depth); +void helper_float_28_recursive(float *buf, int depth) { + if (depth == 7) { + for (int j = 0; j < 128; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 10) { + helper_float_28_recursive(buf + 0, 7); + helper_float_28_recursive(buf + 128, 7); + helper_float_28_recursive(buf + 256, 7); + helper_float_28_recursive(buf + 384, 7); + helper_float_28_recursive(buf + 512, 7); + helper_float_28_recursive(buf + 640, 7); + helper_float_28_recursive(buf + 768, 7); + helper_float_28_recursive(buf + 896, 7); + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_float_28_recursive(buf + 0, 10); + helper_float_28_recursive(buf + 1024, 10); + helper_float_28_recursive(buf + 2048, 10); + helper_float_28_recursive(buf + 3072, 10); + helper_float_28_recursive(buf + 4096, 10); + helper_float_28_recursive(buf + 5120, 10); + helper_float_28_recursive(buf + 6144, 10); + helper_float_28_recursive(buf + 7168, 10); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_28_recursive(buf + 0, 13); + helper_float_28_recursive(buf + 8192, 13); + helper_float_28_recursive(buf + 16384, 13); + helper_float_28_recursive(buf + 24576, 13); + helper_float_28_recursive(buf + 32768, 13); + helper_float_28_recursive(buf + 40960, 13); + helper_float_28_recursive(buf + 49152, 13); + helper_float_28_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_float_28_recursive(buf + 0, 16); + helper_float_28_recursive(buf + 65536, 16); + helper_float_28_recursive(buf + 131072, 16); + helper_float_28_recursive(buf + 196608, 16); + helper_float_28_recursive(buf + 262144, 16); + helper_float_28_recursive(buf + 327680, 16); + helper_float_28_recursive(buf + 393216, 16); + helper_float_28_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_float_28_recursive(buf + 0, 19); + helper_float_28_recursive(buf + 524288, 19); + helper_float_28_recursive(buf + 1048576, 19); + helper_float_28_recursive(buf + 1572864, 19); + helper_float_28_recursive(buf + 2097152, 19); + helper_float_28_recursive(buf + 2621440, 19); + helper_float_28_recursive(buf + 3145728, 19); + helper_float_28_recursive(buf + 3670016, 19); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 524288; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864), "r"(buf + j + k + 2097152), "r"(buf + j + k + 2621440), "r"(buf + j + k + 3145728), "r"(buf + j + k + 3670016) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 25) { + helper_float_28_recursive(buf + 0, 22); + helper_float_28_recursive(buf + 4194304, 22); + helper_float_28_recursive(buf + 8388608, 22); + helper_float_28_recursive(buf + 12582912, 22); + helper_float_28_recursive(buf + 16777216, 22); + helper_float_28_recursive(buf + 20971520, 22); + helper_float_28_recursive(buf + 25165824, 22); + helper_float_28_recursive(buf + 29360128, 22); + for (int j = 0; j < 33554432; j += 33554432) { + for (int k = 0; k < 4194304; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4194304), "r"(buf + j + k + 8388608), "r"(buf + j + k + 12582912), "r"(buf + j + k + 16777216), "r"(buf + j + k + 20971520), "r"(buf + j + k + 25165824), "r"(buf + j + k + 29360128) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 28) { + helper_float_28_recursive(buf + 0, 25); + helper_float_28_recursive(buf + 33554432, 25); + helper_float_28_recursive(buf + 67108864, 25); + helper_float_28_recursive(buf + 100663296, 25); + helper_float_28_recursive(buf + 134217728, 25); + helper_float_28_recursive(buf + 167772160, 25); + helper_float_28_recursive(buf + 201326592, 25); + helper_float_28_recursive(buf + 234881024, 25); + for (int j = 0; j < 268435456; j += 268435456) { + for (int k = 0; k < 33554432; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 33554432), "r"(buf + j + k + 67108864), "r"(buf + j + k + 100663296), "r"(buf + j + k + 134217728), "r"(buf + j + k + 167772160), "r"(buf + j + k + 201326592), "r"(buf + j + k + 234881024) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_28(float *buf); +void helper_float_28(float *buf) { + helper_float_28_recursive(buf, 28); +} +void helper_float_29_recursive(float *buf, int depth); +void helper_float_29_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_29_recursive(buf + 0, 12); + helper_float_29_recursive(buf + 4096, 12); + helper_float_29_recursive(buf + 8192, 12); + helper_float_29_recursive(buf + 12288, 12); + helper_float_29_recursive(buf + 16384, 12); + helper_float_29_recursive(buf + 20480, 12); + helper_float_29_recursive(buf + 24576, 12); + helper_float_29_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_29_recursive(buf + 0, 15); + helper_float_29_recursive(buf + 32768, 15); + helper_float_29_recursive(buf + 65536, 15); + helper_float_29_recursive(buf + 98304, 15); + helper_float_29_recursive(buf + 131072, 15); + helper_float_29_recursive(buf + 163840, 15); + helper_float_29_recursive(buf + 196608, 15); + helper_float_29_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_29_recursive(buf + 0, 18); + helper_float_29_recursive(buf + 262144, 18); + helper_float_29_recursive(buf + 524288, 18); + helper_float_29_recursive(buf + 786432, 18); + helper_float_29_recursive(buf + 1048576, 18); + helper_float_29_recursive(buf + 1310720, 18); + helper_float_29_recursive(buf + 1572864, 18); + helper_float_29_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_29_recursive(buf + 0, 21); + helper_float_29_recursive(buf + 2097152, 21); + helper_float_29_recursive(buf + 4194304, 21); + helper_float_29_recursive(buf + 6291456, 21); + helper_float_29_recursive(buf + 8388608, 21); + helper_float_29_recursive(buf + 10485760, 21); + helper_float_29_recursive(buf + 12582912, 21); + helper_float_29_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_float_29_recursive(buf + 0, 24); + helper_float_29_recursive(buf + 16777216, 24); + helper_float_29_recursive(buf + 33554432, 24); + helper_float_29_recursive(buf + 50331648, 24); + helper_float_29_recursive(buf + 67108864, 24); + helper_float_29_recursive(buf + 83886080, 24); + helper_float_29_recursive(buf + 100663296, 24); + helper_float_29_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 29) { + helper_float_29_recursive(buf + 0, 27); + helper_float_29_recursive(buf + 134217728, 27); + helper_float_29_recursive(buf + 268435456, 27); + helper_float_29_recursive(buf + 402653184, 27); + for (int j = 0; j < 536870912; j += 536870912) { + for (int k = 0; k < 134217728; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vmovups %%ymm0, (%0)\n" + "vmovups %%ymm1, (%1)\n" + "vmovups %%ymm2, (%2)\n" + "vmovups %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_29(float *buf); +void helper_float_29(float *buf) { + helper_float_29_recursive(buf, 29); +} +void helper_float_30_recursive(float *buf, int depth); +void helper_float_30_recursive(float *buf, int depth) { + if (depth == 6) { + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 8; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vpermilps $160, %%ymm0, %%ymm8\n" + "vpermilps $245, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm0\n" + "vpermilps $160, %%ymm1, %%ymm8\n" + "vpermilps $245, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm1\n" + "vpermilps $160, %%ymm2, %%ymm8\n" + "vpermilps $245, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm2\n" + "vpermilps $160, %%ymm3, %%ymm8\n" + "vpermilps $245, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm3\n" + "vpermilps $160, %%ymm4, %%ymm8\n" + "vpermilps $245, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm4\n" + "vpermilps $160, %%ymm5, %%ymm8\n" + "vpermilps $245, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm5\n" + "vpermilps $160, %%ymm6, %%ymm8\n" + "vpermilps $245, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm6\n" + "vpermilps $160, %%ymm7, %%ymm8\n" + "vpermilps $245, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubps %%ymm11, %%ymm8, %%ymm7\n" + "vpermilps $68, %%ymm0, %%ymm8\n" + "vpermilps $238, %%ymm0, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm0\n" + "vpermilps $68, %%ymm1, %%ymm8\n" + "vpermilps $238, %%ymm1, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm1\n" + "vpermilps $68, %%ymm2, %%ymm8\n" + "vpermilps $238, %%ymm2, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm2\n" + "vpermilps $68, %%ymm3, %%ymm8\n" + "vpermilps $238, %%ymm3, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm3\n" + "vpermilps $68, %%ymm4, %%ymm8\n" + "vpermilps $238, %%ymm4, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm4\n" + "vpermilps $68, %%ymm5, %%ymm8\n" + "vpermilps $238, %%ymm5, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm5\n" + "vpermilps $68, %%ymm6, %%ymm8\n" + "vpermilps $238, %%ymm6, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm6\n" + "vpermilps $68, %%ymm7, %%ymm8\n" + "vpermilps $238, %%ymm7, %%ymm9\n" + "vxorps %%ymm10, %%ymm10, %%ymm10\n" + "vsubps %%ymm9, %%ymm10, %%ymm11\n" + "vblendps $204, %%ymm11, %%ymm9, %%ymm12\n" + "vaddps %%ymm8, %%ymm12, %%ymm7\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm0, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm0, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm0\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm1, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm1, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm1\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm2, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm2, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm2\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm3, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm3, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm3\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm4, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm4, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm4\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm5, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm5, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm5\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm6, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm6, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm6\n" + "vxorps %%ymm8, %%ymm8, %%ymm8\n" + "vsubps %%ymm7, %%ymm8, %%ymm9\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm10\n" + "vperm2f128 $49, %%ymm9, %%ymm7, %%ymm11\n" + "vaddps %%ymm10, %%ymm11, %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 9) { + helper_float_30_recursive(buf + 0, 6); + helper_float_30_recursive(buf + 64, 6); + helper_float_30_recursive(buf + 128, 6); + helper_float_30_recursive(buf + 192, 6); + helper_float_30_recursive(buf + 256, 6); + helper_float_30_recursive(buf + 320, 6); + helper_float_30_recursive(buf + 384, 6); + helper_float_30_recursive(buf + 448, 6); + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_float_30_recursive(buf + 0, 9); + helper_float_30_recursive(buf + 512, 9); + helper_float_30_recursive(buf + 1024, 9); + helper_float_30_recursive(buf + 1536, 9); + helper_float_30_recursive(buf + 2048, 9); + helper_float_30_recursive(buf + 2560, 9); + helper_float_30_recursive(buf + 3072, 9); + helper_float_30_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_30_recursive(buf + 0, 12); + helper_float_30_recursive(buf + 4096, 12); + helper_float_30_recursive(buf + 8192, 12); + helper_float_30_recursive(buf + 12288, 12); + helper_float_30_recursive(buf + 16384, 12); + helper_float_30_recursive(buf + 20480, 12); + helper_float_30_recursive(buf + 24576, 12); + helper_float_30_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_30_recursive(buf + 0, 15); + helper_float_30_recursive(buf + 32768, 15); + helper_float_30_recursive(buf + 65536, 15); + helper_float_30_recursive(buf + 98304, 15); + helper_float_30_recursive(buf + 131072, 15); + helper_float_30_recursive(buf + 163840, 15); + helper_float_30_recursive(buf + 196608, 15); + helper_float_30_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_30_recursive(buf + 0, 18); + helper_float_30_recursive(buf + 262144, 18); + helper_float_30_recursive(buf + 524288, 18); + helper_float_30_recursive(buf + 786432, 18); + helper_float_30_recursive(buf + 1048576, 18); + helper_float_30_recursive(buf + 1310720, 18); + helper_float_30_recursive(buf + 1572864, 18); + helper_float_30_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_30_recursive(buf + 0, 21); + helper_float_30_recursive(buf + 2097152, 21); + helper_float_30_recursive(buf + 4194304, 21); + helper_float_30_recursive(buf + 6291456, 21); + helper_float_30_recursive(buf + 8388608, 21); + helper_float_30_recursive(buf + 10485760, 21); + helper_float_30_recursive(buf + 12582912, 21); + helper_float_30_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_float_30_recursive(buf + 0, 24); + helper_float_30_recursive(buf + 16777216, 24); + helper_float_30_recursive(buf + 33554432, 24); + helper_float_30_recursive(buf + 50331648, 24); + helper_float_30_recursive(buf + 67108864, 24); + helper_float_30_recursive(buf + 83886080, 24); + helper_float_30_recursive(buf + 100663296, 24); + helper_float_30_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 30) { + helper_float_30_recursive(buf + 0, 27); + helper_float_30_recursive(buf + 134217728, 27); + helper_float_30_recursive(buf + 268435456, 27); + helper_float_30_recursive(buf + 402653184, 27); + helper_float_30_recursive(buf + 536870912, 27); + helper_float_30_recursive(buf + 671088640, 27); + helper_float_30_recursive(buf + 805306368, 27); + helper_float_30_recursive(buf + 939524096, 27); + for (int j = 0; j < 1073741824; j += 1073741824) { + for (int k = 0; k < 134217728; k += 8) { + __asm__ volatile ( + "vmovups (%0), %%ymm0\n" + "vmovups (%1), %%ymm1\n" + "vmovups (%2), %%ymm2\n" + "vmovups (%3), %%ymm3\n" + "vmovups (%4), %%ymm4\n" + "vmovups (%5), %%ymm5\n" + "vmovups (%6), %%ymm6\n" + "vmovups (%7), %%ymm7\n" + "vaddps %%ymm1, %%ymm0, %%ymm8\n" + "vsubps %%ymm1, %%ymm0, %%ymm9\n" + "vaddps %%ymm3, %%ymm2, %%ymm10\n" + "vsubps %%ymm3, %%ymm2, %%ymm11\n" + "vaddps %%ymm5, %%ymm4, %%ymm12\n" + "vsubps %%ymm5, %%ymm4, %%ymm13\n" + "vaddps %%ymm7, %%ymm6, %%ymm14\n" + "vsubps %%ymm7, %%ymm6, %%ymm15\n" + "vaddps %%ymm10, %%ymm8, %%ymm0\n" + "vsubps %%ymm10, %%ymm8, %%ymm2\n" + "vaddps %%ymm11, %%ymm9, %%ymm1\n" + "vsubps %%ymm11, %%ymm9, %%ymm3\n" + "vaddps %%ymm14, %%ymm12, %%ymm4\n" + "vsubps %%ymm14, %%ymm12, %%ymm6\n" + "vaddps %%ymm15, %%ymm13, %%ymm5\n" + "vsubps %%ymm15, %%ymm13, %%ymm7\n" + "vaddps %%ymm4, %%ymm0, %%ymm8\n" + "vsubps %%ymm4, %%ymm0, %%ymm12\n" + "vaddps %%ymm5, %%ymm1, %%ymm9\n" + "vsubps %%ymm5, %%ymm1, %%ymm13\n" + "vaddps %%ymm6, %%ymm2, %%ymm10\n" + "vsubps %%ymm6, %%ymm2, %%ymm14\n" + "vaddps %%ymm7, %%ymm3, %%ymm11\n" + "vsubps %%ymm7, %%ymm3, %%ymm15\n" + "vmovups %%ymm8, (%0)\n" + "vmovups %%ymm9, (%1)\n" + "vmovups %%ymm10, (%2)\n" + "vmovups %%ymm11, (%3)\n" + "vmovups %%ymm12, (%4)\n" + "vmovups %%ymm13, (%5)\n" + "vmovups %%ymm14, (%6)\n" + "vmovups %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184), "r"(buf + j + k + 536870912), "r"(buf + j + k + 671088640), "r"(buf + j + k + 805306368), "r"(buf + j + k + 939524096) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_float_30(float *buf); +void helper_float_30(float *buf) { + helper_float_30_recursive(buf, 30); +} +int fht_float(float *buf, int log_n) { + if (log_n == 0) { + return 0; + } + if (log_n == 1) { + helper_float_1(buf); + return 0; + } + if (log_n == 2) { + helper_float_2(buf); + return 0; + } + if (log_n == 3) { + helper_float_3(buf); + return 0; + } + if (log_n == 4) { + helper_float_4(buf); + return 0; + } + if (log_n == 5) { + helper_float_5(buf); + return 0; + } + if (log_n == 6) { + helper_float_6(buf); + return 0; + } + if (log_n == 7) { + helper_float_7(buf); + return 0; + } + if (log_n == 8) { + helper_float_8(buf); + return 0; + } + if (log_n == 9) { + helper_float_9(buf); + return 0; + } + if (log_n == 10) { + helper_float_10(buf); + return 0; + } + if (log_n == 11) { + helper_float_11(buf); + return 0; + } + if (log_n == 12) { + helper_float_12(buf); + return 0; + } + if (log_n == 13) { + helper_float_13(buf); + return 0; + } + if (log_n == 14) { + helper_float_14(buf); + return 0; + } + if (log_n == 15) { + helper_float_15(buf); + return 0; + } + if (log_n == 16) { + helper_float_16(buf); + return 0; + } + if (log_n == 17) { + helper_float_17(buf); + return 0; + } + if (log_n == 18) { + helper_float_18(buf); + return 0; + } + if (log_n == 19) { + helper_float_19(buf); + return 0; + } + if (log_n == 20) { + helper_float_20(buf); + return 0; + } + if (log_n == 21) { + helper_float_21(buf); + return 0; + } + if (log_n == 22) { + helper_float_22(buf); + return 0; + } + if (log_n == 23) { + helper_float_23(buf); + return 0; + } + if (log_n == 24) { + helper_float_24(buf); + return 0; + } + if (log_n == 25) { + helper_float_25(buf); + return 0; + } + if (log_n == 26) { + helper_float_26(buf); + return 0; + } + if (log_n == 27) { + helper_float_27(buf); + return 0; + } + if (log_n == 28) { + helper_float_28(buf); + return 0; + } + if (log_n == 29) { + helper_float_29(buf); + return 0; + } + if (log_n == 30) { + helper_float_30(buf); + return 0; + } + return 1; +} +static inline void helper_double_1(double *buf); +static inline void helper_double_1(double *buf) { + for (int j = 0; j < 2; j += 2) { + for (int k = 0; k < 1; ++k) { + double u = buf[j + k]; + double v = buf[j + k + 1]; + buf[j + k] = u + v; + buf[j + k + 1] = u - v; + } + } +} +static inline void helper_double_2(double *buf); +static inline void helper_double_2(double *buf) { + for (int j = 0; j < 4; j += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vmovupd %%ymm0, (%0)\n" + :: "r"(buf + j) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } +} +static inline void helper_double_3(double *buf); +static inline void helper_double_3(double *buf) { + for (int j = 0; j < 8; j += 8) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +void helper_double_4_recursive(double *buf, int depth); +void helper_double_4_recursive(double *buf, int depth) { + if (depth == 4) { + for (int j = 0; j < 16; j += 16) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_4(double *buf); +void helper_double_4(double *buf) { + helper_double_4_recursive(buf, 4); +} +static inline void helper_double_5(double *buf); +static inline void helper_double_5(double *buf) { + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_double_6(double *buf); +static inline void helper_double_6(double *buf) { + for (int j = 0; j < 64; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_double_7(double *buf); +static inline void helper_double_7(double *buf) { + for (int j = 0; j < 128; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_double_8(double *buf); +static inline void helper_double_8(double *buf) { + for (int j = 0; j < 256; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_double_9(double *buf); +static inline void helper_double_9(double *buf) { + for (int j = 0; j < 512; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_double_10(double *buf); +static inline void helper_double_10(double *buf) { + for (int j = 0; j < 1024; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +static inline void helper_double_11(double *buf); +static inline void helper_double_11(double *buf) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } +} +void helper_double_12_recursive(double *buf, int depth); +void helper_double_12_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_12_recursive(buf + 0, 11); + helper_double_12_recursive(buf + 2048, 11); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_12(double *buf); +void helper_double_12(double *buf) { + helper_double_12_recursive(buf, 12); +} +void helper_double_13_recursive(double *buf, int depth); +void helper_double_13_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_double_13_recursive(buf + 0, 11); + helper_double_13_recursive(buf + 2048, 11); + helper_double_13_recursive(buf + 4096, 11); + helper_double_13_recursive(buf + 6144, 11); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_13(double *buf); +void helper_double_13(double *buf) { + helper_double_13_recursive(buf, 13); +} +void helper_double_14_recursive(double *buf, int depth); +void helper_double_14_recursive(double *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_14_recursive(buf + 0, 12); + helper_double_14_recursive(buf + 4096, 12); + helper_double_14_recursive(buf + 8192, 12); + helper_double_14_recursive(buf + 12288, 12); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_14(double *buf); +void helper_double_14(double *buf) { + helper_double_14_recursive(buf, 14); +} +void helper_double_15_recursive(double *buf, int depth); +void helper_double_15_recursive(double *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_15_recursive(buf + 0, 12); + helper_double_15_recursive(buf + 4096, 12); + helper_double_15_recursive(buf + 8192, 12); + helper_double_15_recursive(buf + 12288, 12); + helper_double_15_recursive(buf + 16384, 12); + helper_double_15_recursive(buf + 20480, 12); + helper_double_15_recursive(buf + 24576, 12); + helper_double_15_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_15(double *buf); +void helper_double_15(double *buf) { + helper_double_15_recursive(buf, 15); +} +void helper_double_16_recursive(double *buf, int depth); +void helper_double_16_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_16_recursive(buf + 0, 11); + helper_double_16_recursive(buf + 2048, 11); + helper_double_16_recursive(buf + 4096, 11); + helper_double_16_recursive(buf + 6144, 11); + helper_double_16_recursive(buf + 8192, 11); + helper_double_16_recursive(buf + 10240, 11); + helper_double_16_recursive(buf + 12288, 11); + helper_double_16_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_16_recursive(buf + 0, 14); + helper_double_16_recursive(buf + 16384, 14); + helper_double_16_recursive(buf + 32768, 14); + helper_double_16_recursive(buf + 49152, 14); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_16(double *buf); +void helper_double_16(double *buf) { + helper_double_16_recursive(buf, 16); +} +void helper_double_17_recursive(double *buf, int depth); +void helper_double_17_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_17_recursive(buf + 0, 11); + helper_double_17_recursive(buf + 2048, 11); + helper_double_17_recursive(buf + 4096, 11); + helper_double_17_recursive(buf + 6144, 11); + helper_double_17_recursive(buf + 8192, 11); + helper_double_17_recursive(buf + 10240, 11); + helper_double_17_recursive(buf + 12288, 11); + helper_double_17_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_17_recursive(buf + 0, 14); + helper_double_17_recursive(buf + 16384, 14); + helper_double_17_recursive(buf + 32768, 14); + helper_double_17_recursive(buf + 49152, 14); + helper_double_17_recursive(buf + 65536, 14); + helper_double_17_recursive(buf + 81920, 14); + helper_double_17_recursive(buf + 98304, 14); + helper_double_17_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_17(double *buf); +void helper_double_17(double *buf) { + helper_double_17_recursive(buf, 17); +} +void helper_double_18_recursive(double *buf, int depth); +void helper_double_18_recursive(double *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_18_recursive(buf + 0, 12); + helper_double_18_recursive(buf + 4096, 12); + helper_double_18_recursive(buf + 8192, 12); + helper_double_18_recursive(buf + 12288, 12); + helper_double_18_recursive(buf + 16384, 12); + helper_double_18_recursive(buf + 20480, 12); + helper_double_18_recursive(buf + 24576, 12); + helper_double_18_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_18_recursive(buf + 0, 15); + helper_double_18_recursive(buf + 32768, 15); + helper_double_18_recursive(buf + 65536, 15); + helper_double_18_recursive(buf + 98304, 15); + helper_double_18_recursive(buf + 131072, 15); + helper_double_18_recursive(buf + 163840, 15); + helper_double_18_recursive(buf + 196608, 15); + helper_double_18_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_18(double *buf); +void helper_double_18(double *buf) { + helper_double_18_recursive(buf, 18); +} +void helper_double_19_recursive(double *buf, int depth); +void helper_double_19_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_19_recursive(buf + 0, 11); + helper_double_19_recursive(buf + 2048, 11); + helper_double_19_recursive(buf + 4096, 11); + helper_double_19_recursive(buf + 6144, 11); + helper_double_19_recursive(buf + 8192, 11); + helper_double_19_recursive(buf + 10240, 11); + helper_double_19_recursive(buf + 12288, 11); + helper_double_19_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_19_recursive(buf + 0, 14); + helper_double_19_recursive(buf + 16384, 14); + helper_double_19_recursive(buf + 32768, 14); + helper_double_19_recursive(buf + 49152, 14); + helper_double_19_recursive(buf + 65536, 14); + helper_double_19_recursive(buf + 81920, 14); + helper_double_19_recursive(buf + 98304, 14); + helper_double_19_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_19_recursive(buf + 0, 17); + helper_double_19_recursive(buf + 131072, 17); + helper_double_19_recursive(buf + 262144, 17); + helper_double_19_recursive(buf + 393216, 17); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_19(double *buf); +void helper_double_19(double *buf) { + helper_double_19_recursive(buf, 19); +} +void helper_double_20_recursive(double *buf, int depth); +void helper_double_20_recursive(double *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_20_recursive(buf + 0, 9); + helper_double_20_recursive(buf + 512, 9); + helper_double_20_recursive(buf + 1024, 9); + helper_double_20_recursive(buf + 1536, 9); + helper_double_20_recursive(buf + 2048, 9); + helper_double_20_recursive(buf + 2560, 9); + helper_double_20_recursive(buf + 3072, 9); + helper_double_20_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_20_recursive(buf + 0, 12); + helper_double_20_recursive(buf + 4096, 12); + helper_double_20_recursive(buf + 8192, 12); + helper_double_20_recursive(buf + 12288, 12); + helper_double_20_recursive(buf + 16384, 12); + helper_double_20_recursive(buf + 20480, 12); + helper_double_20_recursive(buf + 24576, 12); + helper_double_20_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_20_recursive(buf + 0, 15); + helper_double_20_recursive(buf + 32768, 15); + helper_double_20_recursive(buf + 65536, 15); + helper_double_20_recursive(buf + 98304, 15); + helper_double_20_recursive(buf + 131072, 15); + helper_double_20_recursive(buf + 163840, 15); + helper_double_20_recursive(buf + 196608, 15); + helper_double_20_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_20_recursive(buf + 0, 18); + helper_double_20_recursive(buf + 262144, 18); + helper_double_20_recursive(buf + 524288, 18); + helper_double_20_recursive(buf + 786432, 18); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_20(double *buf); +void helper_double_20(double *buf) { + helper_double_20_recursive(buf, 20); +} +void helper_double_21_recursive(double *buf, int depth); +void helper_double_21_recursive(double *buf, int depth) { + if (depth == 7) { + for (int j = 0; j < 128; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 10) { + helper_double_21_recursive(buf + 0, 7); + helper_double_21_recursive(buf + 128, 7); + helper_double_21_recursive(buf + 256, 7); + helper_double_21_recursive(buf + 384, 7); + helper_double_21_recursive(buf + 512, 7); + helper_double_21_recursive(buf + 640, 7); + helper_double_21_recursive(buf + 768, 7); + helper_double_21_recursive(buf + 896, 7); + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_double_21_recursive(buf + 0, 10); + helper_double_21_recursive(buf + 1024, 10); + helper_double_21_recursive(buf + 2048, 10); + helper_double_21_recursive(buf + 3072, 10); + helper_double_21_recursive(buf + 4096, 10); + helper_double_21_recursive(buf + 5120, 10); + helper_double_21_recursive(buf + 6144, 10); + helper_double_21_recursive(buf + 7168, 10); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_21_recursive(buf + 0, 13); + helper_double_21_recursive(buf + 8192, 13); + helper_double_21_recursive(buf + 16384, 13); + helper_double_21_recursive(buf + 24576, 13); + helper_double_21_recursive(buf + 32768, 13); + helper_double_21_recursive(buf + 40960, 13); + helper_double_21_recursive(buf + 49152, 13); + helper_double_21_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_21_recursive(buf + 0, 16); + helper_double_21_recursive(buf + 65536, 16); + helper_double_21_recursive(buf + 131072, 16); + helper_double_21_recursive(buf + 196608, 16); + helper_double_21_recursive(buf + 262144, 16); + helper_double_21_recursive(buf + 327680, 16); + helper_double_21_recursive(buf + 393216, 16); + helper_double_21_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_21_recursive(buf + 0, 19); + helper_double_21_recursive(buf + 524288, 19); + helper_double_21_recursive(buf + 1048576, 19); + helper_double_21_recursive(buf + 1572864, 19); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 524288; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_21(double *buf); +void helper_double_21(double *buf) { + helper_double_21_recursive(buf, 21); +} +void helper_double_22_recursive(double *buf, int depth); +void helper_double_22_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_22_recursive(buf + 0, 11); + helper_double_22_recursive(buf + 2048, 11); + helper_double_22_recursive(buf + 4096, 11); + helper_double_22_recursive(buf + 6144, 11); + helper_double_22_recursive(buf + 8192, 11); + helper_double_22_recursive(buf + 10240, 11); + helper_double_22_recursive(buf + 12288, 11); + helper_double_22_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_22_recursive(buf + 0, 14); + helper_double_22_recursive(buf + 16384, 14); + helper_double_22_recursive(buf + 32768, 14); + helper_double_22_recursive(buf + 49152, 14); + helper_double_22_recursive(buf + 65536, 14); + helper_double_22_recursive(buf + 81920, 14); + helper_double_22_recursive(buf + 98304, 14); + helper_double_22_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_22_recursive(buf + 0, 17); + helper_double_22_recursive(buf + 131072, 17); + helper_double_22_recursive(buf + 262144, 17); + helper_double_22_recursive(buf + 393216, 17); + helper_double_22_recursive(buf + 524288, 17); + helper_double_22_recursive(buf + 655360, 17); + helper_double_22_recursive(buf + 786432, 17); + helper_double_22_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_double_22_recursive(buf + 0, 20); + helper_double_22_recursive(buf + 1048576, 20); + helper_double_22_recursive(buf + 2097152, 20); + helper_double_22_recursive(buf + 3145728, 20); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_22(double *buf); +void helper_double_22(double *buf) { + helper_double_22_recursive(buf, 22); +} +void helper_double_23_recursive(double *buf, int depth); +void helper_double_23_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_23_recursive(buf + 0, 11); + helper_double_23_recursive(buf + 2048, 11); + helper_double_23_recursive(buf + 4096, 11); + helper_double_23_recursive(buf + 6144, 11); + helper_double_23_recursive(buf + 8192, 11); + helper_double_23_recursive(buf + 10240, 11); + helper_double_23_recursive(buf + 12288, 11); + helper_double_23_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_23_recursive(buf + 0, 14); + helper_double_23_recursive(buf + 16384, 14); + helper_double_23_recursive(buf + 32768, 14); + helper_double_23_recursive(buf + 49152, 14); + helper_double_23_recursive(buf + 65536, 14); + helper_double_23_recursive(buf + 81920, 14); + helper_double_23_recursive(buf + 98304, 14); + helper_double_23_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_23_recursive(buf + 0, 17); + helper_double_23_recursive(buf + 131072, 17); + helper_double_23_recursive(buf + 262144, 17); + helper_double_23_recursive(buf + 393216, 17); + helper_double_23_recursive(buf + 524288, 17); + helper_double_23_recursive(buf + 655360, 17); + helper_double_23_recursive(buf + 786432, 17); + helper_double_23_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_23_recursive(buf + 0, 20); + helper_double_23_recursive(buf + 1048576, 20); + helper_double_23_recursive(buf + 2097152, 20); + helper_double_23_recursive(buf + 3145728, 20); + helper_double_23_recursive(buf + 4194304, 20); + helper_double_23_recursive(buf + 5242880, 20); + helper_double_23_recursive(buf + 6291456, 20); + helper_double_23_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_23(double *buf); +void helper_double_23(double *buf) { + helper_double_23_recursive(buf, 23); +} +void helper_double_24_recursive(double *buf, int depth); +void helper_double_24_recursive(double *buf, int depth) { + if (depth == 10) { + for (int j = 0; j < 1024; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_double_24_recursive(buf + 0, 10); + helper_double_24_recursive(buf + 1024, 10); + helper_double_24_recursive(buf + 2048, 10); + helper_double_24_recursive(buf + 3072, 10); + helper_double_24_recursive(buf + 4096, 10); + helper_double_24_recursive(buf + 5120, 10); + helper_double_24_recursive(buf + 6144, 10); + helper_double_24_recursive(buf + 7168, 10); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_24_recursive(buf + 0, 13); + helper_double_24_recursive(buf + 8192, 13); + helper_double_24_recursive(buf + 16384, 13); + helper_double_24_recursive(buf + 24576, 13); + helper_double_24_recursive(buf + 32768, 13); + helper_double_24_recursive(buf + 40960, 13); + helper_double_24_recursive(buf + 49152, 13); + helper_double_24_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_24_recursive(buf + 0, 16); + helper_double_24_recursive(buf + 65536, 16); + helper_double_24_recursive(buf + 131072, 16); + helper_double_24_recursive(buf + 196608, 16); + helper_double_24_recursive(buf + 262144, 16); + helper_double_24_recursive(buf + 327680, 16); + helper_double_24_recursive(buf + 393216, 16); + helper_double_24_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_double_24_recursive(buf + 0, 19); + helper_double_24_recursive(buf + 524288, 19); + helper_double_24_recursive(buf + 1048576, 19); + helper_double_24_recursive(buf + 1572864, 19); + helper_double_24_recursive(buf + 2097152, 19); + helper_double_24_recursive(buf + 2621440, 19); + helper_double_24_recursive(buf + 3145728, 19); + helper_double_24_recursive(buf + 3670016, 19); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 524288; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864), "r"(buf + j + k + 2097152), "r"(buf + j + k + 2621440), "r"(buf + j + k + 3145728), "r"(buf + j + k + 3670016) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_24_recursive(buf + 0, 22); + helper_double_24_recursive(buf + 4194304, 22); + helper_double_24_recursive(buf + 8388608, 22); + helper_double_24_recursive(buf + 12582912, 22); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 4194304; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4194304), "r"(buf + j + k + 8388608), "r"(buf + j + k + 12582912) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_24(double *buf); +void helper_double_24(double *buf) { + helper_double_24_recursive(buf, 24); +} +void helper_double_25_recursive(double *buf, int depth); +void helper_double_25_recursive(double *buf, int depth) { + if (depth == 8) { + for (int j = 0; j < 256; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_25_recursive(buf + 0, 8); + helper_double_25_recursive(buf + 256, 8); + helper_double_25_recursive(buf + 512, 8); + helper_double_25_recursive(buf + 768, 8); + helper_double_25_recursive(buf + 1024, 8); + helper_double_25_recursive(buf + 1280, 8); + helper_double_25_recursive(buf + 1536, 8); + helper_double_25_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_25_recursive(buf + 0, 11); + helper_double_25_recursive(buf + 2048, 11); + helper_double_25_recursive(buf + 4096, 11); + helper_double_25_recursive(buf + 6144, 11); + helper_double_25_recursive(buf + 8192, 11); + helper_double_25_recursive(buf + 10240, 11); + helper_double_25_recursive(buf + 12288, 11); + helper_double_25_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_25_recursive(buf + 0, 14); + helper_double_25_recursive(buf + 16384, 14); + helper_double_25_recursive(buf + 32768, 14); + helper_double_25_recursive(buf + 49152, 14); + helper_double_25_recursive(buf + 65536, 14); + helper_double_25_recursive(buf + 81920, 14); + helper_double_25_recursive(buf + 98304, 14); + helper_double_25_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_25_recursive(buf + 0, 17); + helper_double_25_recursive(buf + 131072, 17); + helper_double_25_recursive(buf + 262144, 17); + helper_double_25_recursive(buf + 393216, 17); + helper_double_25_recursive(buf + 524288, 17); + helper_double_25_recursive(buf + 655360, 17); + helper_double_25_recursive(buf + 786432, 17); + helper_double_25_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_25_recursive(buf + 0, 20); + helper_double_25_recursive(buf + 1048576, 20); + helper_double_25_recursive(buf + 2097152, 20); + helper_double_25_recursive(buf + 3145728, 20); + helper_double_25_recursive(buf + 4194304, 20); + helper_double_25_recursive(buf + 5242880, 20); + helper_double_25_recursive(buf + 6291456, 20); + helper_double_25_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 25) { + helper_double_25_recursive(buf + 0, 23); + helper_double_25_recursive(buf + 8388608, 23); + helper_double_25_recursive(buf + 16777216, 23); + helper_double_25_recursive(buf + 25165824, 23); + for (int j = 0; j < 33554432; j += 33554432) { + for (int k = 0; k < 8388608; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_25(double *buf); +void helper_double_25(double *buf) { + helper_double_25_recursive(buf, 25); +} +void helper_double_26_recursive(double *buf, int depth); +void helper_double_26_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_26_recursive(buf + 0, 11); + helper_double_26_recursive(buf + 2048, 11); + helper_double_26_recursive(buf + 4096, 11); + helper_double_26_recursive(buf + 6144, 11); + helper_double_26_recursive(buf + 8192, 11); + helper_double_26_recursive(buf + 10240, 11); + helper_double_26_recursive(buf + 12288, 11); + helper_double_26_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_26_recursive(buf + 0, 14); + helper_double_26_recursive(buf + 16384, 14); + helper_double_26_recursive(buf + 32768, 14); + helper_double_26_recursive(buf + 49152, 14); + helper_double_26_recursive(buf + 65536, 14); + helper_double_26_recursive(buf + 81920, 14); + helper_double_26_recursive(buf + 98304, 14); + helper_double_26_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_26_recursive(buf + 0, 17); + helper_double_26_recursive(buf + 131072, 17); + helper_double_26_recursive(buf + 262144, 17); + helper_double_26_recursive(buf + 393216, 17); + helper_double_26_recursive(buf + 524288, 17); + helper_double_26_recursive(buf + 655360, 17); + helper_double_26_recursive(buf + 786432, 17); + helper_double_26_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_26_recursive(buf + 0, 20); + helper_double_26_recursive(buf + 1048576, 20); + helper_double_26_recursive(buf + 2097152, 20); + helper_double_26_recursive(buf + 3145728, 20); + helper_double_26_recursive(buf + 4194304, 20); + helper_double_26_recursive(buf + 5242880, 20); + helper_double_26_recursive(buf + 6291456, 20); + helper_double_26_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_double_26_recursive(buf + 0, 23); + helper_double_26_recursive(buf + 8388608, 23); + helper_double_26_recursive(buf + 16777216, 23); + helper_double_26_recursive(buf + 25165824, 23); + helper_double_26_recursive(buf + 33554432, 23); + helper_double_26_recursive(buf + 41943040, 23); + helper_double_26_recursive(buf + 50331648, 23); + helper_double_26_recursive(buf + 58720256, 23); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 8388608; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824), "r"(buf + j + k + 33554432), "r"(buf + j + k + 41943040), "r"(buf + j + k + 50331648), "r"(buf + j + k + 58720256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_26(double *buf); +void helper_double_26(double *buf) { + helper_double_26_recursive(buf, 26); +} +void helper_double_27_recursive(double *buf, int depth); +void helper_double_27_recursive(double *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_27_recursive(buf + 0, 9); + helper_double_27_recursive(buf + 512, 9); + helper_double_27_recursive(buf + 1024, 9); + helper_double_27_recursive(buf + 1536, 9); + helper_double_27_recursive(buf + 2048, 9); + helper_double_27_recursive(buf + 2560, 9); + helper_double_27_recursive(buf + 3072, 9); + helper_double_27_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_27_recursive(buf + 0, 12); + helper_double_27_recursive(buf + 4096, 12); + helper_double_27_recursive(buf + 8192, 12); + helper_double_27_recursive(buf + 12288, 12); + helper_double_27_recursive(buf + 16384, 12); + helper_double_27_recursive(buf + 20480, 12); + helper_double_27_recursive(buf + 24576, 12); + helper_double_27_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_27_recursive(buf + 0, 15); + helper_double_27_recursive(buf + 32768, 15); + helper_double_27_recursive(buf + 65536, 15); + helper_double_27_recursive(buf + 98304, 15); + helper_double_27_recursive(buf + 131072, 15); + helper_double_27_recursive(buf + 163840, 15); + helper_double_27_recursive(buf + 196608, 15); + helper_double_27_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_27_recursive(buf + 0, 18); + helper_double_27_recursive(buf + 262144, 18); + helper_double_27_recursive(buf + 524288, 18); + helper_double_27_recursive(buf + 786432, 18); + helper_double_27_recursive(buf + 1048576, 18); + helper_double_27_recursive(buf + 1310720, 18); + helper_double_27_recursive(buf + 1572864, 18); + helper_double_27_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_27_recursive(buf + 0, 21); + helper_double_27_recursive(buf + 2097152, 21); + helper_double_27_recursive(buf + 4194304, 21); + helper_double_27_recursive(buf + 6291456, 21); + helper_double_27_recursive(buf + 8388608, 21); + helper_double_27_recursive(buf + 10485760, 21); + helper_double_27_recursive(buf + 12582912, 21); + helper_double_27_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_double_27_recursive(buf + 0, 24); + helper_double_27_recursive(buf + 16777216, 24); + helper_double_27_recursive(buf + 33554432, 24); + helper_double_27_recursive(buf + 50331648, 24); + helper_double_27_recursive(buf + 67108864, 24); + helper_double_27_recursive(buf + 83886080, 24); + helper_double_27_recursive(buf + 100663296, 24); + helper_double_27_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_27(double *buf); +void helper_double_27(double *buf) { + helper_double_27_recursive(buf, 27); +} +void helper_double_28_recursive(double *buf, int depth); +void helper_double_28_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_28_recursive(buf + 0, 11); + helper_double_28_recursive(buf + 2048, 11); + helper_double_28_recursive(buf + 4096, 11); + helper_double_28_recursive(buf + 6144, 11); + helper_double_28_recursive(buf + 8192, 11); + helper_double_28_recursive(buf + 10240, 11); + helper_double_28_recursive(buf + 12288, 11); + helper_double_28_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_28_recursive(buf + 0, 14); + helper_double_28_recursive(buf + 16384, 14); + helper_double_28_recursive(buf + 32768, 14); + helper_double_28_recursive(buf + 49152, 14); + helper_double_28_recursive(buf + 65536, 14); + helper_double_28_recursive(buf + 81920, 14); + helper_double_28_recursive(buf + 98304, 14); + helper_double_28_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_28_recursive(buf + 0, 17); + helper_double_28_recursive(buf + 131072, 17); + helper_double_28_recursive(buf + 262144, 17); + helper_double_28_recursive(buf + 393216, 17); + helper_double_28_recursive(buf + 524288, 17); + helper_double_28_recursive(buf + 655360, 17); + helper_double_28_recursive(buf + 786432, 17); + helper_double_28_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_28_recursive(buf + 0, 20); + helper_double_28_recursive(buf + 1048576, 20); + helper_double_28_recursive(buf + 2097152, 20); + helper_double_28_recursive(buf + 3145728, 20); + helper_double_28_recursive(buf + 4194304, 20); + helper_double_28_recursive(buf + 5242880, 20); + helper_double_28_recursive(buf + 6291456, 20); + helper_double_28_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_double_28_recursive(buf + 0, 23); + helper_double_28_recursive(buf + 8388608, 23); + helper_double_28_recursive(buf + 16777216, 23); + helper_double_28_recursive(buf + 25165824, 23); + helper_double_28_recursive(buf + 33554432, 23); + helper_double_28_recursive(buf + 41943040, 23); + helper_double_28_recursive(buf + 50331648, 23); + helper_double_28_recursive(buf + 58720256, 23); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 8388608; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824), "r"(buf + j + k + 33554432), "r"(buf + j + k + 41943040), "r"(buf + j + k + 50331648), "r"(buf + j + k + 58720256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 28) { + helper_double_28_recursive(buf + 0, 26); + helper_double_28_recursive(buf + 67108864, 26); + helper_double_28_recursive(buf + 134217728, 26); + helper_double_28_recursive(buf + 201326592, 26); + for (int j = 0; j < 268435456; j += 268435456) { + for (int k = 0; k < 67108864; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vmovupd %%ymm0, (%0)\n" + "vmovupd %%ymm1, (%1)\n" + "vmovupd %%ymm2, (%2)\n" + "vmovupd %%ymm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 67108864), "r"(buf + j + k + 134217728), "r"(buf + j + k + 201326592) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_28(double *buf); +void helper_double_28(double *buf) { + helper_double_28_recursive(buf, 28); +} +void helper_double_29_recursive(double *buf, int depth); +void helper_double_29_recursive(double *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_29_recursive(buf + 0, 11); + helper_double_29_recursive(buf + 2048, 11); + helper_double_29_recursive(buf + 4096, 11); + helper_double_29_recursive(buf + 6144, 11); + helper_double_29_recursive(buf + 8192, 11); + helper_double_29_recursive(buf + 10240, 11); + helper_double_29_recursive(buf + 12288, 11); + helper_double_29_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_29_recursive(buf + 0, 14); + helper_double_29_recursive(buf + 16384, 14); + helper_double_29_recursive(buf + 32768, 14); + helper_double_29_recursive(buf + 49152, 14); + helper_double_29_recursive(buf + 65536, 14); + helper_double_29_recursive(buf + 81920, 14); + helper_double_29_recursive(buf + 98304, 14); + helper_double_29_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_29_recursive(buf + 0, 17); + helper_double_29_recursive(buf + 131072, 17); + helper_double_29_recursive(buf + 262144, 17); + helper_double_29_recursive(buf + 393216, 17); + helper_double_29_recursive(buf + 524288, 17); + helper_double_29_recursive(buf + 655360, 17); + helper_double_29_recursive(buf + 786432, 17); + helper_double_29_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_29_recursive(buf + 0, 20); + helper_double_29_recursive(buf + 1048576, 20); + helper_double_29_recursive(buf + 2097152, 20); + helper_double_29_recursive(buf + 3145728, 20); + helper_double_29_recursive(buf + 4194304, 20); + helper_double_29_recursive(buf + 5242880, 20); + helper_double_29_recursive(buf + 6291456, 20); + helper_double_29_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_double_29_recursive(buf + 0, 23); + helper_double_29_recursive(buf + 8388608, 23); + helper_double_29_recursive(buf + 16777216, 23); + helper_double_29_recursive(buf + 25165824, 23); + helper_double_29_recursive(buf + 33554432, 23); + helper_double_29_recursive(buf + 41943040, 23); + helper_double_29_recursive(buf + 50331648, 23); + helper_double_29_recursive(buf + 58720256, 23); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 8388608; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824), "r"(buf + j + k + 33554432), "r"(buf + j + k + 41943040), "r"(buf + j + k + 50331648), "r"(buf + j + k + 58720256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 29) { + helper_double_29_recursive(buf + 0, 26); + helper_double_29_recursive(buf + 67108864, 26); + helper_double_29_recursive(buf + 134217728, 26); + helper_double_29_recursive(buf + 201326592, 26); + helper_double_29_recursive(buf + 268435456, 26); + helper_double_29_recursive(buf + 335544320, 26); + helper_double_29_recursive(buf + 402653184, 26); + helper_double_29_recursive(buf + 469762048, 26); + for (int j = 0; j < 536870912; j += 536870912) { + for (int k = 0; k < 67108864; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 67108864), "r"(buf + j + k + 134217728), "r"(buf + j + k + 201326592), "r"(buf + j + k + 268435456), "r"(buf + j + k + 335544320), "r"(buf + j + k + 402653184), "r"(buf + j + k + 469762048) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_29(double *buf); +void helper_double_29(double *buf) { + helper_double_29_recursive(buf, 29); +} +void helper_double_30_recursive(double *buf, int depth); +void helper_double_30_recursive(double *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vpermilpd $0, %%ymm0, %%ymm8\n" + "vpermilpd $15, %%ymm0, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm0\n" + "vpermilpd $0, %%ymm1, %%ymm8\n" + "vpermilpd $15, %%ymm1, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm1\n" + "vpermilpd $0, %%ymm2, %%ymm8\n" + "vpermilpd $15, %%ymm2, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm2\n" + "vpermilpd $0, %%ymm3, %%ymm8\n" + "vpermilpd $15, %%ymm3, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm3\n" + "vpermilpd $0, %%ymm4, %%ymm8\n" + "vpermilpd $15, %%ymm4, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm4\n" + "vpermilpd $0, %%ymm5, %%ymm8\n" + "vpermilpd $15, %%ymm5, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm5\n" + "vpermilpd $0, %%ymm6, %%ymm8\n" + "vpermilpd $15, %%ymm6, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm6\n" + "vpermilpd $0, %%ymm7, %%ymm8\n" + "vpermilpd $15, %%ymm7, %%ymm9\n" + "vxorpd %%ymm10, %%ymm10, %%ymm10\n" + "vsubpd %%ymm9, %%ymm10, %%ymm11\n" + "vaddsubpd %%ymm11, %%ymm8, %%ymm7\n" + "vperm2f128 $0, %%ymm0, %%ymm0, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm0, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm0, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm0\n" + "vperm2f128 $0, %%ymm1, %%ymm1, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm1, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm1, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm1\n" + "vperm2f128 $0, %%ymm2, %%ymm2, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm2, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm2, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm2\n" + "vperm2f128 $0, %%ymm3, %%ymm3, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm3, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm3, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm3\n" + "vperm2f128 $0, %%ymm4, %%ymm4, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm4, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm4, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm4\n" + "vperm2f128 $0, %%ymm5, %%ymm5, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm5, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm5, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm5\n" + "vperm2f128 $0, %%ymm6, %%ymm6, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm6, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm6, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm6\n" + "vperm2f128 $0, %%ymm7, %%ymm7, %%ymm8\n" + "vxorpd %%ymm9, %%ymm9, %%ymm9\n" + "vsubpd %%ymm7, %%ymm9, %%ymm10\n" + "vperm2f128 $49, %%ymm10, %%ymm7, %%ymm11\n" + "vaddpd %%ymm11, %%ymm8, %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_30_recursive(buf + 0, 9); + helper_double_30_recursive(buf + 512, 9); + helper_double_30_recursive(buf + 1024, 9); + helper_double_30_recursive(buf + 1536, 9); + helper_double_30_recursive(buf + 2048, 9); + helper_double_30_recursive(buf + 2560, 9); + helper_double_30_recursive(buf + 3072, 9); + helper_double_30_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_30_recursive(buf + 0, 12); + helper_double_30_recursive(buf + 4096, 12); + helper_double_30_recursive(buf + 8192, 12); + helper_double_30_recursive(buf + 12288, 12); + helper_double_30_recursive(buf + 16384, 12); + helper_double_30_recursive(buf + 20480, 12); + helper_double_30_recursive(buf + 24576, 12); + helper_double_30_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_30_recursive(buf + 0, 15); + helper_double_30_recursive(buf + 32768, 15); + helper_double_30_recursive(buf + 65536, 15); + helper_double_30_recursive(buf + 98304, 15); + helper_double_30_recursive(buf + 131072, 15); + helper_double_30_recursive(buf + 163840, 15); + helper_double_30_recursive(buf + 196608, 15); + helper_double_30_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_30_recursive(buf + 0, 18); + helper_double_30_recursive(buf + 262144, 18); + helper_double_30_recursive(buf + 524288, 18); + helper_double_30_recursive(buf + 786432, 18); + helper_double_30_recursive(buf + 1048576, 18); + helper_double_30_recursive(buf + 1310720, 18); + helper_double_30_recursive(buf + 1572864, 18); + helper_double_30_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_30_recursive(buf + 0, 21); + helper_double_30_recursive(buf + 2097152, 21); + helper_double_30_recursive(buf + 4194304, 21); + helper_double_30_recursive(buf + 6291456, 21); + helper_double_30_recursive(buf + 8388608, 21); + helper_double_30_recursive(buf + 10485760, 21); + helper_double_30_recursive(buf + 12582912, 21); + helper_double_30_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_double_30_recursive(buf + 0, 24); + helper_double_30_recursive(buf + 16777216, 24); + helper_double_30_recursive(buf + 33554432, 24); + helper_double_30_recursive(buf + 50331648, 24); + helper_double_30_recursive(buf + 67108864, 24); + helper_double_30_recursive(buf + 83886080, 24); + helper_double_30_recursive(buf + 100663296, 24); + helper_double_30_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } + if (depth == 30) { + helper_double_30_recursive(buf + 0, 27); + helper_double_30_recursive(buf + 134217728, 27); + helper_double_30_recursive(buf + 268435456, 27); + helper_double_30_recursive(buf + 402653184, 27); + helper_double_30_recursive(buf + 536870912, 27); + helper_double_30_recursive(buf + 671088640, 27); + helper_double_30_recursive(buf + 805306368, 27); + helper_double_30_recursive(buf + 939524096, 27); + for (int j = 0; j < 1073741824; j += 1073741824) { + for (int k = 0; k < 134217728; k += 4) { + __asm__ volatile ( + "vmovupd (%0), %%ymm0\n" + "vmovupd (%1), %%ymm1\n" + "vmovupd (%2), %%ymm2\n" + "vmovupd (%3), %%ymm3\n" + "vmovupd (%4), %%ymm4\n" + "vmovupd (%5), %%ymm5\n" + "vmovupd (%6), %%ymm6\n" + "vmovupd (%7), %%ymm7\n" + "vaddpd %%ymm1, %%ymm0, %%ymm8\n" + "vsubpd %%ymm1, %%ymm0, %%ymm9\n" + "vaddpd %%ymm3, %%ymm2, %%ymm10\n" + "vsubpd %%ymm3, %%ymm2, %%ymm11\n" + "vaddpd %%ymm5, %%ymm4, %%ymm12\n" + "vsubpd %%ymm5, %%ymm4, %%ymm13\n" + "vaddpd %%ymm7, %%ymm6, %%ymm14\n" + "vsubpd %%ymm7, %%ymm6, %%ymm15\n" + "vaddpd %%ymm10, %%ymm8, %%ymm0\n" + "vsubpd %%ymm10, %%ymm8, %%ymm2\n" + "vaddpd %%ymm11, %%ymm9, %%ymm1\n" + "vsubpd %%ymm11, %%ymm9, %%ymm3\n" + "vaddpd %%ymm14, %%ymm12, %%ymm4\n" + "vsubpd %%ymm14, %%ymm12, %%ymm6\n" + "vaddpd %%ymm15, %%ymm13, %%ymm5\n" + "vsubpd %%ymm15, %%ymm13, %%ymm7\n" + "vaddpd %%ymm4, %%ymm0, %%ymm8\n" + "vsubpd %%ymm4, %%ymm0, %%ymm12\n" + "vaddpd %%ymm5, %%ymm1, %%ymm9\n" + "vsubpd %%ymm5, %%ymm1, %%ymm13\n" + "vaddpd %%ymm6, %%ymm2, %%ymm10\n" + "vsubpd %%ymm6, %%ymm2, %%ymm14\n" + "vaddpd %%ymm7, %%ymm3, %%ymm11\n" + "vsubpd %%ymm7, %%ymm3, %%ymm15\n" + "vmovupd %%ymm8, (%0)\n" + "vmovupd %%ymm9, (%1)\n" + "vmovupd %%ymm10, (%2)\n" + "vmovupd %%ymm11, (%3)\n" + "vmovupd %%ymm12, (%4)\n" + "vmovupd %%ymm13, (%5)\n" + "vmovupd %%ymm14, (%6)\n" + "vmovupd %%ymm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184), "r"(buf + j + k + 536870912), "r"(buf + j + k + 671088640), "r"(buf + j + k + 805306368), "r"(buf + j + k + 939524096) : "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7", "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15", "memory" + ); + } + } + return; + } +} +void helper_double_30(double *buf); +void helper_double_30(double *buf) { + helper_double_30_recursive(buf, 30); +} +int fht_double(double *buf, int log_n) { + if (log_n == 0) { + return 0; + } + if (log_n == 1) { + helper_double_1(buf); + return 0; + } + if (log_n == 2) { + helper_double_2(buf); + return 0; + } + if (log_n == 3) { + helper_double_3(buf); + return 0; + } + if (log_n == 4) { + helper_double_4(buf); + return 0; + } + if (log_n == 5) { + helper_double_5(buf); + return 0; + } + if (log_n == 6) { + helper_double_6(buf); + return 0; + } + if (log_n == 7) { + helper_double_7(buf); + return 0; + } + if (log_n == 8) { + helper_double_8(buf); + return 0; + } + if (log_n == 9) { + helper_double_9(buf); + return 0; + } + if (log_n == 10) { + helper_double_10(buf); + return 0; + } + if (log_n == 11) { + helper_double_11(buf); + return 0; + } + if (log_n == 12) { + helper_double_12(buf); + return 0; + } + if (log_n == 13) { + helper_double_13(buf); + return 0; + } + if (log_n == 14) { + helper_double_14(buf); + return 0; + } + if (log_n == 15) { + helper_double_15(buf); + return 0; + } + if (log_n == 16) { + helper_double_16(buf); + return 0; + } + if (log_n == 17) { + helper_double_17(buf); + return 0; + } + if (log_n == 18) { + helper_double_18(buf); + return 0; + } + if (log_n == 19) { + helper_double_19(buf); + return 0; + } + if (log_n == 20) { + helper_double_20(buf); + return 0; + } + if (log_n == 21) { + helper_double_21(buf); + return 0; + } + if (log_n == 22) { + helper_double_22(buf); + return 0; + } + if (log_n == 23) { + helper_double_23(buf); + return 0; + } + if (log_n == 24) { + helper_double_24(buf); + return 0; + } + if (log_n == 25) { + helper_double_25(buf); + return 0; + } + if (log_n == 26) { + helper_double_26(buf); + return 0; + } + if (log_n == 27) { + helper_double_27(buf); + return 0; + } + if (log_n == 28) { + helper_double_28(buf); + return 0; + } + if (log_n == 29) { + helper_double_29(buf); + return 0; + } + if (log_n == 30) { + helper_double_30(buf); + return 0; + } + return 1; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_header_only.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_header_only.h new file mode 100644 index 00000000000..76ddc2557e5 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_header_only.h @@ -0,0 +1,38 @@ +#ifndef _FHT_H_ +#define _FHT_H_ + +#define FHT_HEADER_ONLY + +#ifdef __cplusplus +extern "C" { +#endif +int fht_float(float *buf, int log_n); +int fht_double(double *buf, int log_n); +int fht_float_oop(float *in, float *out, int log_n); +int fht_double_oop(double *in, double *out, int log_n); +#ifdef __cplusplus +} +#endif + + +#ifdef __cplusplus +static inline int fht(float *buf, int log_n) { + return fht_float(buf, log_n); +} + +static inline int fht(double *buf, int log_n) { + return fht_double(buf, log_n); +} + +static inline int fht(float *buf, float *out, int log_n) { + return fht_float_oop(buf, out, log_n); +} + +static inline int fht(double *buf, double *out, int log_n) { + return fht_double_oop(buf, out, log_n); +} +#endif // #ifdef __cplusplus + +#include "fht_impl.h" + +#endif diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_impl.h b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_impl.h new file mode 100644 index 00000000000..3bc332d5672 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_impl.h @@ -0,0 +1,32 @@ +#ifndef _FHT_IMPL_H__ +#define _FHT_IMPL_H__ + +#include "fast_copy.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __AVX__ +#include "fht_avx.c" +#define VECTOR_WIDTH (32u) +#else +#include "fht_sse.c" +#define VECTOR_WIDTH (16u) +#endif + +int fht_float_oop(float *in, float *out, int log_n) { + fast_copy(out, in, sizeof(float) << log_n); + return fht_float(out, log_n); +} + +int fht_double_oop(double *in, double *out, int log_n) { + fast_copy(out, in, sizeof(double) << log_n); + return fht_double(out, log_n); +} + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // ifndef _FHT_IMPL_H__ diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_sse.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_sse.c new file mode 100644 index 00000000000..90d0ffc8180 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/fht_sse.c @@ -0,0 +1,26215 @@ +#include "fht.h" +static inline void helper_float_1(float *buf); +static inline void helper_float_1(float *buf) { + for (int j = 0; j < 2; j += 2) { + for (int k = 0; k < 1; ++k) { + float u = buf[j + k]; + float v = buf[j + k + 1]; + buf[j + k] = u + v; + buf[j + k + 1] = u - v; + } + } +} +static inline void helper_float_2(float *buf); +static inline void helper_float_2(float *buf) { + for (int j = 0; j < 4; j += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movups %%xmm0, (%0)\n" + :: "r"(buf + j) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } +} +static inline void helper_float_3(float *buf); +static inline void helper_float_3(float *buf) { + for (int j = 0; j < 8; j += 8) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_float_4(float *buf); +static inline void helper_float_4(float *buf) { + for (int j = 0; j < 16; j += 16) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_float_5(float *buf); +static inline void helper_float_5(float *buf) { + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_float_6(float *buf); +static inline void helper_float_6(float *buf) { + for (int j = 0; j < 64; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +void helper_float_7_recursive(float *buf, int depth); +void helper_float_7_recursive(float *buf, int depth) { + if (depth == 7) { + for (int j = 0; j < 128; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_7(float *buf); +void helper_float_7(float *buf) { + helper_float_7_recursive(buf, 7); +} +static inline void helper_float_8(float *buf); +static inline void helper_float_8(float *buf) { + for (int j = 0; j < 256; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_float_9(float *buf); +static inline void helper_float_9(float *buf) { + for (int j = 0; j < 512; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_float_10(float *buf); +static inline void helper_float_10(float *buf) { + for (int j = 0; j < 1024; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_float_11(float *buf); +static inline void helper_float_11(float *buf) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +void helper_float_12_recursive(float *buf, int depth); +void helper_float_12_recursive(float *buf, int depth) { + if (depth == 7) { + for (int j = 0; j < 128; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 10) { + helper_float_12_recursive(buf + 0, 7); + helper_float_12_recursive(buf + 128, 7); + helper_float_12_recursive(buf + 256, 7); + helper_float_12_recursive(buf + 384, 7); + helper_float_12_recursive(buf + 512, 7); + helper_float_12_recursive(buf + 640, 7); + helper_float_12_recursive(buf + 768, 7); + helper_float_12_recursive(buf + 896, 7); + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_float_12_recursive(buf + 0, 10); + helper_float_12_recursive(buf + 1024, 10); + helper_float_12_recursive(buf + 2048, 10); + helper_float_12_recursive(buf + 3072, 10); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 1024; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_12(float *buf); +void helper_float_12(float *buf) { + helper_float_12_recursive(buf, 12); +} +void helper_float_13_recursive(float *buf, int depth); +void helper_float_13_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_float_13_recursive(buf + 0, 11); + helper_float_13_recursive(buf + 2048, 11); + helper_float_13_recursive(buf + 4096, 11); + helper_float_13_recursive(buf + 6144, 11); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_13(float *buf); +void helper_float_13(float *buf) { + helper_float_13_recursive(buf, 13); +} +void helper_float_14_recursive(float *buf, int depth); +void helper_float_14_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_14_recursive(buf + 0, 11); + helper_float_14_recursive(buf + 2048, 11); + helper_float_14_recursive(buf + 4096, 11); + helper_float_14_recursive(buf + 6144, 11); + helper_float_14_recursive(buf + 8192, 11); + helper_float_14_recursive(buf + 10240, 11); + helper_float_14_recursive(buf + 12288, 11); + helper_float_14_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_14(float *buf); +void helper_float_14(float *buf) { + helper_float_14_recursive(buf, 14); +} +void helper_float_15_recursive(float *buf, int depth); +void helper_float_15_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_15_recursive(buf + 0, 13); + helper_float_15_recursive(buf + 8192, 13); + helper_float_15_recursive(buf + 16384, 13); + helper_float_15_recursive(buf + 24576, 13); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 8192; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_15(float *buf); +void helper_float_15(float *buf) { + helper_float_15_recursive(buf, 15); +} +void helper_float_16_recursive(float *buf, int depth); +void helper_float_16_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_16_recursive(buf + 0, 11); + helper_float_16_recursive(buf + 2048, 11); + helper_float_16_recursive(buf + 4096, 11); + helper_float_16_recursive(buf + 6144, 11); + helper_float_16_recursive(buf + 8192, 11); + helper_float_16_recursive(buf + 10240, 11); + helper_float_16_recursive(buf + 12288, 11); + helper_float_16_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_16_recursive(buf + 0, 14); + helper_float_16_recursive(buf + 16384, 14); + helper_float_16_recursive(buf + 32768, 14); + helper_float_16_recursive(buf + 49152, 14); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_16(float *buf); +void helper_float_16(float *buf) { + helper_float_16_recursive(buf, 16); +} +void helper_float_17_recursive(float *buf, int depth); +void helper_float_17_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_17_recursive(buf + 0, 11); + helper_float_17_recursive(buf + 2048, 11); + helper_float_17_recursive(buf + 4096, 11); + helper_float_17_recursive(buf + 6144, 11); + helper_float_17_recursive(buf + 8192, 11); + helper_float_17_recursive(buf + 10240, 11); + helper_float_17_recursive(buf + 12288, 11); + helper_float_17_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_17_recursive(buf + 0, 14); + helper_float_17_recursive(buf + 16384, 14); + helper_float_17_recursive(buf + 32768, 14); + helper_float_17_recursive(buf + 49152, 14); + helper_float_17_recursive(buf + 65536, 14); + helper_float_17_recursive(buf + 81920, 14); + helper_float_17_recursive(buf + 98304, 14); + helper_float_17_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_17(float *buf); +void helper_float_17(float *buf) { + helper_float_17_recursive(buf, 17); +} +void helper_float_18_recursive(float *buf, int depth); +void helper_float_18_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_18_recursive(buf + 0, 13); + helper_float_18_recursive(buf + 8192, 13); + helper_float_18_recursive(buf + 16384, 13); + helper_float_18_recursive(buf + 24576, 13); + helper_float_18_recursive(buf + 32768, 13); + helper_float_18_recursive(buf + 40960, 13); + helper_float_18_recursive(buf + 49152, 13); + helper_float_18_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_18_recursive(buf + 0, 16); + helper_float_18_recursive(buf + 65536, 16); + helper_float_18_recursive(buf + 131072, 16); + helper_float_18_recursive(buf + 196608, 16); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 65536; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_18(float *buf); +void helper_float_18(float *buf) { + helper_float_18_recursive(buf, 18); +} +void helper_float_19_recursive(float *buf, int depth); +void helper_float_19_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_19_recursive(buf + 0, 13); + helper_float_19_recursive(buf + 8192, 13); + helper_float_19_recursive(buf + 16384, 13); + helper_float_19_recursive(buf + 24576, 13); + helper_float_19_recursive(buf + 32768, 13); + helper_float_19_recursive(buf + 40960, 13); + helper_float_19_recursive(buf + 49152, 13); + helper_float_19_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_float_19_recursive(buf + 0, 16); + helper_float_19_recursive(buf + 65536, 16); + helper_float_19_recursive(buf + 131072, 16); + helper_float_19_recursive(buf + 196608, 16); + helper_float_19_recursive(buf + 262144, 16); + helper_float_19_recursive(buf + 327680, 16); + helper_float_19_recursive(buf + 393216, 16); + helper_float_19_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_19(float *buf); +void helper_float_19(float *buf) { + helper_float_19_recursive(buf, 19); +} +void helper_float_20_recursive(float *buf, int depth); +void helper_float_20_recursive(float *buf, int depth) { + if (depth == 8) { + for (int j = 0; j < 256; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_float_20_recursive(buf + 0, 8); + helper_float_20_recursive(buf + 256, 8); + helper_float_20_recursive(buf + 512, 8); + helper_float_20_recursive(buf + 768, 8); + helper_float_20_recursive(buf + 1024, 8); + helper_float_20_recursive(buf + 1280, 8); + helper_float_20_recursive(buf + 1536, 8); + helper_float_20_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_20_recursive(buf + 0, 11); + helper_float_20_recursive(buf + 2048, 11); + helper_float_20_recursive(buf + 4096, 11); + helper_float_20_recursive(buf + 6144, 11); + helper_float_20_recursive(buf + 8192, 11); + helper_float_20_recursive(buf + 10240, 11); + helper_float_20_recursive(buf + 12288, 11); + helper_float_20_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_20_recursive(buf + 0, 14); + helper_float_20_recursive(buf + 16384, 14); + helper_float_20_recursive(buf + 32768, 14); + helper_float_20_recursive(buf + 49152, 14); + helper_float_20_recursive(buf + 65536, 14); + helper_float_20_recursive(buf + 81920, 14); + helper_float_20_recursive(buf + 98304, 14); + helper_float_20_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_float_20_recursive(buf + 0, 17); + helper_float_20_recursive(buf + 131072, 17); + helper_float_20_recursive(buf + 262144, 17); + helper_float_20_recursive(buf + 393216, 17); + helper_float_20_recursive(buf + 524288, 17); + helper_float_20_recursive(buf + 655360, 17); + helper_float_20_recursive(buf + 786432, 17); + helper_float_20_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_20(float *buf); +void helper_float_20(float *buf) { + helper_float_20_recursive(buf, 20); +} +void helper_float_21_recursive(float *buf, int depth); +void helper_float_21_recursive(float *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_float_21_recursive(buf + 0, 13); + helper_float_21_recursive(buf + 8192, 13); + helper_float_21_recursive(buf + 16384, 13); + helper_float_21_recursive(buf + 24576, 13); + helper_float_21_recursive(buf + 32768, 13); + helper_float_21_recursive(buf + 40960, 13); + helper_float_21_recursive(buf + 49152, 13); + helper_float_21_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_float_21_recursive(buf + 0, 16); + helper_float_21_recursive(buf + 65536, 16); + helper_float_21_recursive(buf + 131072, 16); + helper_float_21_recursive(buf + 196608, 16); + helper_float_21_recursive(buf + 262144, 16); + helper_float_21_recursive(buf + 327680, 16); + helper_float_21_recursive(buf + 393216, 16); + helper_float_21_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_21_recursive(buf + 0, 19); + helper_float_21_recursive(buf + 524288, 19); + helper_float_21_recursive(buf + 1048576, 19); + helper_float_21_recursive(buf + 1572864, 19); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 524288; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_21(float *buf); +void helper_float_21(float *buf) { + helper_float_21_recursive(buf, 21); +} +void helper_float_22_recursive(float *buf, int depth); +void helper_float_22_recursive(float *buf, int depth) { + if (depth == 11) { + for (int j = 0; j < 2048; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_22_recursive(buf + 0, 11); + helper_float_22_recursive(buf + 2048, 11); + helper_float_22_recursive(buf + 4096, 11); + helper_float_22_recursive(buf + 6144, 11); + helper_float_22_recursive(buf + 8192, 11); + helper_float_22_recursive(buf + 10240, 11); + helper_float_22_recursive(buf + 12288, 11); + helper_float_22_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_22_recursive(buf + 0, 14); + helper_float_22_recursive(buf + 16384, 14); + helper_float_22_recursive(buf + 32768, 14); + helper_float_22_recursive(buf + 49152, 14); + helper_float_22_recursive(buf + 65536, 14); + helper_float_22_recursive(buf + 81920, 14); + helper_float_22_recursive(buf + 98304, 14); + helper_float_22_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_float_22_recursive(buf + 0, 17); + helper_float_22_recursive(buf + 131072, 17); + helper_float_22_recursive(buf + 262144, 17); + helper_float_22_recursive(buf + 393216, 17); + helper_float_22_recursive(buf + 524288, 17); + helper_float_22_recursive(buf + 655360, 17); + helper_float_22_recursive(buf + 786432, 17); + helper_float_22_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_float_22_recursive(buf + 0, 20); + helper_float_22_recursive(buf + 1048576, 20); + helper_float_22_recursive(buf + 2097152, 20); + helper_float_22_recursive(buf + 3145728, 20); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_22(float *buf); +void helper_float_22(float *buf) { + helper_float_22_recursive(buf, 22); +} +void helper_float_23_recursive(float *buf, int depth); +void helper_float_23_recursive(float *buf, int depth) { + if (depth == 6) { + for (int j = 0; j < 64; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 9) { + helper_float_23_recursive(buf + 0, 6); + helper_float_23_recursive(buf + 64, 6); + helper_float_23_recursive(buf + 128, 6); + helper_float_23_recursive(buf + 192, 6); + helper_float_23_recursive(buf + 256, 6); + helper_float_23_recursive(buf + 320, 6); + helper_float_23_recursive(buf + 384, 6); + helper_float_23_recursive(buf + 448, 6); + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_float_23_recursive(buf + 0, 9); + helper_float_23_recursive(buf + 512, 9); + helper_float_23_recursive(buf + 1024, 9); + helper_float_23_recursive(buf + 1536, 9); + helper_float_23_recursive(buf + 2048, 9); + helper_float_23_recursive(buf + 2560, 9); + helper_float_23_recursive(buf + 3072, 9); + helper_float_23_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_23_recursive(buf + 0, 12); + helper_float_23_recursive(buf + 4096, 12); + helper_float_23_recursive(buf + 8192, 12); + helper_float_23_recursive(buf + 12288, 12); + helper_float_23_recursive(buf + 16384, 12); + helper_float_23_recursive(buf + 20480, 12); + helper_float_23_recursive(buf + 24576, 12); + helper_float_23_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_23_recursive(buf + 0, 15); + helper_float_23_recursive(buf + 32768, 15); + helper_float_23_recursive(buf + 65536, 15); + helper_float_23_recursive(buf + 98304, 15); + helper_float_23_recursive(buf + 131072, 15); + helper_float_23_recursive(buf + 163840, 15); + helper_float_23_recursive(buf + 196608, 15); + helper_float_23_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_23_recursive(buf + 0, 18); + helper_float_23_recursive(buf + 262144, 18); + helper_float_23_recursive(buf + 524288, 18); + helper_float_23_recursive(buf + 786432, 18); + helper_float_23_recursive(buf + 1048576, 18); + helper_float_23_recursive(buf + 1310720, 18); + helper_float_23_recursive(buf + 1572864, 18); + helper_float_23_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_float_23_recursive(buf + 0, 21); + helper_float_23_recursive(buf + 2097152, 21); + helper_float_23_recursive(buf + 4194304, 21); + helper_float_23_recursive(buf + 6291456, 21); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_23(float *buf); +void helper_float_23(float *buf) { + helper_float_23_recursive(buf, 23); +} +void helper_float_24_recursive(float *buf, int depth); +void helper_float_24_recursive(float *buf, int depth) { + if (depth == 15) { + for (int j = 0; j < 32768; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 32768; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 32768; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 32768; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_24_recursive(buf + 0, 15); + helper_float_24_recursive(buf + 32768, 15); + helper_float_24_recursive(buf + 65536, 15); + helper_float_24_recursive(buf + 98304, 15); + helper_float_24_recursive(buf + 131072, 15); + helper_float_24_recursive(buf + 163840, 15); + helper_float_24_recursive(buf + 196608, 15); + helper_float_24_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_24_recursive(buf + 0, 18); + helper_float_24_recursive(buf + 262144, 18); + helper_float_24_recursive(buf + 524288, 18); + helper_float_24_recursive(buf + 786432, 18); + helper_float_24_recursive(buf + 1048576, 18); + helper_float_24_recursive(buf + 1310720, 18); + helper_float_24_recursive(buf + 1572864, 18); + helper_float_24_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_24_recursive(buf + 0, 21); + helper_float_24_recursive(buf + 2097152, 21); + helper_float_24_recursive(buf + 4194304, 21); + helper_float_24_recursive(buf + 6291456, 21); + helper_float_24_recursive(buf + 8388608, 21); + helper_float_24_recursive(buf + 10485760, 21); + helper_float_24_recursive(buf + 12582912, 21); + helper_float_24_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_24(float *buf); +void helper_float_24(float *buf) { + helper_float_24_recursive(buf, 24); +} +void helper_float_25_recursive(float *buf, int depth); +void helper_float_25_recursive(float *buf, int depth) { + if (depth == 8) { + for (int j = 0; j < 256; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_float_25_recursive(buf + 0, 8); + helper_float_25_recursive(buf + 256, 8); + helper_float_25_recursive(buf + 512, 8); + helper_float_25_recursive(buf + 768, 8); + helper_float_25_recursive(buf + 1024, 8); + helper_float_25_recursive(buf + 1280, 8); + helper_float_25_recursive(buf + 1536, 8); + helper_float_25_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_25_recursive(buf + 0, 11); + helper_float_25_recursive(buf + 2048, 11); + helper_float_25_recursive(buf + 4096, 11); + helper_float_25_recursive(buf + 6144, 11); + helper_float_25_recursive(buf + 8192, 11); + helper_float_25_recursive(buf + 10240, 11); + helper_float_25_recursive(buf + 12288, 11); + helper_float_25_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_25_recursive(buf + 0, 14); + helper_float_25_recursive(buf + 16384, 14); + helper_float_25_recursive(buf + 32768, 14); + helper_float_25_recursive(buf + 49152, 14); + helper_float_25_recursive(buf + 65536, 14); + helper_float_25_recursive(buf + 81920, 14); + helper_float_25_recursive(buf + 98304, 14); + helper_float_25_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_float_25_recursive(buf + 0, 17); + helper_float_25_recursive(buf + 131072, 17); + helper_float_25_recursive(buf + 262144, 17); + helper_float_25_recursive(buf + 393216, 17); + helper_float_25_recursive(buf + 524288, 17); + helper_float_25_recursive(buf + 655360, 17); + helper_float_25_recursive(buf + 786432, 17); + helper_float_25_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_float_25_recursive(buf + 0, 20); + helper_float_25_recursive(buf + 1048576, 20); + helper_float_25_recursive(buf + 2097152, 20); + helper_float_25_recursive(buf + 3145728, 20); + helper_float_25_recursive(buf + 4194304, 20); + helper_float_25_recursive(buf + 5242880, 20); + helper_float_25_recursive(buf + 6291456, 20); + helper_float_25_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 25) { + helper_float_25_recursive(buf + 0, 23); + helper_float_25_recursive(buf + 8388608, 23); + helper_float_25_recursive(buf + 16777216, 23); + helper_float_25_recursive(buf + 25165824, 23); + for (int j = 0; j < 33554432; j += 33554432) { + for (int k = 0; k < 8388608; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_25(float *buf); +void helper_float_25(float *buf) { + helper_float_25_recursive(buf, 25); +} +void helper_float_26_recursive(float *buf, int depth); +void helper_float_26_recursive(float *buf, int depth) { + if (depth == 5) { + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_float_26_recursive(buf + 0, 5); + helper_float_26_recursive(buf + 32, 5); + helper_float_26_recursive(buf + 64, 5); + helper_float_26_recursive(buf + 96, 5); + helper_float_26_recursive(buf + 128, 5); + helper_float_26_recursive(buf + 160, 5); + helper_float_26_recursive(buf + 192, 5); + helper_float_26_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_float_26_recursive(buf + 0, 8); + helper_float_26_recursive(buf + 256, 8); + helper_float_26_recursive(buf + 512, 8); + helper_float_26_recursive(buf + 768, 8); + helper_float_26_recursive(buf + 1024, 8); + helper_float_26_recursive(buf + 1280, 8); + helper_float_26_recursive(buf + 1536, 8); + helper_float_26_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_float_26_recursive(buf + 0, 11); + helper_float_26_recursive(buf + 2048, 11); + helper_float_26_recursive(buf + 4096, 11); + helper_float_26_recursive(buf + 6144, 11); + helper_float_26_recursive(buf + 8192, 11); + helper_float_26_recursive(buf + 10240, 11); + helper_float_26_recursive(buf + 12288, 11); + helper_float_26_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_float_26_recursive(buf + 0, 14); + helper_float_26_recursive(buf + 16384, 14); + helper_float_26_recursive(buf + 32768, 14); + helper_float_26_recursive(buf + 49152, 14); + helper_float_26_recursive(buf + 65536, 14); + helper_float_26_recursive(buf + 81920, 14); + helper_float_26_recursive(buf + 98304, 14); + helper_float_26_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_float_26_recursive(buf + 0, 17); + helper_float_26_recursive(buf + 131072, 17); + helper_float_26_recursive(buf + 262144, 17); + helper_float_26_recursive(buf + 393216, 17); + helper_float_26_recursive(buf + 524288, 17); + helper_float_26_recursive(buf + 655360, 17); + helper_float_26_recursive(buf + 786432, 17); + helper_float_26_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_float_26_recursive(buf + 0, 20); + helper_float_26_recursive(buf + 1048576, 20); + helper_float_26_recursive(buf + 2097152, 20); + helper_float_26_recursive(buf + 3145728, 20); + helper_float_26_recursive(buf + 4194304, 20); + helper_float_26_recursive(buf + 5242880, 20); + helper_float_26_recursive(buf + 6291456, 20); + helper_float_26_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_float_26_recursive(buf + 0, 23); + helper_float_26_recursive(buf + 8388608, 23); + helper_float_26_recursive(buf + 16777216, 23); + helper_float_26_recursive(buf + 25165824, 23); + helper_float_26_recursive(buf + 33554432, 23); + helper_float_26_recursive(buf + 41943040, 23); + helper_float_26_recursive(buf + 50331648, 23); + helper_float_26_recursive(buf + 58720256, 23); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 8388608; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824), "r"(buf + j + k + 33554432), "r"(buf + j + k + 41943040), "r"(buf + j + k + 50331648), "r"(buf + j + k + 58720256) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_26(float *buf); +void helper_float_26(float *buf) { + helper_float_26_recursive(buf, 26); +} +void helper_float_27_recursive(float *buf, int depth); +void helper_float_27_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_27_recursive(buf + 0, 12); + helper_float_27_recursive(buf + 4096, 12); + helper_float_27_recursive(buf + 8192, 12); + helper_float_27_recursive(buf + 12288, 12); + helper_float_27_recursive(buf + 16384, 12); + helper_float_27_recursive(buf + 20480, 12); + helper_float_27_recursive(buf + 24576, 12); + helper_float_27_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_27_recursive(buf + 0, 15); + helper_float_27_recursive(buf + 32768, 15); + helper_float_27_recursive(buf + 65536, 15); + helper_float_27_recursive(buf + 98304, 15); + helper_float_27_recursive(buf + 131072, 15); + helper_float_27_recursive(buf + 163840, 15); + helper_float_27_recursive(buf + 196608, 15); + helper_float_27_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_27_recursive(buf + 0, 18); + helper_float_27_recursive(buf + 262144, 18); + helper_float_27_recursive(buf + 524288, 18); + helper_float_27_recursive(buf + 786432, 18); + helper_float_27_recursive(buf + 1048576, 18); + helper_float_27_recursive(buf + 1310720, 18); + helper_float_27_recursive(buf + 1572864, 18); + helper_float_27_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_27_recursive(buf + 0, 21); + helper_float_27_recursive(buf + 2097152, 21); + helper_float_27_recursive(buf + 4194304, 21); + helper_float_27_recursive(buf + 6291456, 21); + helper_float_27_recursive(buf + 8388608, 21); + helper_float_27_recursive(buf + 10485760, 21); + helper_float_27_recursive(buf + 12582912, 21); + helper_float_27_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_float_27_recursive(buf + 0, 24); + helper_float_27_recursive(buf + 16777216, 24); + helper_float_27_recursive(buf + 33554432, 24); + helper_float_27_recursive(buf + 50331648, 24); + helper_float_27_recursive(buf + 67108864, 24); + helper_float_27_recursive(buf + 83886080, 24); + helper_float_27_recursive(buf + 100663296, 24); + helper_float_27_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_27(float *buf); +void helper_float_27(float *buf) { + helper_float_27_recursive(buf, 27); +} +void helper_float_28_recursive(float *buf, int depth); +void helper_float_28_recursive(float *buf, int depth) { + if (depth == 16) { + for (int j = 0; j < 65536; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 65536; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 65536; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 65536; j += 16384) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 16384; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_float_28_recursive(buf + 0, 16); + helper_float_28_recursive(buf + 65536, 16); + helper_float_28_recursive(buf + 131072, 16); + helper_float_28_recursive(buf + 196608, 16); + helper_float_28_recursive(buf + 262144, 16); + helper_float_28_recursive(buf + 327680, 16); + helper_float_28_recursive(buf + 393216, 16); + helper_float_28_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_float_28_recursive(buf + 0, 19); + helper_float_28_recursive(buf + 524288, 19); + helper_float_28_recursive(buf + 1048576, 19); + helper_float_28_recursive(buf + 1572864, 19); + helper_float_28_recursive(buf + 2097152, 19); + helper_float_28_recursive(buf + 2621440, 19); + helper_float_28_recursive(buf + 3145728, 19); + helper_float_28_recursive(buf + 3670016, 19); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 524288; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864), "r"(buf + j + k + 2097152), "r"(buf + j + k + 2621440), "r"(buf + j + k + 3145728), "r"(buf + j + k + 3670016) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 25) { + helper_float_28_recursive(buf + 0, 22); + helper_float_28_recursive(buf + 4194304, 22); + helper_float_28_recursive(buf + 8388608, 22); + helper_float_28_recursive(buf + 12582912, 22); + helper_float_28_recursive(buf + 16777216, 22); + helper_float_28_recursive(buf + 20971520, 22); + helper_float_28_recursive(buf + 25165824, 22); + helper_float_28_recursive(buf + 29360128, 22); + for (int j = 0; j < 33554432; j += 33554432) { + for (int k = 0; k < 4194304; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4194304), "r"(buf + j + k + 8388608), "r"(buf + j + k + 12582912), "r"(buf + j + k + 16777216), "r"(buf + j + k + 20971520), "r"(buf + j + k + 25165824), "r"(buf + j + k + 29360128) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 28) { + helper_float_28_recursive(buf + 0, 25); + helper_float_28_recursive(buf + 33554432, 25); + helper_float_28_recursive(buf + 67108864, 25); + helper_float_28_recursive(buf + 100663296, 25); + helper_float_28_recursive(buf + 134217728, 25); + helper_float_28_recursive(buf + 167772160, 25); + helper_float_28_recursive(buf + 201326592, 25); + helper_float_28_recursive(buf + 234881024, 25); + for (int j = 0; j < 268435456; j += 268435456) { + for (int k = 0; k < 33554432; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 33554432), "r"(buf + j + k + 67108864), "r"(buf + j + k + 100663296), "r"(buf + j + k + 134217728), "r"(buf + j + k + 167772160), "r"(buf + j + k + 201326592), "r"(buf + j + k + 234881024) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_28(float *buf); +void helper_float_28(float *buf) { + helper_float_28_recursive(buf, 28); +} +void helper_float_29_recursive(float *buf, int depth); +void helper_float_29_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_29_recursive(buf + 0, 12); + helper_float_29_recursive(buf + 4096, 12); + helper_float_29_recursive(buf + 8192, 12); + helper_float_29_recursive(buf + 12288, 12); + helper_float_29_recursive(buf + 16384, 12); + helper_float_29_recursive(buf + 20480, 12); + helper_float_29_recursive(buf + 24576, 12); + helper_float_29_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_29_recursive(buf + 0, 15); + helper_float_29_recursive(buf + 32768, 15); + helper_float_29_recursive(buf + 65536, 15); + helper_float_29_recursive(buf + 98304, 15); + helper_float_29_recursive(buf + 131072, 15); + helper_float_29_recursive(buf + 163840, 15); + helper_float_29_recursive(buf + 196608, 15); + helper_float_29_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_29_recursive(buf + 0, 18); + helper_float_29_recursive(buf + 262144, 18); + helper_float_29_recursive(buf + 524288, 18); + helper_float_29_recursive(buf + 786432, 18); + helper_float_29_recursive(buf + 1048576, 18); + helper_float_29_recursive(buf + 1310720, 18); + helper_float_29_recursive(buf + 1572864, 18); + helper_float_29_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_29_recursive(buf + 0, 21); + helper_float_29_recursive(buf + 2097152, 21); + helper_float_29_recursive(buf + 4194304, 21); + helper_float_29_recursive(buf + 6291456, 21); + helper_float_29_recursive(buf + 8388608, 21); + helper_float_29_recursive(buf + 10485760, 21); + helper_float_29_recursive(buf + 12582912, 21); + helper_float_29_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_float_29_recursive(buf + 0, 24); + helper_float_29_recursive(buf + 16777216, 24); + helper_float_29_recursive(buf + 33554432, 24); + helper_float_29_recursive(buf + 50331648, 24); + helper_float_29_recursive(buf + 67108864, 24); + helper_float_29_recursive(buf + 83886080, 24); + helper_float_29_recursive(buf + 100663296, 24); + helper_float_29_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 29) { + helper_float_29_recursive(buf + 0, 27); + helper_float_29_recursive(buf + 134217728, 27); + helper_float_29_recursive(buf + 268435456, 27); + helper_float_29_recursive(buf + 402653184, 27); + for (int j = 0; j < 536870912; j += 536870912) { + for (int k = 0; k < 134217728; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movups %%xmm0, (%0)\n" + "movups %%xmm1, (%1)\n" + "movups %%xmm2, (%2)\n" + "movups %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_29(float *buf); +void helper_float_29(float *buf) { + helper_float_29_recursive(buf, 29); +} +void helper_float_30_recursive(float *buf, int depth); +void helper_float_30_recursive(float *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 32) { + for (int k = 0; k < 4; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm0, %%xmm0\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm0, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm1, %%xmm1\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm1, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm2, %%xmm2\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm2, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm3, %%xmm3\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm3, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm4, %%xmm4\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm4, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm5, %%xmm5\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm5, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm6, %%xmm6\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm6, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $160, %%xmm8, %%xmm8\n" + "shufps $245, %%xmm7, %%xmm7\n" + "xorps %%xmm9, %%xmm9\n" + "subps %%xmm7, %%xmm9\n" + "addsubps %%xmm9, %%xmm8\n" + "movaps %%xmm8, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm0, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm0, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm0\n" + "movaps %%xmm1, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm1, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm1, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm1\n" + "movaps %%xmm2, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm2\n" + "movaps %%xmm3, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm3, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm3, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm3\n" + "movaps %%xmm4, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm4, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm4, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm4\n" + "movaps %%xmm5, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm5, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm5, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm5\n" + "movaps %%xmm6, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm6, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm6, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm6\n" + "movaps %%xmm7, %%xmm8\n" + "shufps $68, %%xmm8, %%xmm8\n" + "xorps %%xmm9, %%xmm9\n" + "movaps %%xmm7, %%xmm10\n" + "shufps $14, %%xmm9, %%xmm10\n" + "movaps %%xmm7, %%xmm11\n" + "shufps $224, %%xmm11, %%xmm9\n" + "addps %%xmm8, %%xmm10\n" + "subps %%xmm9, %%xmm10\n" + "movaps %%xmm10, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 256) { + for (int k = 0; k < 32; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 2048) { + for (int k = 0; k < 256; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 2048; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_float_30_recursive(buf + 0, 12); + helper_float_30_recursive(buf + 4096, 12); + helper_float_30_recursive(buf + 8192, 12); + helper_float_30_recursive(buf + 12288, 12); + helper_float_30_recursive(buf + 16384, 12); + helper_float_30_recursive(buf + 20480, 12); + helper_float_30_recursive(buf + 24576, 12); + helper_float_30_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_float_30_recursive(buf + 0, 15); + helper_float_30_recursive(buf + 32768, 15); + helper_float_30_recursive(buf + 65536, 15); + helper_float_30_recursive(buf + 98304, 15); + helper_float_30_recursive(buf + 131072, 15); + helper_float_30_recursive(buf + 163840, 15); + helper_float_30_recursive(buf + 196608, 15); + helper_float_30_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_float_30_recursive(buf + 0, 18); + helper_float_30_recursive(buf + 262144, 18); + helper_float_30_recursive(buf + 524288, 18); + helper_float_30_recursive(buf + 786432, 18); + helper_float_30_recursive(buf + 1048576, 18); + helper_float_30_recursive(buf + 1310720, 18); + helper_float_30_recursive(buf + 1572864, 18); + helper_float_30_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_float_30_recursive(buf + 0, 21); + helper_float_30_recursive(buf + 2097152, 21); + helper_float_30_recursive(buf + 4194304, 21); + helper_float_30_recursive(buf + 6291456, 21); + helper_float_30_recursive(buf + 8388608, 21); + helper_float_30_recursive(buf + 10485760, 21); + helper_float_30_recursive(buf + 12582912, 21); + helper_float_30_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_float_30_recursive(buf + 0, 24); + helper_float_30_recursive(buf + 16777216, 24); + helper_float_30_recursive(buf + 33554432, 24); + helper_float_30_recursive(buf + 50331648, 24); + helper_float_30_recursive(buf + 67108864, 24); + helper_float_30_recursive(buf + 83886080, 24); + helper_float_30_recursive(buf + 100663296, 24); + helper_float_30_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 30) { + helper_float_30_recursive(buf + 0, 27); + helper_float_30_recursive(buf + 134217728, 27); + helper_float_30_recursive(buf + 268435456, 27); + helper_float_30_recursive(buf + 402653184, 27); + helper_float_30_recursive(buf + 536870912, 27); + helper_float_30_recursive(buf + 671088640, 27); + helper_float_30_recursive(buf + 805306368, 27); + helper_float_30_recursive(buf + 939524096, 27); + for (int j = 0; j < 1073741824; j += 1073741824) { + for (int k = 0; k < 134217728; k += 4) { + __asm__ volatile ( + "movups (%0), %%xmm0\n" + "movups (%1), %%xmm1\n" + "movups (%2), %%xmm2\n" + "movups (%3), %%xmm3\n" + "movups (%4), %%xmm4\n" + "movups (%5), %%xmm5\n" + "movups (%6), %%xmm6\n" + "movups (%7), %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm9\n" + "addps %%xmm1, %%xmm8\n" + "subps %%xmm1, %%xmm9\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm11\n" + "addps %%xmm3, %%xmm10\n" + "subps %%xmm3, %%xmm11\n" + "movaps %%xmm4, %%xmm12\n" + "movaps %%xmm4, %%xmm13\n" + "addps %%xmm5, %%xmm12\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm6, %%xmm14\n" + "movaps %%xmm6, %%xmm15\n" + "addps %%xmm7, %%xmm14\n" + "subps %%xmm7, %%xmm15\n" + "movaps %%xmm8, %%xmm0\n" + "movaps %%xmm8, %%xmm2\n" + "addps %%xmm10, %%xmm0\n" + "subps %%xmm10, %%xmm2\n" + "movaps %%xmm9, %%xmm1\n" + "movaps %%xmm9, %%xmm3\n" + "addps %%xmm11, %%xmm1\n" + "subps %%xmm11, %%xmm3\n" + "movaps %%xmm12, %%xmm4\n" + "movaps %%xmm12, %%xmm6\n" + "addps %%xmm14, %%xmm4\n" + "subps %%xmm14, %%xmm6\n" + "movaps %%xmm13, %%xmm5\n" + "movaps %%xmm13, %%xmm7\n" + "addps %%xmm15, %%xmm5\n" + "subps %%xmm15, %%xmm7\n" + "movaps %%xmm0, %%xmm8\n" + "movaps %%xmm0, %%xmm12\n" + "addps %%xmm4, %%xmm8\n" + "subps %%xmm4, %%xmm12\n" + "movaps %%xmm1, %%xmm9\n" + "movaps %%xmm1, %%xmm13\n" + "addps %%xmm5, %%xmm9\n" + "subps %%xmm5, %%xmm13\n" + "movaps %%xmm2, %%xmm10\n" + "movaps %%xmm2, %%xmm14\n" + "addps %%xmm6, %%xmm10\n" + "subps %%xmm6, %%xmm14\n" + "movaps %%xmm3, %%xmm11\n" + "movaps %%xmm3, %%xmm15\n" + "addps %%xmm7, %%xmm11\n" + "subps %%xmm7, %%xmm15\n" + "movups %%xmm8, (%0)\n" + "movups %%xmm9, (%1)\n" + "movups %%xmm10, (%2)\n" + "movups %%xmm11, (%3)\n" + "movups %%xmm12, (%4)\n" + "movups %%xmm13, (%5)\n" + "movups %%xmm14, (%6)\n" + "movups %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184), "r"(buf + j + k + 536870912), "r"(buf + j + k + 671088640), "r"(buf + j + k + 805306368), "r"(buf + j + k + 939524096) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_float_30(float *buf); +void helper_float_30(float *buf) { + helper_float_30_recursive(buf, 30); +} +int fht_float(float *buf, int log_n) { + if (log_n == 0) { + return 0; + } + if (log_n == 1) { + helper_float_1(buf); + return 0; + } + if (log_n == 2) { + helper_float_2(buf); + return 0; + } + if (log_n == 3) { + helper_float_3(buf); + return 0; + } + if (log_n == 4) { + helper_float_4(buf); + return 0; + } + if (log_n == 5) { + helper_float_5(buf); + return 0; + } + if (log_n == 6) { + helper_float_6(buf); + return 0; + } + if (log_n == 7) { + helper_float_7(buf); + return 0; + } + if (log_n == 8) { + helper_float_8(buf); + return 0; + } + if (log_n == 9) { + helper_float_9(buf); + return 0; + } + if (log_n == 10) { + helper_float_10(buf); + return 0; + } + if (log_n == 11) { + helper_float_11(buf); + return 0; + } + if (log_n == 12) { + helper_float_12(buf); + return 0; + } + if (log_n == 13) { + helper_float_13(buf); + return 0; + } + if (log_n == 14) { + helper_float_14(buf); + return 0; + } + if (log_n == 15) { + helper_float_15(buf); + return 0; + } + if (log_n == 16) { + helper_float_16(buf); + return 0; + } + if (log_n == 17) { + helper_float_17(buf); + return 0; + } + if (log_n == 18) { + helper_float_18(buf); + return 0; + } + if (log_n == 19) { + helper_float_19(buf); + return 0; + } + if (log_n == 20) { + helper_float_20(buf); + return 0; + } + if (log_n == 21) { + helper_float_21(buf); + return 0; + } + if (log_n == 22) { + helper_float_22(buf); + return 0; + } + if (log_n == 23) { + helper_float_23(buf); + return 0; + } + if (log_n == 24) { + helper_float_24(buf); + return 0; + } + if (log_n == 25) { + helper_float_25(buf); + return 0; + } + if (log_n == 26) { + helper_float_26(buf); + return 0; + } + if (log_n == 27) { + helper_float_27(buf); + return 0; + } + if (log_n == 28) { + helper_float_28(buf); + return 0; + } + if (log_n == 29) { + helper_float_29(buf); + return 0; + } + if (log_n == 30) { + helper_float_30(buf); + return 0; + } + return 1; +} +static inline void helper_double_1(double *buf); +static inline void helper_double_1(double *buf) { + for (int j = 0; j < 2; j += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movupd %%xmm0, (%0)\n" + :: "r"(buf + j) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } +} +void helper_double_2_recursive(double *buf, int depth); +void helper_double_2_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_2(double *buf); +void helper_double_2(double *buf) { + helper_double_2_recursive(buf, 2); +} +void helper_double_3_recursive(double *buf, int depth); +void helper_double_3_recursive(double *buf, int depth) { + if (depth == 3) { + for (int j = 0; j < 8; j += 8) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_3(double *buf); +void helper_double_3(double *buf) { + helper_double_3_recursive(buf, 3); +} +static inline void helper_double_4(double *buf); +static inline void helper_double_4(double *buf) { + for (int j = 0; j < 16; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +void helper_double_5_recursive(double *buf, int depth); +void helper_double_5_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_5_recursive(buf + 0, 2); + helper_double_5_recursive(buf + 4, 2); + helper_double_5_recursive(buf + 8, 2); + helper_double_5_recursive(buf + 12, 2); + helper_double_5_recursive(buf + 16, 2); + helper_double_5_recursive(buf + 20, 2); + helper_double_5_recursive(buf + 24, 2); + helper_double_5_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_5(double *buf); +void helper_double_5(double *buf) { + helper_double_5_recursive(buf, 5); +} +static inline void helper_double_6(double *buf); +static inline void helper_double_6(double *buf) { + for (int j = 0; j < 64; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +static inline void helper_double_7(double *buf); +static inline void helper_double_7(double *buf) { + for (int j = 0; j < 128; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 128; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +void helper_double_8_recursive(double *buf, int depth); +void helper_double_8_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_8_recursive(buf + 0, 2); + helper_double_8_recursive(buf + 4, 2); + helper_double_8_recursive(buf + 8, 2); + helper_double_8_recursive(buf + 12, 2); + helper_double_8_recursive(buf + 16, 2); + helper_double_8_recursive(buf + 20, 2); + helper_double_8_recursive(buf + 24, 2); + helper_double_8_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_8_recursive(buf + 0, 5); + helper_double_8_recursive(buf + 32, 5); + helper_double_8_recursive(buf + 64, 5); + helper_double_8_recursive(buf + 96, 5); + helper_double_8_recursive(buf + 128, 5); + helper_double_8_recursive(buf + 160, 5); + helper_double_8_recursive(buf + 192, 5); + helper_double_8_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_8(double *buf); +void helper_double_8(double *buf) { + helper_double_8_recursive(buf, 8); +} +void helper_double_9_recursive(double *buf, int depth); +void helper_double_9_recursive(double *buf, int depth) { + if (depth == 6) { + for (int j = 0; j < 64; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 9) { + helper_double_9_recursive(buf + 0, 6); + helper_double_9_recursive(buf + 64, 6); + helper_double_9_recursive(buf + 128, 6); + helper_double_9_recursive(buf + 192, 6); + helper_double_9_recursive(buf + 256, 6); + helper_double_9_recursive(buf + 320, 6); + helper_double_9_recursive(buf + 384, 6); + helper_double_9_recursive(buf + 448, 6); + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_9(double *buf); +void helper_double_9(double *buf) { + helper_double_9_recursive(buf, 9); +} +void helper_double_10_recursive(double *buf, int depth); +void helper_double_10_recursive(double *buf, int depth) { + if (depth == 10) { + for (int j = 0; j < 1024; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_10(double *buf); +void helper_double_10(double *buf) { + helper_double_10_recursive(buf, 10); +} +void helper_double_11_recursive(double *buf, int depth); +void helper_double_11_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_11_recursive(buf + 0, 2); + helper_double_11_recursive(buf + 4, 2); + helper_double_11_recursive(buf + 8, 2); + helper_double_11_recursive(buf + 12, 2); + helper_double_11_recursive(buf + 16, 2); + helper_double_11_recursive(buf + 20, 2); + helper_double_11_recursive(buf + 24, 2); + helper_double_11_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_11_recursive(buf + 0, 5); + helper_double_11_recursive(buf + 32, 5); + helper_double_11_recursive(buf + 64, 5); + helper_double_11_recursive(buf + 96, 5); + helper_double_11_recursive(buf + 128, 5); + helper_double_11_recursive(buf + 160, 5); + helper_double_11_recursive(buf + 192, 5); + helper_double_11_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_11_recursive(buf + 0, 8); + helper_double_11_recursive(buf + 256, 8); + helper_double_11_recursive(buf + 512, 8); + helper_double_11_recursive(buf + 768, 8); + helper_double_11_recursive(buf + 1024, 8); + helper_double_11_recursive(buf + 1280, 8); + helper_double_11_recursive(buf + 1536, 8); + helper_double_11_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_11(double *buf); +void helper_double_11(double *buf) { + helper_double_11_recursive(buf, 11); +} +void helper_double_12_recursive(double *buf, int depth); +void helper_double_12_recursive(double *buf, int depth) { + if (depth == 10) { + for (int j = 0; j < 1024; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_12_recursive(buf + 0, 10); + helper_double_12_recursive(buf + 1024, 10); + helper_double_12_recursive(buf + 2048, 10); + helper_double_12_recursive(buf + 3072, 10); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_12(double *buf); +void helper_double_12(double *buf) { + helper_double_12_recursive(buf, 12); +} +static inline void helper_double_13(double *buf); +static inline void helper_double_13(double *buf) { + for (int j = 0; j < 8192; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } +} +void helper_double_14_recursive(double *buf, int depth); +void helper_double_14_recursive(double *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_14_recursive(buf + 0, 9); + helper_double_14_recursive(buf + 512, 9); + helper_double_14_recursive(buf + 1024, 9); + helper_double_14_recursive(buf + 1536, 9); + helper_double_14_recursive(buf + 2048, 9); + helper_double_14_recursive(buf + 2560, 9); + helper_double_14_recursive(buf + 3072, 9); + helper_double_14_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_14_recursive(buf + 0, 12); + helper_double_14_recursive(buf + 4096, 12); + helper_double_14_recursive(buf + 8192, 12); + helper_double_14_recursive(buf + 12288, 12); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_14(double *buf); +void helper_double_14(double *buf) { + helper_double_14_recursive(buf, 14); +} +void helper_double_15_recursive(double *buf, int depth); +void helper_double_15_recursive(double *buf, int depth) { + if (depth == 10) { + for (int j = 0; j < 1024; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_double_15_recursive(buf + 0, 10); + helper_double_15_recursive(buf + 1024, 10); + helper_double_15_recursive(buf + 2048, 10); + helper_double_15_recursive(buf + 3072, 10); + helper_double_15_recursive(buf + 4096, 10); + helper_double_15_recursive(buf + 5120, 10); + helper_double_15_recursive(buf + 6144, 10); + helper_double_15_recursive(buf + 7168, 10); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_15_recursive(buf + 0, 13); + helper_double_15_recursive(buf + 8192, 13); + helper_double_15_recursive(buf + 16384, 13); + helper_double_15_recursive(buf + 24576, 13); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 8192; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_15(double *buf); +void helper_double_15(double *buf) { + helper_double_15_recursive(buf, 15); +} +void helper_double_16_recursive(double *buf, int depth); +void helper_double_16_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_16_recursive(buf + 0, 2); + helper_double_16_recursive(buf + 4, 2); + helper_double_16_recursive(buf + 8, 2); + helper_double_16_recursive(buf + 12, 2); + helper_double_16_recursive(buf + 16, 2); + helper_double_16_recursive(buf + 20, 2); + helper_double_16_recursive(buf + 24, 2); + helper_double_16_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_16_recursive(buf + 0, 5); + helper_double_16_recursive(buf + 32, 5); + helper_double_16_recursive(buf + 64, 5); + helper_double_16_recursive(buf + 96, 5); + helper_double_16_recursive(buf + 128, 5); + helper_double_16_recursive(buf + 160, 5); + helper_double_16_recursive(buf + 192, 5); + helper_double_16_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_16_recursive(buf + 0, 8); + helper_double_16_recursive(buf + 256, 8); + helper_double_16_recursive(buf + 512, 8); + helper_double_16_recursive(buf + 768, 8); + helper_double_16_recursive(buf + 1024, 8); + helper_double_16_recursive(buf + 1280, 8); + helper_double_16_recursive(buf + 1536, 8); + helper_double_16_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_16_recursive(buf + 0, 11); + helper_double_16_recursive(buf + 2048, 11); + helper_double_16_recursive(buf + 4096, 11); + helper_double_16_recursive(buf + 6144, 11); + helper_double_16_recursive(buf + 8192, 11); + helper_double_16_recursive(buf + 10240, 11); + helper_double_16_recursive(buf + 12288, 11); + helper_double_16_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_16_recursive(buf + 0, 14); + helper_double_16_recursive(buf + 16384, 14); + helper_double_16_recursive(buf + 32768, 14); + helper_double_16_recursive(buf + 49152, 14); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 16384; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_16(double *buf); +void helper_double_16(double *buf) { + helper_double_16_recursive(buf, 16); +} +void helper_double_17_recursive(double *buf, int depth); +void helper_double_17_recursive(double *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_17_recursive(buf + 0, 12); + helper_double_17_recursive(buf + 4096, 12); + helper_double_17_recursive(buf + 8192, 12); + helper_double_17_recursive(buf + 12288, 12); + helper_double_17_recursive(buf + 16384, 12); + helper_double_17_recursive(buf + 20480, 12); + helper_double_17_recursive(buf + 24576, 12); + helper_double_17_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_17_recursive(buf + 0, 15); + helper_double_17_recursive(buf + 32768, 15); + helper_double_17_recursive(buf + 65536, 15); + helper_double_17_recursive(buf + 98304, 15); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 32768; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_17(double *buf); +void helper_double_17(double *buf) { + helper_double_17_recursive(buf, 17); +} +void helper_double_18_recursive(double *buf, int depth); +void helper_double_18_recursive(double *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_18_recursive(buf + 0, 12); + helper_double_18_recursive(buf + 4096, 12); + helper_double_18_recursive(buf + 8192, 12); + helper_double_18_recursive(buf + 12288, 12); + helper_double_18_recursive(buf + 16384, 12); + helper_double_18_recursive(buf + 20480, 12); + helper_double_18_recursive(buf + 24576, 12); + helper_double_18_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_18_recursive(buf + 0, 15); + helper_double_18_recursive(buf + 32768, 15); + helper_double_18_recursive(buf + 65536, 15); + helper_double_18_recursive(buf + 98304, 15); + helper_double_18_recursive(buf + 131072, 15); + helper_double_18_recursive(buf + 163840, 15); + helper_double_18_recursive(buf + 196608, 15); + helper_double_18_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_18(double *buf); +void helper_double_18(double *buf) { + helper_double_18_recursive(buf, 18); +} +void helper_double_19_recursive(double *buf, int depth); +void helper_double_19_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_19_recursive(buf + 0, 2); + helper_double_19_recursive(buf + 4, 2); + helper_double_19_recursive(buf + 8, 2); + helper_double_19_recursive(buf + 12, 2); + helper_double_19_recursive(buf + 16, 2); + helper_double_19_recursive(buf + 20, 2); + helper_double_19_recursive(buf + 24, 2); + helper_double_19_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_19_recursive(buf + 0, 5); + helper_double_19_recursive(buf + 32, 5); + helper_double_19_recursive(buf + 64, 5); + helper_double_19_recursive(buf + 96, 5); + helper_double_19_recursive(buf + 128, 5); + helper_double_19_recursive(buf + 160, 5); + helper_double_19_recursive(buf + 192, 5); + helper_double_19_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_19_recursive(buf + 0, 8); + helper_double_19_recursive(buf + 256, 8); + helper_double_19_recursive(buf + 512, 8); + helper_double_19_recursive(buf + 768, 8); + helper_double_19_recursive(buf + 1024, 8); + helper_double_19_recursive(buf + 1280, 8); + helper_double_19_recursive(buf + 1536, 8); + helper_double_19_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_19_recursive(buf + 0, 11); + helper_double_19_recursive(buf + 2048, 11); + helper_double_19_recursive(buf + 4096, 11); + helper_double_19_recursive(buf + 6144, 11); + helper_double_19_recursive(buf + 8192, 11); + helper_double_19_recursive(buf + 10240, 11); + helper_double_19_recursive(buf + 12288, 11); + helper_double_19_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_19_recursive(buf + 0, 14); + helper_double_19_recursive(buf + 16384, 14); + helper_double_19_recursive(buf + 32768, 14); + helper_double_19_recursive(buf + 49152, 14); + helper_double_19_recursive(buf + 65536, 14); + helper_double_19_recursive(buf + 81920, 14); + helper_double_19_recursive(buf + 98304, 14); + helper_double_19_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_19_recursive(buf + 0, 17); + helper_double_19_recursive(buf + 131072, 17); + helper_double_19_recursive(buf + 262144, 17); + helper_double_19_recursive(buf + 393216, 17); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 131072; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_19(double *buf); +void helper_double_19(double *buf) { + helper_double_19_recursive(buf, 19); +} +void helper_double_20_recursive(double *buf, int depth); +void helper_double_20_recursive(double *buf, int depth) { + if (depth == 12) { + for (int j = 0; j < 4096; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_20_recursive(buf + 0, 12); + helper_double_20_recursive(buf + 4096, 12); + helper_double_20_recursive(buf + 8192, 12); + helper_double_20_recursive(buf + 12288, 12); + helper_double_20_recursive(buf + 16384, 12); + helper_double_20_recursive(buf + 20480, 12); + helper_double_20_recursive(buf + 24576, 12); + helper_double_20_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_20_recursive(buf + 0, 15); + helper_double_20_recursive(buf + 32768, 15); + helper_double_20_recursive(buf + 65536, 15); + helper_double_20_recursive(buf + 98304, 15); + helper_double_20_recursive(buf + 131072, 15); + helper_double_20_recursive(buf + 163840, 15); + helper_double_20_recursive(buf + 196608, 15); + helper_double_20_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_20_recursive(buf + 0, 18); + helper_double_20_recursive(buf + 262144, 18); + helper_double_20_recursive(buf + 524288, 18); + helper_double_20_recursive(buf + 786432, 18); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 262144; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_20(double *buf); +void helper_double_20(double *buf) { + helper_double_20_recursive(buf, 20); +} +void helper_double_21_recursive(double *buf, int depth); +void helper_double_21_recursive(double *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_21_recursive(buf + 0, 13); + helper_double_21_recursive(buf + 8192, 13); + helper_double_21_recursive(buf + 16384, 13); + helper_double_21_recursive(buf + 24576, 13); + helper_double_21_recursive(buf + 32768, 13); + helper_double_21_recursive(buf + 40960, 13); + helper_double_21_recursive(buf + 49152, 13); + helper_double_21_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_21_recursive(buf + 0, 16); + helper_double_21_recursive(buf + 65536, 16); + helper_double_21_recursive(buf + 131072, 16); + helper_double_21_recursive(buf + 196608, 16); + helper_double_21_recursive(buf + 262144, 16); + helper_double_21_recursive(buf + 327680, 16); + helper_double_21_recursive(buf + 393216, 16); + helper_double_21_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_21_recursive(buf + 0, 19); + helper_double_21_recursive(buf + 524288, 19); + helper_double_21_recursive(buf + 1048576, 19); + helper_double_21_recursive(buf + 1572864, 19); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 524288; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_21(double *buf); +void helper_double_21(double *buf) { + helper_double_21_recursive(buf, 21); +} +void helper_double_22_recursive(double *buf, int depth); +void helper_double_22_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_22_recursive(buf + 0, 2); + helper_double_22_recursive(buf + 4, 2); + helper_double_22_recursive(buf + 8, 2); + helper_double_22_recursive(buf + 12, 2); + helper_double_22_recursive(buf + 16, 2); + helper_double_22_recursive(buf + 20, 2); + helper_double_22_recursive(buf + 24, 2); + helper_double_22_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_22_recursive(buf + 0, 5); + helper_double_22_recursive(buf + 32, 5); + helper_double_22_recursive(buf + 64, 5); + helper_double_22_recursive(buf + 96, 5); + helper_double_22_recursive(buf + 128, 5); + helper_double_22_recursive(buf + 160, 5); + helper_double_22_recursive(buf + 192, 5); + helper_double_22_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_22_recursive(buf + 0, 8); + helper_double_22_recursive(buf + 256, 8); + helper_double_22_recursive(buf + 512, 8); + helper_double_22_recursive(buf + 768, 8); + helper_double_22_recursive(buf + 1024, 8); + helper_double_22_recursive(buf + 1280, 8); + helper_double_22_recursive(buf + 1536, 8); + helper_double_22_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_22_recursive(buf + 0, 11); + helper_double_22_recursive(buf + 2048, 11); + helper_double_22_recursive(buf + 4096, 11); + helper_double_22_recursive(buf + 6144, 11); + helper_double_22_recursive(buf + 8192, 11); + helper_double_22_recursive(buf + 10240, 11); + helper_double_22_recursive(buf + 12288, 11); + helper_double_22_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_22_recursive(buf + 0, 14); + helper_double_22_recursive(buf + 16384, 14); + helper_double_22_recursive(buf + 32768, 14); + helper_double_22_recursive(buf + 49152, 14); + helper_double_22_recursive(buf + 65536, 14); + helper_double_22_recursive(buf + 81920, 14); + helper_double_22_recursive(buf + 98304, 14); + helper_double_22_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_22_recursive(buf + 0, 17); + helper_double_22_recursive(buf + 131072, 17); + helper_double_22_recursive(buf + 262144, 17); + helper_double_22_recursive(buf + 393216, 17); + helper_double_22_recursive(buf + 524288, 17); + helper_double_22_recursive(buf + 655360, 17); + helper_double_22_recursive(buf + 786432, 17); + helper_double_22_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_double_22_recursive(buf + 0, 20); + helper_double_22_recursive(buf + 1048576, 20); + helper_double_22_recursive(buf + 2097152, 20); + helper_double_22_recursive(buf + 3145728, 20); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 1048576; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_22(double *buf); +void helper_double_22(double *buf) { + helper_double_22_recursive(buf, 22); +} +void helper_double_23_recursive(double *buf, int depth); +void helper_double_23_recursive(double *buf, int depth) { + if (depth == 2) { + for (int j = 0; j < 4; j += 4) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 5) { + helper_double_23_recursive(buf + 0, 2); + helper_double_23_recursive(buf + 4, 2); + helper_double_23_recursive(buf + 8, 2); + helper_double_23_recursive(buf + 12, 2); + helper_double_23_recursive(buf + 16, 2); + helper_double_23_recursive(buf + 20, 2); + helper_double_23_recursive(buf + 24, 2); + helper_double_23_recursive(buf + 28, 2); + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 4; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4), "r"(buf + j + k + 8), "r"(buf + j + k + 12), "r"(buf + j + k + 16), "r"(buf + j + k + 20), "r"(buf + j + k + 24), "r"(buf + j + k + 28) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_23_recursive(buf + 0, 5); + helper_double_23_recursive(buf + 32, 5); + helper_double_23_recursive(buf + 64, 5); + helper_double_23_recursive(buf + 96, 5); + helper_double_23_recursive(buf + 128, 5); + helper_double_23_recursive(buf + 160, 5); + helper_double_23_recursive(buf + 192, 5); + helper_double_23_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_23_recursive(buf + 0, 8); + helper_double_23_recursive(buf + 256, 8); + helper_double_23_recursive(buf + 512, 8); + helper_double_23_recursive(buf + 768, 8); + helper_double_23_recursive(buf + 1024, 8); + helper_double_23_recursive(buf + 1280, 8); + helper_double_23_recursive(buf + 1536, 8); + helper_double_23_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_23_recursive(buf + 0, 11); + helper_double_23_recursive(buf + 2048, 11); + helper_double_23_recursive(buf + 4096, 11); + helper_double_23_recursive(buf + 6144, 11); + helper_double_23_recursive(buf + 8192, 11); + helper_double_23_recursive(buf + 10240, 11); + helper_double_23_recursive(buf + 12288, 11); + helper_double_23_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_23_recursive(buf + 0, 14); + helper_double_23_recursive(buf + 16384, 14); + helper_double_23_recursive(buf + 32768, 14); + helper_double_23_recursive(buf + 49152, 14); + helper_double_23_recursive(buf + 65536, 14); + helper_double_23_recursive(buf + 81920, 14); + helper_double_23_recursive(buf + 98304, 14); + helper_double_23_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_23_recursive(buf + 0, 17); + helper_double_23_recursive(buf + 131072, 17); + helper_double_23_recursive(buf + 262144, 17); + helper_double_23_recursive(buf + 393216, 17); + helper_double_23_recursive(buf + 524288, 17); + helper_double_23_recursive(buf + 655360, 17); + helper_double_23_recursive(buf + 786432, 17); + helper_double_23_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_23_recursive(buf + 0, 20); + helper_double_23_recursive(buf + 1048576, 20); + helper_double_23_recursive(buf + 2097152, 20); + helper_double_23_recursive(buf + 3145728, 20); + helper_double_23_recursive(buf + 4194304, 20); + helper_double_23_recursive(buf + 5242880, 20); + helper_double_23_recursive(buf + 6291456, 20); + helper_double_23_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_23(double *buf); +void helper_double_23(double *buf) { + helper_double_23_recursive(buf, 23); +} +void helper_double_24_recursive(double *buf, int depth); +void helper_double_24_recursive(double *buf, int depth) { + if (depth == 13) { + for (int j = 0; j < 8192; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_24_recursive(buf + 0, 13); + helper_double_24_recursive(buf + 8192, 13); + helper_double_24_recursive(buf + 16384, 13); + helper_double_24_recursive(buf + 24576, 13); + helper_double_24_recursive(buf + 32768, 13); + helper_double_24_recursive(buf + 40960, 13); + helper_double_24_recursive(buf + 49152, 13); + helper_double_24_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_24_recursive(buf + 0, 16); + helper_double_24_recursive(buf + 65536, 16); + helper_double_24_recursive(buf + 131072, 16); + helper_double_24_recursive(buf + 196608, 16); + helper_double_24_recursive(buf + 262144, 16); + helper_double_24_recursive(buf + 327680, 16); + helper_double_24_recursive(buf + 393216, 16); + helper_double_24_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_double_24_recursive(buf + 0, 19); + helper_double_24_recursive(buf + 524288, 19); + helper_double_24_recursive(buf + 1048576, 19); + helper_double_24_recursive(buf + 1572864, 19); + helper_double_24_recursive(buf + 2097152, 19); + helper_double_24_recursive(buf + 2621440, 19); + helper_double_24_recursive(buf + 3145728, 19); + helper_double_24_recursive(buf + 3670016, 19); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 524288; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864), "r"(buf + j + k + 2097152), "r"(buf + j + k + 2621440), "r"(buf + j + k + 3145728), "r"(buf + j + k + 3670016) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_24_recursive(buf + 0, 22); + helper_double_24_recursive(buf + 4194304, 22); + helper_double_24_recursive(buf + 8388608, 22); + helper_double_24_recursive(buf + 12582912, 22); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 4194304; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4194304), "r"(buf + j + k + 8388608), "r"(buf + j + k + 12582912) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_24(double *buf); +void helper_double_24(double *buf) { + helper_double_24_recursive(buf, 24); +} +void helper_double_25_recursive(double *buf, int depth); +void helper_double_25_recursive(double *buf, int depth) { + if (depth == 10) { + for (int j = 0; j < 1024; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 1024; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 13) { + helper_double_25_recursive(buf + 0, 10); + helper_double_25_recursive(buf + 1024, 10); + helper_double_25_recursive(buf + 2048, 10); + helper_double_25_recursive(buf + 3072, 10); + helper_double_25_recursive(buf + 4096, 10); + helper_double_25_recursive(buf + 5120, 10); + helper_double_25_recursive(buf + 6144, 10); + helper_double_25_recursive(buf + 7168, 10); + for (int j = 0; j < 8192; j += 8192) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 16) { + helper_double_25_recursive(buf + 0, 13); + helper_double_25_recursive(buf + 8192, 13); + helper_double_25_recursive(buf + 16384, 13); + helper_double_25_recursive(buf + 24576, 13); + helper_double_25_recursive(buf + 32768, 13); + helper_double_25_recursive(buf + 40960, 13); + helper_double_25_recursive(buf + 49152, 13); + helper_double_25_recursive(buf + 57344, 13); + for (int j = 0; j < 65536; j += 65536) { + for (int k = 0; k < 8192; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192), "r"(buf + j + k + 16384), "r"(buf + j + k + 24576), "r"(buf + j + k + 32768), "r"(buf + j + k + 40960), "r"(buf + j + k + 49152), "r"(buf + j + k + 57344) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 19) { + helper_double_25_recursive(buf + 0, 16); + helper_double_25_recursive(buf + 65536, 16); + helper_double_25_recursive(buf + 131072, 16); + helper_double_25_recursive(buf + 196608, 16); + helper_double_25_recursive(buf + 262144, 16); + helper_double_25_recursive(buf + 327680, 16); + helper_double_25_recursive(buf + 393216, 16); + helper_double_25_recursive(buf + 458752, 16); + for (int j = 0; j < 524288; j += 524288) { + for (int k = 0; k < 65536; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 65536), "r"(buf + j + k + 131072), "r"(buf + j + k + 196608), "r"(buf + j + k + 262144), "r"(buf + j + k + 327680), "r"(buf + j + k + 393216), "r"(buf + j + k + 458752) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 22) { + helper_double_25_recursive(buf + 0, 19); + helper_double_25_recursive(buf + 524288, 19); + helper_double_25_recursive(buf + 1048576, 19); + helper_double_25_recursive(buf + 1572864, 19); + helper_double_25_recursive(buf + 2097152, 19); + helper_double_25_recursive(buf + 2621440, 19); + helper_double_25_recursive(buf + 3145728, 19); + helper_double_25_recursive(buf + 3670016, 19); + for (int j = 0; j < 4194304; j += 4194304) { + for (int k = 0; k < 524288; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 524288), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1572864), "r"(buf + j + k + 2097152), "r"(buf + j + k + 2621440), "r"(buf + j + k + 3145728), "r"(buf + j + k + 3670016) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 25) { + helper_double_25_recursive(buf + 0, 22); + helper_double_25_recursive(buf + 4194304, 22); + helper_double_25_recursive(buf + 8388608, 22); + helper_double_25_recursive(buf + 12582912, 22); + helper_double_25_recursive(buf + 16777216, 22); + helper_double_25_recursive(buf + 20971520, 22); + helper_double_25_recursive(buf + 25165824, 22); + helper_double_25_recursive(buf + 29360128, 22); + for (int j = 0; j < 33554432; j += 33554432) { + for (int k = 0; k < 4194304; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4194304), "r"(buf + j + k + 8388608), "r"(buf + j + k + 12582912), "r"(buf + j + k + 16777216), "r"(buf + j + k + 20971520), "r"(buf + j + k + 25165824), "r"(buf + j + k + 29360128) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_25(double *buf); +void helper_double_25(double *buf) { + helper_double_25_recursive(buf, 25); +} +void helper_double_26_recursive(double *buf, int depth); +void helper_double_26_recursive(double *buf, int depth) { + if (depth == 5) { + for (int j = 0; j < 32; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 32; j += 32) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 8) { + helper_double_26_recursive(buf + 0, 5); + helper_double_26_recursive(buf + 32, 5); + helper_double_26_recursive(buf + 64, 5); + helper_double_26_recursive(buf + 96, 5); + helper_double_26_recursive(buf + 128, 5); + helper_double_26_recursive(buf + 160, 5); + helper_double_26_recursive(buf + 192, 5); + helper_double_26_recursive(buf + 224, 5); + for (int j = 0; j < 256; j += 256) { + for (int k = 0; k < 32; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32), "r"(buf + j + k + 64), "r"(buf + j + k + 96), "r"(buf + j + k + 128), "r"(buf + j + k + 160), "r"(buf + j + k + 192), "r"(buf + j + k + 224) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 11) { + helper_double_26_recursive(buf + 0, 8); + helper_double_26_recursive(buf + 256, 8); + helper_double_26_recursive(buf + 512, 8); + helper_double_26_recursive(buf + 768, 8); + helper_double_26_recursive(buf + 1024, 8); + helper_double_26_recursive(buf + 1280, 8); + helper_double_26_recursive(buf + 1536, 8); + helper_double_26_recursive(buf + 1792, 8); + for (int j = 0; j < 2048; j += 2048) { + for (int k = 0; k < 256; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 256), "r"(buf + j + k + 512), "r"(buf + j + k + 768), "r"(buf + j + k + 1024), "r"(buf + j + k + 1280), "r"(buf + j + k + 1536), "r"(buf + j + k + 1792) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 14) { + helper_double_26_recursive(buf + 0, 11); + helper_double_26_recursive(buf + 2048, 11); + helper_double_26_recursive(buf + 4096, 11); + helper_double_26_recursive(buf + 6144, 11); + helper_double_26_recursive(buf + 8192, 11); + helper_double_26_recursive(buf + 10240, 11); + helper_double_26_recursive(buf + 12288, 11); + helper_double_26_recursive(buf + 14336, 11); + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 2048; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2048), "r"(buf + j + k + 4096), "r"(buf + j + k + 6144), "r"(buf + j + k + 8192), "r"(buf + j + k + 10240), "r"(buf + j + k + 12288), "r"(buf + j + k + 14336) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_26_recursive(buf + 0, 14); + helper_double_26_recursive(buf + 16384, 14); + helper_double_26_recursive(buf + 32768, 14); + helper_double_26_recursive(buf + 49152, 14); + helper_double_26_recursive(buf + 65536, 14); + helper_double_26_recursive(buf + 81920, 14); + helper_double_26_recursive(buf + 98304, 14); + helper_double_26_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_26_recursive(buf + 0, 17); + helper_double_26_recursive(buf + 131072, 17); + helper_double_26_recursive(buf + 262144, 17); + helper_double_26_recursive(buf + 393216, 17); + helper_double_26_recursive(buf + 524288, 17); + helper_double_26_recursive(buf + 655360, 17); + helper_double_26_recursive(buf + 786432, 17); + helper_double_26_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_26_recursive(buf + 0, 20); + helper_double_26_recursive(buf + 1048576, 20); + helper_double_26_recursive(buf + 2097152, 20); + helper_double_26_recursive(buf + 3145728, 20); + helper_double_26_recursive(buf + 4194304, 20); + helper_double_26_recursive(buf + 5242880, 20); + helper_double_26_recursive(buf + 6291456, 20); + helper_double_26_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_double_26_recursive(buf + 0, 23); + helper_double_26_recursive(buf + 8388608, 23); + helper_double_26_recursive(buf + 16777216, 23); + helper_double_26_recursive(buf + 25165824, 23); + helper_double_26_recursive(buf + 33554432, 23); + helper_double_26_recursive(buf + 41943040, 23); + helper_double_26_recursive(buf + 50331648, 23); + helper_double_26_recursive(buf + 58720256, 23); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 8388608; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824), "r"(buf + j + k + 33554432), "r"(buf + j + k + 41943040), "r"(buf + j + k + 50331648), "r"(buf + j + k + 58720256) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_26(double *buf); +void helper_double_26(double *buf) { + helper_double_26_recursive(buf, 26); +} +void helper_double_27_recursive(double *buf, int depth); +void helper_double_27_recursive(double *buf, int depth) { + if (depth == 6) { + for (int j = 0; j < 64; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 9) { + helper_double_27_recursive(buf + 0, 6); + helper_double_27_recursive(buf + 64, 6); + helper_double_27_recursive(buf + 128, 6); + helper_double_27_recursive(buf + 192, 6); + helper_double_27_recursive(buf + 256, 6); + helper_double_27_recursive(buf + 320, 6); + helper_double_27_recursive(buf + 384, 6); + helper_double_27_recursive(buf + 448, 6); + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_27_recursive(buf + 0, 9); + helper_double_27_recursive(buf + 512, 9); + helper_double_27_recursive(buf + 1024, 9); + helper_double_27_recursive(buf + 1536, 9); + helper_double_27_recursive(buf + 2048, 9); + helper_double_27_recursive(buf + 2560, 9); + helper_double_27_recursive(buf + 3072, 9); + helper_double_27_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_27_recursive(buf + 0, 12); + helper_double_27_recursive(buf + 4096, 12); + helper_double_27_recursive(buf + 8192, 12); + helper_double_27_recursive(buf + 12288, 12); + helper_double_27_recursive(buf + 16384, 12); + helper_double_27_recursive(buf + 20480, 12); + helper_double_27_recursive(buf + 24576, 12); + helper_double_27_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_27_recursive(buf + 0, 15); + helper_double_27_recursive(buf + 32768, 15); + helper_double_27_recursive(buf + 65536, 15); + helper_double_27_recursive(buf + 98304, 15); + helper_double_27_recursive(buf + 131072, 15); + helper_double_27_recursive(buf + 163840, 15); + helper_double_27_recursive(buf + 196608, 15); + helper_double_27_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_27_recursive(buf + 0, 18); + helper_double_27_recursive(buf + 262144, 18); + helper_double_27_recursive(buf + 524288, 18); + helper_double_27_recursive(buf + 786432, 18); + helper_double_27_recursive(buf + 1048576, 18); + helper_double_27_recursive(buf + 1310720, 18); + helper_double_27_recursive(buf + 1572864, 18); + helper_double_27_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_27_recursive(buf + 0, 21); + helper_double_27_recursive(buf + 2097152, 21); + helper_double_27_recursive(buf + 4194304, 21); + helper_double_27_recursive(buf + 6291456, 21); + helper_double_27_recursive(buf + 8388608, 21); + helper_double_27_recursive(buf + 10485760, 21); + helper_double_27_recursive(buf + 12582912, 21); + helper_double_27_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_double_27_recursive(buf + 0, 24); + helper_double_27_recursive(buf + 16777216, 24); + helper_double_27_recursive(buf + 33554432, 24); + helper_double_27_recursive(buf + 50331648, 24); + helper_double_27_recursive(buf + 67108864, 24); + helper_double_27_recursive(buf + 83886080, 24); + helper_double_27_recursive(buf + 100663296, 24); + helper_double_27_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_27(double *buf); +void helper_double_27(double *buf) { + helper_double_27_recursive(buf, 27); +} +void helper_double_28_recursive(double *buf, int depth); +void helper_double_28_recursive(double *buf, int depth) { + if (depth == 14) { + for (int j = 0; j < 16384; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 16384; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 16384; j += 1024) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384), "r"(buf + j + k + 512), "r"(buf + j + k + 640), "r"(buf + j + k + 768), "r"(buf + j + k + 896) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 16384; j += 8192) { + for (int k = 0; k < 1024; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1024), "r"(buf + j + k + 2048), "r"(buf + j + k + 3072), "r"(buf + j + k + 4096), "r"(buf + j + k + 5120), "r"(buf + j + k + 6144), "r"(buf + j + k + 7168) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 16384; j += 16384) { + for (int k = 0; k < 8192; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8192) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 17) { + helper_double_28_recursive(buf + 0, 14); + helper_double_28_recursive(buf + 16384, 14); + helper_double_28_recursive(buf + 32768, 14); + helper_double_28_recursive(buf + 49152, 14); + helper_double_28_recursive(buf + 65536, 14); + helper_double_28_recursive(buf + 81920, 14); + helper_double_28_recursive(buf + 98304, 14); + helper_double_28_recursive(buf + 114688, 14); + for (int j = 0; j < 131072; j += 131072) { + for (int k = 0; k < 16384; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16384), "r"(buf + j + k + 32768), "r"(buf + j + k + 49152), "r"(buf + j + k + 65536), "r"(buf + j + k + 81920), "r"(buf + j + k + 98304), "r"(buf + j + k + 114688) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 20) { + helper_double_28_recursive(buf + 0, 17); + helper_double_28_recursive(buf + 131072, 17); + helper_double_28_recursive(buf + 262144, 17); + helper_double_28_recursive(buf + 393216, 17); + helper_double_28_recursive(buf + 524288, 17); + helper_double_28_recursive(buf + 655360, 17); + helper_double_28_recursive(buf + 786432, 17); + helper_double_28_recursive(buf + 917504, 17); + for (int j = 0; j < 1048576; j += 1048576) { + for (int k = 0; k < 131072; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 131072), "r"(buf + j + k + 262144), "r"(buf + j + k + 393216), "r"(buf + j + k + 524288), "r"(buf + j + k + 655360), "r"(buf + j + k + 786432), "r"(buf + j + k + 917504) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 23) { + helper_double_28_recursive(buf + 0, 20); + helper_double_28_recursive(buf + 1048576, 20); + helper_double_28_recursive(buf + 2097152, 20); + helper_double_28_recursive(buf + 3145728, 20); + helper_double_28_recursive(buf + 4194304, 20); + helper_double_28_recursive(buf + 5242880, 20); + helper_double_28_recursive(buf + 6291456, 20); + helper_double_28_recursive(buf + 7340032, 20); + for (int j = 0; j < 8388608; j += 8388608) { + for (int k = 0; k < 1048576; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 1048576), "r"(buf + j + k + 2097152), "r"(buf + j + k + 3145728), "r"(buf + j + k + 4194304), "r"(buf + j + k + 5242880), "r"(buf + j + k + 6291456), "r"(buf + j + k + 7340032) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 26) { + helper_double_28_recursive(buf + 0, 23); + helper_double_28_recursive(buf + 8388608, 23); + helper_double_28_recursive(buf + 16777216, 23); + helper_double_28_recursive(buf + 25165824, 23); + helper_double_28_recursive(buf + 33554432, 23); + helper_double_28_recursive(buf + 41943040, 23); + helper_double_28_recursive(buf + 50331648, 23); + helper_double_28_recursive(buf + 58720256, 23); + for (int j = 0; j < 67108864; j += 67108864) { + for (int k = 0; k < 8388608; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8388608), "r"(buf + j + k + 16777216), "r"(buf + j + k + 25165824), "r"(buf + j + k + 33554432), "r"(buf + j + k + 41943040), "r"(buf + j + k + 50331648), "r"(buf + j + k + 58720256) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 28) { + helper_double_28_recursive(buf + 0, 26); + helper_double_28_recursive(buf + 67108864, 26); + helper_double_28_recursive(buf + 134217728, 26); + helper_double_28_recursive(buf + 201326592, 26); + for (int j = 0; j < 268435456; j += 268435456) { + for (int k = 0; k < 67108864; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 67108864), "r"(buf + j + k + 134217728), "r"(buf + j + k + 201326592) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_28(double *buf); +void helper_double_28(double *buf) { + helper_double_28_recursive(buf, 28); +} +void helper_double_29_recursive(double *buf, int depth); +void helper_double_29_recursive(double *buf, int depth) { + if (depth == 9) { + for (int j = 0; j < 512; j += 16) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm4, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm4, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm4\n" + "movapd %%xmm5, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm5, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm5\n" + "movapd %%xmm6, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm6, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm6\n" + "movapd %%xmm7, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm7, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6), "r"(buf + j + k + 8), "r"(buf + j + k + 10), "r"(buf + j + k + 12), "r"(buf + j + k + 14) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 128) { + for (int k = 0; k < 16; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16), "r"(buf + j + k + 32), "r"(buf + j + k + 48), "r"(buf + j + k + 64), "r"(buf + j + k + 80), "r"(buf + j + k + 96), "r"(buf + j + k + 112) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 128; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 128), "r"(buf + j + k + 256), "r"(buf + j + k + 384) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_29_recursive(buf + 0, 9); + helper_double_29_recursive(buf + 512, 9); + helper_double_29_recursive(buf + 1024, 9); + helper_double_29_recursive(buf + 1536, 9); + helper_double_29_recursive(buf + 2048, 9); + helper_double_29_recursive(buf + 2560, 9); + helper_double_29_recursive(buf + 3072, 9); + helper_double_29_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_29_recursive(buf + 0, 12); + helper_double_29_recursive(buf + 4096, 12); + helper_double_29_recursive(buf + 8192, 12); + helper_double_29_recursive(buf + 12288, 12); + helper_double_29_recursive(buf + 16384, 12); + helper_double_29_recursive(buf + 20480, 12); + helper_double_29_recursive(buf + 24576, 12); + helper_double_29_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_29_recursive(buf + 0, 15); + helper_double_29_recursive(buf + 32768, 15); + helper_double_29_recursive(buf + 65536, 15); + helper_double_29_recursive(buf + 98304, 15); + helper_double_29_recursive(buf + 131072, 15); + helper_double_29_recursive(buf + 163840, 15); + helper_double_29_recursive(buf + 196608, 15); + helper_double_29_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_29_recursive(buf + 0, 18); + helper_double_29_recursive(buf + 262144, 18); + helper_double_29_recursive(buf + 524288, 18); + helper_double_29_recursive(buf + 786432, 18); + helper_double_29_recursive(buf + 1048576, 18); + helper_double_29_recursive(buf + 1310720, 18); + helper_double_29_recursive(buf + 1572864, 18); + helper_double_29_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_29_recursive(buf + 0, 21); + helper_double_29_recursive(buf + 2097152, 21); + helper_double_29_recursive(buf + 4194304, 21); + helper_double_29_recursive(buf + 6291456, 21); + helper_double_29_recursive(buf + 8388608, 21); + helper_double_29_recursive(buf + 10485760, 21); + helper_double_29_recursive(buf + 12582912, 21); + helper_double_29_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_double_29_recursive(buf + 0, 24); + helper_double_29_recursive(buf + 16777216, 24); + helper_double_29_recursive(buf + 33554432, 24); + helper_double_29_recursive(buf + 50331648, 24); + helper_double_29_recursive(buf + 67108864, 24); + helper_double_29_recursive(buf + 83886080, 24); + helper_double_29_recursive(buf + 100663296, 24); + helper_double_29_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 29) { + helper_double_29_recursive(buf + 0, 27); + helper_double_29_recursive(buf + 134217728, 27); + helper_double_29_recursive(buf + 268435456, 27); + helper_double_29_recursive(buf + 402653184, 27); + for (int j = 0; j < 536870912; j += 536870912) { + for (int k = 0; k < 134217728; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_29(double *buf); +void helper_double_29(double *buf) { + helper_double_29_recursive(buf, 29); +} +void helper_double_30_recursive(double *buf, int depth); +void helper_double_30_recursive(double *buf, int depth) { + if (depth == 3) { + for (int j = 0; j < 8; j += 8) { + for (int k = 0; k < 2; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm0\n" + "movapd %%xmm1, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm1, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm2, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm2, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm2\n" + "movapd %%xmm3, %%xmm8\n" + "haddpd %%xmm8, %%xmm8\n" + "movapd %%xmm3, %%xmm9\n" + "hsubpd %%xmm9, %%xmm9\n" + "blendpd $1, %%xmm8, %%xmm9\n" + "movapd %%xmm9, %%xmm3\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movupd %%xmm0, (%0)\n" + "movupd %%xmm1, (%1)\n" + "movupd %%xmm2, (%2)\n" + "movupd %%xmm3, (%3)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2), "r"(buf + j + k + 4), "r"(buf + j + k + 6) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 6) { + helper_double_30_recursive(buf + 0, 3); + helper_double_30_recursive(buf + 8, 3); + helper_double_30_recursive(buf + 16, 3); + helper_double_30_recursive(buf + 24, 3); + helper_double_30_recursive(buf + 32, 3); + helper_double_30_recursive(buf + 40, 3); + helper_double_30_recursive(buf + 48, 3); + helper_double_30_recursive(buf + 56, 3); + for (int j = 0; j < 64; j += 64) { + for (int k = 0; k < 8; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 8), "r"(buf + j + k + 16), "r"(buf + j + k + 24), "r"(buf + j + k + 32), "r"(buf + j + k + 40), "r"(buf + j + k + 48), "r"(buf + j + k + 56) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 9) { + helper_double_30_recursive(buf + 0, 6); + helper_double_30_recursive(buf + 64, 6); + helper_double_30_recursive(buf + 128, 6); + helper_double_30_recursive(buf + 192, 6); + helper_double_30_recursive(buf + 256, 6); + helper_double_30_recursive(buf + 320, 6); + helper_double_30_recursive(buf + 384, 6); + helper_double_30_recursive(buf + 448, 6); + for (int j = 0; j < 512; j += 512) { + for (int k = 0; k < 64; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 64), "r"(buf + j + k + 128), "r"(buf + j + k + 192), "r"(buf + j + k + 256), "r"(buf + j + k + 320), "r"(buf + j + k + 384), "r"(buf + j + k + 448) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 12) { + helper_double_30_recursive(buf + 0, 9); + helper_double_30_recursive(buf + 512, 9); + helper_double_30_recursive(buf + 1024, 9); + helper_double_30_recursive(buf + 1536, 9); + helper_double_30_recursive(buf + 2048, 9); + helper_double_30_recursive(buf + 2560, 9); + helper_double_30_recursive(buf + 3072, 9); + helper_double_30_recursive(buf + 3584, 9); + for (int j = 0; j < 4096; j += 4096) { + for (int k = 0; k < 512; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 512), "r"(buf + j + k + 1024), "r"(buf + j + k + 1536), "r"(buf + j + k + 2048), "r"(buf + j + k + 2560), "r"(buf + j + k + 3072), "r"(buf + j + k + 3584) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 15) { + helper_double_30_recursive(buf + 0, 12); + helper_double_30_recursive(buf + 4096, 12); + helper_double_30_recursive(buf + 8192, 12); + helper_double_30_recursive(buf + 12288, 12); + helper_double_30_recursive(buf + 16384, 12); + helper_double_30_recursive(buf + 20480, 12); + helper_double_30_recursive(buf + 24576, 12); + helper_double_30_recursive(buf + 28672, 12); + for (int j = 0; j < 32768; j += 32768) { + for (int k = 0; k < 4096; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 4096), "r"(buf + j + k + 8192), "r"(buf + j + k + 12288), "r"(buf + j + k + 16384), "r"(buf + j + k + 20480), "r"(buf + j + k + 24576), "r"(buf + j + k + 28672) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 18) { + helper_double_30_recursive(buf + 0, 15); + helper_double_30_recursive(buf + 32768, 15); + helper_double_30_recursive(buf + 65536, 15); + helper_double_30_recursive(buf + 98304, 15); + helper_double_30_recursive(buf + 131072, 15); + helper_double_30_recursive(buf + 163840, 15); + helper_double_30_recursive(buf + 196608, 15); + helper_double_30_recursive(buf + 229376, 15); + for (int j = 0; j < 262144; j += 262144) { + for (int k = 0; k < 32768; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 32768), "r"(buf + j + k + 65536), "r"(buf + j + k + 98304), "r"(buf + j + k + 131072), "r"(buf + j + k + 163840), "r"(buf + j + k + 196608), "r"(buf + j + k + 229376) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 21) { + helper_double_30_recursive(buf + 0, 18); + helper_double_30_recursive(buf + 262144, 18); + helper_double_30_recursive(buf + 524288, 18); + helper_double_30_recursive(buf + 786432, 18); + helper_double_30_recursive(buf + 1048576, 18); + helper_double_30_recursive(buf + 1310720, 18); + helper_double_30_recursive(buf + 1572864, 18); + helper_double_30_recursive(buf + 1835008, 18); + for (int j = 0; j < 2097152; j += 2097152) { + for (int k = 0; k < 262144; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 262144), "r"(buf + j + k + 524288), "r"(buf + j + k + 786432), "r"(buf + j + k + 1048576), "r"(buf + j + k + 1310720), "r"(buf + j + k + 1572864), "r"(buf + j + k + 1835008) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 24) { + helper_double_30_recursive(buf + 0, 21); + helper_double_30_recursive(buf + 2097152, 21); + helper_double_30_recursive(buf + 4194304, 21); + helper_double_30_recursive(buf + 6291456, 21); + helper_double_30_recursive(buf + 8388608, 21); + helper_double_30_recursive(buf + 10485760, 21); + helper_double_30_recursive(buf + 12582912, 21); + helper_double_30_recursive(buf + 14680064, 21); + for (int j = 0; j < 16777216; j += 16777216) { + for (int k = 0; k < 2097152; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 2097152), "r"(buf + j + k + 4194304), "r"(buf + j + k + 6291456), "r"(buf + j + k + 8388608), "r"(buf + j + k + 10485760), "r"(buf + j + k + 12582912), "r"(buf + j + k + 14680064) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 27) { + helper_double_30_recursive(buf + 0, 24); + helper_double_30_recursive(buf + 16777216, 24); + helper_double_30_recursive(buf + 33554432, 24); + helper_double_30_recursive(buf + 50331648, 24); + helper_double_30_recursive(buf + 67108864, 24); + helper_double_30_recursive(buf + 83886080, 24); + helper_double_30_recursive(buf + 100663296, 24); + helper_double_30_recursive(buf + 117440512, 24); + for (int j = 0; j < 134217728; j += 134217728) { + for (int k = 0; k < 16777216; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 16777216), "r"(buf + j + k + 33554432), "r"(buf + j + k + 50331648), "r"(buf + j + k + 67108864), "r"(buf + j + k + 83886080), "r"(buf + j + k + 100663296), "r"(buf + j + k + 117440512) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } + if (depth == 30) { + helper_double_30_recursive(buf + 0, 27); + helper_double_30_recursive(buf + 134217728, 27); + helper_double_30_recursive(buf + 268435456, 27); + helper_double_30_recursive(buf + 402653184, 27); + helper_double_30_recursive(buf + 536870912, 27); + helper_double_30_recursive(buf + 671088640, 27); + helper_double_30_recursive(buf + 805306368, 27); + helper_double_30_recursive(buf + 939524096, 27); + for (int j = 0; j < 1073741824; j += 1073741824) { + for (int k = 0; k < 134217728; k += 2) { + __asm__ volatile ( + "movupd (%0), %%xmm0\n" + "movupd (%1), %%xmm1\n" + "movupd (%2), %%xmm2\n" + "movupd (%3), %%xmm3\n" + "movupd (%4), %%xmm4\n" + "movupd (%5), %%xmm5\n" + "movupd (%6), %%xmm6\n" + "movupd (%7), %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm9\n" + "addpd %%xmm1, %%xmm8\n" + "subpd %%xmm1, %%xmm9\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm11\n" + "addpd %%xmm3, %%xmm10\n" + "subpd %%xmm3, %%xmm11\n" + "movapd %%xmm4, %%xmm12\n" + "movapd %%xmm4, %%xmm13\n" + "addpd %%xmm5, %%xmm12\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm6, %%xmm14\n" + "movapd %%xmm6, %%xmm15\n" + "addpd %%xmm7, %%xmm14\n" + "subpd %%xmm7, %%xmm15\n" + "movapd %%xmm8, %%xmm0\n" + "movapd %%xmm8, %%xmm2\n" + "addpd %%xmm10, %%xmm0\n" + "subpd %%xmm10, %%xmm2\n" + "movapd %%xmm9, %%xmm1\n" + "movapd %%xmm9, %%xmm3\n" + "addpd %%xmm11, %%xmm1\n" + "subpd %%xmm11, %%xmm3\n" + "movapd %%xmm12, %%xmm4\n" + "movapd %%xmm12, %%xmm6\n" + "addpd %%xmm14, %%xmm4\n" + "subpd %%xmm14, %%xmm6\n" + "movapd %%xmm13, %%xmm5\n" + "movapd %%xmm13, %%xmm7\n" + "addpd %%xmm15, %%xmm5\n" + "subpd %%xmm15, %%xmm7\n" + "movapd %%xmm0, %%xmm8\n" + "movapd %%xmm0, %%xmm12\n" + "addpd %%xmm4, %%xmm8\n" + "subpd %%xmm4, %%xmm12\n" + "movapd %%xmm1, %%xmm9\n" + "movapd %%xmm1, %%xmm13\n" + "addpd %%xmm5, %%xmm9\n" + "subpd %%xmm5, %%xmm13\n" + "movapd %%xmm2, %%xmm10\n" + "movapd %%xmm2, %%xmm14\n" + "addpd %%xmm6, %%xmm10\n" + "subpd %%xmm6, %%xmm14\n" + "movapd %%xmm3, %%xmm11\n" + "movapd %%xmm3, %%xmm15\n" + "addpd %%xmm7, %%xmm11\n" + "subpd %%xmm7, %%xmm15\n" + "movupd %%xmm8, (%0)\n" + "movupd %%xmm9, (%1)\n" + "movupd %%xmm10, (%2)\n" + "movupd %%xmm11, (%3)\n" + "movupd %%xmm12, (%4)\n" + "movupd %%xmm13, (%5)\n" + "movupd %%xmm14, (%6)\n" + "movupd %%xmm15, (%7)\n" + :: "r"(buf + j + k + 0), "r"(buf + j + k + 134217728), "r"(buf + j + k + 268435456), "r"(buf + j + k + 402653184), "r"(buf + j + k + 536870912), "r"(buf + j + k + 671088640), "r"(buf + j + k + 805306368), "r"(buf + j + k + 939524096) : "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15", "memory" + ); + } + } + return; + } +} +void helper_double_30(double *buf); +void helper_double_30(double *buf) { + helper_double_30_recursive(buf, 30); +} +int fht_double(double *buf, int log_n) { + if (log_n == 0) { + return 0; + } + if (log_n == 1) { + helper_double_1(buf); + return 0; + } + if (log_n == 2) { + helper_double_2(buf); + return 0; + } + if (log_n == 3) { + helper_double_3(buf); + return 0; + } + if (log_n == 4) { + helper_double_4(buf); + return 0; + } + if (log_n == 5) { + helper_double_5(buf); + return 0; + } + if (log_n == 6) { + helper_double_6(buf); + return 0; + } + if (log_n == 7) { + helper_double_7(buf); + return 0; + } + if (log_n == 8) { + helper_double_8(buf); + return 0; + } + if (log_n == 9) { + helper_double_9(buf); + return 0; + } + if (log_n == 10) { + helper_double_10(buf); + return 0; + } + if (log_n == 11) { + helper_double_11(buf); + return 0; + } + if (log_n == 12) { + helper_double_12(buf); + return 0; + } + if (log_n == 13) { + helper_double_13(buf); + return 0; + } + if (log_n == 14) { + helper_double_14(buf); + return 0; + } + if (log_n == 15) { + helper_double_15(buf); + return 0; + } + if (log_n == 16) { + helper_double_16(buf); + return 0; + } + if (log_n == 17) { + helper_double_17(buf); + return 0; + } + if (log_n == 18) { + helper_double_18(buf); + return 0; + } + if (log_n == 19) { + helper_double_19(buf); + return 0; + } + if (log_n == 20) { + helper_double_20(buf); + return 0; + } + if (log_n == 21) { + helper_double_21(buf); + return 0; + } + if (log_n == 22) { + helper_double_22(buf); + return 0; + } + if (log_n == 23) { + helper_double_23(buf); + return 0; + } + if (log_n == 24) { + helper_double_24(buf); + return 0; + } + if (log_n == 25) { + helper_double_25(buf); + return 0; + } + if (log_n == 26) { + helper_double_26(buf); + return 0; + } + if (log_n == 27) { + helper_double_27(buf); + return 0; + } + if (log_n == 28) { + helper_double_28(buf); + return 0; + } + if (log_n == 29) { + helper_double_29(buf); + return 0; + } + if (log_n == 30) { + helper_double_30(buf); + return 0; + } + return 1; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/gen.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/gen.py new file mode 100644 index 00000000000..b0d92368b3d --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/gen.py @@ -0,0 +1,595 @@ +import csv +import os +import sys +import subprocess + +max_log_n = 30 + + +def is_distinct(l): + return len(set(l)) == len(l) + + +def float_avx_0(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 4: + raise Exception('float_avx_0 needs at least four auxiliary registers') + res = ident + '"vpermilps $160, %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[0]) + res += ident + '"vpermilps $245, %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[1]) + res += ident + '"vxorps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[2], aux_registers[2], aux_registers[2]) + res += ident + '"vsubps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[1], aux_registers[2], aux_registers[3]) + res += ident + '"vaddsubps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[3], aux_registers[0], register) + return res + + +def float_avx_1(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 5: + raise Exception('float_avx_1 needs at least five auxiliary registers') + res = ident + '"vpermilps $68, %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[0]) + res += ident + '"vpermilps $238, %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[1]) + res += ident + '"vxorps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[2], aux_registers[2], aux_registers[2]) + res += ident + '"vsubps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[1], aux_registers[2], aux_registers[3]) + res += ident + '"vblendps $204, %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[3], aux_registers[1], aux_registers[4]) + res += ident + '"vaddps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[0], aux_registers[4], register) + return res + + +def float_avx_2(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 4: + raise Exception('float_avx_2 needs at least four auxiliary registers') + res = ident + '"vxorps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[0], aux_registers[0], aux_registers[0]) + res += ident + '"vsubps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + register, aux_registers[0], aux_registers[1]) + res += ident + '"vperm2f128 $0, %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + register, register, aux_registers[2]) + res += ident + '"vperm2f128 $49, %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[1], register, aux_registers[3]) + res += ident + '"vaddps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[2], aux_registers[3], register) + return res + + +def float_avx_3_etc(from_register_0, + from_register_1, + to_register_0, + to_register_1, + ident=''): + if not is_distinct( + [from_register_0, from_register_1, to_register_0, to_register_1]): + raise Exception('four registers must be distinct') + res = ident + '"vaddps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + from_register_1, from_register_0, to_register_0) + res += ident + '"vsubps %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + from_register_1, from_register_0, to_register_1) + return res + + +def double_avx_0(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 4: + raise Exception('double_avx_0 needs at least four auxiliary registers') + res = ident + '"vpermilpd $0, %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[0]) + res += ident + '"vpermilpd $15, %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[1]) + res += ident + '"vxorpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[2], aux_registers[2], aux_registers[2]) + res += ident + '"vsubpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[1], aux_registers[2], aux_registers[3]) + res += ident + '"vaddsubpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[3], aux_registers[0], register) + return res + + +def double_avx_1(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 4: + raise Exception('double_avx_1 needs at least four auxiliary registers') + res = ident + '"vperm2f128 $0, %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + register, register, aux_registers[0]) + res += ident + '"vxorpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[1], aux_registers[1], aux_registers[1]) + res += ident + '"vsubpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + register, aux_registers[1], aux_registers[2]) + res += ident + '"vperm2f128 $49, %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[2], register, aux_registers[3]) + res += ident + '"vaddpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + aux_registers[3], aux_registers[0], register) + return res + + +def double_avx_2_etc(from_register_0, + from_register_1, + to_register_0, + to_register_1, + ident=''): + if not is_distinct( + [from_register_0, from_register_1, to_register_0, to_register_1]): + raise Exception('four registers must be distinct') + res = ident + '"vaddpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + from_register_1, from_register_0, to_register_0) + res += ident + '"vsubpd %%%%%s, %%%%%s, %%%%%s\\n"\n' % ( + from_register_1, from_register_0, to_register_1) + return res + + +def float_sse_0(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 2: + raise Exception('float_sse_0 needs at least two auxiliary registers') + res = ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (register, aux_registers[0]) + res += ident + '"shufps $160, %%%%%s, %%%%%s\\n"\n' % (aux_registers[0], + aux_registers[0]) + res += ident + '"shufps $245, %%%%%s, %%%%%s\\n"\n' % (register, register) + res += ident + '"xorps %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + aux_registers[1]) + res += ident + '"subps %%%%%s, %%%%%s\\n"\n' % (register, aux_registers[1]) + res += ident + '"addsubps %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + aux_registers[0]) + res += ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (aux_registers[0], + register) + return res + + +def float_sse_1(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 4: + raise Exception('float_sse_1 needs at least four auxiliary registers') + res = ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (register, aux_registers[0]) + res += ident + '"shufps $68, %%%%%s, %%%%%s\\n"\n' % (aux_registers[0], + aux_registers[0]) + res += ident + '"xorps %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + aux_registers[1]) + res += ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[2]) + res += ident + '"shufps $14, %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + aux_registers[2]) + res += ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[3]) + res += ident + '"shufps $224, %%%%%s, %%%%%s\\n"\n' % (aux_registers[3], + aux_registers[1]) + res += ident + '"addps %%%%%s, %%%%%s\\n"\n' % (aux_registers[0], + aux_registers[2]) + res += ident + '"subps %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + aux_registers[2]) + res += ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (aux_registers[2], + register) + return res + + +def float_sse_2_etc(from_register_0, + from_register_1, + to_register_0, + to_register_1, + ident=''): + if not is_distinct( + [from_register_0, from_register_1, to_register_0, to_register_1]): + raise Exception('four registers must be distinct') + res = ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (from_register_0, + to_register_0) + res += ident + '"movaps %%%%%s, %%%%%s\\n"\n' % (from_register_0, + to_register_1) + res += ident + '"addps %%%%%s, %%%%%s\\n"\n' % (from_register_1, + to_register_0) + res += ident + '"subps %%%%%s, %%%%%s\\n"\n' % (from_register_1, + to_register_1) + return res + + +def double_sse_0(register, aux_registers, ident=''): + if not is_distinct(aux_registers): + raise Exception('auxiliary registers must be distinct') + if register in aux_registers: + raise Exception( + 'the main register can\'t be one of the auxiliary ones') + if len(aux_registers) < 2: + raise Exception('double_sse_0 needs at least two auxiliary registers') + res = ident + '"movapd %%%%%s, %%%%%s\\n"\n' % (register, aux_registers[0]) + res += ident + '"haddpd %%%%%s, %%%%%s\\n"\n' % (aux_registers[0], + aux_registers[0]) + res += ident + '"movapd %%%%%s, %%%%%s\\n"\n' % (register, + aux_registers[1]) + res += ident + '"hsubpd %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + aux_registers[1]) + res += ident + '"blendpd $1, %%%%%s, %%%%%s\\n"\n' % (aux_registers[0], + aux_registers[1]) + res += ident + '"movapd %%%%%s, %%%%%s\\n"\n' % (aux_registers[1], + register) + return res + + +def double_sse_1_etc(from_register_0, + from_register_1, + to_register_0, + to_register_1, + ident=''): + if not is_distinct( + [from_register_0, from_register_1, to_register_0, to_register_1]): + raise Exception('four registers must be distinct') + res = ident + '"movapd %%%%%s, %%%%%s\\n"\n' % (from_register_0, + to_register_0) + res += ident + '"movapd %%%%%s, %%%%%s\\n"\n' % (from_register_0, + to_register_1) + res += ident + '"addpd %%%%%s, %%%%%s\\n"\n' % (from_register_1, + to_register_0) + res += ident + '"subpd %%%%%s, %%%%%s\\n"\n' % (from_register_1, + to_register_1) + return res + + +def plain_step(type_name, buf_name, log_n, it, ident=''): + if log_n <= 0: + raise Exception("log_n must be positive") + if it < 0: + raise Exception("it must be non-negative") + if it >= log_n: + raise Exception("it must be smaller than log_n") + n = 1 << log_n + res = ident + "for (int j = 0; j < %d; j += %d) {\n" % (n, 1 << (it + 1)) + res += ident + " for (int k = 0; k < %d; ++k) {\n" % (1 << it) + res += ident + " %s u = %s[j + k];\n" % (type_name, buf_name) + res += ident + " %s v = %s[j + k + %d];\n" % (type_name, buf_name, + 1 << it) + res += ident + " %s[j + k] = u + v;\n" % buf_name + res += ident + " %s[j + k + %d] = u - v;\n" % (buf_name, 1 << it) + res += ident + " }\n" + res += ident + "}\n" + return res + + +def composite_step(buf_name, + log_n, + from_it, + to_it, + log_w, + registers, + move_instruction, + special_steps, + main_step, + ident=''): + if log_n < log_w: + raise Exception('need at least %d elements' % (1 << log_w)) + num_registers = len(registers) + if num_registers % 2 == 1: + raise Exception('odd number of registers: %d' % num_registers) + num_nontrivial_levels = 0 + if to_it > log_w: + first_nontrivial = max(from_it, log_w) + num_nontrivial_levels = to_it - first_nontrivial + if 1 << num_nontrivial_levels > num_registers / 2: + raise Exception('not enough registers') + n = 1 << log_n + input_registers = [] + output_registers = [] + for i in range(num_registers): + if i < num_registers / 2: + input_registers.append(registers[i]) + else: + output_registers.append(registers[i]) + clobber = ', '.join(['"%%%s"' % x for x in registers]) + if num_nontrivial_levels == 0: + res = ident + 'for (int j = 0; j < %d; j += %d) {\n' % (n, 1 << log_w) + res += ident + ' __asm__ volatile (\n' + res += ident + ' "%s (%%0), %%%%%s\\n"\n' % (move_instruction, + input_registers[0]) + for it in range(from_it, to_it): + res += special_steps[it](input_registers[0], output_registers, + ident + ' ') + res += ident + ' "%s %%%%%s, (%%0)\\n"\n' % (move_instruction, + input_registers[0]) + res += ident + ' :: "r"(%s + j) : %s, "memory"\n' % (buf_name, + clobber) + res += ident + ' );\n' + res += ident + '}\n' + return res + res = ident + 'for (int j = 0; j < %d; j += %d) {\n' % (n, 1 << to_it) + res += ident + ' for (int k = 0; k < %d; k += %d) {\n' % ( + 1 << (to_it - num_nontrivial_levels), 1 << log_w) + subcube = [] + for l in range(1 << num_nontrivial_levels): + subcube.append('j + k + ' + str(l * (1 << + (to_it - num_nontrivial_levels)))) + res += ident + ' __asm__ volatile (\n' + for l in range(1 << num_nontrivial_levels): + res += ident + ' "%s (%%%d), %%%%%s\\n"\n' % (move_instruction, l, + input_registers[l]) + for it in range(from_it, log_w): + for ii in range(1 << num_nontrivial_levels): + res += special_steps[it](input_registers[ii], output_registers, + ident + ' ') + for it in range(num_nontrivial_levels): + for ii in range(0, 1 << num_nontrivial_levels, 1 << (it + 1)): + for jj in range(1 << it): + res += main_step(input_registers[ii + jj], + input_registers[ii + jj + (1 << it)], + output_registers[ii + jj], + output_registers[ii + jj + + (1 << it)], ident + ' ') + tmp = input_registers + input_registers = output_registers + output_registers = tmp + for l in range(1 << num_nontrivial_levels): + res += ident + ' "%s %%%%%s, (%%%d)\\n"\n' % ( + move_instruction, input_registers[l], l) + res += ident + ' :: %s : %s, "memory"\n' % ( + ', '.join(['"r"(%s + %s)' % (buf_name, x) for x in subcube]), clobber) + res += ident + ' );\n' + res += ident + ' }\n' + res += ident + '}\n' + return res + + +def float_avx_composite_step(buf_name, log_n, from_it, to_it, ident=''): + return composite_step(buf_name, log_n, from_it, to_it, 3, + ['ymm%d' % x for x in range(16)], 'vmovups', + [float_avx_0, float_avx_1, + float_avx_2], float_avx_3_etc, ident) + + +def double_avx_composite_step(buf_name, log_n, from_it, to_it, ident=''): + return composite_step(buf_name, log_n, from_it, to_it, 2, [ + 'ymm%d' % x for x in range(16) + ], 'vmovupd', [double_avx_0, double_avx_1], double_avx_2_etc, ident) + + +def float_sse_composite_step(buf_name, log_n, from_it, to_it, ident=''): + return composite_step(buf_name, log_n, from_it, to_it, 2, + ['xmm%d' % x for x in range(16)], 'movups', + [float_sse_0, float_sse_1], float_sse_2_etc, ident) + + +def double_sse_composite_step(buf_name, log_n, from_it, to_it, ident=''): + return composite_step(buf_name, log_n, from_it, to_it, 1, + ['xmm%d' % x for x in range(16)], 'movupd', + [double_sse_0], double_sse_1_etc, ident) + + +def plain_unmerged(type_name, log_n): + signature = "static inline void helper_%s_%d(%s *buf)" % (type_name, log_n, + type_name) + res = '%s;\n' % signature + res += '%s {\n' % signature + for i in range(log_n): + res += plain_step(type_name, 'buf', log_n, i, ' ') + res += "}\n" + return res + + +def greedy_merged(type_name, log_n, composite_step): + try: + composite_step('buf', log_n, 0, 0) + except Exception: + raise Exception('log_n is too small: %d' % log_n) + signature = 'static inline void helper_%s_%d(%s *buf)' % (type_name, log_n, + type_name) + res = '%s;\n' % signature + res += '%s {\n' % signature + cur_it = 0 + while cur_it < log_n: + cur_to_it = log_n + while True: + try: + composite_step('buf', log_n, cur_it, cur_to_it) + break + except Exception: + cur_to_it -= 1 + continue + res += composite_step('buf', log_n, cur_it, cur_to_it, ' ') + cur_it = cur_to_it + res += '}\n' + return res + + +def greedy_merged_recursive(type_name, log_n, threshold_step, composite_step): + if threshold_step > log_n: + raise Exception('threshold_step must be at most log_n') + try: + composite_step('buf', threshold_step, 0, 0) + except Exception: + raise Exception('threshold_step is too small: %d' % threshold_step) + signature = 'void helper_%s_%d_recursive(%s *buf, int depth)' % (type_name, + log_n, + type_name) + res = '%s;\n' % signature + res += '%s {\n' % signature + res += ' if (depth == %d) {\n' % threshold_step + cur_it = 0 + while cur_it < threshold_step: + cur_to_it = threshold_step + while True: + try: + composite_step('buf', threshold_step, cur_it, cur_to_it) + break + except Exception: + cur_to_it -= 1 + continue + res += composite_step('buf', threshold_step, cur_it, cur_to_it, ' ') + cur_it = cur_to_it + res += ' return;\n' + res += ' }\n' + cur_it = threshold_step + while cur_it < log_n: + cur_to_it = log_n + while True: + try: + composite_step('buf', cur_to_it, cur_it, cur_to_it) + break + except Exception: + cur_to_it -= 1 + continue + res += ' if (depth == %d) {\n' % cur_to_it + for i in range(1 << (cur_to_it - cur_it)): + res += ' helper_%s_%d_recursive(buf + %d, %d);\n' % ( + type_name, log_n, i * (1 << cur_it), cur_it) + res += composite_step('buf', cur_to_it, cur_it, cur_to_it, ' ') + res += ' return;\n' + res += ' }\n' + cur_it = cur_to_it + res += '}\n' + signature = 'void helper_%s_%d(%s *buf)' % (type_name, log_n, type_name) + res += '%s;\n' % signature + res += '%s {\n' % signature + res += ' helper_%s_%d_recursive(buf, %d);\n' % (type_name, log_n, log_n) + res += '}\n' + return res + + +def extract_time(data): + cpu_time = float(data['cpu_time']) + time_unit = data['time_unit'] + if time_unit != 'ns': + raise Exception('nanoseconds expected') + return cpu_time / 1e9 + + +def get_mean_stddev(): + with open('measurements/output.csv', 'r') as csvfile: + reader = csv.reader(csvfile) + first = True + for row in reader: + if first: + header = row + first = False + else: + data = {} + for (x, y) in zip(header, row): + data[x] = y + if data['name'] == 'benchmark_fht_mean': + mean = extract_time(data) + elif data['name'] == 'benchmark_fht_stddev': + stddev = extract_time(data) + return mean + + +def measure_time(code, log_n, type_name, method_name, num_it=3): + if num_it % 2 == 0: + raise Exception('even number of runs: %d' % num_it) + with open('measurements/to_run.h', 'w') as output: + output.write(code) + output.write('const int log_n = %d;\n' % log_n) + signature = 'void run(%s *buf)' % type_name + output.write('%s;\n' % signature) + output.write('%s {\n' % signature) + output.write(' %s(buf);\n' % method_name) + output.write('}\n') + with open('/dev/null', 'wb') as devnull: + code = subprocess.call( + 'cd measurements && make run_%s' % type_name, + shell=True, + stdout=devnull) + if code != 0: + raise Exception('bad exit code') + code = subprocess.call( + './measurements/run_%s --benchmark_repetitions=%d --benchmark_format=csv > ./measurements/output.csv' + % (type_name, num_it), + shell=True, + stderr=devnull) + if code != 0: + raise Exception('bad exit code') + return get_mean_stddev() + + +if __name__ == '__main__': + final_code = '#include "fht.h"\n' + hall_of_fame = [] + for (type_name, + composite_step_generator) in [('float', float_avx_composite_step), + ('double', double_avx_composite_step)]: + for log_n in range(1, max_log_n + 1): + sys.stdout.write('log_n = %d\n' % log_n) + times = [] + try: + (res, desc) = (greedy_merged(type_name, log_n, + composite_step_generator), + 'greedy_merged') + except Exception: + (res, desc) = (plain_unmerged(type_name, log_n), + 'plain_unmerged') + time = measure_time(res, log_n, type_name, + 'helper_%s_%d' % (type_name, log_n)) + times.append((time, res, desc)) + sys.stdout.write('log_n = %d; iterative; time = %.10e\n' % (log_n, + time)) + for threshold_step in range(1, log_n + 1): + try: + res = greedy_merged_recursive(type_name, log_n, + threshold_step, + composite_step_generator) + time = measure_time(res, log_n, type_name, + 'helper_%s_%d' % (type_name, log_n)) + times.append( + (time, res, + 'greedy_merged_recursive %d' % threshold_step)) + sys.stdout.write( + 'log_n = %d; threshold_step = %d; time = %.10e\n' % + (log_n, threshold_step, time)) + except Exception: + sys.stdout.write('FAIL: %d\n' % threshold_step) + (best_time, best_code, best_desc) = min(times) + hall_of_fame.append((type_name, log_n, best_time, best_desc)) + final_code += best_code + sys.stdout.write('log_n = %d; best_time = %.10e; %s\n' % + (log_n, best_time, best_desc)) + final_code += 'int fht_%s(%s *buf, int log_n) {\n' % (type_name, + type_name) + final_code += ' if (log_n == 0) {\n' + final_code += ' return 0;\n' + final_code += ' }\n' + for i in range(1, max_log_n + 1): + final_code += ' if (log_n == %d) {\n' % i + final_code += ' helper_%s_%d(buf);\n' % (type_name, i) + final_code += ' return 0;\n' + final_code += ' }\n' + final_code += ' return 1;\n' + final_code += '}\n' + with open('fht_avx.c', 'w') as output: + output.write(final_code) + sys.stdout.write('hall of fame\n') + with open('hall_of_fame_avx.txt', 'w') as hof: + for (type_name, log_n, best_time, best_desc) in hall_of_fame: + s = 'type_name = %s; log_n = %d; best_time = %.10e; best_desc = %s\n' % ( + type_name, log_n, best_time, best_desc) + sys.stdout.write(s) + hof.write(s) diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/hall_of_fame_avx.txt b/extension/llm/custom_ops/spinquant/third-party/FFHT/hall_of_fame_avx.txt new file mode 100644 index 00000000000..316ac08e5bc --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/hall_of_fame_avx.txt @@ -0,0 +1,60 @@ +type_name = float; log_n = 1; best_time = 3.8273900000e-08; best_desc = plain_unmerged +type_name = float; log_n = 2; best_time = 3.8694400000e-08; best_desc = plain_unmerged +type_name = float; log_n = 3; best_time = 4.4120100000e-08; best_desc = greedy_merged +type_name = float; log_n = 4; best_time = 4.6617800000e-08; best_desc = greedy_merged +type_name = float; log_n = 5; best_time = 4.8970800000e-08; best_desc = greedy_merged +type_name = float; log_n = 6; best_time = 5.3648500000e-08; best_desc = greedy_merged +type_name = float; log_n = 7; best_time = 7.1866600000e-08; best_desc = greedy_merged_recursive 7 +type_name = float; log_n = 8; best_time = 1.0585600000e-07; best_desc = greedy_merged_recursive 6 +type_name = float; log_n = 9; best_time = 1.7403800000e-07; best_desc = greedy_merged +type_name = float; log_n = 10; best_time = 3.4412700000e-07; best_desc = greedy_merged_recursive 10 +type_name = float; log_n = 11; best_time = 6.5679200000e-07; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 12; best_time = 1.3143800000e-06; best_desc = greedy_merged +type_name = float; log_n = 13; best_time = 2.8488300000e-06; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 14; best_time = 6.1163700000e-06; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 15; best_time = 1.3664400000e-05; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 16; best_time = 3.0120900000e-05; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 17; best_time = 6.5561000000e-05; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 18; best_time = 1.4176100000e-04; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 19; best_time = 3.0320000000e-04; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 20; best_time = 6.7070400000e-04; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 21; best_time = 1.4708400000e-03; best_desc = greedy_merged_recursive 9 +type_name = float; log_n = 22; best_time = 3.9836500000e-03; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 23; best_time = 8.8893400000e-03; best_desc = greedy_merged_recursive 9 +type_name = float; log_n = 24; best_time = 1.9483500000e-02; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 25; best_time = 4.5779600000e-02; best_desc = greedy_merged_recursive 7 +type_name = float; log_n = 26; best_time = 9.7643700000e-02; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 27; best_time = 2.1200800000e-01; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 28; best_time = 4.9995900000e-01; best_desc = greedy_merged_recursive 7 +type_name = float; log_n = 29; best_time = 1.0615600000e+00; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 30; best_time = 2.2532100000e+00; best_desc = greedy_merged_recursive 6 +type_name = double; log_n = 1; best_time = 3.8275100000e-08; best_desc = plain_unmerged +type_name = double; log_n = 2; best_time = 4.1286300000e-08; best_desc = greedy_merged +type_name = double; log_n = 3; best_time = 4.3412600000e-08; best_desc = greedy_merged +type_name = double; log_n = 4; best_time = 4.5500500000e-08; best_desc = greedy_merged_recursive 4 +type_name = double; log_n = 5; best_time = 4.9231800000e-08; best_desc = greedy_merged +type_name = double; log_n = 6; best_time = 6.2857100000e-08; best_desc = greedy_merged +type_name = double; log_n = 7; best_time = 8.9013300000e-08; best_desc = greedy_merged +type_name = double; log_n = 8; best_time = 1.4163900000e-07; best_desc = greedy_merged +type_name = double; log_n = 9; best_time = 2.7611500000e-07; best_desc = greedy_merged +type_name = double; log_n = 10; best_time = 5.2217100000e-07; best_desc = greedy_merged +type_name = double; log_n = 11; best_time = 1.0466200000e-06; best_desc = greedy_merged +type_name = double; log_n = 12; best_time = 2.3401300000e-06; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 13; best_time = 5.0560300000e-06; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 14; best_time = 1.1394900000e-05; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 15; best_time = 2.5470800000e-05; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 16; best_time = 5.7387600000e-05; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 17; best_time = 1.2497400000e-04; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 18; best_time = 2.6934700000e-04; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 19; best_time = 6.0233800000e-04; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 20; best_time = 1.3345100000e-03; best_desc = greedy_merged_recursive 9 +type_name = double; log_n = 21; best_time = 3.6883500000e-03; best_desc = greedy_merged_recursive 7 +type_name = double; log_n = 22; best_time = 8.6217800000e-03; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 23; best_time = 1.9016200000e-02; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 24; best_time = 4.5002100000e-02; best_desc = greedy_merged_recursive 10 +type_name = double; log_n = 25; best_time = 9.7061600000e-02; best_desc = greedy_merged_recursive 8 +type_name = double; log_n = 26; best_time = 2.1355500000e-01; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 27; best_time = 4.8974200000e-01; best_desc = greedy_merged_recursive 9 +type_name = double; log_n = 28; best_time = 1.0586200000e+00; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 29; best_time = 2.1763100000e+00; best_desc = greedy_merged_recursive 11 +type_name = double; log_n = 30; best_time = 4.8801600000e+00; best_desc = greedy_merged_recursive 9 diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/hall_of_fame_sse.txt b/extension/llm/custom_ops/spinquant/third-party/FFHT/hall_of_fame_sse.txt new file mode 100644 index 00000000000..67c1d5cfe0d --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/hall_of_fame_sse.txt @@ -0,0 +1,60 @@ +type_name = float; log_n = 1; best_time = 3.8234100000e-08; best_desc = plain_unmerged +type_name = float; log_n = 2; best_time = 3.9592000000e-08; best_desc = greedy_merged +type_name = float; log_n = 3; best_time = 4.2633300000e-08; best_desc = greedy_merged +type_name = float; log_n = 4; best_time = 4.5965700000e-08; best_desc = greedy_merged +type_name = float; log_n = 5; best_time = 5.2128100000e-08; best_desc = greedy_merged +type_name = float; log_n = 6; best_time = 6.9547900000e-08; best_desc = greedy_merged +type_name = float; log_n = 7; best_time = 1.0289400000e-07; best_desc = greedy_merged_recursive 7 +type_name = float; log_n = 8; best_time = 1.7410400000e-07; best_desc = greedy_merged +type_name = float; log_n = 9; best_time = 3.5127900000e-07; best_desc = greedy_merged +type_name = float; log_n = 10; best_time = 6.8896800000e-07; best_desc = greedy_merged +type_name = float; log_n = 11; best_time = 1.3963700000e-06; best_desc = greedy_merged +type_name = float; log_n = 12; best_time = 3.0889100000e-06; best_desc = greedy_merged_recursive 7 +type_name = float; log_n = 13; best_time = 6.3768900000e-06; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 14; best_time = 1.3732600000e-05; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 15; best_time = 2.8962800000e-05; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 16; best_time = 6.2055900000e-05; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 17; best_time = 1.3487500000e-04; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 18; best_time = 2.7989100000e-04; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 19; best_time = 5.9871200000e-04; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 20; best_time = 1.3111100000e-03; best_desc = greedy_merged_recursive 8 +type_name = float; log_n = 21; best_time = 2.7614800000e-03; best_desc = greedy_merged_recursive 13 +type_name = float; log_n = 22; best_time = 6.4353000000e-03; best_desc = greedy_merged_recursive 11 +type_name = float; log_n = 23; best_time = 1.4471700000e-02; best_desc = greedy_merged_recursive 6 +type_name = float; log_n = 24; best_time = 3.1766100000e-02; best_desc = greedy_merged_recursive 15 +type_name = float; log_n = 25; best_time = 6.9094300000e-02; best_desc = greedy_merged_recursive 8 +type_name = float; log_n = 26; best_time = 1.4882800000e-01; best_desc = greedy_merged_recursive 5 +type_name = float; log_n = 27; best_time = 3.1941300000e-01; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 28; best_time = 6.9037700000e-01; best_desc = greedy_merged_recursive 16 +type_name = float; log_n = 29; best_time = 1.4692400000e+00; best_desc = greedy_merged_recursive 12 +type_name = float; log_n = 30; best_time = 3.0698600000e+00; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 1; best_time = 3.8948100000e-08; best_desc = greedy_merged +type_name = double; log_n = 2; best_time = 4.0811300000e-08; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 3; best_time = 4.3109200000e-08; best_desc = greedy_merged_recursive 3 +type_name = double; log_n = 4; best_time = 4.9621200000e-08; best_desc = greedy_merged +type_name = double; log_n = 5; best_time = 6.3119400000e-08; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 6; best_time = 9.2256300000e-08; best_desc = greedy_merged +type_name = double; log_n = 7; best_time = 1.5220200000e-07; best_desc = greedy_merged +type_name = double; log_n = 8; best_time = 2.9771700000e-07; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 9; best_time = 6.0368400000e-07; best_desc = greedy_merged_recursive 6 +type_name = double; log_n = 10; best_time = 1.2246100000e-06; best_desc = greedy_merged_recursive 10 +type_name = double; log_n = 11; best_time = 2.6907000000e-06; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 12; best_time = 5.6900800000e-06; best_desc = greedy_merged_recursive 10 +type_name = double; log_n = 13; best_time = 1.2392900000e-05; best_desc = greedy_merged +type_name = double; log_n = 14; best_time = 2.6329500000e-05; best_desc = greedy_merged_recursive 9 +type_name = double; log_n = 15; best_time = 5.6564400000e-05; best_desc = greedy_merged_recursive 10 +type_name = double; log_n = 16; best_time = 1.2357300000e-04; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 17; best_time = 2.5763800000e-04; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 18; best_time = 5.5563300000e-04; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 19; best_time = 1.2115600000e-03; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 20; best_time = 2.5899100000e-03; best_desc = greedy_merged_recursive 12 +type_name = double; log_n = 21; best_time = 6.0839900000e-03; best_desc = greedy_merged_recursive 13 +type_name = double; log_n = 22; best_time = 1.3738100000e-02; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 23; best_time = 3.0164700000e-02; best_desc = greedy_merged_recursive 2 +type_name = double; log_n = 24; best_time = 6.6689900000e-02; best_desc = greedy_merged_recursive 13 +type_name = double; log_n = 25; best_time = 1.4307200000e-01; best_desc = greedy_merged_recursive 10 +type_name = double; log_n = 26; best_time = 3.0875700000e-01; best_desc = greedy_merged_recursive 5 +type_name = double; log_n = 27; best_time = 6.7026700000e-01; best_desc = greedy_merged_recursive 6 +type_name = double; log_n = 28; best_time = 1.4210300000e+00; best_desc = greedy_merged_recursive 14 +type_name = double; log_n = 29; best_time = 3.0175300000e+00; best_desc = greedy_merged_recursive 9 +type_name = double; log_n = 30; best_time = 6.4575800000e+00; best_desc = greedy_merged_recursive 3 diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/Makefile b/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/Makefile new file mode 100644 index 00000000000..c0b6daff716 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/Makefile @@ -0,0 +1,13 @@ +CXX=g++ +CXX_FLAGS=-O3 -Wall -march=native -std=c++11 -I../external/benchmark/include -L../external/benchmark/src -lbenchmark -lpthread + +.PHONY: run_float run_double clean + +run_float: + $(CXX) run_float.cpp -o run_float $(CXX_FLAGS) + +run_double: + $(CXX) run_double.cpp -o run_double $(CXX_FLAGS) + +clean: + rm -rf run_float run_double diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/run_double.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/run_double.cpp new file mode 100644 index 00000000000..711456a1f7a --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/run_double.cpp @@ -0,0 +1,27 @@ +#include "to_run.h" + +#include + +#include +#include + +#include + +static void benchmark_fht(benchmark::State &state) { + double *buf; + if (posix_memalign((void**)&buf, 32, sizeof(double) * (1 << log_n))) { + throw std::runtime_error("posix_memalign failed"); + } + while (state.KeepRunning()) { + auto start = std::chrono::high_resolution_clock::now(); + run(buf); + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + } + free(buf); +} + +BENCHMARK(benchmark_fht); + +BENCHMARK_MAIN(); diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/run_float.cpp b/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/run_float.cpp new file mode 100644 index 00000000000..d84159d6ca7 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/measurements/run_float.cpp @@ -0,0 +1,27 @@ +#include "to_run.h" + +#include + +#include +#include + +#include + +static void benchmark_fht(benchmark::State &state) { + float *buf; + if (posix_memalign((void**)&buf, 32, sizeof(float) * (1 << log_n))) { + throw std::runtime_error("posix_memalign failed"); + } + while (state.KeepRunning()) { + auto start = std::chrono::high_resolution_clock::now(); + run(buf); + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds = std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + } + free(buf); +} + +BENCHMARK(benchmark_fht); + +BENCHMARK_MAIN(); diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/setup.py b/extension/llm/custom_ops/spinquant/third-party/FFHT/setup.py new file mode 100644 index 00000000000..f4841cb7397 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/setup.py @@ -0,0 +1,46 @@ +import sys + +try: + import pypandoc + long_description = pypandoc.convert('README.md', 'rst') +except(IOError, ImportError): + long_description = open('README.md').read() + +try: + from setuptools import setup, find_packages, Extension +except ImportError: + sys.stderr.write('Setuptools not found!\n') + raise + +try: + import numpy as np +except ImportError: + sys.stderr.write('NumPy not found!\n') + raise + +if sys.version_info[0] == 2: + arr_sources = ['_ffht_2.c', 'fht.c'] + +if sys.version_info[0] == 3: + arr_sources = ['_ffht_3.c', 'fht.c'] + +module = Extension('ffht', + sources= arr_sources, + extra_compile_args=['-march=native', '-O3', '-Wall', '-Wextra', '-pedantic', + '-Wshadow', '-Wpointer-arith', '-Wcast-qual', + '-Wstrict-prototypes', '-Wmissing-prototypes', + '-std=c99', '-DFHT_HEADER_ONLY'], + include_dirs=[np.get_include()]) + +setup(name='FFHT', + version='1.1', + author='Ilya Razenshteyn, Ludwig Schmidt', + author_email='falconn.lib@gmail.com', + url='https://github.com/FALCONN-LIB/FFHT', + description='Fast implementation of the Fast Hadamard Transform (FHT)', + long_description=long_description, + license='MIT', + keywords='fast Fourier Hadamard transform butterfly', + packages=find_packages(), + include_package_data=True, + ext_modules=[module]) diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/test_double.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_double.c new file mode 100644 index 00000000000..f532ae6e2ff --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_double.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include "fht.h" + +void dumb_fht(double *buf, int log_n); +void dumb_fht(double *buf, int log_n) { + int n = 1 << log_n; + for (int i = 0; i < log_n; ++i) { + int s1 = 1 << i; + int s2 = s1 << 1; + for (int j = 0; j < n; j += s2) { + for (int k = 0; k < s1; ++k) { + double u = buf[j + k]; + double v = buf[j + k + s1]; + buf[j + k] = u + v; + buf[j + k + s1] = u - v; + } + } + } +} + +int main(void) { + srand(4057218); + for (int log_n = 1; log_n <= 30; ++log_n) { + printf("%d ", log_n); + int n = 1 << log_n; + void *buf = malloc(sizeof(double) * n + 32); + char *start = buf; + while ((size_t)start % 32 != 0) start = start + 1; + double *a = (double*)start; + double *aux = (double*)malloc(sizeof(double) * n); + for (int i = 0; i < n; ++i) { + a[i] = 1.0 - 2.0 * (rand() & 1); + aux[i] = a[i]; + } + fht_double(a, log_n); + dumb_fht(aux, log_n); + double max_error = 0.0; + for (int i = 0; i < n; ++i) { + double error = fabs(a[i] - aux[i]); + if (error > max_error) { + max_error = error; + } + } + if (max_error > 1e-5) { + printf("ERROR: %.10lf\n", max_error); + return 1; + } + for (int num_it = 10;; num_it *= 2) { + clock_t tt1 = clock(); + for (int it = 0; it < num_it; ++it) { + fht_double(a, log_n); + } + clock_t tt2 = clock(); + double sec = (tt2 - tt1) / (CLOCKS_PER_SEC + 0.0); + if (sec >= 1.0) { + printf("%.10e\n", sec / (num_it + 0.0)); + break; + } + } + free(buf); + free(aux); + } + return 0; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/test_double_header_only.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_double_header_only.c new file mode 100644 index 00000000000..081dca1d560 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_double_header_only.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include "fht_header_only.h" + +void dumb_fht(double *buf, int log_n); +void dumb_fht(double *buf, int log_n) { + int n = 1 << log_n; + for (int i = 0; i < log_n; ++i) { + int s1 = 1 << i; + int s2 = s1 << 1; + for (int j = 0; j < n; j += s2) { + for (int k = 0; k < s1; ++k) { + double u = buf[j + k]; + double v = buf[j + k + s1]; + buf[j + k] = u + v; + buf[j + k + s1] = u - v; + } + } + } +} + +int main(void) { + srand(4057218); + for (int log_n = 1; log_n <= 30; ++log_n) { + printf("%d ", log_n); + int n = 1 << log_n; + void *buf = malloc(sizeof(double) * n + 32); + char *start = buf; + while ((size_t)start % 32 != 0) start = start + 1; + double *a = (double*)start; + double *aux = (double*)malloc(sizeof(double) * n); + for (int i = 0; i < n; ++i) { + a[i] = 1.0 - 2.0 * (rand() & 1); + aux[i] = a[i]; + } + fht_double(a, log_n); + dumb_fht(aux, log_n); + double max_error = 0.0; + for (int i = 0; i < n; ++i) { + double error = fabs(a[i] - aux[i]); + if (error > max_error) { + max_error = error; + } + } + if (max_error > 1e-5) { + printf("ERROR: %.10lf\n", max_error); + return 1; + } + for (int num_it = 10;; num_it *= 2) { + clock_t tt1 = clock(); + for (int it = 0; it < num_it; ++it) { + fht_double(a, log_n); + } + clock_t tt2 = clock(); + double sec = (tt2 - tt1) / (CLOCKS_PER_SEC + 0.0); + if (sec >= 1.0) { + printf("%.10e\n", sec / (num_it + 0.0)); + break; + } + } + free(buf); + free(aux); + } + return 0; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/test_float.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_float.c new file mode 100644 index 00000000000..db8c60ddddb --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_float.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include "fht.h" + +void dumb_fht(float *buf, int log_n); +void dumb_fht(float *buf, int log_n) { + int n = 1 << log_n; + for (int i = 0; i < log_n; ++i) { + int s1 = 1 << i; + int s2 = s1 << 1; + for (int j = 0; j < n; j += s2) { + for (int k = 0; k < s1; ++k) { + float u = buf[j + k]; + float v = buf[j + k + s1]; + buf[j + k] = u + v; + buf[j + k + s1] = u - v; + } + } + } +} + +int main(void) { + srand(4057218); + for (int log_n = 1; log_n <= 30; ++log_n) { + printf("%d ", log_n); + int n = 1 << log_n; + void *buf = malloc(sizeof(float) * n + 32); + char *start = buf; + while ((size_t)start % 32 != 0) start = start + 1; + float *a = (float*)start; + float *aux = (float*)malloc(sizeof(double) * n); + for (int i = 0; i < n; ++i) { + a[i] = 1.0 - 2.0 * (rand() & 1); + aux[i] = a[i]; + } + fht_float(a, log_n); + dumb_fht(aux, log_n); + double max_error = 0.0; + for (int i = 0; i < n; ++i) { + double error = fabs(a[i] - aux[i]); + if (error > max_error) { + max_error = error; + } + } + if (max_error > 1e-5) { + printf("ERROR: %.10lf\n", max_error); + return 1; + } + for (int num_it = 10;; num_it *= 2) { + clock_t tt1 = clock(); + for (int it = 0; it < num_it; ++it) { + fht_float(a, log_n); + } + clock_t tt2 = clock(); + double sec = (tt2 - tt1) / (CLOCKS_PER_SEC + 0.0); + if (sec >= 1.0) { + printf("%.10e\n", sec / (num_it + 0.0)); + break; + } + } + free(buf); + free(aux); + } + return 0; +} diff --git a/extension/llm/custom_ops/spinquant/third-party/FFHT/test_float_header_only.c b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_float_header_only.c new file mode 100644 index 00000000000..d069b0c6571 --- /dev/null +++ b/extension/llm/custom_ops/spinquant/third-party/FFHT/test_float_header_only.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include "fht_header_only.h" + +void dumb_fht(float *buf, int log_n); +void dumb_fht(float *buf, int log_n) { + int n = 1 << log_n; + for (int i = 0; i < log_n; ++i) { + int s1 = 1 << i; + int s2 = s1 << 1; + for (int j = 0; j < n; j += s2) { + for (int k = 0; k < s1; ++k) { + float u = buf[j + k]; + float v = buf[j + k + s1]; + buf[j + k] = u + v; + buf[j + k + s1] = u - v; + } + } + } +} + +int main(void) { + srand(4057218); + for (int log_n = 1; log_n <= 30; ++log_n) { + printf("%d ", log_n); + int n = 1 << log_n; + void *buf = malloc(sizeof(float) * n + 32); + char *start = buf; + while ((size_t)start % 32 != 0) start = start + 1; + float *a = (float*)start; + float *aux = (float*)malloc(sizeof(double) * n); + for (int i = 0; i < n; ++i) { + a[i] = 1.0 - 2.0 * (rand() & 1); + aux[i] = a[i]; + } + fht_float(a, log_n); + dumb_fht(aux, log_n); + double max_error = 0.0; + for (int i = 0; i < n; ++i) { + double error = fabs(a[i] - aux[i]); + if (error > max_error) { + max_error = error; + } + } + if (max_error > 1e-5) { + printf("ERROR: %.10lf\n", max_error); + return 1; + } + for (int num_it = 10;; num_it *= 2) { + clock_t tt1 = clock(); + for (int it = 0; it < num_it; ++it) { + fht_float(a, log_n); + } + clock_t tt2 = clock(); + double sec = (tt2 - tt1) / (CLOCKS_PER_SEC + 0.0); + if (sec >= 1.0) { + printf("%.10e\n", sec / (num_it + 0.0)); + break; + } + } + free(buf); + free(aux); + } + return 0; +}