A leaner gesv_rbt_async with improved execution speed and accuracy

CMakeLists.txt

@@ -11,6 +11,13 @@ else()
     project( MAGMA LANGUAGES C CXX )
 endif()
 
+FIND_PROGRAM(PROGRAM_CCACHE ccache)
+IF (PROGRAM_CCACHE)
+    SET(CMAKE_CXX_COMPILER_LAUNCHER ${PROGRAM_CCACHE})
+    SET(CMAKE_C_COMPILER_LAUNCHER ${PROGRAM_CCACHE})
+    SET(CMAKE_CUDA_COMPILER_LAUNCHER ${PROGRAM_CCACHE})
+    SET(CMAKE_OPENCL_COMPILER_LAUNCHER ${PROGRAM_CCACHE})
+ENDIF ()
 
 # ----------------------------------------
 # to show compile commands, set this here or use 'make VERBOSE=1'

Makefile

@@ -261,7 +261,7 @@ else ifeq ($(BACKEND),hip)
 
     ## Suggestion by Mark (from SLATE)
     # Valid architecture numbers
-    # TODO: remove very old ones?
+    # TODO: remove veryold ones?
     VALID_GFXS = 600 601 602 700 701 702 703 704 705 801 802 803 805 810 900 902 904 906 908 909 90a 940 941 942 90c 1010 1011 1012 1030 1031 1032 1033
 
 

include/magma_auxiliary.h

@@ -81,6 +81,9 @@ magma_int_t magma_get_smlsize_divideconquer();
 magma_int_t
 magma_malloc( magma_ptr *ptr_ptr, size_t bytes );
 
+magma_int_t
+magma_malloc_async( magma_ptr* ptrPtr, size_t size, magma_queue_t queue);
+
 magma_int_t
 magma_malloc_cpu( void **ptr_ptr, size_t bytes );
 
@@ -93,6 +96,9 @@ magma_free_cpu( void *ptr );
 #define magma_free( ptr ) \
         magma_free_internal( ptr, __func__, __FILE__, __LINE__ )
 
+#define magma_free_async( ptr, queue ) \
+        magma_free_internal_async( ptr, __func__, __FILE__, __LINE__, queue )
+
 #define magma_free_pinned( ptr ) \
         magma_free_pinned_internal( ptr, __func__, __FILE__, __LINE__ )
 
@@ -101,6 +107,11 @@ magma_free_internal(
     magma_ptr ptr,
     const char* func, const char* file, int line );
 
+magma_int_t
+magma_free_internal_async(
+    magma_ptr ptr,
+    const char* func, const char* file, int line, magma_queue_t queue );
+
 magma_int_t
 magma_free_pinned_internal(
     void *ptr,
@@ -128,24 +139,45 @@ magma_memset_async(void * ptr, int value, size_t count, magma_queue_t queue);
 /// Type-safe version of magma_malloc(), for magma_int_t arrays. Allocates n*sizeof(magma_int_t) bytes.
 static inline magma_int_t magma_imalloc( magmaInt_ptr           *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(magma_int_t)        ); }
 
+/// Type-safe asynchronous version of magma_malloc(), for magma_int_t arrays. Allocates n*sizeof(magma_int_t) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_imalloc_async( magmaInt_ptr           *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(magma_int_t), queue ); }
+
 /// Type-safe version of magma_malloc(), for magma_index_t arrays. Allocates n*sizeof(magma_index_t) bytes.
 static inline magma_int_t magma_index_malloc( magmaIndex_ptr    *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(magma_index_t)      ); }
 
+/// Type-safe asynchronous version of magma_malloc(), for magma_index_t arrays. Allocates n*sizeof(magma_index_t) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_index_malloc_async( magmaIndex_ptr    *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(magma_index_t), queue ); }
+
 /// Type-safe version of magma_malloc(), for magma_uindex_t arrays. Allocates n*sizeof(magma_uindex_t) bytes.
 static inline magma_int_t magma_uindex_malloc( magmaUIndex_ptr    *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(magma_uindex_t)      ); }
 
+/// Type-safe asynchronous version of magma_malloc(), for magma_uindex_t arrays. Allocates n*sizeof(magma_uindex_t) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_uindex_malloc_async( magmaUIndex_ptr    *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(magma_uindex_t), queue); }
+
 /// Type-safe version of magma_malloc(), for float arrays. Allocates n*sizeof(float) bytes.
 static inline magma_int_t magma_smalloc( magmaFloat_ptr         *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(float)              ); }
 
+/// Type-safe asynchronous version of magma_malloc(), for float arrays. Allocates n*sizeof(float) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_smalloc_async( magmaFloat_ptr         *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(float), queue); }
+
 /// Type-safe version of magma_malloc(), for double arrays. Allocates n*sizeof(double) bytes.
 static inline magma_int_t magma_dmalloc( magmaDouble_ptr        *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(double)             ); }
 
+/// Type-safe asynchronous version of magma_malloc(), for double arrays. Allocates n*sizeof(double) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_dmalloc_async( magmaDouble_ptr        *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(double), queue); }
+
 /// Type-safe version of magma_malloc(), for magmaFloatComplex arrays. Allocates n*sizeof(magmaFloatComplex) bytes.
 static inline magma_int_t magma_cmalloc( magmaFloatComplex_ptr  *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(magmaFloatComplex)  ); }
 
+/// Type-safe asynchronous version of magma_malloc(), for magmaFloatComplex arrays. Allocates n*sizeof(magmaFloatComplex) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_cmalloc_async( magmaFloatComplex_ptr  *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(magmaFloatComplex), queue ); }
+
 /// Type-safe version of magma_malloc(), for magmaDoubleComplex arrays. Allocates n*sizeof(magmaDoubleComplex) bytes.
 static inline magma_int_t magma_zmalloc( magmaDoubleComplex_ptr *ptr_ptr, size_t n ) { return magma_malloc( (magma_ptr*) ptr_ptr, n*sizeof(magmaDoubleComplex) ); }
 
+/// Type-safe asynchronous version of magma_malloc_async(), for magmaDoubleComplex arrays. Allocates n*sizeof(magmaDoubleComplex) bytes using CUDA stream specified in queue.
+static inline magma_int_t magma_zmalloc_async( magmaDoubleComplex_ptr *ptr_ptr, size_t n, magma_queue_t queue ) { return magma_malloc_async( (magma_ptr*) ptr_ptr, n*sizeof(magmaDoubleComplex), queue ); }
+
 /// @}
 
 

include/magma_z.h

@@ -477,6 +477,16 @@ magma_zgerbt_gpu(
     magmaDoubleComplex *U, magmaDoubleComplex *V,
     magma_int_t *info);
 
+// CUDA MAGMA only
+magma_int_t
+magma_zgerbt_gpu_async(
+        const magma_bool_t gen, const magma_int_t n, const magma_int_t nrhs,
+        magmaDoubleComplex_ptr const dA, magma_int_t const ldda,
+        magmaDoubleComplex_ptr const dB, magma_int_t const lddb,
+        magmaDoubleComplex_ptr const dU, magmaDoubleComplex_ptr const dV,
+        magma_int_t *info,
+        magma_queue_t queue);
+
 // CUDA MAGMA only
 magma_int_t
 magma_zgerfs_nopiv_gpu(
@@ -488,6 +498,20 @@ magma_zgerfs_nopiv_gpu(
     magma_int_t *iter,
     magma_int_t *info);
 
+// CUDA MAGMA only
+magma_int_t
+magma_zgerfs_nopiv_gpu_async(
+        magma_trans_t trans, magma_int_t n, magma_int_t nrhs,
+        magmaDoubleComplex_ptr dA, magma_int_t ldda,
+        magmaDoubleComplex_ptr dB, magma_int_t lddb,
+        magmaDoubleComplex_ptr dX, magma_int_t lddx,
+        magmaDoubleComplex_ptr dworkd, magmaDoubleComplex_ptr dAF,
+        magma_int_t *iter,
+        magma_int_t *info,
+        magma_int_t iter_max,
+        double bwdmax,
+        magma_queue_t queue);
+
 magma_int_t
 magma_zgesdd(
     magma_vec_t jobz, magma_int_t m, magma_int_t n,
@@ -525,6 +549,13 @@ magma_zgesv_nopiv_gpu(
     magmaDoubleComplex_ptr dB, magma_int_t lddb,
     magma_int_t *info);
 
+magma_int_t
+magma_zgesv_nopiv_gpu_async(
+        magma_int_t n, magma_int_t nrhs,
+        magmaDoubleComplex_ptr dA, magma_int_t ldda,
+        magmaDoubleComplex_ptr dB, magma_int_t lddb,
+        magma_int_t *info, magma_queue_t queue );
+
 // CUDA MAGMA only
 magma_int_t
 magma_zgesv_rbt(
@@ -533,6 +564,26 @@ magma_zgesv_rbt(
     magmaDoubleComplex *B, magma_int_t ldb,
     magma_int_t *info);
 
+// CUDA MAGMA only
+magma_int_t
+magma_zgesv_rbt_async(
+        const magma_bool_t refine, const magma_int_t n, const magma_int_t nrhs,
+        const magmaDoubleComplex *const dA, const magma_int_t lda,
+        magmaDoubleComplex *const dB, const magma_int_t ldb,
+        magma_int_t *info,
+        const magma_int_t iter_max, const double bwdmax,
+        magma_queue_t queue );
+
+// CUDA MAGMA only
+magma_int_t
+magma_zgesv_rbt_refine_async(
+        const magma_int_t n, const magma_int_t nrhs,
+        const magmaDoubleComplex *const dA_, const magma_int_t lda,
+        magmaDoubleComplex *const dB_, const magma_int_t ldb,
+        magma_int_t *info,
+        const magma_int_t iter_max, const double bwdmax,
+        magma_queue_t queue);
+
 magma_int_t
 magma_zgesvd(
     magma_vec_t jobu, magma_vec_t jobvt, magma_int_t m, magma_int_t n,
@@ -676,6 +727,13 @@ magma_zgetrf_nopiv_gpu(
     magmaDoubleComplex_ptr dA, magma_int_t ldda,
     magma_int_t *info);
 
+magma_int_t
+magma_zgetrf_nopiv_gpu_async(
+        magma_int_t m, magma_int_t n,
+        magmaDoubleComplex_ptr dA, magma_int_t ldda,
+        magma_int_t *info,
+        magma_queue_t queue);
+
 magma_int_t
 magma_zgetri_gpu(
     magma_int_t n,
@@ -721,6 +779,13 @@ magma_zgetrs_nopiv_gpu(
     magmaDoubleComplex_ptr dB, magma_int_t lddb,
     magma_int_t *info);
 
+magma_int_t
+magma_zgetrs_nopiv_gpu_async(
+        magma_trans_t trans, magma_int_t n, magma_int_t nrhs,
+        magmaDoubleComplex_ptr dA, magma_int_t ldda,
+        magmaDoubleComplex_ptr dB, magma_int_t lddb,
+        magma_int_t *info, magma_queue_t queue);
+
 // ------------------------------------------------------------ zhe routines
 magma_int_t
 magma_zheevd(

include/magmablas_z.h

@@ -496,6 +496,13 @@ magmablas_ztrtri_diag(
     magmaDoubleComplex_ptr d_dinvA,
     magma_queue_t queue );
 
+void
+magmablas_ztrtri_diag_async(
+    magma_uplo_t uplo, magma_diag_t diag, magma_int_t n,
+    magmaDoubleComplex_const_ptr dA, magma_int_t ldda,
+    magmaDoubleComplex_ptr d_dinvA,
+    magma_queue_t queue );
+
   /*
    * to cleanup (alphabetical order)
    */
@@ -757,6 +764,15 @@ magmablas_ztrsm(
     magmaDoubleComplex_ptr       dB, magma_int_t lddb,
     magma_queue_t queue );
 
+void
+magmablas_ztrsm_async(
+    magma_side_t side, magma_uplo_t uplo, magma_trans_t transA, magma_diag_t diag,
+    magma_int_t m, magma_int_t n,
+    magmaDoubleComplex alpha,
+    magmaDoubleComplex_const_ptr dA, magma_int_t ldda,
+    magmaDoubleComplex_ptr       dB, magma_int_t lddb,
+    magma_queue_t queue );
+
 void
 magmablas_ztrsm_outofplace(
     magma_side_t side, magma_uplo_t uplo, magma_trans_t transA, magma_diag_t diag,