Skip to content

Commit b161ac2

Browse files
committed
Merge branch 'develop' of github.com:xianyi/OpenBLAS into develop
2 parents dd2d3e6 + b20ee69 commit b161ac2

File tree

8 files changed

+181
-31
lines changed

8 files changed

+181
-31
lines changed

driver/others/blas_server.c

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
7474
#include <sys/resource.h>
7575
#endif
7676

77+
#ifndef likely
78+
#ifdef __GNUC__
79+
#define likely(x) __builtin_expect(!!(x), 1)
80+
#else
81+
#define likely(x) (x)
82+
#endif
83+
#endif
84+
#ifndef unlikely
85+
#ifdef __GNUC__
86+
#define unlikely(x) __builtin_expect(!!(x), 0)
87+
#else
88+
#define unlikely(x) (x)
89+
#endif
90+
#endif
91+
7792
#ifdef SMP_SERVER
7893

7994
#undef MONITOR
@@ -83,8 +98,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
8398

8499
#define ATTRIBUTE_SIZE 128
85100

86-
extern void openblas_warning(int verbose, const char * msg);
87-
88101
/* This is a thread server model implementation. The threads are */
89102
/* spawned at first access to blas library, and still remains until */
90103
/* destruction routine is called. The number of threads are */
@@ -586,6 +599,10 @@ static BLASULONG exec_queue_lock = 0;
586599

587600
int exec_blas_async(BLASLONG pos, blas_queue_t *queue){
588601

602+
#ifdef SMP_SERVER
603+
// Handle lazy re-init of the thread-pool after a POSIX fork
604+
if (unlikely(blas_server_avail == 0)) blas_thread_init();
605+
#endif
589606
BLASLONG i = 0;
590607
blas_queue_t *current = queue;
591608
#if defined(OS_LINUX) && !defined(NO_AFFINITY) && !defined(PARAMTEST)
@@ -710,7 +727,11 @@ int exec_blas_async_wait(BLASLONG num, blas_queue_t *queue){
710727
/* Execute Threads */
711728
int exec_blas(BLASLONG num, blas_queue_t *queue){
712729

713-
int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG);
730+
#ifdef SMP_SERVER
731+
// Handle lazy re-init of the thread-pool after a POSIX fork
732+
if (unlikely(blas_server_avail == 0)) blas_thread_init();
733+
#endif
734+
int (*routine)(blas_arg_t *, void *, void *, double *, double *, BLASLONG);
714735

715736
#ifdef TIMING_DEBUG
716737
BLASULONG start, stop;
@@ -923,17 +944,5 @@ int BLASFUNC(blas_thread_shutdown)(void){
923944
return 0;
924945
}
925946

926-
/*
927-
https://github.com/xianyi/OpenBLAS/issues/294
928-
Use pthread_atfork to close blas_thread_server before fork.
929-
Then, re-init blas_thread_server after fork at child and parent.
930-
*/
931-
void openblas_fork_handler()
932-
{
933-
int err;
934-
err = pthread_atfork (BLASFUNC(blas_thread_shutdown), blas_thread_init, blas_thread_init);
935-
if(err != 0)
936-
openblas_warning(0, "OpenBLAS cannot install fork handler. You may meet hang after fork.\n");
937-
}
938947
#endif
939948

driver/others/blas_server_omp.c

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -315,9 +315,4 @@ int exec_blas(BLASLONG num, blas_queue_t *queue){
315315
return 0;
316316
}
317317

318-
void openblas_fork_handler()
319-
{
320-
321-
}
322-
323318
#endif

driver/others/blas_server_win32.c

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -498,8 +498,3 @@ void openblas_set_num_threads(int num)
498498
{
499499
goto_set_num_threads(num);
500500
}
501-
502-
void openblas_fork_handler()
503-
{
504-
505-
}

driver/others/memory.c

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
143143
gotoblas_t *gotoblas = NULL;
144144
#endif
145145

146+
extern void openblas_warning(int verbose, const char * msg);
147+
146148
#ifndef SMP
147149

148150
#define blas_cpu_number 1
@@ -253,6 +255,23 @@ int goto_get_num_procs (void) {
253255
return blas_cpu_number;
254256
}
255257

258+
void openblas_fork_handler()
259+
{
260+
// This handler shuts down the OpenBLAS-managed PTHREAD pool when OpenBLAS is
261+
// built with "make USE_OPENMP=0".
262+
// Hanging can still happen when OpenBLAS is built against the libgomp
263+
// implementation of OpenMP. The problem is tracked at:
264+
// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=60035
265+
// In the mean time build with USE_OPENMP=0 or link against another
266+
// implementation of OpenMP.
267+
#ifndef OS_WINDOWS
268+
int err;
269+
err = pthread_atfork (BLASFUNC(blas_thread_shutdown), NULL, NULL);
270+
if(err != 0)
271+
openblas_warning(0, "OpenBLAS Warning ... cannot install fork handler. You may meet hang after fork.\n");
272+
#endif
273+
}
274+
256275
int blas_get_cpu_number(void){
257276
char *p;
258277
#if defined(OS_LINUX) || defined(OS_WINDOWS) || defined(OS_FREEBSD) || defined(OS_DARWIN)
@@ -1268,6 +1287,9 @@ void CONSTRUCTOR gotoblas_init(void) {
12681287

12691288
if (gotoblas_initialized) return;
12701289

1290+
#ifdef SMP
1291+
openblas_fork_handler();
1292+
#endif
12711293

12721294
#ifdef PROFILE
12731295
moncontrol (0);
@@ -1288,11 +1310,7 @@ void CONSTRUCTOR gotoblas_init(void) {
12881310
#ifdef SMP
12891311
if (blas_cpu_number == 0) blas_get_cpu_number();
12901312
#ifdef SMP_SERVER
1291-
if (blas_server_avail == 0) {
1292-
blas_thread_init();
1293-
//deal with pthread and fork.
1294-
openblas_fork_handler();
1295-
}
1313+
if (blas_server_avail == 0) blas_thread_init();
12961314
#endif
12971315
#endif
12981316

utest/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ CUNIT_LIB=$(CUNIT_DIR)/lib/libcunit.a
1111

1212
CFLAGS+=-I$(CUNIT_DIR)/include
1313

14-
OBJS=main.o test_rot.o test_swap.o test_axpy.o test_dotu.o test_rotmg.o test_dsdot.o test_amax.o
14+
OBJS=main.o test_rot.o test_swap.o test_axpy.o test_dotu.o test_rotmg.o test_dsdot.o test_amax.o test_fork.o
1515

1616
all : run_test
1717

utest/common_utest.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,6 @@ void test_dsdot_n_1(void);
6363

6464
void test_samax(void);
6565

66+
void test_fork_safety(void);
67+
6668
#endif

utest/main.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ CU_TestInfo test_level1[]={
6060
{"Testing dsdot with n == 1",test_dsdot_n_1},
6161

6262
{"Testing samax", test_samax},
63+
64+
#if !defined(USE_OPENMP) && !defined(OS_WINDOWS)
65+
// The GNU OpenMP implementation libgomp is not fork-safe (as of 4.8.2):
66+
// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=60035
67+
// Hence skip this test when OpenBLAS is built with OpenMP.
68+
{"Testing fork safety", test_fork_safety},
69+
#endif
70+
6371
CU_TEST_INFO_NULL,
6472
};
6573

utest/test_fork.c

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*****************************************************************************
2+
Copyright (c) 2014, Lab of Parallel Software and Computational Science,ICSAS
3+
All rights reserved.
4+
5+
Redistribution and use in source and binary forms, with or without
6+
modification, are permitted provided that the following conditions are
7+
met:
8+
9+
1. Redistributions of source code must retain the above copyright
10+
notice, this list of conditions and the following disclaimer.
11+
12+
2. Redistributions in binary form must reproduce the above copyright
13+
notice, this list of conditions and the following disclaimer in
14+
the documentation and/or other materials provided with the
15+
distribution.
16+
3. Neither the name of the ISCAS nor the names of its contributors may
17+
be used to endorse or promote products derived from this software
18+
without specific prior written permission.
19+
20+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
29+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30+
31+
**********************************************************************************/
32+
33+
#ifndef OS_WINDOWS
34+
#include "common_utest.h"
35+
#include <sys/wait.h>
36+
#include <cblas.h>
37+
38+
void* xmalloc(size_t n)
39+
{
40+
void* tmp;
41+
tmp = malloc(n);
42+
if (tmp == NULL) {
43+
fprintf(stderr, "You are about to die\n");
44+
exit(1);
45+
} else {
46+
return tmp;
47+
}
48+
}
49+
50+
void check_dgemm(double *a, double *b, double *result, double *expected, int n)
51+
{
52+
int i;
53+
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, n, n, n,
54+
1.0, a, n, b, n, 0.0, result, n);
55+
for(i = 0; i < n * n; ++i) {
56+
CU_ASSERT_DOUBLE_EQUAL(expected[i], result[i], CHECK_EPS);
57+
}
58+
}
59+
60+
void test_fork_safety(void)
61+
{
62+
int n = 1000;
63+
int i;
64+
65+
double *a, *b, *c, *d;
66+
size_t n_bytes;
67+
68+
pid_t fork_pid;
69+
pid_t fork_pid_nested;
70+
71+
n_bytes = sizeof(*a) * n * n;
72+
73+
a = xmalloc(n_bytes);
74+
b = xmalloc(n_bytes);
75+
c = xmalloc(n_bytes);
76+
d = xmalloc(n_bytes);
77+
78+
// Put ones in a and b
79+
for(i = 0; i < n * n; ++i) {
80+
a[i] = 1;
81+
b[i] = 1;
82+
}
83+
84+
// Compute a DGEMM product in the parent process prior to forking to
85+
// ensure that the OpenBLAS thread pool is initialized.
86+
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, n, n, n,
87+
1.0, a, n, b, n, 0.0, c, n);
88+
89+
fork_pid = fork();
90+
if (fork_pid == -1) {
91+
CU_FAIL("Failed to fork process.");
92+
} else if (fork_pid == 0) {
93+
// Compute a DGEMM product in the child process to check that the
94+
// thread pool as been properly been reinitialized after the fork.
95+
check_dgemm(a, b, d, c, n);
96+
97+
// Nested fork to check that the pthread_atfork protection can work
98+
// recursively
99+
fork_pid_nested = fork();
100+
if (fork_pid_nested == -1) {
101+
CU_FAIL("Failed to fork process.");
102+
exit(1);
103+
} else if (fork_pid_nested == 0) {
104+
check_dgemm(a, b, d, c, n);
105+
exit(0);
106+
} else {
107+
check_dgemm(a, b, d, c, n);
108+
int child_status = 0;
109+
pid_t wait_pid = wait(&child_status);
110+
CU_ASSERT(wait_pid == fork_pid_nested);
111+
CU_ASSERT(WEXITSTATUS (child_status) == 0);
112+
exit(0);
113+
}
114+
} else {
115+
check_dgemm(a, b, d, c, n);
116+
// Wait for the child to finish and check the exit code.
117+
int child_status = 0;
118+
pid_t wait_pid = wait(&child_status);
119+
CU_ASSERT(wait_pid == fork_pid);
120+
CU_ASSERT(WEXITSTATUS (child_status) == 0);
121+
}
122+
}
123+
#endif

0 commit comments

Comments
 (0)