Skip to content

Topic/external32 v2 #6697

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 21, 2019
Merged

Topic/external32 v2 #6697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a96efb7
common/ompio: add comm_ompio_read_all/write_all functions
edgargabriel Apr 22, 2019
35be18b
common/ompio: rename ompio_cuda* to ompio_buffer*
edgargabriel May 20, 2019
d955753
common/ompio: abstraction for different convertor types
edgargabriel May 20, 2019
f6b3a0a
common/ompio: individual write of external32 works
edgargabriel May 20, 2019
ab56e6f
common/ompio: make individual read operations work.
edgargabriel May 20, 2019
27b2ec7
common/ompio: add support for read operations and collective I/O
edgargabriel May 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions ompi/mca/common/ompio/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ headers = \
common_ompio_aggregators.h \
common_ompio_print_queue.h \
common_ompio_request.h \
common_ompio_buffer.h \
common_ompio.h

sources = \
Expand All @@ -34,6 +35,7 @@ sources = \
common_ompio_file_open.c \
common_ompio_file_view.c \
common_ompio_file_read.c \
common_ompio_buffer.c \
common_ompio_file_write.c


Expand Down Expand Up @@ -74,10 +76,6 @@ else
ompidir = $(includedir)
endif

if OPAL_cuda_support
headers += common_ompio_cuda.h
sources += common_ompio_cuda.c
endif

# These two rules will sym link the "noinst" libtool library filename
# to the installable libtool library filename in the case where we are
Expand Down
20 changes: 17 additions & 3 deletions ompi/mca/common/ompio/common_ompio.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2016 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* Copyright (c) 2018 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2018 DataDirect Networks. All rights reserved.
Expand Down Expand Up @@ -66,7 +66,7 @@
#define OMPIO_LOCK_ENTIRE_FILE 0x00000080
#define OMPIO_LOCK_NEVER 0x00000100
#define OMPIO_LOCK_NOT_THIS_OP 0x00000200

#define OMPIO_DATAREP_NATIVE 0x00000400

#define OMPIO_ROOT 0

Expand Down Expand Up @@ -157,7 +157,8 @@ struct ompio_file_t {
ompi_communicator_t *f_comm;
const char *f_filename;
char *f_datarep;
opal_convertor_t *f_convertor;
opal_convertor_t *f_mem_convertor;
opal_convertor_t *f_file_convertor;
opal_info_t *f_info;
int32_t f_flags;
void *f_fs_ptr;
Expand Down Expand Up @@ -253,10 +254,16 @@ OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at (ompio_file_t *fh, OMPI_MPI_O
const void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);

OMPI_DECLSPEC int mca_common_ompio_file_write_all (ompio_file_t *fh, const void *buf,
int count, struct ompi_datatype_t *datatype,
ompi_status_public_t *status);

OMPI_DECLSPEC int mca_common_ompio_file_write_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
int count, struct ompi_datatype_t *datatype,
ompi_status_public_t *status);

OMPI_DECLSPEC int mca_common_ompio_file_iwrite_all (ompio_file_t *fp, const void *buf,
int count, struct ompi_datatype_t *datatype, ompi_request_t **request);

OMPI_DECLSPEC int mca_common_ompio_file_iwrite_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset, const void *buf,
int count, struct ompi_datatype_t *datatype, ompi_request_t **request);
Expand All @@ -282,10 +289,16 @@ OMPI_DECLSPEC int mca_common_ompio_file_iread_at (ompio_file_t *fh, OMPI_MPI_OFF
void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);

OMPI_DECLSPEC int mca_common_ompio_file_read_all (ompio_file_t *fh, void *buf, int count, struct ompi_datatype_t *datatype,
ompi_status_public_t * status);

OMPI_DECLSPEC int mca_common_ompio_file_read_at_all (ompio_file_t *fh, OMPI_MPI_OFFSET_TYPE offset,
void *buf, int count, struct ompi_datatype_t *datatype,
ompi_status_public_t * status);

OMPI_DECLSPEC int mca_common_ompio_file_iread_all (ompio_file_t *fp, void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);

OMPI_DECLSPEC int mca_common_ompio_file_iread_at_all (ompio_file_t *fp, OMPI_MPI_OFFSET_TYPE offset,
void *buf, int count, struct ompi_datatype_t *datatype,
ompi_request_t **request);
Expand Down Expand Up @@ -318,6 +331,7 @@ OMPI_DECLSPEC int mca_common_ompio_decode_datatype (struct ompio_file_t *fh,
int count,
const void *buf,
size_t *max_data,
opal_convertor_t *convertor,
struct iovec **iov,
uint32_t *iov_count);

Expand Down
63 changes: 34 additions & 29 deletions ompi/mca/common/ompio/common_ompio_cuda.c → ompi/mca/common/ompio/common_ompio_buffer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -27,18 +27,19 @@
#include "opal/mca/allocator/allocator.h"
#include "opal/mca/allocator/base/base.h"
#include "common_ompio.h"
#include "common_ompio_cuda.h"
#include "common_ompio_buffer.h"


static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread safety */
static opal_mutex_t mca_common_ompio_buffer_mutex; /* lock for thread safety */
static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;

static opal_atomic_int32_t mca_common_ompio_cuda_init = 0;
static opal_atomic_int32_t mca_common_ompio_buffer_init = 0;
static int32_t mca_common_ompio_pagesize=4096;
static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );
static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size );
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf );

#if OPAL_CUDA_SUPPORT
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is_gpu,
int *is_managed)
{
Expand All @@ -57,8 +58,9 @@ void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is

return;
}
#endif

static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
{
char *buf=NULL;
size_t realsize, numpages;
Expand All @@ -67,64 +69,67 @@ static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
realsize = numpages * mca_common_ompio_pagesize;

buf = malloc ( realsize);
#if OPAL_CUDA_SUPPORT
if ( NULL != buf ) {
mca_common_cuda_register ( ( char *)buf, realsize, NULL );
}
#endif
*size = realsize;
return buf;
}

static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf )
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf )
{
if ( NULL != buf ) {
#if OPAL_CUDA_SUPPORT
mca_common_cuda_unregister ( (char *) buf, NULL );
#endif
free ( buf );
}
return;
}

int mca_common_ompio_cuda_alloc_init ( void )
int mca_common_ompio_buffer_alloc_init ( void )
{
bool thread_safe=true;

if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_cuda_init, 1) > 1)
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_buffer_init, 1) > 1)
return OMPI_SUCCESS;

/* initialize static objects */
OBJ_CONSTRUCT(&mca_common_ompio_cuda_mutex, opal_mutex_t);
OBJ_CONSTRUCT(&mca_common_ompio_buffer_mutex, opal_mutex_t);

OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex );
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex );
/* lookup name of the allocator to use */
if(NULL == (mca_common_ompio_allocator_component = mca_allocator_component_lookup("basic"))) {
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
return OMPI_ERR_BUFFER;
}

/* create an instance of the allocator */
mca_common_ompio_allocator = mca_common_ompio_allocator_component->allocator_init(thread_safe,
mca_common_ompio_cuda_alloc_seg,
mca_common_ompio_cuda_free_seg,
mca_common_ompio_buffer_alloc_seg,
mca_common_ompio_buffer_free_seg,
NULL);
if(NULL == mca_common_ompio_allocator) {
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
return OMPI_ERR_BUFFER;
}

// mca_common_ompio_pagesize = sysconf(_SC_PAGESIZE);
mca_common_ompio_pagesize = opal_getpagesize();

OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
return OMPI_SUCCESS;
}

int mca_common_ompio_cuda_alloc_fini ( void )
int mca_common_ompio_buffer_alloc_fini ( void )
{
if ( NULL != mca_common_ompio_allocator ) {
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
mca_common_ompio_allocator->alc_finalize(mca_common_ompio_allocator);
mca_common_ompio_allocator=NULL;
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
OBJ_DESTRUCT (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
OBJ_DESTRUCT (&mca_common_ompio_buffer_mutex);
}

return OMPI_SUCCESS;
Expand All @@ -134,31 +139,31 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize )
{
char *tmp=NULL;

if ( !mca_common_ompio_cuda_init ){
mca_common_ompio_cuda_alloc_init ();
if ( !mca_common_ompio_buffer_init ){
mca_common_ompio_buffer_alloc_init ();
}

OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
tmp = mca_common_ompio_allocator->alc_alloc (mca_common_ompio_allocator,
bufsize, 0 );
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
return tmp;
}

void mca_common_ompio_release_buf ( ompio_file_t *fh, void *buf )
{

if ( !mca_common_ompio_cuda_init ){
if ( !mca_common_ompio_buffer_init ){
/* Should not happen. You can not release a buf without
** having it allocated first.
*/
opal_output (1, "error in mca_common_ompio_release_buf: allocator not initialized\n");
}

OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
mca_common_ompio_allocator->alc_free (mca_common_ompio_allocator,
buf);
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);

return;
}
Expand Down
31 changes: 25 additions & 6 deletions ompi/mca/common/ompio/common_ompio_cuda.h → ompi/mca/common/ompio/common_ompio_buffer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
Expand All @@ -22,9 +22,9 @@
#define MCA_COMMON_OMPIO_CUDA_H


#define OMPIO_CUDA_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
opal_convertor_clone ( _fh->f_convertor, _convertor, 0); \
opal_convertor_prepare_for_send ( _convertor, &(_datatype->super), _count, _buf );\
#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
OBJ_CONSTRUCT( _convertor, opal_convertor_t); \
opal_convertor_copy_and_prepare_for_send ( _fh->f_file_convertor, &(_datatype->super), _count, _buf, CONVERTOR_SEND_CONVERSION, _convertor ); \
opal_convertor_get_packed_size( _convertor, &_max_data ); \
_tbuf = mca_common_ompio_alloc_buf (_fh, _max_data); \
if ( NULL == _tbuf ) { \
Expand All @@ -40,11 +40,30 @@
_decoded_iov->iov_len = _max_data; \
_iov_count=1;}

#define OMPIO_PREPARE_READ_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
OBJ_CONSTRUCT( _convertor, opal_convertor_t); \
opal_convertor_copy_and_prepare_for_recv ( _fh->f_file_convertor, &(_datatype->super), _count, _buf, 0, _convertor ); \
opal_convertor_get_packed_size( _convertor, &_max_data ); \
_tbuf = mca_common_ompio_alloc_buf (_fh, _max_data); \
if ( NULL == _tbuf ) { \
opal_output(1, "common_ompio: error allocating memory\n"); \
return OMPI_ERR_OUT_OF_RESOURCE; \
} \
_decoded_iov = (struct iovec *) malloc ( sizeof ( struct iovec )); \
if ( NULL == _decoded_iov ) { \
opal_output(1, "common_ompio: could not allocate memory.\n"); \
return OMPI_ERR_OUT_OF_RESOURCE; \
} \
_decoded_iov->iov_base = _tbuf; \
_decoded_iov->iov_len = _max_data; \
_iov_count=1;}

#if OPAL_CUDA_SUPPORT
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf,
int *is_gpu, int *is_managed);
int mca_common_ompio_cuda_alloc_init ( void );
int mca_common_ompio_cuda_alloc_fini ( void );
#endif
int mca_common_ompio_buffer_alloc_init ( void );
int mca_common_ompio_buffer_alloc_fini ( void );


void* mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize);
Expand Down
Loading