Skip to content

Commit ce9da7c

Browse files
hppritchahppritcha
authored
Merge pull request #3490 from ggouaillardet/topic/v3.x/romio_ad_nfs
v3.x: romio314: ad_nfs fixes for large files from upstream mpich
2 parents ddc2652 + 0619797 commit ce9da7c

File tree

2 files changed

+93
-110
lines changed

2 files changed

+93
-110
lines changed

‎ompi/mca/io/romio314/romio/adio/ad_nfs/ad_nfs_read.c‎

Lines changed: 47 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -7,79 +7,68 @@
77

88
#include "ad_nfs.h"
99
#include "adio_extern.h"
10+
#ifdef HAVE_UNISTD_H
11+
#include <unistd.h>
12+
#endif
1013

1114
void ADIOI_NFS_ReadContig(ADIO_File fd, void *buf, int count,
1215
MPI_Datatype datatype, int file_ptr_type,
1316
ADIO_Offset offset, ADIO_Status *status, int *error_code)
1417
{
15-
int err=-1;
18+
ssize_t err=-1;
1619
MPI_Count datatype_size, len;
20+
ADIO_Offset bytes_xfered=0;
21+
size_t rd_count;
1722
static char myname[] = "ADIOI_NFS_READCONTIG";
23+
char *p;
1824

1925
MPI_Type_size_x(datatype, &datatype_size);
2026
len = datatype_size * count;
2127

22-
if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
23-
if (fd->fp_sys_posn != offset) {
24-
#ifdef ADIOI_MPE_LOGGING
25-
MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
26-
#endif
27-
lseek(fd->fd_sys, offset, SEEK_SET);
28-
#ifdef ADIOI_MPE_LOGGING
29-
MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
30-
#endif
31-
}
32-
if (fd->atomicity)
33-
ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
34-
else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
35-
#ifdef ADIOI_MPE_LOGGING
36-
MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
37-
#endif
38-
err = read(fd->fd_sys, buf, len);
39-
#ifdef ADIOI_MPE_LOGGING
40-
MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
41-
#endif
42-
ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
43-
fd->fp_sys_posn = offset + err;
44-
/* individual file pointer not updated */
45-
}
46-
else { /* read from curr. location of ind. file pointer */
28+
if (file_ptr_type == ADIO_INDIVIDUAL) {
4729
offset = fd->fp_ind;
48-
if (fd->fp_sys_posn != fd->fp_ind) {
49-
#ifdef ADIOI_MPE_LOGGING
50-
MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
51-
#endif
52-
lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
53-
#ifdef ADIOI_MPE_LOGGING
54-
MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
55-
#endif
56-
}
30+
}
31+
32+
p = buf;
33+
while (bytes_xfered < len ) {
34+
rd_count = len - bytes_xfered;
35+
/* FreeBSD and Darwin workaround: bigger than INT_MAX is an error */
36+
if (rd_count > INT_MAX)
37+
rd_count = INT_MAX;
5738
if (fd->atomicity)
58-
ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
59-
else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
39+
ADIOI_WRITE_LOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
40+
else ADIOI_READ_LOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
6041
#ifdef ADIOI_MPE_LOGGING
6142
MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
6243
#endif
63-
err = read(fd->fd_sys, buf, len);
44+
err = pread(fd->fd_sys, p, rd_count, offset+bytes_xfered);
45+
/* --BEGIN ERROR HANDLING-- */
46+
if (err == -1) {
47+
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
48+
MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO,
49+
"**io", "**io %s", strerror(errno));
50+
}
51+
/* --END ERROR HANDLING-- */
6452
#ifdef ADIOI_MPE_LOGGING
6553
MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
6654
#endif
67-
ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
68-
fd->fp_ind += err;
69-
fd->fp_sys_posn = fd->fp_ind;
55+
ADIOI_UNLOCK(fd, offset+bytes_xfered, SEEK_SET, rd_count);
56+
if (err == 0) {
57+
/* end of file */
58+
break;
59+
}
60+
bytes_xfered += err;
61+
p += err;
7062
}
7163

72-
/* --BEGIN ERROR HANDLING-- */
73-
if (err == -1) {
74-
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
75-
myname, __LINE__, MPI_ERR_IO,
76-
"**io", "**io %s", strerror(errno));
77-
return;
64+
fd->fp_sys_posn = offset + bytes_xfered;
65+
if (file_ptr_type == ADIO_INDIVIDUAL) {
66+
fd->fp_ind += bytes_xfered;
7867
}
7968
/* --END ERROR HANDLING-- */
8069

8170
#ifdef HAVE_STATUS_SET_BYTES
82-
MPIR_Status_set_bytes(status, datatype, err);
71+
if (err != -1) MPIR_Status_set_bytes(status, datatype, bytes_xfered);
8372
#endif
8473

8574
*error_code = MPI_SUCCESS;
@@ -168,8 +157,9 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
168157
/* offset is in units of etype relative to the filetype. */
169158

170159
ADIOI_Flatlist_node *flat_buf, *flat_file;
171-
int i, j, k, err=-1, brd_size, frd_size=0, st_index=0;
172-
int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;
160+
int i, j, k, err=-1, brd_size, st_index=0;
161+
int num, size, sum, n_etypes_in_filetype, size_in_filetype;
162+
MPI_Count bufsize;
173163
int n_filetypes, etype_in_filetype;
174164
ADIO_Offset abs_off_in_filetype=0;
175165
int req_len, partial_read;
@@ -179,8 +169,9 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
179169
ADIO_Offset userbuf_off;
180170
ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;
181171
char *readbuf, *tmp_buf, *value;
182-
int st_frd_size, st_n_filetypes, readbuf_len;
183-
int new_brd_size, new_frd_size, err_flag=0, info_flag, max_bufsize;
172+
int st_n_filetypes, readbuf_len;
173+
ADIO_Offset frd_size=0, new_frd_size, st_frd_size;
174+
int new_brd_size, err_flag=0, info_flag, max_bufsize;
184175

185176
static char myname[] = "ADIOI_NFS_READSTRIDED";
186177

@@ -460,12 +451,13 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
460451
else {
461452
/* noncontiguous in memory as well as in file */
462453

454+
ADIO_Offset i;
463455
ADIOI_Flatten_datatype(datatype);
464456
flat_buf = ADIOI_Flatlist;
465457
while (flat_buf->type != datatype) flat_buf = flat_buf->next;
466458

467459
k = num = buf_count = 0;
468-
i = (int) (flat_buf->indices[0]);
460+
i = flat_buf->indices[0];
469461
j = st_index;
470462
off = offset;
471463
n_filetypes = st_n_filetypes;
@@ -510,8 +502,8 @@ void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
510502

511503
k = (k + 1)%flat_buf->count;
512504
buf_count++;
513-
i = (int) (buftype_extent*(buf_count/flat_buf->count) +
514-
flat_buf->indices[k]);
505+
i = buftype_extent*(buf_count/flat_buf->count) +
506+
flat_buf->indices[k];
515507
new_brd_size = flat_buf->blocklens[k];
516508
if (size != frd_size) {
517509
off += size;

‎ompi/mca/io/romio314/romio/adio/ad_nfs/ad_nfs_write.c‎

Lines changed: 46 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -7,76 +7,64 @@
77

88
#include "ad_nfs.h"
99
#include "adio_extern.h"
10+
#ifdef HAVE_UNISTD_H
11+
#include <unistd.h>
12+
#endif
1013

1114
void ADIOI_NFS_WriteContig(ADIO_File fd, const void *buf, int count,
1215
MPI_Datatype datatype, int file_ptr_type,
1316
ADIO_Offset offset, ADIO_Status *status, int *error_code)
1417
{
15-
int err=-1;
18+
ssize_t err=-1;
1619
MPI_Count datatype_size, len;
20+
ADIO_Offset bytes_xfered=0;
21+
size_t wr_count;
1722
static char myname[] = "ADIOI_NFS_WRITECONTIG";
23+
char *p;
1824

1925
MPI_Type_size_x(datatype, &datatype_size);
20-
len = datatype_size * count;
26+
len = datatype_size * (ADIO_Offset)count;
2127

22-
if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
23-
if (fd->fp_sys_posn != offset) {
24-
#ifdef ADIOI_MPE_LOGGING
25-
MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
26-
#endif
27-
lseek(fd->fd_sys, offset, SEEK_SET);
28-
#ifdef ADIOI_MPE_LOGGING
29-
MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
30-
#endif
31-
}
32-
ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
33-
#ifdef ADIOI_MPE_LOGGING
34-
MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
35-
#endif
36-
err = write(fd->fd_sys, buf, len);
37-
#ifdef ADIOI_MPE_LOGGING
38-
MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
39-
#endif
40-
ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
41-
fd->fp_sys_posn = offset + err;
42-
/* individual file pointer not updated */
43-
}
44-
else { /* write from curr. location of ind. file pointer */
28+
if (file_ptr_type == ADIO_INDIVIDUAL) {
4529
offset = fd->fp_ind;
46-
if (fd->fp_sys_posn != fd->fp_ind) {
47-
#ifdef ADIOI_MPE_LOGGING
48-
MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
49-
#endif
50-
lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
51-
#ifdef ADIOI_MPE_LOGGING
52-
MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
53-
#endif
54-
}
55-
ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
30+
}
31+
32+
p = (char *)buf;
33+
while (bytes_xfered < len) {
5634
#ifdef ADIOI_MPE_LOGGING
5735
MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
5836
#endif
59-
err = write(fd->fd_sys, buf, len);
37+
wr_count = len - bytes_xfered;
38+
/* work around FreeBSD and OS X defects*/
39+
if (wr_count > INT_MAX)
40+
wr_count = INT_MAX;
41+
42+
ADIOI_WRITE_LOCK(fd, offset+bytes_xfered, SEEK_SET, wr_count);
43+
err = pwrite(fd->fd_sys, p, wr_count, offset+bytes_xfered);
44+
/* --BEGIN ERROR HANDLING-- */
45+
if (err == -1) {
46+
*error_code = MPIO_Err_create_code(MPI_SUCCESS,
47+
MPIR_ERR_RECOVERABLE,
48+
myname, __LINE__, MPI_ERR_IO, "**io",
49+
"**io %s", strerror(errno));
50+
fd->fp_sys_posn = -1;
51+
return;
52+
}
53+
/* --END ERROR HANDLING-- */
6054
#ifdef ADIOI_MPE_LOGGING
6155
MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
6256
#endif
63-
ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
64-
fd->fp_ind += err;
65-
fd->fp_sys_posn = fd->fp_ind;
57+
ADIOI_UNLOCK(fd, offset+bytes_xfered, SEEK_SET, wr_count);
58+
bytes_xfered += err;
59+
p += err;
6660
}
6761

68-
/* --BEGIN ERROR HANDLING-- */
69-
if (err == -1) {
70-
*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
71-
myname, __LINE__, MPI_ERR_IO,
72-
"**io",
73-
"**io %s", strerror(errno));
74-
return;
62+
if (file_ptr_type == ADIO_INDIVIDUAL) {
63+
fd->fp_ind += bytes_xfered;
7564
}
76-
/* --END ERROR HANDLING-- */
7765

7866
#ifdef HAVE_STATUS_SET_BYTES
79-
MPIR_Status_set_bytes(status, datatype, err);
67+
MPIR_Status_set_bytes(status, datatype, bytes_xfered);
8068
#endif
8169

8270
*error_code = MPI_SUCCESS;
@@ -272,8 +260,9 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
272260
/* offset is in units of etype relative to the filetype. */
273261

274262
ADIOI_Flatlist_node *flat_buf, *flat_file;
275-
int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
276-
int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;
263+
int i, j, k, err=-1, bwr_size, st_index=0;
264+
int num, size, sum, n_etypes_in_filetype, size_in_filetype;
265+
MPI_Count bufsize;
277266
int n_filetypes, etype_in_filetype;
278267
ADIO_Offset abs_off_in_filetype=0;
279268
int req_len;
@@ -283,8 +272,9 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
283272
ADIO_Offset userbuf_off;
284273
ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
285274
char *writebuf=NULL, *value;
286-
int st_fwr_size, st_n_filetypes, writebuf_len, write_sz;
287-
int new_bwr_size, new_fwr_size, err_flag=0, info_flag, max_bufsize;
275+
int st_n_filetypes, writebuf_len, write_sz;
276+
ADIO_Offset fwr_size = 0, new_fwr_size, st_fwr_size;
277+
int new_bwr_size, err_flag=0, info_flag, max_bufsize;
288278
static char myname[] = "ADIOI_NFS_WRITESTRIDED";
289279

290280
ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
@@ -565,12 +555,13 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
565555
else {
566556
/* noncontiguous in memory as well as in file */
567557

558+
ADIO_Offset i;
568559
ADIOI_Flatten_datatype(datatype);
569560
flat_buf = ADIOI_Flatlist;
570561
while (flat_buf->type != datatype) flat_buf = flat_buf->next;
571562

572563
k = num = buf_count = 0;
573-
i = (int) (flat_buf->indices[0]);
564+
i = flat_buf->indices[0];
574565
j = st_index;
575566
off = offset;
576567
n_filetypes = st_n_filetypes;
@@ -616,8 +607,8 @@ void ADIOI_NFS_WriteStrided(ADIO_File fd, const void *buf, int count,
616607

617608
k = (k + 1)%flat_buf->count;
618609
buf_count++;
619-
i = (int) (buftype_extent*(buf_count/flat_buf->count) +
620-
flat_buf->indices[k]);
610+
i = buftype_extent*(buf_count/flat_buf->count) +
611+
flat_buf->indices[k];
621612
new_bwr_size = flat_buf->blocklens[k];
622613
if (size != fwr_size) {
623614
off += size;

0 commit comments

Comments
 (0)