Skip to content

Commit 9dcddc6

Browse files
houjunhoujun
authored
Merge pull request #2 from hpc-io/develop
Update linux.yml
2 parents 990c456 + b06d31c commit 9dcddc6

File tree

4 files changed

+95
-103
lines changed

4 files changed

+95
-103
lines changed

‎.github/workflows/linux.yml‎

Lines changed: 20 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -42,20 +42,21 @@ jobs:
4242
sudo apt-get install libtool
4343
sudo apt install cmake
4444
# hdf5
45-
git clone --depth 1 https://github.com/HDFGroup/hdf5.git
45+
git clone https://github.com/HDFGroup/hdf5.git
4646
# libfabric 1.12.1
4747
wget https://github.com/ofiwg/libfabric/archive/refs/tags/v1.12.1.tar.gz
4848
tar xf v1.12.1.tar.gz
4949
# mercury
50-
git clone https://github.com/mercury-hpc/mercury.git
50+
git clone https://github.com/mercury-hpc/mercury.git --recursive
5151
# mpi
5252
sudo apt-get install libopenmpi-dev
5353
# zlib
5454
# sudo apt-get install zlib1g-dev
5555
# python3
5656
sudo apt-get install python3
5757
# pdc
58-
git clone --depth 1 https://github.com/hpc-io/pdc.git
58+
git clone https://github.com/hpc-io/pdc.git
59+
5960
6061
- name: Installation
6162
run: |
@@ -66,17 +67,17 @@ jobs:
6667
export VOL_DIR=$mydir/vol-pdc
6768
export MERCURY_DIR=$mydir/mercury/install
6869
export LIBFABRIC_DIR=$mydir/libfabric-1.12.1/install
69-
export PDC_DIR=$mydir/pdc/src/install
70-
export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
70+
export PDC_DIR=$mydir/pdc/install
71+
export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$PDC_DIR/lib:$VOL_DIR/install/lib:$LD_LIBRARY_PATH"
7172
export PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"
7273
7374
# Compile HDF5
7475
mkdir $HDF5_DIR
7576
cd hdf5
77+
git checkout hdf5-1_14_1-2
7678
export HDF5_LIBTOOL=/usr/bin/libtoolize
7779
./autogen.sh
78-
./configure --prefix=$HDF5_DIR --enable-parallel --enable-threadsafe --enable-unsupported \
79-
--disable-tests --disable-hl --disable-fortran
80+
./configure CC=mpicc --prefix=$HDF5_DIR --enable-parallel --disable-tests --disable-hl --disable-fortran
8081
make -j2 && make install
8182
8283
# Compile libfabric
@@ -91,45 +92,34 @@ jobs:
9192
# Compile Mercury
9293
mkdir $MERCURY_DIR
9394
cd $MERCURY_DIR
94-
git submodule update --init
95-
cmake ../ -DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF
95+
git checkout v2.1.0
96+
cmake ../ -DCMAKE_C_COMPILER=gcc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DMERCURY_USE_CHECKSUMS=OFF
9697
make -j2 && sudo make install
9798
ctest
9899
99100
# Compile PDC
100101
mkdir $PDC_DIR
101102
cd $PDC_DIR
102-
cmake ../ -DBUILD_MPI_TESTING=OFF -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc
103+
git checkout develop
104+
cmake ../ -DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc
103105
make -j2 && sudo make install
104106
105107
# Compile VOL-PDC
106108
cd $VOL_DIR
107109
mkdir build
108110
cd build
109-
cmake ../ -DHDF5_INCLUDE_DIR=$HDF5_INCLUDE_DIR -DHDF5_LIBRARY=$HDF5_LIBRARY -DBUILD_SHARED_LIBS=ON -DHDF5_DIR=$HDF5_DIR
111+
cmake ../ -DCMAKE_INSTALL_PREFIX=$VOL_DIR/install -DHDF5_INCLUDE_DIR=$HDF5_INCLUDE_DIR -DHDF5_LIBRARY=$HDF5_LIBRARY -DBUILD_SHARED_LIBS=ON -DHDF5_DIR=$HDF5_DIR -DPDC_DIR=$PDC_DIR/share/cmake/pdc -DBUILD_EXAMPLES=ON
110112
make && sudo make install
111-
112-
- name: Compile and Test VOL-PDC
113-
run: |
114-
mydir="$PWD"
115113
116-
# Set Environmental Variables
117-
export VOL_DIR=$mydir/vol-pdc
118-
export H5_DIR=$mydir/hdf5
119-
export HDF5_DIR=$mydir/hdf5/install
120-
export MERCURY_DIR=$mydir/mercury/install
121-
export LIBFABRIC_DIR=$mydir/libfabric-1.12.1/install
122-
export PDC_DIR=$mydir/pdc/src/install
123-
export LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
124-
export HDF5_PLUGIN_PATH="$VOL_DIR/build/bin"
114+
export HDF5_PLUGIN_PATH=$VOL_DIR/install/lib
125115
export HDF5_VOL_CONNECTOR="pdc under_vol=0;under_info={}"
126-
127-
# Compile example
128-
cd $VOL_DIR/examples
129-
cmake .
130-
make
116+
export LD_PRELOAD=$VOL_DIR/install/lib/libhdf5_vol_pdc.so
131117
132118
# Run test
133119
mpirun -N 1 -n 1 -c 1 $PDC_DIR/bin/pdc_server.exe &
134-
mpirun -N 1 -n 1 -c 1 ./h5pdc_vpicio test
120+
mpirun -N 1 -n 1 -c 1 ./bin/h5pdc_vpicio test
135121
mpirun -N 1 -n 1 -c 1 $PDC_DIR/bin/close_server
122+
123+
- name: Setup tmate session
124+
if: ${{ failure() }}
125+
uses: mxschmitt/action-tmate@v3

‎CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -300,8 +300,8 @@ install(
300300

301301
install(
302302
FILES
303-
${CMAKE_CURRENT_SOURCE_DIR}/examples/h5pdc_vpicio
304-
${CMAKE_CURRENT_SOURCE_DIR}/examples/h5pdc_vpicio_open
303+
${HDF5_VOL_PDC_BINARY_DIR}/bin/h5pdc_vpicio
304+
${HDF5_VOL_PDC_BINARY_DIR}/bin/h5pdc_vpicio_open
305305
DESTINATION
306306
${CMAKE_INSTALL_PREFIX}/bin
307307
)

‎examples/h5pdc_vpicio.c‎

Lines changed: 64 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ uniform_random_number()
1919
int
2020
main(int argc, char *argv[])
2121
{
22-
hid_t file_id, group_id, fapl_id, lcpl, gcpl;
22+
hid_t file_id, group_id, fapl_id, dxpl_id;
2323
hid_t dset_id1, dset_id2, dset_id3, dset_id4, dset_id5, dset_id6, dset_id7, dset_id8;
24-
hid_t r_dset_id1, r_dset_id2, r_dset_id3, r_dset_id4, r_dset_id5, r_dset_id6, r_dset_id7, r_dset_id8;
24+
hid_t r_dset_id1, r_dset_id2;
2525
hid_t filespace, memspace;
2626
hid_t attr1;
2727
hid_t aid1;
@@ -34,24 +34,18 @@ main(int argc, char *argv[])
3434
my_rank = 0;
3535
num_procs = 1;
3636
// Variables and dimensions
37-
// long numparticles = 8388608; // 8 meg particles per process
38-
long numparticles = 4;
37+
long numparticles = 8388608; // 8 meg particles per process
38+
/* long numparticles = 4; */
3939
long long total_particles, offset;
4040

4141
float *x, *y, *z;
42-
float *xr;
4342
float *px, *py, *pz;
4443
int * id1, *id2;
4544
int x_dim = 64;
4645
int y_dim = 64;
4746
int z_dim = 64;
48-
int i, j;
49-
50-
float matrix[ADIM1][ADIM2]; /* Attribute data */
51-
for (i = 0; i < ADIM1; i++) { /* Values of the array attribute */
52-
for (j = 0; j < ADIM2; j++)
53-
matrix[i][j] = -1.;
54-
}
47+
int i;
48+
double stime, etime;
5549

5650
MPI_Init(&argc, &argv);
5751
MPI_Comm_dup(MPI_COMM_WORLD, &comm);
@@ -66,16 +60,12 @@ main(int argc, char *argv[])
6660
if (my_rank == 0)
6761
printf("Number of paritcles: %ld \n", numparticles);
6862

69-
x = (float *)malloc(numparticles * sizeof(float));
70-
y = (float *)malloc(numparticles * sizeof(float));
71-
z = (float *)malloc(numparticles * sizeof(float));
72-
73-
xr = (float *)malloc(numparticles * sizeof(float));
74-
75-
px = (float *)malloc(numparticles * sizeof(float));
76-
py = (float *)malloc(numparticles * sizeof(float));
77-
pz = (float *)malloc(numparticles * sizeof(float));
78-
63+
x = (float *)malloc(numparticles * sizeof(float));
64+
y = (float *)malloc(numparticles * sizeof(float));
65+
z = (float *)malloc(numparticles * sizeof(float));
66+
px = (float *)malloc(numparticles * sizeof(float));
67+
py = (float *)malloc(numparticles * sizeof(float));
68+
pz = (float *)malloc(numparticles * sizeof(float));
7969
id1 = (int *)malloc(numparticles * sizeof(int));
8070
id2 = (int *)malloc(numparticles * sizeof(int));
8171

@@ -95,24 +85,18 @@ main(int argc, char *argv[])
9585
printf("H5Pcreate() error\n");
9686
H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL);
9787

98-
/* Initialize VOL */
99-
// pdc_vol_id = H5VLregister_connector_by_name("pdc", H5P_DEFAULT); // not used if choosing to use
100-
// environmental variable
88+
memspace = H5Screate_simple(1, (hsize_t *)&numparticles, NULL);
89+
filespace = H5Screate_simple(1, (hsize_t *)&total_particles, NULL);
90+
91+
MPI_Barrier(MPI_COMM_WORLD);
92+
stime = MPI_Wtime();
10193

10294
/* Create file */
10395
if ((file_id = H5Fcreate(argv[1], H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0)
10496
printf("H5Fcreate() error\n");
10597

106-
memspace = H5Screate_simple(1, (hsize_t *)&numparticles, NULL);
107-
filespace = H5Screate_simple(1, (hsize_t *)&total_particles, NULL);
108-
109-
if ((lcpl = H5Pcreate(H5P_LINK_CREATE)) < 0)
110-
printf("lcpl H5Pcreate() error\n");
111-
if ((gcpl = H5Pcreate(H5P_GROUP_CREATE)) < 0)
112-
printf("gcpl H5Pcreate() error\n");
113-
11498
/* Create group */
115-
if ((group_id = H5Gcreate2(file_id, "group1", lcpl, gcpl, H5P_DEFAULT)) < 0)
99+
if ((group_id = H5Gcreate2(file_id, "group1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
116100
printf("H5Gcreate() error\n");
117101

118102
/* Create dataset */
@@ -125,64 +109,79 @@ main(int argc, char *argv[])
125109
dset_id7 = H5Dcreate(file_id, "id1", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
126110
dset_id8 = H5Dcreate(file_id, "id2", H5T_NATIVE_INT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
127111

128-
fapl_id = H5Pcreate(H5P_DATASET_XFER);
129-
H5Pset_dxpl_mpio(fapl_id, H5FD_MPIO_COLLECTIVE);
112+
MPI_Barrier(MPI_COMM_WORLD);
113+
etime = MPI_Wtime();
114+
if (my_rank == 0)
115+
printf("File/Group/Dataset create took %.2f seconds\n", etime - stime);
116+
117+
dxpl_id = H5Pcreate(H5P_DATASET_XFER);
118+
H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE);
130119
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t *)&offset, NULL, (hsize_t *)&numparticles, NULL);
131120

132121
MPI_Barrier(comm);
133-
ierr = H5Dwrite(dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, x);
122+
stime = MPI_Wtime();
123+
124+
ierr = H5Dwrite(dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, x);
134125
if (ierr < 0)
135126
printf("write x failed\n");
136-
ierr = H5Dwrite(dset_id2, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, y);
127+
ierr = H5Dwrite(dset_id2, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, y);
137128
if (ierr < 0)
138129
printf("write y failed\n");
139-
ierr = H5Dwrite(dset_id3, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, z);
130+
ierr = H5Dwrite(dset_id3, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, z);
140131
if (ierr < 0)
141132
printf("write z failed\n");
142-
ierr = H5Dwrite(dset_id4, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, px);
133+
ierr = H5Dwrite(dset_id4, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, px);
143134
if (ierr < 0)
144135
printf("write px failed\n");
145-
ierr = H5Dwrite(dset_id5, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, py);
136+
ierr = H5Dwrite(dset_id5, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, py);
146137
if (ierr < 0)
147138
printf("write py failed\n");
148-
ierr = H5Dwrite(dset_id6, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, pz);
139+
ierr = H5Dwrite(dset_id6, H5T_NATIVE_FLOAT, memspace, filespace, dxpl_id, pz);
149140
if (ierr < 0)
150141
printf("write pz failed\n");
151-
ierr = H5Dwrite(dset_id7, H5T_NATIVE_INT, memspace, filespace, fapl_id, id1);
142+
ierr = H5Dwrite(dset_id7, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id1);
152143
if (ierr < 0)
153144
printf("write id1 failed\n");
154-
ierr = H5Dwrite(dset_id8, H5T_NATIVE_INT, memspace, filespace, fapl_id, id2);
145+
ierr = H5Dwrite(dset_id8, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id2);
155146
if (ierr < 0)
156147
printf("write id2 failed\n");
157148

158-
r_dset_id1 = H5Dopen2(file_id, "x", H5P_DEFAULT);
159-
// r_dset_id2 = H5Dopen2(file_id, "y", H5P_DEFAULT);
160-
// r_dset_id3 = H5Dopen2(file_id, "z", H5P_DEFAULT);
161-
// r_dset_id4 = H5Dopen2(file_id, "px", H5P_DEFAULT);
162-
// r_dset_id5 = H5Dopen2(file_id, "py", H5P_DEFAULT);
163-
// r_dset_id6 = H5Dopen2(file_id, "pz", H5P_DEFAULT);
164-
// r_dset_id7 = H5Dopen2(file_id, "id1", H5P_DEFAULT);
165-
// r_dset_id8 = H5Dopen2(file_id, "id2", H5P_DEFAULT);
149+
MPI_Barrier(MPI_COMM_WORLD);
150+
etime = MPI_Wtime();
151+
if (my_rank == 0)
152+
printf("Dataset write took %.2f seconds\n", etime - stime);
153+
154+
r_dset_id1 = H5Dopen2(file_id, "id1", H5P_DEFAULT);
155+
r_dset_id2 = H5Dopen2(file_id, "id2", H5P_DEFAULT);
166156

167-
ierr = H5Dread(dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, xr);
157+
ierr = H5Dread(r_dset_id1, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id1);
168158
if (ierr < 0)
169159
printf("read dset1 failed\n");
170160

171-
fprintf(stderr, "\nxr vals:\n");
161+
ierr = H5Dread(r_dset_id2, H5T_NATIVE_INT, memspace, filespace, dxpl_id, id2);
162+
if (ierr < 0)
163+
printf("read r_dset2 failed\n");
164+
165+
if (H5Dclose(r_dset_id1) < 0)
166+
printf("H5Dclose dataset1 error\n");
167+
168+
if (H5Dclose(r_dset_id2) < 0)
169+
printf("H5Dclose dataset2 error\n");
170+
172171
for (loop = 0; loop < numparticles; loop++) {
173-
fprintf(stderr, "%f ", xr[loop]);
172+
if (id1[loop] != loop) {
173+
fprintf(stderr, "Error id1: %d / %d\n", id1[loop], loop);
174+
break;
175+
}
174176
}
175-
fprintf(stderr, "\nx vals:\n");
177+
176178
for (loop = 0; loop < numparticles; loop++) {
177-
fprintf(stderr, "%f ", x[loop]);
179+
if (id2[loop] != loop * 2) {
180+
fprintf(stderr, "Error id2: %d / %d\n", id2[loop], loop * 2);
181+
break;
182+
}
178183
}
179184

180-
ierr = H5Dread(r_dset_id1, H5T_NATIVE_FLOAT, memspace, filespace, fapl_id, xr);
181-
if (ierr < 0)
182-
printf("read r_dset1 failed\n");
183-
if (H5Dclose(r_dset_id1) < 0)
184-
printf("H5Dclose dataset8 error\n");
185-
186185
/* create attribute */
187186
aid1 = H5Screate(H5S_SCALAR);
188187
attr1 = H5Acreate2(dset_id1, "Integer attribute", H5T_NATIVE_INT, aid1, H5P_DEFAULT, H5P_DEFAULT);
@@ -234,9 +233,7 @@ main(int argc, char *argv[])
234233
printf("H5Fclose error\n");
235234
if (H5Pclose(fapl_id) < 0)
236235
printf("H5Pclose error\n");
237-
if (H5Pclose(lcpl) < 0)
238-
printf("H5Pclose error\n");
239-
if (H5Pclose(gcpl) < 0)
236+
if (H5Pclose(dxpl_id) < 0)
240237
printf("H5Pclose error\n");
241238

242239
if (my_rank == 0) {
@@ -252,7 +249,6 @@ main(int argc, char *argv[])
252249
free(id1);
253250
free(id2);
254251

255-
free(xr);
256-
(void)MPI_Finalize();
252+
MPI_Finalize();
257253
return 0;
258254
}

‎src/H5VLpdc.c‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
#define H5VL_PDC_SEQ_LIST_LEN 128
3838
/* (Uncomment to enable) */
3939
//#define ENABLE_LOGGING
40-
#define USE_REGION_TRANSFER
40+
#define USE_REGION_TRANSFER 1
4141

4242
/* Remove warnings when connector does not use callback arguments */
4343
#if defined(__cplusplus)
@@ -1312,7 +1312,7 @@ H5VL_pdc_dataset_write(size_t count, void *_dset[], hid_t mem_type_id[], hid_t m
13121312
#endif
13131313

13141314
H5VL_pdc_obj_t *dset;
1315-
uint64_t * offset;
1315+
uint64_t * offset = NULL;
13161316
size_t type_size;
13171317
int ndim;
13181318
pdcid_t region_x, region_xx;
@@ -1349,15 +1349,19 @@ H5VL_pdc_dataset_write(size_t count, void *_dset[], hid_t mem_type_id[], hid_t m
13491349
}
13501350
else
13511351
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "data dimension not supported");
1352+
1353+
/* printf("Rank %d: mem offset %lu\n", dset->my_rank, offset[0]); */
1354+
/* printf("Rank %d: mem count %lu\n", dset->my_rank, dims[0]); */
13521355
region_x = PDCregion_create(ndim, offset, dims);
13531356
dset->reg_id_from = region_x;
13541357

13551358
type_size = H5Tget_size(mem_type_id[u]);
13561359
H5VL__pdc_sel_to_recx_iov(file_space_id[u], type_size, offset);
13571360

1361+
/* printf("Rank %d: file offset %lu\n", dset->my_rank, offset[0]); */
1362+
/* printf("Rank %d: file count %lu\n", dset->my_rank, dims[0]); */
13581363
region_xx = PDCregion_create(ndim, offset, dims);
13591364
dset->reg_id_to = region_xx;
1360-
free(offset);
13611365

13621366
#ifdef USE_REGION_TRANSFER
13631367
transfer_request =
@@ -1397,6 +1401,8 @@ H5VL_pdc_dataset_write(size_t count, void *_dset[], hid_t mem_type_id[], hid_t m
13971401
HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't unmap object");
13981402
}
13991403
#endif
1404+
if (offset)
1405+
free(offset);
14001406
}
14011407

14021408
done:

0 commit comments

Comments
 (0)