1818void ADIOI_LUSTRE_Open (ADIO_File fd , int * error_code )
1919{
2020 int perm , old_mask , amode , amode_direct ;
21- int lumlen ;
21+ int lumlen , myrank , flag , set_layout = 0 , err ;
2222 struct lov_user_md * lum = NULL ;
2323 char * value ;
24+ ADIO_Offset str_factor = -1 , str_unit = 0 , start_iodev = -1 ;
2425
2526#if defined(MPICH ) || !defined(PRINT_ERR_MSG )
2627 static char myname [] = "ADIOI_LUSTRE_OPEN" ;
2728#endif
2829
30+ MPI_Comm_rank (fd -> comm , & myrank );
31+
2932 if (fd -> perm == ADIO_PERM_NULL ) {
3033 old_mask = umask (022 );
3134 umask (old_mask );
@@ -47,46 +50,103 @@ void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
4750
4851 amode_direct = amode | O_DIRECT ;
4952
50- fd -> fd_sys = open (fd -> filename , amode |O_CREAT , perm );
51-
52- if (fd -> fd_sys != -1 ) {
53- int err ;
54-
55- /* get file striping information and set it in info */
56- /* odd malloc here because lov_user_md contains some fixed data and
57- * then a list of 'lmm_objects' representing stripe */
58- lumlen = sizeof (struct lov_user_md ) +
59- MAX_LOV_UUID_COUNT * sizeof (struct lov_user_ost_data );
60- /* furthermore, Pascal Deveze reports that, even though we pass a
61- * "GETSTRIPE" (read) flag to the ioctl, if some of the values of this
62- * struct are uninitialzed, the call can give an error. calloc in case
63- * there are other members that must be initialized and in case
64- * lov_user_md struct changes in future */
65- lum = (struct lov_user_md * )ADIOI_Calloc (1 ,lumlen );
66- lum -> lmm_magic = LOV_USER_MAGIC ;
67- err = ioctl (fd -> fd_sys , LL_IOC_LOV_GETSTRIPE , (void * )lum );
68- if (!err ) {
69- value = (char * ) ADIOI_Malloc ((MPI_MAX_INFO_VAL + 1 )* sizeof (char ));
70-
71- fd -> hints -> striping_unit = lum -> lmm_stripe_size ;
72- sprintf (value , "%d" , lum -> lmm_stripe_size );
73- ADIOI_Info_set (fd -> info , "striping_unit" , value );
74-
75- fd -> hints -> striping_factor = lum -> lmm_stripe_count ;
76- sprintf (value , "%d" , lum -> lmm_stripe_count );
77- ADIOI_Info_set (fd -> info , "striping_factor" , value );
78-
79- fd -> hints -> fs_hints .lustre .start_iodevice = lum -> lmm_stripe_offset ;
80- sprintf (value , "%d" , lum -> lmm_stripe_offset );
81- ADIOI_Info_set (fd -> info , "romio_lustre_start_iodevice" , value );
82-
83- ADIOI_Free (value );
84- }
85- ADIOI_Free (lum );
86-
87- if (fd -> access_mode & ADIO_APPEND )
88- fd -> fp_ind = fd -> fp_sys_posn = lseek (fd -> fd_sys , 0 , SEEK_END );
8953 }
54+ /* odd length here because lov_user_md contains some fixed data and
55+ * then a list of 'lmm_objects' representing stripe */
56+ lumlen = sizeof (struct lov_user_md ) +
57+ MAX_LOV_UUID_COUNT * sizeof (struct lov_user_ost_data );
58+ lum = (struct lov_user_md * )ADIOI_Calloc (1 ,lumlen );
59+
60+ value = (char * ) ADIOI_Malloc ((MPI_MAX_INFO_VAL + 1 )* sizeof (char ));
61+ /* we already validated in LUSTRE_SetInfo that these are going to be the same */
62+ if (fd -> info != MPI_INFO_NULL ) {
63+ /* striping information */
64+ ADIOI_Info_get (fd -> info , "striping_unit" , MPI_MAX_INFO_VAL ,
65+ value , & flag );
66+ if (flag )
67+ str_unit = atoll (value );
68+
69+ ADIOI_Info_get (fd -> info , "striping_factor" , MPI_MAX_INFO_VAL ,
70+ value , & flag );
71+ if (flag )
72+ str_factor = atoll (value );
73+
74+ ADIOI_Info_get (fd -> info , "romio_lustre_start_iodevice" ,
75+ MPI_MAX_INFO_VAL , value , & flag );
76+ if (flag )
77+ start_iodev = atoll (value );
78+ }
79+ if ((str_factor > 0 ) || (str_unit > 0 ) || (start_iodev >= 0 ))
80+ set_layout = 1 ;
81+
82+ /* if hints were set, we need to delay creation of any lustre objects.
83+ * However, if we open the file with O_LOV_DELAY_CREATE and don't call the
84+ * follow-up ioctl, subsequent writes will fail */
85+ if (myrank == 0 && set_layout )
86+ amode = amode | O_LOV_DELAY_CREATE ;
87+
88+ fd -> fd_sys = open (fd -> filename , amode , perm );
89+ if (fd -> fd_sys == -1 ) goto fn_exit ;
90+
91+ /* we can only set these hints on new files */
92+ /* It was strange and buggy to open the file in the hint path. Instead,
93+ * we'll apply the file tunings at open time */
94+ if ((amode & O_CREAT ) && set_layout ) {
95+ /* if user has specified striping info, process 0 tries to set it */
96+ if (!myrank ) {
97+ lum -> lmm_magic = LOV_USER_MAGIC ;
98+ lum -> lmm_pattern = 0 ;
99+ /* crude check for overflow of lustre internal datatypes.
100+ * Silently cap to large value if user provides a value
101+ * larger than lustre supports */
102+ if (str_unit > UINT_MAX )
103+ lum -> lmm_stripe_size = UINT_MAX ;
104+ else
105+ lum -> lmm_stripe_size = str_unit ;
106+
107+ if (str_factor > USHRT_MAX )
108+ lum -> lmm_stripe_count = USHRT_MAX ;
109+ else
110+ lum -> lmm_stripe_count = str_factor ;
111+
112+ if (start_iodev > USHRT_MAX )
113+ lum -> lmm_stripe_offset = USHRT_MAX ;
114+ else
115+ lum -> lmm_stripe_offset = start_iodev ;
116+ err = ioctl (fd -> fd_sys , LL_IOC_LOV_SETSTRIPE , lum );
117+ if (err == -1 && errno != EEXIST ) {
118+ fprintf (stderr , "Failure to set stripe info %s \n" , strerror (errno ));
119+ /* not a fatal error, but user might care to know */
120+ }
121+ } /* End of striping parameters validation */
122+ }
123+
124+ /* Pascal Deveze reports that, even though we pass a
125+ * "GETSTRIPE" (read) flag to the ioctl, if some of the values of this
126+ * struct are uninitialzed, the call can give an error. zero it out in case
127+ * there are other members that must be initialized and in case
128+ * lov_user_md struct changes in future */
129+ memset (lum , 0 , lumlen );
130+ lum -> lmm_magic = LOV_USER_MAGIC ;
131+ err = ioctl (fd -> fd_sys , LL_IOC_LOV_GETSTRIPE , (void * )lum );
132+ if (!err ) {
133+
134+ fd -> hints -> striping_unit = lum -> lmm_stripe_size ;
135+ sprintf (value , "%d" , lum -> lmm_stripe_size );
136+ ADIOI_Info_set (fd -> info , "striping_unit" , value );
137+
138+ fd -> hints -> striping_factor = lum -> lmm_stripe_count ;
139+ sprintf (value , "%d" , lum -> lmm_stripe_count );
140+ ADIOI_Info_set (fd -> info , "striping_factor" , value );
141+
142+ fd -> hints -> fs_hints .lustre .start_iodevice = lum -> lmm_stripe_offset ;
143+ sprintf (value , "%d" , lum -> lmm_stripe_offset );
144+ ADIOI_Info_set (fd -> info , "romio_lustre_start_iodevice" , value );
145+
146+ }
147+
148+ if (fd -> access_mode & ADIO_APPEND )
149+ fd -> fp_ind = fd -> fp_sys_posn = lseek (fd -> fd_sys , 0 , SEEK_END );
90150
91151 if ((fd -> fd_sys != -1 ) && (fd -> access_mode & ADIO_APPEND ))
92152 fd -> fp_ind = fd -> fp_sys_posn = lseek (fd -> fd_sys , 0 , SEEK_END );
@@ -101,6 +161,9 @@ void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
101161 fd -> direct_write = fd -> direct_read = 0 ;
102162 }
103163 }
164+ fn_exit :
165+ ADIOI_Free (lum );
166+ ADIOI_Free (value );
104167
105168 /* --BEGIN ERROR HANDLING-- */
106169 if (fd -> fd_sys == -1 || ((fd -> fd_direct == -1 ) &&
0 commit comments