Skip to content

Commit bfe22b5

Browse files
oehmkeoehmke
committed
Merge fix improving the memory scalability of Fields.
Merge branch 'fix/array-scaling' into develop
2 parents 82f8052 + 5c7af32 commit bfe22b5

File tree

2 files changed

+55
-74
lines changed

2 files changed

+55
-74
lines changed

‎src/Infrastructure/Array/include/ESMCI_Array.h‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -203,13 +203,6 @@ namespace ESMCI {
203203
// entry of 0 indicates replicated dim
204204
// distr. Array dims as 1, 2, 3, .. only
205205
int *contiguousFlag; // [ssiLocalDeCount]
206-
int *exclusiveElementCountPDe; // [deCount] number of elements in
207-
// exclusive region only considering
208-
// DistGrid dims that are associated with
209-
// the Array dims.
210-
// Multiply with tensorElementCount to get
211-
// total number of elements in exclusive
212-
// Array region.
213206
int *totalElementCountPLocalDe; // [ssiLocalDeCount] number of elements in
214207
// total region only considering
215208
// DistGrid dims that are associated with
@@ -272,7 +265,6 @@ namespace ESMCI {
272265
distgridToPackedArrayMap = NULL;
273266
contiguousFlag = NULL;
274267
tensorElementCount = 0;
275-
exclusiveElementCountPDe = NULL;
276268
totalElementCountPLocalDe = NULL;
277269
sizeSuperUndist = NULL;
278270
sizeDist = NULL;
@@ -311,7 +303,6 @@ namespace ESMCI {
311303
distgridToPackedArrayMap = NULL;
312304
contiguousFlag = NULL;
313305
tensorElementCount = 0;
314-
exclusiveElementCountPDe = NULL;
315306
totalElementCountPLocalDe = NULL;
316307
sizeSuperUndist = NULL;
317308
sizeDist = NULL;
@@ -392,8 +383,6 @@ namespace ESMCI {
392383
int getReplicatedDimCount() const {return replicatedDimCount;}
393384
const int *getUndistLBound() const {return undistLBound;}
394385
const int *getUndistUBound() const {return undistUBound;}
395-
const int *getExclusiveElementCountPDe()const
396-
{return exclusiveElementCountPDe;}
397386
const int *getTotalElementCountPLocalDe()const
398387
{return totalElementCountPLocalDe;}
399388
int getTensorElementCount() const {return tensorElementCount;}

‎src/Infrastructure/Array/src/ESMCI_Array.C‎

Lines changed: 55 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -273,21 +273,6 @@ Array::Array(
273273
dimCount * sizeof(int));
274274
// indexflag
275275
indexflag = indexflagArg;
276-
// exclusiveElementCountPDe
277-
int deCount = delayout->getDeCount();
278-
exclusiveElementCountPDe = new int[deCount];
279-
const int *indexCountPDimPDe = distgrid->getIndexCountPDimPDe();
280-
for (int i=0; i<deCount; i++){
281-
exclusiveElementCountPDe[i] = 1; // prime exclusiveElementCountPDe element
282-
for (int jj=0; jj<rank; jj++){
283-
int j = arrayToDistGridMap[jj];// j is dimIndex basis 1, or 0 for tensor d
284-
if (j){
285-
// decomposed dimension
286-
--j; // shift to basis 0
287-
exclusiveElementCountPDe[i] *= indexCountPDimPDe[i*dimCount+j];
288-
}
289-
}
290-
}
291276
// totalElementCountPLocalDe
292277
totalElementCountPLocalDe = new int[ssiLocalDeCount];
293278
for (int i=0; i<ssiLocalDeCount; i++){
@@ -439,8 +424,6 @@ void Array::destruct(bool followCreator, bool noGarbage){
439424
delete [] distgridToPackedArrayMap;
440425
if (contiguousFlag != NULL)
441426
delete [] contiguousFlag;
442-
if (exclusiveElementCountPDe != NULL)
443-
delete [] exclusiveElementCountPDe;
444427
if (totalElementCountPLocalDe != NULL)
445428
delete [] totalElementCountPLocalDe;
446429
if (sizeSuperUndist != NULL)
@@ -2412,11 +2395,6 @@ Array *Array::create(
24122395
arrayOut->distgridToPackedArrayMap = new int[dimCount];
24132396
memcpy(arrayOut->distgridToPackedArrayMap,
24142397
arrayIn->distgridToPackedArrayMap, dimCount * sizeof(int));
2415-
// exclusiveElementCountPDe
2416-
int deCount = arrayIn->delayout->getDeCount();
2417-
arrayOut->exclusiveElementCountPDe = new int[deCount];
2418-
memcpy(arrayOut->exclusiveElementCountPDe,
2419-
arrayIn->exclusiveElementCountPDe, deCount * sizeof(int));
24202398

24212399
// Set up rim members and fill with canonical seqIndex values
24222400
arrayOut->setRimMembers();
@@ -2623,12 +2601,6 @@ Array *Array::create(
26232601
if (ssiLocalDeCount)
26242602
memcpy(arrayOut->contiguousFlag, arrayIn->contiguousFlag,
26252603
ssiLocalDeCount * sizeof(int));
2626-
// exclusiveElementCountPDe
2627-
int deCount = arrayIn->delayout->getDeCount();
2628-
arrayOut->exclusiveElementCountPDe = new int[deCount];
2629-
if (deCount)
2630-
memcpy(arrayOut->exclusiveElementCountPDe,
2631-
arrayIn->exclusiveElementCountPDe, deCount * sizeof(int));
26322604
// totalElementCountPLocalDe
26332605
arrayOut->totalElementCountPLocalDe = new int[ssiLocalDeCount];
26342606
if (ssiLocalDeCount)
@@ -4233,30 +4205,6 @@ int Array::print()const{
42334205
int de = localDeToDeMap[i];
42344206
printf("~ local data in LocalArray for DE %d ~\n", de);
42354207
larrayList[i]->print();
4236-
if (exclusiveElementCountPDe[de]){
4237-
// associated DE
4238-
int j=0; // reset
4239-
int jjj=0; // reset
4240-
for (int jj=0; jj<rank; jj++){
4241-
if (arrayToDistGridMap[jj]){
4242-
// distributed dimension
4243-
printf("dim %d: [%d]: [%d [%d [%d, %d] %d] %d]\n",
4244-
jj+1, j,
4245-
totalLBound[i*redDimCount+j], computationalLBound[i*redDimCount+j],
4246-
exclusiveLBound[i*redDimCount+j], exclusiveUBound[i*redDimCount+j],
4247-
computationalUBound[i*redDimCount+j], totalUBound[i*redDimCount+j]);
4248-
++j;
4249-
}else{
4250-
// non-distributed dimension
4251-
printf("dim %d: undistLBound[%d]=%d undistUBound[%d]=%d\n",
4252-
jj+1, jjj, undistLBound[jjj], jjj, undistUBound[jjj]);
4253-
++jjj;
4254-
}
4255-
}
4256-
}else{
4257-
// unassociated DE
4258-
printf("this DE is not associated with DistGrid points\n");
4259-
}
42604208
}
42614209
printf("--- ESMCI::Array::print end ---\n");
42624210

@@ -4535,11 +4483,8 @@ int Array::serialize(
45354483
*ip++ = distgridToPackedArrayMap[i];
45364484
*ip++ = tensorElementCount;
45374485
*ip++ = replicatedDimCount;
4538-
for (int i=0; i<delayout->getDeCount(); i++)
4539-
*ip++ = exclusiveElementCountPDe[i];
45404486
} else
4541-
ip += 3 + 2*tensorCount + 2*distgrid->getDimCount () +
4542-
rank + delayout->getDeCount ();
4487+
ip += 3 + 2*tensorCount + 2*distgrid->getDimCount () + rank;
45434488

45444489
// fix offset
45454490
cp = (char *)ip;
@@ -4631,9 +4576,6 @@ int Array::deserialize(
46314576
distgridToPackedArrayMap[i] = *ip++;
46324577
tensorElementCount = *ip++;
46334578
replicatedDimCount = *ip++;
4634-
exclusiveElementCountPDe = new int[delayout->getDeCount()];
4635-
for (int i=0; i<delayout->getDeCount(); i++)
4636-
exclusiveElementCountPDe[i] = *ip++;
46374579

46384580
// fix offset
46394581
cp = (char *)ip;
@@ -4791,6 +4733,20 @@ int Array::gather(
47914733

47924734
int redDimCount = rank - tensorCount;
47934735

4736+
// setup exclusiveElementCountPDe
4737+
vector<int> exclusiveElementCountPDe(deCount);
4738+
for (int i=0; i<deCount; i++){
4739+
exclusiveElementCountPDe[i] = 1; // prime exclusiveElementCountPDe element
4740+
for (int jj=0; jj<rank; jj++){
4741+
int j = arrayToDistGridMap[jj];// j is dimIndex basis 1, or 0 for tensor d
4742+
if (j){
4743+
// decomposed dimension
4744+
--j; // shift to basis 0
4745+
exclusiveElementCountPDe[i] *= indexCountPDimPDe[i*dimCount+j];
4746+
}
4747+
}
4748+
}
4749+
47944750
// prepare for comms
47954751
VMK::commhandle *commh; // used by all comm calls
47964752
vector<VMK::commhandle*> commhList(dimCount); // used for indexList comm
@@ -5188,6 +5144,20 @@ int Array::scatter(
51885144

51895145
int redDimCount = rank - tensorCount;
51905146

5147+
// setup exclusiveElementCountPDe
5148+
vector<int> exclusiveElementCountPDe(deCount);
5149+
for (int i=0; i<deCount; i++){
5150+
exclusiveElementCountPDe[i] = 1; // prime exclusiveElementCountPDe element
5151+
for (int jj=0; jj<rank; jj++){
5152+
int j = arrayToDistGridMap[jj];// j is dimIndex basis 1, or 0 for tensor d
5153+
if (j){
5154+
// decomposed dimension
5155+
--j; // shift to basis 0
5156+
exclusiveElementCountPDe[i] *= indexCountPDimPDe[i*dimCount+j];
5157+
}
5158+
}
5159+
}
5160+
51915161
// prepare for comms
51925162
VMK::commhandle *commh; // used by all comm calls
51935163
vector<VMK::commhandle*> commhList(dimCount); // used for indexList comm
@@ -9378,17 +9348,29 @@ template<typename SIT, typename DIT>
93789348
const int srcLocalDeCount = srcArray->delayout->getLocalDeCount();
93799349
const int *srcLocalDeToDeMap = srcArray->delayout->getLocalDeToDeMap();
93809350
int *srcLocalDeElementCount = new int[srcLocalDeCount];
9351+
const int *srcIndexCountPDimPDe = srcArray->distgrid->getIndexCountPDimPDe();
93819352
int srcElementCount = 0; // initialize
93829353
for (int i=0; i<srcLocalDeCount; i++){
93839354
int de = srcLocalDeToDeMap[i]; // global DE index
9384-
srcLocalDeElementCount[i] = srcArray->exclusiveElementCountPDe[de]
9385-
* srcArray->tensorElementCount;
9355+
srcLocalDeElementCount[i] = 1; // prime the element count
9356+
for (int jj=0; jj<srcArray->rank; jj++){
9357+
int j = srcArray->arrayToDistGridMap[jj]; // j is dimIndex basis 1,
9358+
// or 0 for tensor dim
9359+
if (j){
9360+
// decomposed dimension
9361+
--j; // shift to basis 0
9362+
srcLocalDeElementCount[i] *=
9363+
srcIndexCountPDimPDe[de*(srcArray->distgrid->getDimCount())+j];
9364+
}
9365+
}
9366+
srcLocalDeElementCount[i] *= srcArray->tensorElementCount;
93869367
srcElementCount += srcLocalDeElementCount[i];
93879368
}
93889369
// determine local dstElementCount
93899370
const int dstLocalDeCount = dstArray->delayout->getLocalDeCount();
93909371
const int *dstLocalDeToDeMap = dstArray->delayout->getLocalDeToDeMap();
93919372
int *dstLocalDeElementCount = new int[dstLocalDeCount];
9373+
const int *dstIndexCountPDimPDe = dstArray->distgrid->getIndexCountPDimPDe();
93929374
int dstElementCount = 0; // initialize
93939375
for (int i=0; i<dstLocalDeCount; i++){
93949376
if (haloFlag){
@@ -9405,8 +9387,18 @@ template<typename SIT, typename DIT>
94059387
}
94069388
}else{
94079389
int de = dstLocalDeToDeMap[i]; // global DE index
9408-
dstLocalDeElementCount[i] = dstArray->exclusiveElementCountPDe[de]
9409-
* dstArray->tensorElementCount;
9390+
dstLocalDeElementCount[i] = 1; // prime the element count
9391+
for (int jj=0; jj<dstArray->rank; jj++){
9392+
int j = dstArray->arrayToDistGridMap[jj]; // j is dimIndex basis 1,
9393+
// or 0 for tensor dim
9394+
if (j){
9395+
// decomposed dimension
9396+
--j; // shift to basis 0
9397+
dstLocalDeElementCount[i] *=
9398+
dstIndexCountPDimPDe[de*(dstArray->distgrid->getDimCount())+j];
9399+
}
9400+
}
9401+
dstLocalDeElementCount[i] *= dstArray->tensorElementCount;
94109402
}
94119403
dstElementCount += dstLocalDeElementCount[i];
94129404
}

0 commit comments

Comments
 (0)