Skip to content

Commit 958a718

Browse files
author
Y
committed
Quicksilver code updates for SYCL and CUDA versions
1 parent b2aff3f commit 958a718

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2088
-626
lines changed

‎QuickSilver/CUDA/src/CollisionEvent.hh‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,13 @@ inline HOST_DEVICE void updateTrajectory(double energy, double angle, MC_Particl
8888
}
8989
HOST_DEVICE_END
9090

91-
inline HOST_DEVICE bool CollisionEvent(MonteCarlo *monteCarlo, MC_Particle &mc_particle, unsigned int tally_index, int particle_index, int *tallyArray)
91+
inline HOST_DEVICE bool CollisionEvent(MonteCarlo_d *monteCarlo, MC_Particle &mc_particle, unsigned int tally_index, int particle_index, int *tallyArray)
9292
{
93-
#ifdef __CUDA_ARCH__
93+
//#ifdef __CUDA_ARCH__
9494
const MC_Cell_State &cell = monteCarlo->domain_d[mc_particle.domain].cell_state[mc_particle.cell];
95-
#else
96-
const MC_Cell_State &cell = monteCarlo->domain[mc_particle.domain].cell_state[mc_particle.cell];
97-
#endif
95+
//#else
96+
// const MC_Cell_State &cell = monteCarlo->domain[mc_particle.domain].cell_state[mc_particle.cell];
97+
//#endif
9898

9999
int globalMatIndex = cell._material;
100100

@@ -107,21 +107,21 @@ inline HOST_DEVICE bool CollisionEvent(MonteCarlo *monteCarlo, MC_Particle &mc_p
107107
int selectedIso = -1;
108108
int selectedUniqueNumber = -1;
109109
int selectedReact = -1;
110-
#ifdef __CUDA_ARCH__
110+
//#ifdef __CUDA_ARCH__
111111
int numIsos = (int)monteCarlo->_material_d[globalMatIndex]._isosize;
112-
#else
113-
int numIsos = (int)monteCarlo->_materialDatabase->_mat[globalMatIndex]._iso.size();
114-
#endif
112+
//#else
113+
// int numIsos = (int)monteCarlo->_materialDatabase->_mat[globalMatIndex]._iso.size();
114+
//#endif
115115

116116
for (int isoIndex = 0; isoIndex < numIsos && currentCrossSection >= 0; isoIndex++)
117117
{
118-
#ifdef __CUDA_ARCH__
118+
//#ifdef __CUDA_ARCH__
119119
int uniqueNumber = monteCarlo->_material_d[globalMatIndex]._iso[isoIndex]._gid;
120120
int numReacts = monteCarlo->_nuclearData_d->getNumberReactions(uniqueNumber);
121-
#else
122-
int uniqueNumber = monteCarlo->_materialDatabase->_mat[globalMatIndex]._iso[isoIndex]._gid;
123-
int numReacts = monteCarlo->_nuclearData->getNumberReactions(uniqueNumber);
124-
#endif
121+
//#else
122+
// int uniqueNumber = monteCarlo->_materialDatabase->_mat[globalMatIndex]._iso[isoIndex]._gid;
123+
// int numReacts = monteCarlo->_nuclearData->getNumberReactions(uniqueNumber);
124+
//#endif
125125
for (int reactIndex = 0; reactIndex < numReacts; reactIndex++)
126126
{
127127
currentCrossSection -= macroscopicCrossSection(monteCarlo, reactIndex, mc_particle.domain, mc_particle.cell,
@@ -145,58 +145,58 @@ inline HOST_DEVICE bool CollisionEvent(MonteCarlo *monteCarlo, MC_Particle &mc_p
145145
double energyOut[MAX_PRODUCTION_SIZE];
146146
double angleOut[MAX_PRODUCTION_SIZE];
147147
int nOut = 0;
148-
#ifdef __CUDA_ARCH__
148+
//#ifdef __CUDA_ARCH__
149149
double mat_mass = monteCarlo->_material_d[globalMatIndex]._mass;
150150
monteCarlo->_nuclearData_d->_isotopes[selectedUniqueNumber]._species[0]._reactions[selectedReact].sampleCollision(
151151
mc_particle.kinetic_energy, mat_mass, &energyOut[0], &angleOut[0], nOut, &(mc_particle.random_number_seed), MAX_PRODUCTION_SIZE);
152-
#else
153-
double mat_mass = monteCarlo->_materialDatabase->_mat[globalMatIndex]._mass;
154-
monteCarlo->_nuclearData->_isotopes[selectedUniqueNumber]._species[0]._reactions[selectedReact].sampleCollision(
155-
mc_particle.kinetic_energy, mat_mass, &energyOut[0], &angleOut[0], nOut, &(mc_particle.random_number_seed), MAX_PRODUCTION_SIZE);
156-
#endif
152+
//#else
153+
// double mat_mass = monteCarlo->_materialDatabase->_mat[globalMatIndex]._mass;
154+
// monteCarlo->_nuclearData->_isotopes[selectedUniqueNumber]._species[0]._reactions[selectedReact].sampleCollision(
155+
// mc_particle.kinetic_energy, mat_mass, &energyOut[0], &angleOut[0], nOut, &(mc_particle.random_number_seed), MAX_PRODUCTION_SIZE);
156+
//#endif
157157

158158
//--------------------------------------------------------------------------------------------------------------
159159
// Post-Collision Phase 1:
160160
// Tally the collision
161161
//--------------------------------------------------------------------------------------------------------------
162162

163163
// Set the reaction for this particle.
164-
#ifdef __CUDA_ARCH__
164+
//#ifdef __CUDA_ARCH__
165165
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 3]);
166-
#else
167-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._collision);
168-
#endif
166+
//#else
167+
// ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._collision);
168+
//#endif
169169

170-
#ifdef __CUDA_ARCH__
170+
//#ifdef __CUDA_ARCH__
171171
NuclearDataReaction::Enum reactionType = (NuclearDataReaction::Enum)monteCarlo->_nuclearData_d->_isotopes[selectedUniqueNumber]._species[0]._reactions[selectedReact]._reactionType;
172-
#else
173-
NuclearDataReaction::Enum reactionType = monteCarlo->_nuclearData->_isotopes[selectedUniqueNumber]._species[0]._reactions[selectedReact]._reactionType;
174-
#endif
172+
//#else
173+
// NuclearDataReaction::Enum reactionType = monteCarlo->_nuclearData->_isotopes[selectedUniqueNumber]._species[0]._reactions[selectedReact]._reactionType;
174+
//#endif
175175

176176
switch (reactionType)
177177
{
178178
case NuclearDataReaction::Scatter:
179-
#ifdef __CUDA_ARCH__
179+
//#ifdef __CUDA_ARCH__
180180
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 4]);
181-
#else
182-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._scatter);
183-
#endif
181+
//#else
182+
// ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._scatter);
183+
//#endif
184184
break;
185185
case NuclearDataReaction::Absorption:
186-
#ifdef __CUDA_ARCH__
186+
//#ifdef __CUDA_ARCH__
187187
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 5]);
188-
#else
189-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._absorb);
190-
#endif
188+
//#else
189+
// ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._absorb);
190+
//#endif
191191
break;
192192
case NuclearDataReaction::Fission:
193-
#ifdef __CUDA_ARCH__
193+
//#ifdef __CUDA_ARCH__
194194
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 6]);
195195
ATOMIC_ADD(tallyArray[tally_index * NUM_TALLIES + 7], nOut);
196-
#else
197-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._fission);
198-
ATOMIC_ADD(monteCarlo->_tallies->_balanceTask[tally_index]._produce, nOut);
199-
#endif
196+
//#else
197+
// ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._fission);
198+
// ATOMIC_ADD(monteCarlo->_tallies->_balanceTask[tally_index]._produce, nOut);
199+
//#endif
200200
break;
201201
case NuclearDataReaction::Undefined:
202202
#ifdef DEBUG
@@ -219,7 +219,7 @@ inline HOST_DEVICE bool CollisionEvent(MonteCarlo *monteCarlo, MC_Particle &mc_p
219219
updateTrajectory(energyOut[secondaryIndex], angleOut[secondaryIndex], secondaryParticle);
220220

221221
// Atomic capture will be called here
222-
monteCarlo->_particleVaultContainer->addExtraParticle(secondaryParticle);
222+
monteCarlo->_particleVaultContainer_d->addExtraParticle(secondaryParticle);
223223
}
224224

225225
updateTrajectory(energyOut[0], angleOut[0], mc_particle);
@@ -231,15 +231,15 @@ inline HOST_DEVICE bool CollisionEvent(MonteCarlo *monteCarlo, MC_Particle &mc_p
231231
if (nOut > 1)
232232
{
233233
// Atomic capture will be called here
234-
monteCarlo->_particleVaultContainer->addExtraParticle(mc_particle);
234+
monteCarlo->_particleVaultContainer_d->addExtraParticle(mc_particle);
235235
}
236236

237237
// If we are still tracking this particle the update its energy group
238-
#ifdef __CUDA_ARCH__
238+
//#ifdef __CUDA_ARCH__
239239
mc_particle.energy_group = monteCarlo->_nuclearData_d->getEnergyGroup(mc_particle.kinetic_energy);
240-
#else
241-
mc_particle.energy_group = monteCarlo->_nuclearData->getEnergyGroup(mc_particle.kinetic_energy);
242-
#endif
240+
//#else
241+
// mc_particle.energy_group = monteCarlo->_nuclearData->getEnergyGroup(mc_particle.kinetic_energy);
242+
//#endif
243243

244244
return nOut == 1;
245245
}

‎QuickSilver/CUDA/src/CycleTracking.hh‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,12 @@ class MC_Particle;
7373
#endif
7474
const int NIt = MaxIt;
7575

76-
inline HOST_DEVICE_CUDA void CycleTrackingFunction(MonteCarlo *monteCarlo, MC_Particle &mc_particle, int particle_index, ParticleVault *processingVault, ParticleVault *processedVault, int *tallyArray)
76+
inline HOST_DEVICE_CUDA void CycleTrackingFunction(MonteCarlo_d *monteCarlo, MC_Particle &mc_particle, int particle_index, ParticleVault_d *processingVault, ParticleVault_d *processedVault, int *tallyArray)
7777
{
7878
bool keepTrackingThisParticle = true;
79-
unsigned int tally_index = (particle_index) % monteCarlo->_tallies->GetNumBalanceReplications();
80-
unsigned int flux_tally_index = (particle_index) % monteCarlo->_tallies->GetNumFluxReplications();
81-
unsigned int cell_tally_index = (particle_index) % monteCarlo->_tallies->GetNumCellTallyReplications();
79+
unsigned int tally_index = (particle_index) % monteCarlo->_tallies_d->GetNumBalanceReplications();
80+
unsigned int flux_tally_index = (particle_index) % monteCarlo->_tallies_d->GetNumFluxReplications();
81+
unsigned int cell_tally_index = (particle_index) % monteCarlo->_tallies_d->GetNumCellTallyReplications();
8282

8383
int i1 = 0;
8484
// The while loop will exit after a particle reaches census or goes through MaxIters iterations, whichever comes first. If a particle reaches MaxIters it will be added to the ExtraVaults and processed in a later kernel. MaxIt can be defined in the makefile, otherwise it defaults to a large number that should ensure that it is never reached.
@@ -98,15 +98,15 @@ inline HOST_DEVICE_CUDA void CycleTrackingFunction(MonteCarlo *monteCarlo, MC_Pa
9898
{
9999

100100
#ifdef EXPONENTIAL_TALLY
101-
monteCarlo->_tallies->TallyCellValue(exp(rngSample(&mc_particle.random_number_seed)), mc_particle.domain, cell_tally_index, mc_particle.cell);
101+
monteCarlo->_tallies_d->TallyCellValue(exp(rngSample(&mc_particle.random_number_seed)), mc_particle.domain, cell_tally_index, mc_particle.cell);
102102
#endif
103103
segment_outcome = MC_Segment_Outcome(monteCarlo, mc_particle, flux_tally_index);
104104

105-
#ifdef __CUDA_ARCH__
105+
//#ifdef __CUDA_ARCH__
106106
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 0]);
107-
#else
108-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._numSegments);
109-
#endif
107+
//#else
108+
// ATOMIC_UPDATE(monteCarlo->_tallies_d->_balanceTask[tally_index]._numSegments);
109+
//#endif
110110

111111

112112
mc_particle.num_segments += 1.; /* Track the number of segments this particle has
@@ -143,11 +143,11 @@ inline HOST_DEVICE_CUDA void CycleTrackingFunction(MonteCarlo *monteCarlo, MC_Pa
143143
}
144144
else if (facet_crossing_type == MC_Tally_Event::Facet_Crossing_Escape)
145145
{
146-
#ifdef __CUDA_ARCH__
146+
//#ifdef __CUDA_ARCH__
147147
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 1]);
148-
#else
149-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._escape);
150-
#endif
148+
//#else
149+
//ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._escape);
150+
//#endif
151151

152152
mc_particle.last_event = MC_Tally_Event::Facet_Crossing_Escape;
153153
mc_particle.species = -1;
@@ -170,11 +170,11 @@ inline HOST_DEVICE_CUDA void CycleTrackingFunction(MonteCarlo *monteCarlo, MC_Pa
170170
{
171171
// The particle has reached the end of the time step.
172172
processedVault->pushParticle(mc_particle);
173-
#ifdef __CUDA_ARCH__
173+
//#ifdef __CUDA_ARCH__
174174
ATOMIC_UPDATE(tallyArray[tally_index * NUM_TALLIES + 2]);
175-
#else
176-
ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._census);
177-
#endif
175+
//#else
176+
//ATOMIC_UPDATE(monteCarlo->_tallies->_balanceTask[tally_index]._census);
177+
//#endif
178178

179179
keepTrackingThisParticle = false;
180180
}
@@ -200,11 +200,11 @@ inline HOST_DEVICE_CUDA void CycleTrackingFunction(MonteCarlo *monteCarlo, MC_Pa
200200
}
201201
else
202202
{
203-
monteCarlo->_particleVaultContainer->addExtraParticle(mc_particle);
203+
monteCarlo->_particleVaultContainer_d->addExtraParticle(mc_particle);
204204
}
205205
}
206206

207-
inline HOST_DEVICE_CUDA void CycleTrackingGuts(MonteCarlo *monteCarlo, int particle_index, ParticleVault *processingVault, ParticleVault *processedVault, int *tallyArray)
207+
inline HOST_DEVICE_CUDA void CycleTrackingGuts(MonteCarlo_d *monteCarlo, int particle_index, ParticleVault_d *processingVault, ParticleVault_d *processedVault, int *tallyArray)
208208
{
209209
MC_Particle mc_particle;
210210

‎QuickSilver/CUDA/src/EnergySpectrum.cc‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,3 +74,55 @@ void EnergySpectrum::PrintSpectrum(MonteCarlo* monteCarlo)
7474
}
7575
delete []sumHist;
7676
}
77+
//------------------------------------------------------------------------------------------------------
78+
void EnergySpectrum_d::UpdateSpectrum(MonteCarlo* monteCarlo)
79+
{
80+
if( _fileName == "" ) return;
81+
82+
for( uint64_t ii = 0; ii < monteCarlo->_particleVaultContainer->processingSize(); ii++)
83+
{
84+
ParticleVault* processing = monteCarlo->_particleVaultContainer->getTaskProcessingVault( ii );
85+
for( uint64_t jj = 0; jj < processing->size(); jj++ )
86+
{
87+
MC_Particle mc_particle;
88+
MC_Load_Particle(monteCarlo, mc_particle, processing, jj);
89+
_censusEnergySpectrum[mc_particle.energy_group]++;
90+
}
91+
}
92+
for( uint64_t ii = 0; ii < monteCarlo->_particleVaultContainer->processedSize(); ii++)
93+
{
94+
ParticleVault* processed = monteCarlo->_particleVaultContainer->getTaskProcessedVault( ii );
95+
for( uint64_t jj = 0; jj < processed->size(); jj++ )
96+
{
97+
MC_Particle mc_particle;
98+
MC_Load_Particle(monteCarlo, mc_particle, processed, jj);
99+
_censusEnergySpectrum[mc_particle.energy_group]++;
100+
}
101+
}
102+
}
103+
104+
void EnergySpectrum_d::PrintSpectrum(MonteCarlo* monteCarlo)
105+
{
106+
if( _fileName == "" ) return;
107+
108+
const int count = monteCarlo->_nuclearData->_energies.size();
109+
uint64_t *sumHist = new uint64_t[ count ]();
110+
111+
//mpiAllreduce( _censusEnergySpectrum.data(), sumHist, count, MPI_INT64_T, MPI_SUM, monteCarlo->processor_info->comm_mc_world );
112+
mpiAllreduce( _censusEnergySpectrum, sumHist, count, MPI_INT64_T, MPI_SUM, monteCarlo->processor_info->comm_mc_world );
113+
114+
if( monteCarlo->processor_info->rank == 0 )
115+
{
116+
_fileName += ".dat";
117+
FILE* spectrumFile;
118+
spectrumFile = fopen( _fileName.c_str(), "w" );
119+
120+
for( int ii = 0; ii < 230; ii++ )
121+
{
122+
fprintf( spectrumFile, "%d\t%g\t%lu\n", ii, monteCarlo->_nuclearData->_energies[ii], sumHist[ii] );
123+
}
124+
125+
fclose( spectrumFile );
126+
}
127+
delete []sumHist;
128+
}

‎QuickSilver/CUDA/src/EnergySpectrum.hh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,20 @@ private:
1717
std::string _fileName;
1818
std::vector<uint64_t> _censusEnergySpectrum;
1919
};
20+
//------------------------------------------------------------------------------------------------------------
21+
class EnergySpectrum_d
22+
{
23+
public:
24+
EnergySpectrum_d(std::string name, uint64_t size) : _fileName(std::move(name))/*, _censusEnergySpectrum(size, 0)*/{
25+
_censusEnergySpectrum = new uint64_t[size];
26+
};
27+
void UpdateSpectrum(MonteCarlo *monteCarlo);
28+
void PrintSpectrum(MonteCarlo *monteCarlo);
2029

30+
private:
31+
std::string _fileName;
32+
//std::vector<uint64_t> _censusEnergySpectrum;
33+
uint64_t* _censusEnergySpectrum;
34+
int _censusEnergySpectrumSize;
35+
};
2136
#endif

0 commit comments

Comments
 (0)