Skip to content

Commit 3596b61

Browse files
franzpoeschelfranzpoeschel
committed
wip: use variable-based encoding in tests
1 parent 7372467 commit 3596b61

File tree

3 files changed

+92
-50
lines changed

3 files changed

+92
-50
lines changed

‎src/Series.cpp‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2144,10 +2144,29 @@ creating new iterations.
21442144
case IterationEncoding::variableBased: {
21452145
if (!currentSteps.has_value() || currentSteps.value().empty())
21462146
{
2147-
currentSteps = std::vector<IterationIndex_t>{
2148-
read_only_this_single_iteration.has_value()
2149-
? *read_only_this_single_iteration
2150-
: 0};
2147+
if (!read_only_this_single_iteration.has_value())
2148+
{
2149+
Parameter<Operation::LIST_DATASETS> ld;
2150+
Parameter<Operation::LIST_PATHS> lp;
2151+
IOHandler()->enqueue(IOTask(&iterations, ld));
2152+
IOHandler()->enqueue(IOTask(&iterations, lp));
2153+
IOHandler()->flush(internal::defaultFlushParams);
2154+
if (ld.datasets->empty() && lp.paths->empty())
2155+
{
2156+
return {}; // no iterations, just global attributes
2157+
}
2158+
else
2159+
{
2160+
// there is data, defaulting to calling this Iteration idx 0
2161+
// when no further info is available
2162+
currentSteps = std::vector<IterationIndex_t>{0};
2163+
}
2164+
}
2165+
else
2166+
{
2167+
currentSteps = std::vector<IterationIndex_t>{
2168+
*read_only_this_single_iteration};
2169+
}
21512170
}
21522171
else if (read_only_this_single_iteration.has_value())
21532172
{
@@ -3242,7 +3261,10 @@ Series::snapshots(std::optional<SnapshotWorkflow> const snapshot_workflow)
32423261
access::writeOnly(access) &&
32433262
// 5. The backend is ADIOS2 in a recent enough version to support
32443263
// modifiable attributes (v2.9).
3245-
IOHandler()->fullSupportForVariableBasedEncoding())
3264+
IOHandler()->fullSupportForVariableBasedEncoding() &&
3265+
// 6. The Series must not yet be written, otherwise we're too late
3266+
// for this
3267+
!this->written())
32463268
{
32473269
setIterationEncoding_internal(
32483270
IterationEncoding::variableBased,

‎test/ParallelIOTest.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ TEST_CASE("parallel_multi_series_test", "[parallel]")
5757
.append(".")
5858
.append(file_ending),
5959
Access::CREATE,
60-
MPI_COMM_WORLD);
60+
MPI_COMM_WORLD,
61+
R"(iteration_encoding = "variable_based")");
6162
allSeries.back().iterations[sn].setAttribute("wululu", sn);
6263
allSeries.back().flush();
6364
}
@@ -110,7 +111,7 @@ void write_test_zero_extent(
110111

111112
for (int step = 0; step <= max_step; step += 20)
112113
{
113-
Iteration it = o.iterations[step];
114+
Iteration it = o.writeIterations()[step];
114115
it.setAttribute("yolo", "yo");
115116

116117
if (rank != 0 || declareFromAll)
@@ -467,7 +468,7 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig)
467468

468469
// array record component -> array record component
469470
// should work
470-
auto E_x = write.iterations[0].meshes["E"]["x"];
471+
auto E_x = write.writeIterations()[0].meshes["E"]["x"];
471472
E_x.resetDataset(ds1);
472473
E_x.storeChunk(data1, {mpi_rank, 0}, {1, 25});
473474
write.flush();
@@ -673,7 +674,7 @@ void write_4D_test(std::string const &file_ending)
673674
std::string name = "../samples/parallel_write_4d." + file_ending;
674675
Series o = Series(name, Access::CREATE, MPI_COMM_WORLD);
675676

676-
auto it = o.iterations[1];
677+
auto it = o.writeIterations()[1];
677678
auto E_x = it.meshes["E"]["x"];
678679

679680
// every rank out of mpi_size MPI ranks contributes two writes:
@@ -708,7 +709,7 @@ void write_makeconst_some(std::string const &file_ending)
708709
std::cout << name << std::endl;
709710
Series o = Series(name, Access::CREATE, MPI_COMM_WORLD);
710711

711-
auto it = o.iterations[1];
712+
auto it = o.writeIterations()[1];
712713
// I would have expected we need this, since the first call that writes
713714
// data below (makeConstant) is not executed in MPI collective manner
714715
// it.open();
@@ -1138,7 +1139,7 @@ TEST_CASE("independent_write_with_collective_flush", "[parallel]")
11381139
int size, rank;
11391140
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
11401141
MPI_Comm_size(MPI_COMM_WORLD, &size);
1141-
auto iteration = write.iterations[0];
1142+
auto iteration = write.writeIterations()[0];
11421143
auto E_x = iteration.meshes["E"]["x"];
11431144
E_x.resetDataset({Datatype::DOUBLE, {10}});
11441145
write.flush();

‎test/SerialIOTest.cpp‎

Lines changed: 58 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ namespace detail
131131
template <typename Char>
132132
void writeChar(Series &series, std::string const &component_name)
133133
{
134-
auto component = series.iterations[0].meshes["E"][component_name];
134+
auto component = series.writeIterations()[0].meshes["E"][component_name];
135135
std::vector<Char> data(10);
136136
component.resetDataset({determineDatatype<Char>(), {10}});
137137
component.storeChunk(data, {0}, {10});
@@ -288,7 +288,7 @@ TEST_CASE("multi_series_test", "[serial]")
288288
.append(".")
289289
.append(file_ending),
290290
Access::CREATE);
291-
allSeries.back().iterations[sn].setAttribute("wululu", sn);
291+
allSeries.back().writeIterations()[sn].setAttribute("wululu", sn);
292292
allSeries.back().flush();
293293
}
294294
}
@@ -647,12 +647,13 @@ void close_and_copy_attributable_test(std::string const &file_ending)
647647
{
648648
if (iteration_ptr)
649649
{
650-
*iteration_ptr = series.iterations[i];
650+
*iteration_ptr = series.writeIterations()[i];
651651
}
652652
else
653653
{
654654
// use copy constructor
655-
iteration_ptr = std::make_unique<Iteration>(series.iterations[i]);
655+
iteration_ptr =
656+
std::make_unique<Iteration>(series.writeIterations()[i]);
656657
}
657658
Record electronPositions = iteration_ptr->particles["e"]["position"];
658659
// TODO set this automatically to zero if not provided
@@ -784,17 +785,20 @@ inline void empty_dataset_test(std::string const &file_ending)
784785
"../samples/empty_datasets." + file_ending, Access::CREATE);
785786

786787
auto makeEmpty_dim_7_int =
787-
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int"];
788+
series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_int"];
788789
auto makeEmpty_dim_7_long =
789-
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool"];
790+
series.writeIterations()[1].meshes["rho"]["makeEmpty_dim_7_bool"];
790791
auto makeEmpty_dim_7_int_alt =
791-
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_int_alt"];
792+
series.writeIterations()[1]
793+
.meshes["rho"]["makeEmpty_dim_7_int_alt"];
792794
auto makeEmpty_dim_7_long_alt =
793-
series.iterations[1].meshes["rho"]["makeEmpty_dim_7_bool_alt"];
795+
series.writeIterations()[1]
796+
.meshes["rho"]["makeEmpty_dim_7_bool_alt"];
794797
auto makeEmpty_resetDataset_dim3 =
795-
series.iterations[1].meshes["rho"]["makeEmpty_resetDataset_dim3"];
798+
series.writeIterations()[1]
799+
.meshes["rho"]["makeEmpty_resetDataset_dim3"];
796800
auto makeEmpty_resetDataset_dim3_notallzero =
797-
series.iterations[1]
801+
series.writeIterations()[1]
798802
.meshes["rho"]["makeEmpty_resetDataset_dim3_notallzero"];
799803
makeEmpty_dim_7_int.makeEmpty<int>(7);
800804
makeEmpty_dim_7_long.makeEmpty<long>(7);
@@ -899,17 +903,18 @@ inline void constant_scalar(std::string const &file_ending)
899903
Series s =
900904
Series("../samples/constant_scalar." + file_ending, Access::CREATE);
901905
s.setOpenPMD("2.0.0");
902-
auto rho = s.iterations[1].meshes["rho"][MeshRecordComponent::SCALAR];
903-
REQUIRE(s.iterations[1].meshes["rho"].scalar());
906+
auto rho =
907+
s.writeIterations()[1].meshes["rho"][MeshRecordComponent::SCALAR];
908+
REQUIRE(s.writeIterations()[1].meshes["rho"].scalar());
904909
rho.resetDataset(Dataset(Datatype::CHAR, {1, 2, 3}));
905910
rho.makeConstant(static_cast<char>('a'));
906911
REQUIRE(rho.constant());
907912

908913
// mixed constant/non-constant
909-
auto E_x = s.iterations[1].meshes["E"]["x"];
914+
auto E_x = s.writeIterations()[1].meshes["E"]["x"];
910915
E_x.resetDataset(Dataset(Datatype::FLOAT, {1, 2, 3}));
911916
E_x.makeConstant(static_cast<float>(13.37));
912-
auto E_y = s.iterations[1].meshes["E"]["y"];
917+
auto E_y = s.writeIterations()[1].meshes["E"]["y"];
913918
E_y.resetDataset(Dataset(Datatype::UINT, {1, 2, 3}));
914919
UniquePtrWithLambda<unsigned int> E(
915920
new unsigned int[6], [](unsigned int const *p) { delete[] p; });
@@ -918,7 +923,7 @@ inline void constant_scalar(std::string const &file_ending)
918923
E_y.storeChunk(std::move(E), {0, 0, 0}, {1, 2, 3});
919924

920925
// store a number of predefined attributes in E
921-
Mesh &E_mesh = s.iterations[1].meshes["E"];
926+
Mesh &E_mesh = s.writeIterations()[1].meshes["E"];
922927
// test that these can be defined successively
923928
E_mesh.setGridUnitDimension({{{UnitDimension::L, 1}}, {}, {}});
924929
E_mesh.setGridUnitDimension(
@@ -943,21 +948,21 @@ inline void constant_scalar(std::string const &file_ending)
943948
std::vector<double>{gridUnitSI, gridUnitSI, gridUnitSI});
944949

945950
// constant scalar
946-
auto pos =
947-
s.iterations[1].particles["e"]["position"][RecordComponent::SCALAR];
951+
auto pos = s.writeIterations()[1]
952+
.particles["e"]["position"][RecordComponent::SCALAR];
948953
pos.resetDataset(Dataset(Datatype::DOUBLE, {3, 2, 1}));
949954
pos.makeConstant(static_cast<double>(42.));
950955
auto posOff =
951-
s.iterations[1]
956+
s.writeIterations()[1]
952957
.particles["e"]["positionOffset"][RecordComponent::SCALAR];
953958
posOff.resetDataset(Dataset(Datatype::INT, {3, 2, 1}));
954959
posOff.makeConstant(static_cast<int>(-42));
955960

956961
// mixed constant/non-constant
957-
auto vel_x = s.iterations[1].particles["e"]["velocity"]["x"];
962+
auto vel_x = s.writeIterations()[1].particles["e"]["velocity"]["x"];
958963
vel_x.resetDataset(Dataset(Datatype::SHORT, {3, 2, 1}));
959964
vel_x.makeConstant(static_cast<short>(-1));
960-
auto vel_y = s.iterations[1].particles["e"]["velocity"]["y"];
965+
auto vel_y = s.writeIterations()[1].particles["e"]["velocity"]["y"];
961966
vel_y.resetDataset(Dataset(Datatype::ULONGLONG, {3, 2, 1}));
962967
UniquePtrWithLambda<unsigned long long> vel(
963968
new unsigned long long[6],
@@ -1146,7 +1151,7 @@ TEST_CASE("flush_without_position_positionOffset", "[serial]")
11461151
Series s = Series(
11471152
"../samples/flush_without_position_positionOffset." + file_ending,
11481153
Access::CREATE);
1149-
ParticleSpecies e = s.iterations[0].particles["e"];
1154+
ParticleSpecies e = s.writeIterations()[0].particles["e"];
11501155
RecordComponent weighting = e["weighting"][RecordComponent::SCALAR];
11511156
weighting.resetDataset(Dataset(Datatype::FLOAT, Extent{2, 2}));
11521157
weighting.storeChunk(
@@ -1407,7 +1412,7 @@ inline void dtype_test(
14071412
// should be possible to parse without error upon opening
14081413
// the series for reading
14091414
{
1410-
auto E = s.iterations[0].meshes["E"];
1415+
auto E = s.writeIterations()[0].meshes["E"];
14111416
E.setGridSpacing(std::vector<long double>{1.0, 1.0});
14121417
auto E_x = E["x"];
14131418
E_x.makeEmpty<double>(1);
@@ -1622,7 +1627,7 @@ inline void write_test(
16221627
Series o =
16231628
Series("../samples/serial_write." + backend, Access::CREATE, jsonCfg);
16241629

1625-
ParticleSpecies &e_1 = o.iterations[1].particles["e"];
1630+
ParticleSpecies &e_1 = o.writeIterations()[1].particles["e"];
16261631

16271632
std::vector<double> position_global(4);
16281633
double pos{0.};
@@ -1657,7 +1662,7 @@ inline void write_test(
16571662
e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1});
16581663
}
16591664

1660-
ParticleSpecies &e_2 = o.iterations[2].particles["e"];
1665+
ParticleSpecies &e_2 = o.writeIterations()[2].particles["e"];
16611666

16621667
std::generate(position_global.begin(), position_global.end(), [&pos] {
16631668
return pos++;
@@ -1688,7 +1693,7 @@ inline void write_test(
16881693

16891694
o.flush();
16901695

1691-
ParticleSpecies &e_3 = o.iterations[3].particles["e"];
1696+
ParticleSpecies &e_3 = o.writeIterations()[3].particles["e"];
16921697

16931698
std::generate(position_global.begin(), position_global.end(), [&pos] {
16941699
return pos++;
@@ -1797,15 +1802,17 @@ void test_complex(const std::string &backend)
17971802
o.setAttribute(
17981803
"longDoublesYouSay", std::complex<long double>(5.5, -4.55));
17991804

1800-
auto Cflt = o.iterations[0].meshes["Cflt"][RecordComponent::SCALAR];
1805+
auto Cflt =
1806+
o.writeIterations()[0].meshes["Cflt"][RecordComponent::SCALAR];
18011807
std::vector<std::complex<float>> cfloats(3);
18021808
cfloats.at(0) = {1., 2.};
18031809
cfloats.at(1) = {-3., 4.};
18041810
cfloats.at(2) = {5., -6.};
18051811
Cflt.resetDataset(Dataset(Datatype::CFLOAT, {cfloats.size()}));
18061812
Cflt.storeChunk(cfloats, {0});
18071813

1808-
auto Cdbl = o.iterations[0].meshes["Cdbl"][RecordComponent::SCALAR];
1814+
auto Cdbl =
1815+
o.writeIterations()[0].meshes["Cdbl"][RecordComponent::SCALAR];
18091816
std::vector<std::complex<double>> cdoubles(3);
18101817
cdoubles.at(0) = {2., 1.};
18111818
cdoubles.at(1) = {-4., 3.};
@@ -1817,7 +1824,7 @@ void test_complex(const std::string &backend)
18171824
if (o.backend() != "ADIOS2")
18181825
{
18191826
auto Cldbl =
1820-
o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR];
1827+
o.writeIterations()[0].meshes["Cldbl"][RecordComponent::SCALAR];
18211828
cldoubles.at(0) = {3., 2.};
18221829
cldoubles.at(1) = {-5., 4.};
18231830
cldoubles.at(2) = {7., -6.};
@@ -2427,7 +2434,10 @@ TEST_CASE("sample_write_thetaMode", "[serial][thetaMode]")
24272434
inline void bool_test(const std::string &backend)
24282435
{
24292436
{
2430-
Series o = Series("../samples/serial_bool." + backend, Access::CREATE);
2437+
Series o = Series(
2438+
"../samples/serial_bool." + backend,
2439+
Access::CREATE,
2440+
R"({"iteration_encoding": "variable_based"})");
24312441

24322442
o.setAttribute("Bool attribute true", true);
24332443
o.setAttribute("Bool attribute false", false);
@@ -2463,7 +2473,7 @@ inline void patch_test(const std::string &backend)
24632473
{
24642474
Series o = Series("../samples/serial_patch." + backend, Access::CREATE);
24652475

2466-
auto e = o.iterations[1].particles["e"];
2476+
auto e = o.writeIterations()[1].particles["e"];
24672477

24682478
uint64_t const num_particles = 1u;
24692479
auto dset_d = Dataset(Datatype::DOUBLE, {num_particles});
@@ -4647,7 +4657,12 @@ TEST_CASE("adios2_engines_and_file_endings")
46474657
Series write(
46484658
name,
46494659
Access::CREATE,
4650-
json::merge("backend = \"adios2\"", jsonCfg));
4660+
json::merge(
4661+
R"(
4662+
backend = "adios2"
4663+
iteration_encoding = "variable_based"
4664+
)",
4665+
jsonCfg));
46514666
}
46524667
if (directory)
46534668
{
@@ -4712,7 +4727,11 @@ TEST_CASE("adios2_engines_and_file_endings")
47124727
auto filesystemname =
47134728
filesystemExt.empty() ? name : basename + filesystemExt;
47144729
{
4715-
Series write(name, Access::CREATE, jsonCfg);
4730+
Series write(
4731+
name,
4732+
Access::CREATE,
4733+
json::merge(
4734+
R"(iteration_encoding = "variable_based")", jsonCfg));
47164735
write.close();
47174736
}
47184737
if (directory)
@@ -6118,8 +6137,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]")
61186137
// automatically (de)activate span-based storeChunking
61196138
{
61206139
Series write("../samples/span_based.bp", Access::CREATE);
6121-
auto E_uncompressed = write.iterations[0].meshes["E"]["x"];
6122-
auto E_compressed = write.iterations[0].meshes["E"]["y"];
6140+
auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"];
6141+
auto E_compressed = write.writeIterations()[0].meshes["E"]["y"];
61236142

61246143
Dataset ds{Datatype::INT, {10}};
61256144

@@ -6169,8 +6188,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]")
61696188
}
61706189
})END";
61716190
Series write("../samples/span_based.bp", Access::CREATE, enable);
6172-
auto E_uncompressed = write.iterations[0].meshes["E"]["x"];
6173-
auto E_compressed = write.iterations[0].meshes["E"]["y"];
6191+
auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"];
6192+
auto E_compressed = write.writeIterations()[0].meshes["E"]["y"];
61746193

61756194
Dataset ds{Datatype::INT, {10}};
61766195

@@ -6233,8 +6252,8 @@ TEST_CASE("automatically_deactivate_span", "[serial][adios2]")
62336252
}
62346253
})END";
62356254
Series write("../samples/span_based.bp", Access::CREATE, disable);
6236-
auto E_uncompressed = write.iterations[0].meshes["E"]["x"];
6237-
auto E_compressed = write.iterations[0].meshes["E"]["y"];
6255+
auto E_uncompressed = write.writeIterations()[0].meshes["E"]["x"];
6256+
auto E_compressed = write.writeIterations()[0].meshes["E"]["y"];
62386257

62396258
Dataset ds{Datatype::INT, {10}};
62406259

@@ -6689,7 +6708,7 @@ void deferred_parsing(std::string const &extension)
66896708
Series series(basename + "%06T." + extension, Access::CREATE);
66906709
std::vector<float> buffer(20);
66916710
std::iota(buffer.begin(), buffer.end(), 0.f);
6692-
auto dataset = series.iterations[0].meshes["E"]["x"];
6711+
auto dataset = series.writeIterations()[0].meshes["E"]["x"];
66936712
dataset.resetDataset({Datatype::FLOAT, {20}});
66946713
dataset.storeChunk(buffer, {0}, {20});
66956714
series.flush();

0 commit comments

Comments
 (0)