Skip to content

Commit d6a4b4a

Browse files
imikejacksonJDuffeyBQ
imikejackson
and
JDuffeyBQ
authored
ENH: Support reading of legacy statsgenerator data (#1584)
* Added initial version of legacy StatsDataArray reading * Update to read legacy synthetic files --------- Signed-off-by: Jared Duffey <jared.duffey@bluequartz.net> Signed-off-by: Michael Jackson <mike.jackson@bluequartz.net> Co-authored-by: Jared Duffey <jared.duffey@bluequartz.net>
1 parent 1a42ec6 commit d6a4b4a

1 file changed

Lines changed: 329 additions & 31 deletions

File tree

‎src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp‎

Lines changed: 329 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <nlohmann/json.hpp>
2929

3030
#include <fstream>
31+
#include <set>
3132
#include <sstream>
3233
#include <stdexcept>
3334
#include <string_view>
@@ -1282,6 +1283,296 @@ Result<> finishImportingLegacyArray(DataStructure& dataStructure, const nx::core
12821283
}
12831284
}
12841285

1286+
Result<> readDatasetAsDataArray(DataStructure& dataStructure, const HDF5::DatasetIO& datasetIO, DataObject::IdType parentId, bool preflight)
1287+
{
1288+
ShapeType tDims = datasetIO.getDimensions();
1289+
if(tDims.empty())
1290+
{
1291+
return MakeErrorResult(-13345, "Unable to read dataset dimensions");
1292+
}
1293+
ShapeType cDims = {1};
1294+
Result<DataType> dataTypeResult = datasetIO.getDataType();
1295+
if(dataTypeResult.invalid())
1296+
{
1297+
hid_t datasetId = datasetIO.getId();
1298+
hid_t typeId = H5Dget_type(datasetId);
1299+
H5T_class_t classType = H5Tget_class(typeId);
1300+
H5Tclose(typeId);
1301+
if(classType == H5T_STRING)
1302+
{
1303+
usize size = std::accumulate(tDims.cbegin(), tDims.cend(), static_cast<usize>(1), std::multiplies<>());
1304+
StringArray* stringArray = StringArray::CreateWithValues(dataStructure, datasetIO.getName(), tDims, std::vector<std::string>(size), parentId);
1305+
return {};
1306+
}
1307+
}
1308+
1309+
DataType dataType = dataTypeResult.value();
1310+
switch(dataType)
1311+
{
1312+
case DataType::float32: {
1313+
return ConvertResult(createLegacyDataArray<float32>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1314+
}
1315+
case DataType::float64: {
1316+
return ConvertResult(createLegacyDataArray<float64>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1317+
}
1318+
case DataType::int8: {
1319+
return ConvertResult(createLegacyDataArray<int8>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1320+
}
1321+
case DataType::int16: {
1322+
return ConvertResult(createLegacyDataArray<int16>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1323+
}
1324+
case DataType::int32: {
1325+
return ConvertResult(createLegacyDataArray<int32>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1326+
}
1327+
case DataType::int64: {
1328+
return ConvertResult(createLegacyDataArray<int64>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1329+
}
1330+
case DataType::boolean: {
1331+
return ConvertResult(createLegacyDataArray<bool>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1332+
}
1333+
case DataType::uint8: {
1334+
return ConvertResult(createLegacyDataArray<uint8>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1335+
}
1336+
case DataType::uint16: {
1337+
return ConvertResult(createLegacyDataArray<uint16>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1338+
}
1339+
case DataType::uint32: {
1340+
return ConvertResult(createLegacyDataArray<uint32>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1341+
}
1342+
case DataType::uint64: {
1343+
return ConvertResult(createLegacyDataArray<uint64>(dataStructure, parentId, datasetIO, tDims, cDims, preflight));
1344+
}
1345+
}
1346+
return MakeErrorResult(-456345, fmt::format("StatsReader: Unsupported array type: {}", to_underlying(dataType)));
1347+
}
1348+
1349+
Result<> readLegacyStatsDataArrayDatasetChild(DataStructure& dataStructure, const nx::core::HDF5::DatasetIO& datasetIO, DataObject::IdType parentId, bool preflight)
1350+
{
1351+
Result<std::string> objectTypeResult = datasetIO.readStringAttribute(Constants::k_ObjectTypeTag);
1352+
if(objectTypeResult.invalid())
1353+
{
1354+
return readDatasetAsDataArray(dataStructure, datasetIO, parentId, preflight);
1355+
}
1356+
std::string objectType = std::move(objectTypeResult.value());
1357+
if(objectType.starts_with("DataArray<"))
1358+
{
1359+
return ConvertResult(readLegacyDataArray(dataStructure, datasetIO, parentId, preflight));
1360+
}
1361+
return MakeErrorResult(-343254, fmt::format("Unable to read dataset \"{}\"", datasetIO.getName()));
1362+
}
1363+
1364+
template <typename T>
1365+
Result<IDataArray*> CreateDataArrayFromAttribute(DataStructure& dataStructure, DataObject::IdType parentId, const HDF5::ObjectIO& objectIO, const std::string& attributeName,
1366+
const std::string& dataArrayName, bool preflight)
1367+
{
1368+
Result<std::vector<T>> result = objectIO.readVectorAttribute<T>(attributeName);
1369+
if(result.invalid())
1370+
{
1371+
return nx::core::MakeErrorResult<IDataArray*>(Legacy::k_FailedReadingDataArrayData_Code, fmt::format("Error reading HDF5 attribute: {}", attributeName));
1372+
}
1373+
std::vector<T> data = std::move(result.value());
1374+
1375+
std::vector<usize> tDims = {data.size()};
1376+
std::vector<usize> cDims = {1};
1377+
1378+
DataArray<T>* dataArray = nullptr;
1379+
1380+
if(preflight)
1381+
{
1382+
dataArray = DataArray<T>::template CreateWithStore<EmptyDataStore<T>>(dataStructure, dataArrayName, tDims, cDims, parentId);
1383+
}
1384+
else
1385+
{
1386+
auto dataStore = std::make_unique<DataStore<T>>(tDims, cDims, static_cast<T>(0));
1387+
std::copy(data.begin(), data.end(), dataStore->begin());
1388+
dataArray = DataArray<T>::Create(dataStructure, dataArrayName, std::move(dataStore), parentId);
1389+
}
1390+
1391+
if(nullptr == dataArray)
1392+
{
1393+
return nx::core::MakeErrorResult<IDataArray*>(Legacy::k_FailedCreatingArray_Code, fmt::format("Failed to create DataArray: '{}'", dataArrayName));
1394+
}
1395+
1396+
return {dataArray};
1397+
}
1398+
1399+
Result<> ReadAttributeAsDataArray(HDF5::ObjectIO& objectIO, const std::string& attributeName, DataStructure& dataStructure, DataObject::IdType parentId, bool preflight, std::string_view prefix)
1400+
{
1401+
HDF_ERROR_HANDLER_OFF
1402+
hid_t attribId = H5Aopen(objectIO.getId(), attributeName.c_str(), H5P_DEFAULT);
1403+
HDF_ERROR_HANDLER_ON
1404+
if(attribId < 0)
1405+
{
1406+
return MakeErrorResult(-16565, fmt::format("Unable to open attribute \"\"", attributeName));
1407+
}
1408+
hid_t typeId = H5Aget_type(attribId);
1409+
1410+
std::string daName = fmt::format("{}{}", prefix, attributeName);
1411+
1412+
Result<> result;
1413+
1414+
H5T_class_t classType = H5Tget_class(typeId);
1415+
if(classType == H5T_STRING)
1416+
{
1417+
Result<std::string> stringResult = objectIO.readStringAttribute(attributeName);
1418+
if(result.valid())
1419+
{
1420+
auto* stringArray = StringArray::CreateWithValues(dataStructure, daName, {1}, std::vector<std::string>{std::move(stringResult.value())}, parentId);
1421+
if(stringArray == nullptr)
1422+
{
1423+
result = MakeErrorResult(-16566, "Unable to create StringArray");
1424+
}
1425+
}
1426+
else
1427+
{
1428+
result = ConvertResult(std::move(stringResult));
1429+
}
1430+
}
1431+
else
1432+
{
1433+
HDF5::Type type = HDF5::getTypeFromId(typeId);
1434+
1435+
switch(type)
1436+
{
1437+
case HDF5::Type::int8: {
1438+
result = ConvertResult(CreateDataArrayFromAttribute<int8>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1439+
break;
1440+
}
1441+
case HDF5::Type::int16: {
1442+
result = ConvertResult(CreateDataArrayFromAttribute<int16>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1443+
break;
1444+
}
1445+
case HDF5::Type::int32: {
1446+
result = ConvertResult(CreateDataArrayFromAttribute<int32>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1447+
break;
1448+
}
1449+
case HDF5::Type::int64: {
1450+
result = ConvertResult(CreateDataArrayFromAttribute<int64>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1451+
break;
1452+
}
1453+
case HDF5::Type::uint8: {
1454+
result = ConvertResult(CreateDataArrayFromAttribute<uint8>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1455+
break;
1456+
}
1457+
case HDF5::Type::uint16: {
1458+
result = ConvertResult(CreateDataArrayFromAttribute<uint16>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1459+
break;
1460+
}
1461+
case HDF5::Type::uint32: {
1462+
result = ConvertResult(CreateDataArrayFromAttribute<uint32>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1463+
break;
1464+
}
1465+
case HDF5::Type::uint64: {
1466+
result = ConvertResult(CreateDataArrayFromAttribute<uint64>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1467+
break;
1468+
}
1469+
case HDF5::Type::float32: {
1470+
result = ConvertResult(CreateDataArrayFromAttribute<float32>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1471+
break;
1472+
}
1473+
case HDF5::Type::float64: {
1474+
result = ConvertResult(CreateDataArrayFromAttribute<float64>(dataStructure, parentId, objectIO, attributeName, daName, preflight));
1475+
break;
1476+
}
1477+
default: {
1478+
result = MakeErrorResult(-16567, "Invalid HDF5 for DataArray");
1479+
break;
1480+
}
1481+
}
1482+
}
1483+
1484+
H5Aclose(attribId);
1485+
H5Tclose(typeId);
1486+
1487+
return ConvertResult(std::move(result));
1488+
}
1489+
1490+
Result<> ReadAllAttributesAsDataArrays(DataStructure& dataStructure, nx::core::HDF5::ObjectIO& objectIO, DataObject::IdType parentId, bool preflight, std::string_view prefix,
1491+
const std::set<std::string>& exclusions)
1492+
{
1493+
auto attributeNames = objectIO.getAttributeNames();
1494+
for(const auto& name : attributeNames)
1495+
{
1496+
if(exclusions.contains(name))
1497+
{
1498+
continue;
1499+
}
1500+
auto result = ReadAttributeAsDataArray(objectIO, name, dataStructure, parentId, preflight, prefix);
1501+
if(result.invalid())
1502+
{
1503+
return result;
1504+
}
1505+
}
1506+
return {};
1507+
}
1508+
1509+
Result<> readLegacyStatsDataArrayChild(DataStructure& dataStructure, const nx::core::HDF5::GroupIO& parentReader, const std::string& name, DataObject::IdType parentId, bool preflight)
1510+
{
1511+
if(parentReader.isDataset(name))
1512+
{
1513+
HDF5::DatasetIO datasetIO = parentReader.openDataset(name);
1514+
std::string prefix = fmt::format("{}_", datasetIO.getName());
1515+
static const std::set<std::string> exclusions = {Legacy::CompDims, Legacy::TupleDims, "ObjectType", "Tuple Axis Dimensions", "DataArrayVersion"};
1516+
Result<> attributeResult = ReadAllAttributesAsDataArrays(dataStructure, datasetIO, parentId, false, prefix, exclusions);
1517+
if(attributeResult.invalid())
1518+
{
1519+
return attributeResult;
1520+
}
1521+
return readLegacyStatsDataArrayDatasetChild(dataStructure, datasetIO, parentId, preflight);
1522+
}
1523+
if(parentReader.isGroup(name))
1524+
{
1525+
HDF5::GroupIO groupIO = parentReader.openGroup(name);
1526+
DataGroup* dataGroup = DataGroup::Create(dataStructure, name, parentId);
1527+
if(dataGroup == nullptr)
1528+
{
1529+
return MakeErrorResult(-1434535, fmt::format("Unable to create group \"{}\"", name));
1530+
}
1531+
DataObject::IdType groupId = dataGroup->getId();
1532+
Result<> attributeResult = ReadAllAttributesAsDataArrays(dataStructure, groupIO, groupId, false, "", {});
1533+
if(attributeResult.invalid())
1534+
{
1535+
return attributeResult;
1536+
}
1537+
std::vector<std::string> groupChildren = groupIO.getChildNames();
1538+
for(const auto& childName : groupChildren)
1539+
{
1540+
Result<> result = readLegacyStatsDataArrayChild(dataStructure, groupIO, childName, groupId, preflight);
1541+
if(result.invalid())
1542+
{
1543+
return result;
1544+
}
1545+
}
1546+
return {};
1547+
}
1548+
return MakeErrorResult(-769634, fmt::format("StatsReader: Unsupported object type for \"{}\"", name));
1549+
}
1550+
1551+
Result<> readLegacyStatsDataArray(DataStructure& dataStructure, const nx::core::HDF5::GroupIO& statsReader, DataObject::IdType parentId, bool /*preflight*/)
1552+
{
1553+
// Always fully import Statistics data (ignoring the preflight parameter) because
1554+
// the Statistics hierarchy produces DataPaths of depth > 3 (up to depth 6) which
1555+
// FinishImportingLegacyDataObject cannot handle. Since StatsDataArray data is
1556+
// relatively small, fully importing during the initial read is safe and avoids
1557+
// the need for a separate finish-importing step.
1558+
std::string statsGroupName = "Statistics";
1559+
DataGroup* dataGroup = DataGroup::Create(dataStructure, statsGroupName, parentId);
1560+
if(dataGroup == nullptr)
1561+
{
1562+
return MakeErrorResult(-1434547, fmt::format("Unable to create group \"{}\"", statsGroupName));
1563+
}
1564+
std::vector<std::string> childNames = statsReader.getChildNames();
1565+
for(const auto& name : childNames)
1566+
{
1567+
Result<> result = readLegacyStatsDataArrayChild(dataStructure, statsReader, name, dataGroup->getId(), false);
1568+
if(result.invalid())
1569+
{
1570+
return result;
1571+
}
1572+
}
1573+
return {};
1574+
}
1575+
12851576
Result<> readLegacyAttributeMatrix(DataStructure& dataStructure, const nx::core::HDF5::GroupIO& amGroupReader, DataObject& parent, bool preflight = false, bool importChildren = true)
12861577
{
12871578
DataObject::IdType parentId = parent.getId();
@@ -1300,42 +1591,40 @@ Result<> readLegacyAttributeMatrix(DataStructure& dataStructure, const nx::core:
13001591
std::vector<Result<>> daResults;
13011592
if(importChildren)
13021593
{
1303-
auto dataArrayNames = amGroupReader.getChildNames();
1304-
for(const auto& daName : dataArrayNames)
1594+
auto childNames = amGroupReader.getChildNames();
1595+
for(const auto& childName : childNames)
13051596
{
1306-
// Check if child is a Dataset.
1307-
// Stats data is not supported in simplnx.
1308-
// TODO: LEGACY SYNTHETIC MICROSTRUCTURE SUPPORT
1309-
if(!amGroupReader.isDataset(daName))
1597+
if(!amGroupReader.isDataset(childName))
13101598
{
1311-
Result<> unsupportedDataResult = MakeWarningVoidResult(-298012, fmt::format("DataObject '{}' is not a supported simplnx data type", daName));
1312-
daResults.push_back(unsupportedDataResult);
1313-
continue;
1314-
}
1315-
1316-
auto dataArraySet = amGroupReader.openDataset(daName);
1317-
#if 0
1318-
if(!dataArraySet.isValid())
1319-
{
1320-
// Could not open HDF5 DataSet. Could be stats array
1321-
std::string ss = fmt::format("Could not open array '{}'", daName);
1322-
daResults.push_back(nx::core::MakeWarningVoidResult(Legacy::k_LegacyDataArrayH5_Code, ss));
1323-
continue;
1324-
}
1325-
#endif
1326-
1327-
if(isLegacyNeighborList(dataArraySet))
1328-
{
1329-
daResults.push_back(readLegacyNeighborList(dataStructure, amGroupReader, dataArraySet, attributeMatrix->getId()));
1330-
}
1331-
else if(isLegacyStringArray(dataArraySet))
1332-
{
1333-
daResults.push_back(readLegacyStringArray(dataStructure, dataArraySet, attributeMatrix->getId(), preflight));
1599+
auto groupReader = amGroupReader.openGroup(childName);
1600+
auto objectTypeResult = groupReader.readStringAttribute(Constants::k_ObjectTypeTag);
1601+
if(objectTypeResult.valid() && objectTypeResult.value() == "Statistics")
1602+
{
1603+
daResults.push_back(readLegacyStatsDataArray(dataStructure, groupReader, parentId, preflight));
1604+
}
1605+
else
1606+
{
1607+
Result<> unsupportedDataResult = MakeWarningVoidResult(-298012, fmt::format("DataObject '{}' is not a supported simplnx data type", childName));
1608+
daResults.push_back(unsupportedDataResult);
1609+
}
13341610
}
13351611
else
13361612
{
1337-
Result<> result = ConvertResult(readLegacyDataArray(dataStructure, dataArraySet, attributeMatrix->getId(), preflight));
1338-
daResults.push_back(result);
1613+
auto dataArraySet = amGroupReader.openDataset(childName);
1614+
1615+
if(isLegacyNeighborList(dataArraySet))
1616+
{
1617+
daResults.push_back(readLegacyNeighborList(dataStructure, amGroupReader, dataArraySet, attributeMatrix->getId()));
1618+
}
1619+
else if(isLegacyStringArray(dataArraySet))
1620+
{
1621+
daResults.push_back(readLegacyStringArray(dataStructure, dataArraySet, attributeMatrix->getId(), preflight));
1622+
}
1623+
else
1624+
{
1625+
Result<> result = ConvertResult(readLegacyDataArray(dataStructure, dataArraySet, attributeMatrix->getId(), preflight));
1626+
daResults.push_back(result);
1627+
}
13391628
}
13401629
}
13411630
}
@@ -1842,6 +2131,15 @@ Result<std::vector<std::shared_ptr<DataObject>>> ImportLegacyDataObjectFromFile(
18422131

18432132
Result<> FinishImportingLegacyDataObject(DataStructure& dataStructure, const nx::core::HDF5::GroupIO& parentReader, const DataPath& dataPath)
18442133
{
2134+
// Statistics data is fully imported during the initial read (readLegacyStatsDataArray
2135+
// always imports with preflight=false), so skip the finish-importing step for all
2136+
// Statistics paths. The Statistics group is placed as a sibling of the AttributeMatrix
2137+
// under the DataContainer, so any path with "Statistics" at index 1 is part of this hierarchy.
2138+
if(dataPath.getLength() >= 2 && dataPath[1] == "Statistics")
2139+
{
2140+
return {};
2141+
}
2142+
18452143
switch(dataPath.getLength())
18462144
{
18472145
case 1:

0 commit comments

Comments
 (0)