Merge branch 'feature/handlers' of github.com:lisataldir/TCLB into feature/handlers

Author: lisataldir

src/ArbConnectivity.hpp

@@ -16,8 +16,10 @@ struct ArbLatticeConnectivity {
     std::unique_ptr<Index[]> og_index;
     std::unique_ptr<Index[]> nbrs;
     std::unique_ptr<ZoneIndex[]> zones_per_node;
+    std::vector<Index> cart_index;
     std::vector<ZoneIndex> zones;
     double grid_size{};
+    int nx, ny, nz; // total region
 
     ArbLatticeConnectivity() = default;
     ArbLatticeConnectivity(size_t chunk_begin_, size_t chunk_end_, size_t num_nodes_global_, size_t Q_)
@@ -28,7 +30,8 @@ struct ArbLatticeConnectivity {
           coords(std::make_unique<double[]>(3 * (chunk_end_ - chunk_begin_))),
           og_index(std::make_unique<Index[]>(chunk_end_ - chunk_begin_)),
           nbrs(std::make_unique<Index[]>((chunk_end_ - chunk_begin_) * Q)),
-          zones_per_node(std::make_unique<ZoneIndex[]>(chunk_end_ - chunk_begin_)) {
+          zones_per_node(std::make_unique<ZoneIndex[]>(chunk_end_ - chunk_begin_)),
+          cart_index(chunk_end_ - chunk_begin_) {
         zones.reserve(getLocalSize());
     }
 
@@ -53,6 +56,8 @@ struct ArbLatticeConnectivity {
 
     double& coord(size_t dim, size_t local_node_ind) { return coords[local_node_ind + dim * getLocalSize()]; }
     double coord(size_t dim, size_t local_node_ind) const { return coords[local_node_ind + dim * getLocalSize()]; }
+    Index& cartesian_ind(size_t local_node_ind) { return cart_index[local_node_ind]; }
+    Index cartesian_ind(size_t local_node_ind) const { return cart_index[local_node_ind]; }
     Index& neighbor(size_t q, size_t local_node_ind) { return nbrs[local_node_ind + q * getLocalSize()]; }
     Index neighbor(size_t q, size_t local_node_ind) const { return nbrs[local_node_ind + q * getLocalSize()]; }
 

src/ArbLattice.cpp

@@ -167,6 +167,14 @@ void ArbLattice::readFromCxn(const std::string& cxn_path) {
     file >> grid_size;
     check_file_ok("Failed to read section: GRID_SIZE");
 
+    // Total region
+    file >> word;
+    check_file_ok("Failed to read section header: TOTAL_REGION");
+    check_expected_word("TOTAL_REGION", word);
+    int nx, ny, nz;
+    file >> nx >> ny >> nz;
+    check_file_ok("Failed to read section: TOTAL_REGION");
+
     // Labels present in the .cxn file
     const auto labels = process_section("NODE_LABELS", [&file](size_t n_labels) {
         std::vector<std::string> retval(n_labels);
@@ -177,24 +185,30 @@ void ArbLattice::readFromCxn(const std::string& cxn_path) {
 
     // Nodes header
     process_section("NODES", [&](size_t num_nodes_global) {
-        // Compute the current rank's offset and number of nodes to read
+        // Compute the current ranks offset and number of nodes to read
         const auto chunk_offsets = computeInitialNodeDist(num_nodes_global, static_cast<size_t>(comm_size));
         const auto chunk_begin = static_cast<size_t>(chunk_offsets[comm_rank]), chunk_end = static_cast<size_t>(chunk_offsets[comm_rank + 1]);
         const auto num_nodes_local = chunk_end - chunk_begin;
 
         connect = ArbLatticeConnectivity(chunk_begin, chunk_end, num_nodes_global, Q);
         connect.grid_size = grid_size;
+        connect.nx = nx;
+        connect.ny = ny;
+        connect.nz = nz;
 
         // Skip chunk_begin + 1 (header) newlines
         for (size_t i = 0; i != chunk_begin + 1; ++i) file.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
-        check_file_ok("Failed to skip ahead to this rank's chunk");
+        check_file_ok("Failed to skip ahead to this ranks chunk");
 
         // Parse node data
         std::vector<long> nbrs_in_file(q_provided.size());
         for (size_t local_node_ind = 0; local_node_ind != num_nodes_local; ++local_node_ind) {
             // Read coords
             file >> connect.coord(0, local_node_ind) >> connect.coord(1, local_node_ind) >> connect.coord(2, local_node_ind);
 
+            // Read cartesian index
+            file >> connect.cartesian_ind(local_node_ind);
+
             // Read neighbors, map to local (required) ones
             for (auto& nbr : nbrs_in_file) file >> nbr;
             size_t j = 0;

src/CartLatticeLauncher.hpp.Rt

@@ -255,9 +255,9 @@ public:
 
   CudaDeviceFunction void Execute() const {
     using LA = CartLatticeAccessAll;
-    int x = CudaBlock.x + small.dx;
-    int y = CudaBlock.y + small.dy;
-    int z = CudaBlock.z + small.dz;
+    int x = (CudaBlock.x + small.dx) %container.nx;
+    int y = (CudaBlock.y + small.dy) %container.ny;
+    int z = (CudaBlock.z + small.dz) %container.nz;
     LA acc(x, y, z, container); <?R
     if (q$adjoint) { ?>
       Node< LA, Adjoint, NoGlobals, Get > now(acc, data); <?R

src/toArb.cpp

@@ -61,6 +61,7 @@ static int writeArbLatticeHeader(std::fstream& file, const lbRegion& region, siz
     file << "OFFSET_DIRECTIONS " << Model_m::offset_directions.size() << '\n';
     for (const auto [x, y, z] : Model_m::offset_directions) file << x << ' ' << y << ' ' << z << '\n';
     file << "GRID_SIZE " << grid_size << '\n';
+    file << "TOTAL_REGION " << region.nx << ' ' << region.ny << ' ' << region.nz << '\n';
     file << "NODE_LABELS " << model.nodetypeflags.size() + zone_map.size() << '\n';
     for (const auto& ntf : model.nodetypeflags) file << ntf.name << '\n';
     for (const auto& [name, zf] : zone_map) file << "_Z_" << name << '\n';
@@ -93,6 +94,9 @@ static int writeArbLatticeNodes(const Geometry& geo,
                 const double z_coord = (static_cast<double>(z) + .5) * spacing;
                 file << x_coord << ' ' << y_coord << ' ' << z_coord << ' ';
 
+                // Cartesian index
+                file << current_lin_pos << ' ';
+
                 // Neighbors
                 for (const auto [dx, dy, dz] : Model_m::offset_directions) {
                     const auto lin_pos_offset = linPosBoundschecked(x + dx, y + dy, z + dz, nx, ny, nz);