New view constructor and deepCopy method for Collections

Author: leonardo beltrame

DataFormats/Portable/README.md

@@ -98,6 +98,10 @@ SET_PORTABLEHOSTCOLLECTION_READ_RULES(portabletest::TestHostCollection);
 They have no implicit or explicit references to alpaka (neither as part of the class signature nor as part of its name).
 This could make it possible to read them back with different portability solutions in the future.
 
+The member function `void deepCopy(ConstView const& view)` copies the content of all scalars and columns from `view`
+(pointing to data in host memory and potentially to multiple buffers) into the `PortableHostCollection` contiguous buffer.
+See the [`View` section](../../DataFormats/SoATemplate/README.md#view) of [`DataFormats/SoATemplate/README.md`](../../DataFormats/SoATemplate/README.md) for more details.
+
 ### `PortableDeviceCollection<T, TDev>`
 
 `PortableDeviceCollection<T, TDev>` is a class template that wraps a SoA type `T` and an alpaka device buffer, which

DataFormats/Portable/interface/PortableHostCollection.h

@@ -94,6 +94,10 @@ class PortableHostCollection {
     layout.ROOTStreamerCleaner();
   }
 
+  // Copy column by column the content of the given view into this PortableHostCollection.
+  // The view must point to data in host memory.
+  void deepCopy(ConstView const& view) { layout_.deepCopy(view); }
+
 private:
   std::optional<Buffer> buffer_;  //!
   Layout layout_;                 //

DataFormats/Portable/test/BuildFile.xml

@@ -2,4 +2,5 @@
   <use name="DataFormats/Portable"/>
   <use name="DataFormats/SoATemplate"/>
   <use name="catch2"/>
+  <use name="eigen"/>
 </bin>

DataFormats/Portable/test/test_catch2_deepCopyOnHost.cc

@@ -0,0 +1,126 @@
+#include <Eigen/Core>
+#include <Eigen/Dense>
+
+#include <catch.hpp>
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+GENERATE_SOA_LAYOUT(SoAPositionTemplate,
+                    SOA_COLUMN(float, x),
+                    SOA_COLUMN(float, y),
+                    SOA_COLUMN(float, z),
+                    SOA_SCALAR(int, detectorType))
+
+using SoAPosition = SoAPositionTemplate<>;
+using SoAPositionView = SoAPosition::View;
+using SoAPositionConstView = SoAPosition::ConstView;
+
+GENERATE_SOA_LAYOUT(SoAPCATemplate,
+                    SOA_COLUMN(float, eigenvalues),
+                    SOA_COLUMN(float, eigenvector_1),
+                    SOA_COLUMN(float, eigenvector_2),
+                    SOA_COLUMN(float, eigenvector_3),
+                    SOA_EIGEN_COLUMN(Eigen::Vector3d, candidateDirection))
+
+using SoAPCA = SoAPCATemplate<>;
+using SoAPCAView = SoAPCA::View;
+using SoAPCAConstView = SoAPCA::ConstView;
+
+GENERATE_SOA_LAYOUT(GenericSoATemplate,
+                    SOA_COLUMN(float, xPos),
+                    SOA_COLUMN(float, yPos),
+                    SOA_COLUMN(float, zPos),
+                    SOA_EIGEN_COLUMN(Eigen::Vector3d, candidateDirection))
+
+using GenericSoA = GenericSoATemplate<>;
+using GenericSoAView = GenericSoA::View;
+using GenericSoAConstView = GenericSoA::ConstView;
+
+TEST_CASE("Deep copy from SoA Generic View") {
+  // common number of elements for the SoAs
+  const std::size_t elems = 10;
+
+  // Portable Collections
+  PortableHostCollection<SoAPosition> positionCollection(elems, cms::alpakatools::host());
+  PortableHostCollection<SoAPCA> pcaCollection(elems, cms::alpakatools::host());
+
+  // Portable Collection Views
+  SoAPositionView& positionCollectionView = positionCollection.view();
+  SoAPCAView& pcaCollectionView = pcaCollection.view();
+  // Portable Collection ConstViews
+  const SoAPositionConstView& positionCollectionConstView = positionCollection.const_view();
+  const SoAPCAConstView& pcaCollectionConstView = pcaCollection.const_view();
+
+  // fill up
+  for (size_t i = 0; i < elems; i++) {
+    positionCollectionView[i] = {i * 1.0f, i * 2.0f, i * 3.0f};
+  }
+  positionCollectionView.detectorType() = 1;
+
+  float time = 0.01;
+  for (size_t i = 0; i < elems; i++) {
+    pcaCollectionView[i].eigenvector_1() = positionCollectionView[i].x() / time;
+    pcaCollectionView[i].eigenvector_2() = positionCollectionView[i].y() / time;
+    pcaCollectionView[i].eigenvector_3() = positionCollectionView[i].z() / time;
+    pcaCollectionView[i].candidateDirection()(0) = positionCollectionView[i].x() / time;
+    pcaCollectionView[i].candidateDirection()(1) = positionCollectionView[i].y() / time;
+    pcaCollectionView[i].candidateDirection()(2) = positionCollectionView[i].z() / time;
+  }
+
+  SECTION("Deep copy the View") {
+    // addresses and size of the SoA columns
+    const auto posRecs = positionCollectionView.records();
+    const auto pcaRecs = pcaCollectionView.records();
+
+    // building the View with runtime check for the size
+    GenericSoAView genericView(posRecs.x(), posRecs.y(), posRecs.z(), pcaRecs.candidateDirection());
+
+    // Check for equality of memory addresses
+    REQUIRE(genericView.metadata().addressOf_xPos() == positionCollectionView.metadata().addressOf_x());
+    REQUIRE(genericView.metadata().addressOf_yPos() == positionCollectionView.metadata().addressOf_y());
+    REQUIRE(genericView.metadata().addressOf_zPos() == positionCollectionView.metadata().addressOf_z());
+    REQUIRE(genericView.metadata().addressOf_candidateDirection() ==
+            pcaCollectionView.metadata().addressOf_candidateDirection());
+
+    // PortableHostCollection that will host the aggregated columns
+    PortableHostCollection<GenericSoA> genericCollection(elems, cms::alpakatools::host());
+    genericCollection.deepCopy(genericView);
+
+    // Check for inequality of memory addresses
+    REQUIRE(genericCollection.view().metadata().addressOf_xPos() != positionCollectionView.metadata().addressOf_x());
+    REQUIRE(genericCollection.view().metadata().addressOf_yPos() != positionCollectionView.metadata().addressOf_y());
+    REQUIRE(genericCollection.view().metadata().addressOf_zPos() != positionCollectionView.metadata().addressOf_z());
+    REQUIRE(genericCollection.view().metadata().addressOf_candidateDirection() !=
+            pcaCollectionView.metadata().addressOf_candidateDirection());
+  }
+
+  SECTION("Deep copy the ConstView") {
+    // addresses and size of the SoA columns
+    const auto posRecs = positionCollectionConstView.records();
+    const auto pcaRecs = pcaCollectionConstView.records();
+
+    // building the View with runtime check for the size
+    GenericSoAConstView genericConstView(posRecs.x(), posRecs.y(), posRecs.z(), pcaRecs.candidateDirection());
+
+    // Check for equality of memory addresses
+    REQUIRE(genericConstView.metadata().addressOf_xPos() == positionCollectionView.metadata().addressOf_x());
+    REQUIRE(genericConstView.metadata().addressOf_yPos() == positionCollectionView.metadata().addressOf_y());
+    REQUIRE(genericConstView.metadata().addressOf_zPos() == positionCollectionView.metadata().addressOf_z());
+    REQUIRE(genericConstView.metadata().addressOf_candidateDirection() ==
+            pcaCollectionView.metadata().addressOf_candidateDirection());
+
+    // PortableHostCollection that will host the aggregated columns
+    PortableHostCollection<GenericSoA> genericCollection(elems, cms::alpakatools::host());
+    genericCollection.deepCopy(genericConstView);
+
+    // Check for inequality of memory addresses
+    REQUIRE(genericCollection.view().metadata().addressOf_xPos() != positionCollectionView.metadata().addressOf_x());
+    REQUIRE(genericCollection.view().metadata().addressOf_yPos() != positionCollectionView.metadata().addressOf_y());
+    REQUIRE(genericCollection.view().metadata().addressOf_zPos() != positionCollectionView.metadata().addressOf_z());
+    REQUIRE(genericCollection.view().metadata().addressOf_candidateDirection() !=
+            pcaCollectionView.metadata().addressOf_candidateDirection());
+  }
+}

DataFormats/SoATemplate/README.md

@@ -44,6 +44,9 @@ provided: `ViewTemplate`, `ViewViewTemplateFreeParams` and respectively `ConstVi
 `ConstViewTemplateFreeParams`. The parametrization of those templates is explained in the [Template
 parameters section](#template-parameters).
 
+It is also possible to build a generic `View` or `ConstView` passing from the [Metarecords sublass](#metarecords-subclass). This
+view can point to data belonging to different SoAs and thus not contiguous in memory.
+
 ## Metadata subclass
 
 In order to no clutter the namespace of the generated class, a subclass name `Metadata` is generated. It is
@@ -52,6 +55,13 @@ of elements in the SoA), `byteSize()`, `byteAlignment()`, `data()` (a pointer to
 function computes the first byte of a structure right after a layout, allowing using a single buffer for multiple
 layouts.
 
+## Metarecords subclass
+
+The nested type `Metarecords` describes the elements of the SoA. It can be instantiated by the `records()` member 
+function of a `View` or `ConstView`. Every object contains the address of the first element of the column, the number
+of elements per column, and the stride for the Eigen columns. These are used to validate the columns size at run time 
+and to build a generic `View` as described in [View](#view).
+
 ## ROOT serialization and de-serialization
 
 Layouts can be serialized and de-serialized with ROOT. In order to generate the ROOT dictionary, separate

DataFormats/SoATemplate/interface/SoACommon.h

@@ -7,6 +7,8 @@
 
 #include <cstdint>
 #include <cassert>
+#include <cstring>
+#include <memory>
 #include <ostream>
 #include <tuple>
 #include <type_traits>
@@ -123,12 +125,13 @@ namespace cms::soa {
     // default constructor
     SoAConstParametersImpl() = default;
 
-    // constructor from an address
-    SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ValueType const* addr) : addr_(addr) {}
+    // constructor from address and size
+    SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ValueType const* addr, size_type size)
+        : addr_(addr), size_{size} {}
 
     // constructor from a non-const parameter set
     SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(SoAParametersImpl<columnType, ValueType> const& o)
-        : addr_{o.addr_} {}
+        : addr_{o.addr_}, size_{o.size_} {}
 
     static constexpr bool checkAlignment(ValueType* addr, byte_size_type alignment) {
       return reinterpret_cast<intptr_t>(addr) % alignment;
@@ -139,6 +142,7 @@ namespace cms::soa {
   public:
     // scalar or column
     ValueType const* addr_ = nullptr;
+    size_type size_ = 0;
   };
 
   // Templated const parameter specialisation for Eigen columns
@@ -153,17 +157,19 @@ namespace cms::soa {
     // default constructor
     SoAConstParametersImpl() = default;
 
-    // constructor from individual address and stride
-    SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ScalarType const* addr, byte_size_type stride)
-        : addr_(addr), stride_(stride) {}
+    // constructor from individual address, stride and size
+    SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(ScalarType const* addr,
+                                                                byte_size_type stride,
+                                                                size_type size)
+        : addr_(addr), stride_(stride), size_{size} {}
 
     // constructor from address and stride packed in a tuple
     SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(TupleOrPointerType const& tuple)
         : addr_(std::get<0>(tuple)), stride_(std::get<1>(tuple)) {}
 
     // constructor from a non-const parameter set
     SOA_HOST_DEVICE SOA_INLINE constexpr SoAConstParametersImpl(SoAParametersImpl<columnType, ValueType> const& o)
-        : addr_{o.addr_}, stride_{o.stride_} {}
+        : addr_{o.addr_}, stride_{o.stride_}, size_{o.size_} {}
 
     static constexpr bool checkAlignment(TupleOrPointerType const& tuple, byte_size_type alignment) {
       const auto& [addr, stride] = tuple;
@@ -173,9 +179,10 @@ namespace cms::soa {
     TupleOrPointerType tupleOrPointer() { return {addr_, stride_}; }
 
   public:
-    // address and stride
+    // address, stride and size
     ScalarType const* addr_ = nullptr;
     byte_size_type stride_ = 0;
+    size_type size_ = 0;
   };
 
   // Matryoshka template to avoid commas inside macros
@@ -200,8 +207,9 @@ namespace cms::soa {
     // default constructor
     SoAParametersImpl() = default;
 
-    // constructor from an address
-    SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ValueType* addr) : addr_(addr) {}
+    // constructor from address and size
+    SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ValueType* addr, size_type size)
+        : addr_(addr), size_{size} {}
 
     static constexpr bool checkAlignment(ValueType* addr, byte_size_type alignment) {
       return reinterpret_cast<intptr_t>(addr) % alignment;
@@ -212,6 +220,7 @@ namespace cms::soa {
   public:
     // scalar or column
     ValueType* addr_ = nullptr;
+    size_type size_ = 0;
   };
 
   // Templated parameter specialisation for Eigen columns
@@ -229,9 +238,9 @@ namespace cms::soa {
     // default constructor
     SoAParametersImpl() = default;
 
-    // constructor from individual address and stride
-    SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ScalarType* addr, byte_size_type stride)
-        : addr_(addr), stride_(stride) {}
+    // constructor from individual address, stride and size
+    SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(ScalarType* addr, byte_size_type stride, size_type size)
+        : addr_(addr), stride_(stride), size_(size) {}
 
     // constructor from address and stride packed in a tuple
     SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl(TupleOrPointerType const& tuple)
@@ -245,9 +254,10 @@ namespace cms::soa {
     TupleOrPointerType tupleOrPointer() { return {addr_, stride_}; }
 
   public:
-    // address and stride
+    // address, stride and size
     ScalarType* addr_ = nullptr;
     byte_size_type stride_ = 0;
+    size_type size_ = 0;
   };
 
   // Matryoshka template to avoid commas inside macros
@@ -268,13 +278,13 @@ namespace cms::soa {
   template <SoAColumnType COLUMN_TYPE, typename T>
   SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl<COLUMN_TYPE, T> const_cast_SoAParametersImpl(
       SoAConstParametersImpl<COLUMN_TYPE, T> const& o) {
-    return SoAParametersImpl<COLUMN_TYPE, T>{non_const_ptr(o.addr_)};
+    return SoAParametersImpl<COLUMN_TYPE, T>{non_const_ptr(o.addr_), o.size_};
   }
 
   template <typename T>
   SOA_HOST_DEVICE SOA_INLINE constexpr SoAParametersImpl<SoAColumnType::eigen, T> const_cast_SoAParametersImpl(
       SoAConstParametersImpl<SoAColumnType::eigen, T> const& o) {
-    return SoAParametersImpl<SoAColumnType::eigen, T>{non_const_ptr(o.addr_), o.stride_};
+    return SoAParametersImpl<SoAColumnType::eigen, T>{non_const_ptr(o.addr_), o.stride_, o.size_};
   }
 
   // Helper template managing the value at index idx within a column.
@@ -554,6 +564,7 @@ namespace cms::soa {
 #endif
 
   // Helper function to compute aligned size
+  //this is an integer division -> it rounds size to the next multiple of alignment
   constexpr inline byte_size_type alignSize(byte_size_type size, byte_size_type alignment) {
     return ((size + alignment - 1) / alignment) * alignment;
   }
@@ -564,11 +575,12 @@ namespace cms::soa {
 #define SOA_COLUMN(TYPE, NAME) (_VALUE_TYPE_COLUMN, TYPE, NAME)
 #define SOA_EIGEN_COLUMN(TYPE, NAME) (_VALUE_TYPE_EIGEN_COLUMN, TYPE, NAME)
 
-/* Iterate on the macro MACRO and return the result as a comma separated list */
+/* Iterate on the macro MACRO and return the result as a comma separated list, converting
+   the boost sequence into tuples and then into list */
 #define _ITERATE_ON_ALL_COMMA(MACRO, DATA, ...) \
   BOOST_PP_TUPLE_ENUM(BOOST_PP_SEQ_TO_TUPLE(_ITERATE_ON_ALL(MACRO, DATA, __VA_ARGS__)))
 
-/* Iterate MACRO on all elements */
+/* Iterate MACRO on all elements of the boost sequence */
 #define _ITERATE_ON_ALL(MACRO, DATA, ...) BOOST_PP_SEQ_FOR_EACH(MACRO, DATA, BOOST_PP_VARIADIC_TO_SEQ(__VA_ARGS__))
 
 /* Switch on macros depending on scalar / column type */
@@ -599,6 +611,7 @@ namespace cms::soa {
     SOA_HOST_DEVICE SOA_INLINE SoAColumnAccessorsImpl(const SoAParametersImpl<SoAColumnType::column, T>& params)
         : params_(params) {}
     SOA_HOST_DEVICE SOA_INLINE T* operator()() { return params_.addr_; }
+
     using NoParamReturnType = T*;
     using ParamReturnType = T&;
     SOA_HOST_DEVICE SOA_INLINE T& operator()(size_type index) { return params_.addr_[index]; }