[ntuple] DAOS support: required changes after rebase

Author: jalopezg-git

tree/ntuple/v7/inc/ROOT/RPageStorageDaos.hxx

@@ -6,7 +6,7 @@
 /// is welcome!
 
 /*************************************************************************
- * Copyright (C) 1995-2020, Rene Brun and Fons Rademakers.               *
+ * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers.               *
  * All rights reserved.                                                  *
  *                                                                       *
  * For the licensing terms see $ROOTSYS/LICENSE.                         *
@@ -41,14 +41,14 @@ class RDaosContainer;
 
 // clang-format off
 /**
-\class ROOT::Experimental::RDaosNTuple
+\class ROOT::Experimental::Detail::RDaosNTupleAnchor
 \ingroup NTuple
 \brief Entry point for an RNTuple in a DAOS container. It encodes essential
 information to read the ntuple; currently, it contains (un)compressed size of
 the header/footer blobs.
 */
 // clang-format on
-struct RDaosNTuple {
+struct RDaosNTupleAnchor {
    /// Allows for evolving the struct in future versions
    std::uint32_t fVersion = 0;
    /// The size of the compressed ntuple header
@@ -59,16 +59,13 @@ struct RDaosNTuple {
    std::uint32_t fNBytesFooter = 0;
    /// The size of the uncompressed ntuple footer
    std::uint32_t fLenFooter = 0;
-   /// Currently unused, reserved for later use
-   std::uint64_t fReserved = 0;
 
-   bool operator ==(const RDaosNTuple &other) const {
+   bool operator ==(const RDaosNTupleAnchor &other) const {
       return fVersion == other.fVersion &&
          fNBytesHeader == other.fNBytesHeader &&
          fLenHeader == other.fLenHeader &&
          fNBytesFooter == other.fNBytesFooter &&
-         fLenFooter == other.fLenFooter &&
-         fReserved == other.fReserved;
+         fLenFooter == other.fLenFooter;
    }
 };
 
@@ -88,31 +85,29 @@ private:
    RNTupleMetrics fMetrics;
    std::unique_ptr<RPageAllocatorHeap> fPageAllocator;
 
-   std::shared_ptr<RDaosPool> fDaosPool;
+   /// \brief Underlying DAOS container. An internal `std::shared_ptr` keep the pool connection alive.
+   /// ISO C++ ensures the correct destruction order, i.e., `~RDaosContainer` is invoked first
+   /// (which calls `daos_cont_close()`; the destructor for the `std::shared_ptr<RDaosPool>` is invoked
+   /// after (which calls `daos_pool_disconect()`).
    std::unique_ptr<RDaosContainer> fDaosContainer;
-   /// A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'
-   std::string fLocator;
+   /// \brief A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'
+   std::string fURI;
 
-   RDaosNTuple fNTupleAnchor;
-   // FIXME: do we really need these data members?
-   /// Byte offset of the first page of the current cluster
-   std::uint64_t fClusterMinOffset = std::uint64_t(-1);
-   /// Byte offset of the end of the last page of the current cluster
-   std::uint64_t fClusterMaxOffset = 0;
-   /// Helper for zipping keys and header / footer; comprises a 16MB zip buffer
-   RNTupleCompressor fCompressor;
+   RDaosNTupleAnchor fNTupleAnchor;
 
 protected:
    void CreateImpl(const RNTupleModel &model) final;
    RClusterDescriptor::RLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final;
+   RClusterDescriptor::RLocator CommitSealedPageImpl(DescriptorId_t columnId,
+                                                     const RPageStorage::RSealedPage &sealedPage) final;
    RClusterDescriptor::RLocator CommitClusterImpl(NTupleSize_t nEntries) final;
    void CommitDatasetImpl() final;
    void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
    void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
    void WriteNTupleAnchor();
 
 public:
-   RPageSinkDaos(std::string_view ntupleName, std::string_view locator, const RNTupleWriteOptions &options);
+   RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options);
    virtual ~RPageSinkDaos();
 
    RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements = 0) final;
@@ -144,11 +139,6 @@ public:
 */
 // clang-format on
 class RPageSourceDaos : public RPageSource {
-public:
-   // FIXME: this value probably needs to match DAOS object size limit.
-   /// Cannot process pages larger than 1MB
-   static constexpr std::size_t kMaxPageSize = 1024 * 1024;
-
 private:
    /// I/O performance counters that get registered in fMetrics
    struct RCounters {
@@ -168,32 +158,29 @@ private:
    /// Wraps the I/O counters and is observed by the RNTupleReader metrics
    RNTupleMetrics fMetrics;
 
-   /// Populated pages might be shared; there memory buffer is managed by the RPageAllocatorDaos
+   /// Populated pages might be shared; the memory buffer is managed by the RPageAllocatorDaos
    std::unique_ptr<RPageAllocatorDaos> fPageAllocator;
    // TODO: the page pool should probably be handled by the base class.
    /// The page pool might, at some point, be used by multiple page sources
    std::shared_ptr<RPagePool> fPagePool;
    /// The last cluster from which a page got populated.  Points into fClusterPool->fPool
    RCluster *fCurrentCluster = nullptr;
-   /// Helper to unzip pages and header/footer; comprises a 16MB unzip buffer
-   RNTupleDecompressor fDecompressor;
-   /// A connection to a DAOS pool
-   std::shared_ptr<RDaosPool> fDaosPool;
    /// A container that stores object data (header/footer, pages, etc.)
    std::unique_ptr<RDaosContainer> fDaosContainer;
    /// A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'
-   std::string fLocator;
+   std::string fURI;
    /// The cluster pool asynchronously preloads the next few clusters
    std::unique_ptr<RClusterPool> fClusterPool;
 
    RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterDescriptor &clusterDescriptor,
-                                 ClusterSize_t::ValueType clusterIndex);
+                                 ClusterSize_t::ValueType idxInCluster);
 
 protected:
    RNTupleDescriptor AttachImpl() final;
+   void UnzipClusterImpl(RCluster *cluster) final;
 
 public:
-   RPageSourceDaos(std::string_view ntupleName, std::string_view locator, const RNTupleReadOptions &options);
+   RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options);
    /// The cloned page source creates a new connection to the pool/container.
    /// The meta-data (header and footer) is reread and parsed by the clone.
    std::unique_ptr<RPageSource> Clone() const final;
@@ -203,6 +190,9 @@ public:
    RPage PopulatePage(ColumnHandle_t columnHandle, const RClusterIndex &clusterIndex) final;
    void ReleasePage(RPage &page) final;
 
+   void LoadSealedPage(DescriptorId_t columnId, const RClusterIndex &clusterIndex,
+                       RSealedPage &sealedPage) final;
+
    std::unique_ptr<RCluster> LoadCluster(DescriptorId_t clusterId, const ColumnSet_t &columns) final;
 
    RNTupleMetrics &GetMetrics() final { return fMetrics; }