Merge pull request #289 from DUNE-DAQ/johnfreeman/prep_release_merged

Johnfreeman/prep release merged

Author: jcfreeman2

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(fdreadoutlibs VERSION 2.6.3)
+project(fdreadoutlibs VERSION 2.7.0)
 
 find_package(daq-cmake REQUIRED)
 

include/fdreadoutlibs/DAPHNEEthStreamTypeAdapter.hpp

@@ -0,0 +1,101 @@
+#ifndef FDREADOUTLIBS_INCLUDE_FDREADOUTLIBS_DAPHNEETHSTREAMTYPEADAPTER_
+#define FDREADOUTLIBS_INCLUDE_FDREADOUTLIBS_DAPHNEETHSTREAMTYPEADAPTER_
+
+#include "daqdataformats/FragmentHeader.hpp"
+#include "daqdataformats/SourceID.hpp"
+#include "fddetdataformats/DAPHNEEthStreamFrame.hpp"
+
+
+
+namespace dunedaq::fdreadoutlibs::types {
+
+  /**                                                                                                                       
+   */
+  const constexpr std::size_t kDAPHNEEthStreamNumFrames = 1;
+  const constexpr std::size_t kDAPHNEEthStreamSize = kDAPHNEEthStreamNumFrames * sizeof(dunedaq::fddetdataformats::DAPHNEEthStreamFrame); 
+
+  struct DAPHNEEthStreamTypeAdapter {
+
+    using FrameType = dunedaq::fddetdataformats::DAPHNEEthStreamFrame;
+
+    char data[kDAPHNEEthStreamSize];
+
+    // comparable based on first timestamp
+    bool operator<(const DAPHNEEthStreamTypeAdapter& other) const
+    {
+      auto thisptr = reinterpret_cast<const FrameType*>(&data);        // NOLINT
+      auto otherptr = reinterpret_cast<const FrameType*>(&other.data); // NOLINT
+      return thisptr->get_timestamp() < otherptr->get_timestamp() ? true : false;
+    }
+
+    uint64_t get_timestamp() const // NOLINT(build/unsigned)                                                          
+    {
+      return reinterpret_cast<const FrameType*>(&data)->daq_header.get_timestamp(); // NOLINT                                                                                                               
+    }
+
+    void set_timestamp(uint64_t ts) // NOLINT(build/unsigned)                                                         
+    {
+      auto frame = reinterpret_cast<FrameType*>(&data); // NOLINT                  
+      frame->set_timestamp(ts);
+    }
+
+    void fake_timestamps(uint64_t first_timestamp, uint64_t offset = 280) // NOLINT(build/unsigned)                          
+    {
+      uint64_t ts_next = first_timestamp; // NOLINT(build/unsigned)                                                         
+      for (unsigned int i = 0; i < get_num_frames(); ++i) {
+        auto df = reinterpret_cast<FrameType*>((reinterpret_cast<uint8_t*>(&data)) + i * get_frame_size());
+        df->daq_header.timestamp = ts_next;
+        ts_next += offset;
+      }
+    }
+
+  void fake_geoid(uint16_t crate_id, uint16_t slot_id, uint16_t stream_id) {
+      for (unsigned int i = 0; i < get_num_frames(); ++i) {
+        auto df = reinterpret_cast<FrameType*>((reinterpret_cast<uint8_t*>(&data)) + i * get_frame_size());
+        df->daq_header.crate_id = crate_id;
+        df->daq_header.slot_id = slot_id;
+        df->daq_header.stream_id = stream_id;
+      }
+  }
+
+  void fake_adc_pattern(int channel) {
+    // Set the ADC for the first sample to the 14-bit max value 
+    auto frame = reinterpret_cast<FrameType*>(&data); // NOLINT
+    frame->set_adc(channel, 0, 0x3FFF);
+  }
+
+
+    void fake_frame_errors(std::vector<uint16_t>* /*fake_errors*/) // NOLINT                                                
+    {
+      // Set frame error bits in header                                                                                     
+    }
+
+    FrameType* begin()
+    {
+      return reinterpret_cast<FrameType*>(&data[0]); // NOLINT                                                              
+    }
+
+    FrameType* end()
+    {
+      return reinterpret_cast<FrameType*>(data + kDAPHNEEthStreamSize); // NOLINT                                  
+    }
+
+    constexpr size_t get_payload_size() const { return kDAPHNEEthStreamSize; }
+
+    constexpr size_t get_num_frames() const { return kDAPHNEEthStreamNumFrames; }
+
+    constexpr size_t get_frame_size() const { return sizeof(dunedaq::fddetdataformats::DAPHNEEthStreamFrame); }
+
+    static const constexpr daqdataformats::SourceID::Subsystem subsystem = daqdataformats::SourceID::Subsystem::kDetectorReadout;
+    static const constexpr daqdataformats::FragmentType fragment_type = daqdataformats::FragmentType::kDAPHNEEthStream;
+    static const constexpr uint64_t expected_tick_difference = 280; // NOLINT(build/unsigned)    
+  };
+
+  static_assert(sizeof(struct DAPHNEEthStreamTypeAdapter) == kDAPHNEEthStreamSize,
+                "Check your assumptions on DAPHNEEthStreamTypeAdapter");
+
+
+} // namespace dunedaq::fdreadoutlibs::types
+
+
+#endif // FDREADOUTLIBS_INCLUDE_FDREADOUTLIBS_DAPHNEETHSTREAMTYPEADAPTER_ 

include/fdreadoutlibs/daphneeth/DAPHNEEthStreamFrameProcessor.hpp

@@ -0,0 +1,72 @@
+/**
+ * @file DAPHNEEthStreamFrameProcessor.hpp DAPHNE specific Task based raw processor
+ * for DAPHNE Eth Streaming mode
+ *
+ * This is part of the DUNE DAQ , copyright 2020.
+ * Licensing/copyright details are in the COPYING file that you should have
+ * received with this code.
+ */
+#ifndef FDREADOUTLIBS_INCLUDE_FDREADOUTLIBS_DAPHNEETH_DAPHNEETHSTREAMFRAMEPROCESSOR_HPP_
+#define FDREADOUTLIBS_INCLUDE_FDREADOUTLIBS_DAPHNEETH_DAPHNEETHSTREAMFRAMEPROCESSOR_HPP_
+
+#include "logging/Logging.hpp"
+
+#include "datahandlinglibs/FrameErrorRegistry.hpp"
+#include "datahandlinglibs/DataHandlingIssues.hpp"
+#include "datahandlinglibs/ReadoutLogging.hpp"
+#include "datahandlinglibs/models/TaskRawDataProcessorModel.hpp"
+
+#include "fddetdataformats/DAPHNEEthStreamFrame.hpp"
+#include "fdreadoutlibs/DAPHNEEthStreamTypeAdapter.hpp"
+
+#include <atomic>
+#include <functional>
+#include <memory>
+#include <string>
+
+using dunedaq::datahandlinglibs::logging::TLVL_BOOKKEEPING;
+
+namespace dunedaq {
+namespace fdreadoutlibs {
+
+class DAPHNEEthStreamFrameProcessor : public datahandlinglibs::TaskRawDataProcessorModel<types::DAPHNEEthStreamTypeAdapter>
+{
+
+public:
+  using inherited = datahandlinglibs::TaskRawDataProcessorModel<types::DAPHNEEthStreamTypeAdapter>;
+  using frameptr = types::DAPHNEEthStreamTypeAdapter*;
+  using daphneframeptr = dunedaq::fddetdataformats::DAPHNEEthStreamFrame*;
+  using timestamp_t = std::uint64_t; // NOLINT(build/unsigned)
+
+  explicit DAPHNEEthStreamFrameProcessor(std::unique_ptr<datahandlinglibs::FrameErrorRegistry>& error_registry, bool post_processing_enabled)
+    : datahandlinglibs::TaskRawDataProcessorModel<types::DAPHNEEthStreamTypeAdapter>(error_registry, post_processing_enabled)
+  {}
+
+  // Override config for pipeline setup
+  void conf(const appmodel::DataHandlerModule* conf) override;
+
+protected:
+  /**
+   * Pipeline Stage 1.: Check proper timestamp increments in DAPHNE frame
+   * */
+  void timestamp_check(frameptr /*fp*/);
+
+  /**
+   * Pipeline Stage 2.: Check DAPHNE headers for error flags
+   * */
+  void frame_error_check(frameptr /*fp*/);
+
+  // Internals
+  timestamp_t m_previous_ts = 0;
+  timestamp_t m_current_ts = 0;
+  bool m_first_ts_fake = true;
+  bool m_first_ts_missmatch = true;
+  bool m_problem_reported = false;
+  std::atomic<int> m_ts_error_ctr{ 0 };
+
+};
+
+} // namespace fdreadoutlibs
+} // namespace dunedaq
+
+#endif // FDREADOUTLIBS_INCLUDE_FDREADOUTLIBS_DAPHNEETH_DAPHNEETHSTREAMFRAMEPROCESSOR_HPP_

src/daphneeth/DAPHNEEthStreamFrameProcessor.cpp

@@ -0,0 +1,86 @@
+/**
+ * @file DAPHNEEthStreamFrameProcessor.hpp DAPHNE specific Task based raw processor
+ * implementation for streaming mode
+ *
+ * This is part of the DUNE DAQ , copyright 2020.
+ * Licensing/copyright details are in the COPYING file that you should have
+ * received with this code.
+ */
+#include "fddetdataformats/DAPHNEEthStreamFrame.hpp"
+#include "fdreadoutlibs/daphneeth/DAPHNEEthStreamFrameProcessor.hpp"
+
+#include <atomic>
+#include <functional>
+#include <memory>
+#include <string>
+
+using dunedaq::datahandlinglibs::logging::TLVL_BOOKKEEPING;
+using dunedaq::datahandlinglibs::logging::TLVL_FRAME_RECEIVED;
+
+namespace dunedaq {
+namespace fdreadoutlibs {
+
+void 
+DAPHNEEthStreamFrameProcessor::conf(const appmodel::DataHandlerModule* conf)
+{
+  datahandlinglibs::TaskRawDataProcessorModel<types::DAPHNEEthStreamTypeAdapter>::add_preprocess_task(
+    std::bind(&DAPHNEEthStreamFrameProcessor::timestamp_check, this, std::placeholders::_1));
+  // m_tasklist.push_back( std::bind(&DAPHNEStreamFrameProcessor::frame_error_check, this, std::placeholders::_1) );
+  TaskRawDataProcessorModel<types::DAPHNEEthStreamTypeAdapter>::conf(conf);
+}
+
+/**
+ * Pipeline Stage 1.: Check proper timestamp increments in DAPHNE frame
+ * */
+void 
+DAPHNEEthStreamFrameProcessor::timestamp_check(frameptr fp)
+{
+/* Let Source Emulator deal with this
+  // If EMU data, emulate perfectly incrementing timestamp
+  if (inherited::m_emulator_mode) { // emulate perfectly incrementing timestamp
+    uint64_t ts_next = m_previous_ts + 64; // NOLINT(build/unsigned)
+    auto df = reinterpret_cast<daphneframeptr>(((uint8_t*)fp));  // NOLINT
+    for (unsigned int i = 0; i < fp->get_num_frames(); ++i) { // NOLINT(build/unsigned)
+      //auto wfh = const_cast<dunedaq::fddetdataformats::WIB2Header*>(wf->get_wib_header());
+      df->set_timestamp(ts_next);
+      ts_next += 64;
+      df++;
+    }
+  }
+*/
+  // Acquire timestamp
+  m_current_ts = fp->get_timestamp();
+  uint64_t k_clock_frequency = 62500000; // NOLINT(build/unsigned)
+  uint16_t tick_difference = types::DAPHNEEthStreamTypeAdapter::expected_tick_difference;
+  uint16_t frame_tick_difference = tick_difference * fp->get_num_frames();
+  TLOG_DEBUG(TLVL_FRAME_RECEIVED) << "Received DAPHNEEthStream frame timestamp value of " << m_current_ts << " ticks (..." << std::fixed << std::setprecision(8) << (static_cast<double>(m_current_ts % (k_clock_frequency*1000)) / static_cast<double>(k_clock_frequency)) << " sec)"; // NOLINT
+
+  // Check timestamp
+  // RS warning : not fixed rate!
+  if (m_current_ts - m_previous_ts != frame_tick_difference) {
+    ++m_ts_error_ctr;
+  }
+
+  if (m_ts_error_ctr > 1000) {
+    if (!m_problem_reported) {
+      TLOG() << "*** Data Integrity ERROR *** Timestamp continuity is completely broken! "
+             << "Something is wrong with the FE source or with the configuration!";
+      m_problem_reported = true;
+    }
+  }
+
+  m_previous_ts = m_current_ts;
+  m_last_processed_daq_ts = m_current_ts;
+}
+
+/**
+ * Pipeline Stage 2.: Check DAPHNE headers for error flags
+ * */
+void 
+DAPHNEEthStreamFrameProcessor::frame_error_check(frameptr /*fp*/)
+{
+  // check error fields
+}
+
+} // namespace fdreadoutlibs
+} // namespace dunedaq

unittest/FDReadoutRequestHandlers_test.cxx

@@ -9,6 +9,7 @@
 #define BOOST_TEST_MODULE FDReadoutRequestHandlers_test // NOLINT
 
 #include "fdreadoutlibs/DAPHNEStreamSuperChunkTypeAdapter.hpp"
+#include "fdreadoutlibs/DAPHNEEthStreamTypeAdapter.hpp"
 #include "fdreadoutlibs/DAPHNESuperChunkTypeAdapter.hpp"
 #include "fdreadoutlibs/DAPHNEEthTypeAdapter.hpp"
 #include "fdreadoutlibs/DUNEWIBEthTypeAdapter.hpp"
@@ -56,7 +57,19 @@ BOOST_AUTO_TEST_CASE(BinarySearchQueueModel_DAPHNEStreamSuperChunk)
     dunedaq::datahandlinglibs::BinarySearchQueueModel,
     dunedaq::fdreadoutlibs::types::DAPHNEStreamSuperChunkTypeAdapter>();
 }
+BOOST_AUTO_TEST_CASE(FixedRateQueueModel_DAPHNEEthStream)
+{
+  dunedaq::datahandlinglibs::test::test_request_model<
+    dunedaq::datahandlinglibs::FixedRateQueueModel,
+    dunedaq::fdreadoutlibs::types::DAPHNEEthStreamTypeAdapter>();
+}
 
+BOOST_AUTO_TEST_CASE(BinarySearchQueueModel_DAPHNEEthStream)
+{
+  dunedaq::datahandlinglibs::test::test_request_model<
+    dunedaq::datahandlinglibs::BinarySearchQueueModel,
+    dunedaq::fdreadoutlibs::types::DAPHNEEthStreamTypeAdapter>();
+}
 BOOST_AUTO_TEST_CASE(SkipListLatencyBufferModel_DAPHNESuperChunk)
 {
   dunedaq::datahandlinglibs::test::test_request_model<dunedaq::datahandlinglibs::SkipListLatencyBufferModel,

unittest/FDTypeAdaptersBuffers_test.cxx

@@ -9,6 +9,7 @@
 #define BOOST_TEST_MODULE FDTypeAdaptersBuffers_test // NOLINT
 
 #include "fdreadoutlibs/DAPHNEStreamSuperChunkTypeAdapter.hpp"
+#include "fdreadoutlibs/DAPHNEEthStreamTypeAdapter.hpp"
 #include "fdreadoutlibs/DAPHNEEthTypeAdapter.hpp"
 #include "fdreadoutlibs/DAPHNESuperChunkTypeAdapter.hpp"
 #include "fdreadoutlibs/DUNEWIBEthTypeAdapter.hpp"
@@ -56,7 +57,11 @@ BOOST_AUTO_TEST_CASE(SkipListLatencyBufferModel_DAPHNEEth)
   dunedaq::datahandlinglibs::test::test_queue_model<dunedaq::datahandlinglibs::SkipListLatencyBufferModel,
                                                     dunedaq::fdreadoutlibs::types::DAPHNEEthTypeAdapter>();
 }
-BOOST_AUTO_TEST_CASE(FixedRateQueueModel_TDEEth)
+BOOST_AUTO_TEST_CASE(FixedRateQueueModel_DAPHNEEthStream)
+{
+  dunedaq::datahandlinglibs::test::test_queue_model<dunedaq::datahandlinglibs::FixedRateQueueModel,
+                                                    dunedaq::fdreadoutlibs::types::DAPHNEEthStreamTypeAdapter>();
+}BOOST_AUTO_TEST_CASE(FixedRateQueueModel_TDEEth)
 {
   dunedaq::datahandlinglibs::test::test_queue_model<dunedaq::datahandlinglibs::FixedRateQueueModel,
                                                     dunedaq::fdreadoutlibs::types::TDEEthTypeAdapter>();