added orbit consistency check with timestamp

Author: sy-c

src/DataBlock.h

@@ -62,6 +62,7 @@ struct DataBlockHeader {
   uint32_t timeframeOrbitLast;  ///< from timeframe
   uint8_t flagEndOfTimeframe;   ///< flag to signal this is the last TF block
   uint8_t isRdhFormat;          ///< flag set when payload is RDH-formatted
+  uint32_t orbitFirstInBlock;   ///< the first orbit in this block
   uint32_t orbitOffset;         ///< set when RDH orbits should be added given offset to match TFid
   uint32_t memorySize;          ///< size in memory of current block
 
@@ -70,10 +71,10 @@ struct DataBlockHeader {
 
 // Version of this header
 // with DB marker for DataBlock start, 1st byte in header little-endian
-const uint32_t DataBlockVersion = 0x0004DBDB;
+const uint32_t DataBlockVersion = 0x0005DBDB;
 
 // DataBlockHeader instance with all default fields
-const DataBlockHeader defaultDataBlockHeader = { .headerVersion = DataBlockVersion, .headerSize = sizeof(DataBlockHeader), .dataSize = 0, .blockId = undefinedBlockId, .pipelineId = undefinedBlockId, .timeframeId = undefinedTimeframeId, .runNumber = undefinedRunNumber, .systemId = undefinedSystemId, .feeId = undefinedFeeId, .equipmentId = undefinedEquipmentId, .linkId = undefinedLinkId, .timeframeOrbitFirst = undefinedOrbit, .timeframeOrbitLast = undefinedOrbit, .flagEndOfTimeframe = 0, .isRdhFormat = 1, .orbitOffset = undefinedOrbit, .memorySize = 0, .userSpace = { 0 } };
+const DataBlockHeader defaultDataBlockHeader = { .headerVersion = DataBlockVersion, .headerSize = sizeof(DataBlockHeader), .dataSize = 0, .blockId = undefinedBlockId, .pipelineId = undefinedBlockId, .timeframeId = undefinedTimeframeId, .runNumber = undefinedRunNumber, .systemId = undefinedSystemId, .feeId = undefinedFeeId, .equipmentId = undefinedEquipmentId, .linkId = undefinedLinkId, .timeframeOrbitFirst = undefinedOrbit, .timeframeOrbitLast = undefinedOrbit, .flagEndOfTimeframe = 0, .isRdhFormat = 1, .orbitFirstInBlock = undefinedOrbit, .orbitOffset = undefinedOrbit, .memorySize = 0, .userSpace = { 0 } };
 
 // DataBlock
 // Pair of header + payload data

src/ReadoutEquipment.cxx

@@ -487,12 +487,20 @@ Thread::CallbackResult ReadoutEquipment::threadCallback(void* arg)
 	}
 
         static InfoLogger::AutoMuteToken logTFdiscontinuityToken(LogWarningSupport_(3004), 10, 60);
+        static InfoLogger::AutoMuteToken logTFdiscontinuityTokenError(LogErrorSupport_(3004), 10, 60);
 
 	ptr->statsNumberOfTimeframes++;
 	// detect gaps in TF id continuity
 	if (tfId != ptr->lastTimeframe + 1) {
 	  if (ptr->cfgRdhDumpWarningEnabled) {
             theLog.log(logTFdiscontinuityToken, "Non-contiguous timeframe IDs %llu ... %llu", (unsigned long long)ptr->lastTimeframe, (unsigned long long)tfId);
+            // check if difference is large and orbit consistant with timestamp
+            double now = ptr->firstTimeframeTimestamp.getTime();
+            double dt = (nextBlock->getData()->header.orbitFirstInBlock - ptr->firstTimeframeHbOrbitBegin) * 1.0 / ptr->LHCOrbitRate; // diff in orbit / orbit rate = should be close to current timestamp
+            uint32_t expected = ptr->firstTimeframeHbOrbitBegin + (uint32_t)(now * ptr->LHCOrbitRate);
+            if (fabs(dt - now) > 10) {
+              theLog.log(logTFdiscontinuityTokenError, "Orbit 0x%X seems inconsistent from expected ~0x%X (orbit rate %u, elapsed time %.1fs)", (int)nextBlock->getData()->header.orbitFirstInBlock, expected, ptr->LHCOrbitRate, now);
+            }
 	  }
 	}
 	ptr->lastTimeframe = tfId;
@@ -640,6 +648,7 @@ uint64_t ReadoutEquipment::getTimeframeFromOrbit(uint32_t hbOrbit)
 {
   if (!isDefinedFirstTimeframeHbOrbitBegin) {
     firstTimeframeHbOrbitBegin = hbOrbit;
+    firstTimeframeTimestamp.reset();
     isDefinedFirstTimeframeHbOrbitBegin = 1;
     bool isOk = true;
     gReadoutStats.mutex.lock();
@@ -730,6 +739,7 @@ int ReadoutEquipment::tagDatablockFromRdh(RdhHandle& h, DataBlockHeader& bh)
     }
   }
   getTimeframeOrbitRange(tfId, bh.timeframeOrbitFirst, bh.timeframeOrbitLast);
+  bh.orbitFirstInBlock = hbOrbit;
   bh.timeframeOrbitFirst -= bh.orbitOffset;
   bh.timeframeOrbitLast -= bh.orbitOffset;
   // printf("TF %d eq %d link %d : orbits %X - %X\n", (int)bh.timeframeId, (int)bh.equipmentId, (int)bh.linkId, (int)bh.timeframeOrbitFirst, (int)bh.timeframeOrbitLast);

src/ReadoutEquipment.h

@@ -156,6 +156,7 @@ class ReadoutEquipment
   unsigned long long statsNumberOfTimeframes = 0; // number of timeframes read out
   uint32_t firstTimeframeHbOrbitBegin = 0;        // HbOrbit of beginning of first timeframe
   bool isDefinedFirstTimeframeHbOrbitBegin = 0;
+  AliceO2::Common::Timer firstTimeframeTimestamp; // timestamp of first timeframe/orbit received, for consistency checks
 
   AliceO2::Common::Timer timeframeClock; // timeframe id should be increased at each clock cycle
   uint64_t currentTimeframe = 0;         // id of current timeframe