valid range of Energy values for LHCInfoPerFill O2O + refacotring and

testing of perFill O2O

Energy 0 check

dont upload empty payklod to empty tags in duringfill mode of perFill O2O

extract findFillToProcess from populateIovs in perFill O2O

simple catch2 test of LHCInfoPerFill PopCon

test: LHCInfoPerFillPopConSourceHandler.isPayloadValid works

test: addPayloadToBuffer works

refactor getLumiData, add inclusiveSearchTime arg to findFillToProcess

catch2 tests code-format

remove dead code: add empty payload in duringfill

adjust expected value in test_lhcInfoNewPopCon.sh for perFill duringFill mode

extract createSubsystemDbSessions from populateIovs

refactor: extract getNextFillSearchTimestamp from populateIovs

refactor: extract handleIfNewTagAndGetLastSince and fetchLastPayload from populateIovs

add TestLHCInfoPerFillPopConSourceHandler with mocks

Successfull populateIovs test

mock execution time, successful duringFill test

print additional info on invalid energy, rename throwIfInvalidPayload

throw exc or filter out invalid payloads depending on throwOnInvalid in LHCInfoPer O2O duringFill mode

Author: JanChyczynski

CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "CondCore/PopCon/interface/PopConSourceHandler.h"
 #include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
 #include "CondTools/RunInfo/interface/OMSAccess.h"
@@ -11,31 +13,64 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
   void getNewObjects() override;
   std::string id() const override;
 
-private:
+  bool isPayloadValid(const LHCInfoPerFill& payload) const;
+
+protected:
+  virtual std::unique_ptr<LHCInfoPerFill> findFillToProcess(cond::OMSService& oms,
+                                                            const boost::posix_time::ptime& nextFillSearchTime,
+                                                            bool inclusiveSearchTime);
   void addEmptyPayload(cond::Time_t iov);
 
   // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map
   void addPayloadToBuffer(cond::OMSServiceResultRef& row);
   void convertBufferedIovsToLumiid(std::map<cond::Time_t, cond::Time_t> timestampToLumiid);
 
-  size_t getLumiData(const cond::OMSService& oms,
-                     unsigned short fillId,
-                     const boost::posix_time::ptime& beginFillTime,
-                     const boost::posix_time::ptime& endFillTime);
+  /**
+   * @return A tuple containing:
+   *         - cond::OMSServiceResult: The result of the luminosity query.
+   *         - bool: Indicates whether the query was successful.
+   *         - std::unique_ptr<cond::OMSServiceQuery>: Owner object for the query.
+   *           Query result resources are tied to its lifetime so it needs to be kept in the same scope.
+   */
+  virtual std::tuple<cond::OMSServiceResult, bool, std::unique_ptr<cond::OMSServiceQuery>> executeLumiQuery(
+      const cond::OMSService& oms,
+      unsigned short fillId,
+      const boost::posix_time::ptime& beginFillTime,
+      const boost::posix_time::ptime& endFillTime) const;
+
+  virtual void getLumiData(const cond::OMSService& oms,
+                           unsigned short fillId,
+                           const boost::posix_time::ptime& beginFillTime,
+                           const boost::posix_time::ptime& endFillTime);
+
+  virtual void getDipData(const cond::OMSService& oms,
+                          const boost::posix_time::ptime& beginFillTime,
+                          const boost::posix_time::ptime& endFillTime);
 
-  void getDipData(const cond::OMSService& oms,
-                  const boost::posix_time::ptime& beginFillTime,
-                  const boost::posix_time::ptime& endFillTime);
+  virtual bool getCTPPSData(cond::persistency::Session& session,
+                            const boost::posix_time::ptime& beginFillTime,
+                            const boost::posix_time::ptime& endFillTime);
 
-  bool getCTPPSData(cond::persistency::Session& session,
-                    const boost::posix_time::ptime& beginFillTime,
-                    const boost::posix_time::ptime& endFillTime);
+  virtual bool getEcalData(cond::persistency::Session& session,
+                           const boost::posix_time::ptime& lowerTime,
+                           const boost::posix_time::ptime& upperTime);
 
-  bool getEcalData(cond::persistency::Session& session,
-                   const boost::posix_time::ptime& lowerTime,
-                   const boost::posix_time::ptime& upperTime);
+  bool getCTPPSDataImpl(cond::persistency::Session& session,
+                        const boost::posix_time::ptime& beginFillTime,
+                        const boost::posix_time::ptime& endFillTime);
 
-private:
+  bool getEcalDataImpl(cond::persistency::Session& session,
+                       const boost::posix_time::ptime& lowerTime,
+                       const boost::posix_time::ptime& upperTime);
+
+protected:
+  virtual std::tuple<cond::persistency::Session, cond::persistency::Session> createSubsystemDbSessions() const;
+  virtual cond::Time_t getNextFillSearchTimestamp(cond::Time_t lastSince) const;
+  virtual cond::Time_t handleIfNewTagAndGetLastSince();
+  virtual void fetchLastPayload();
+  virtual boost::posix_time::ptime getExecutionTime() const;
+  void populateIovs();
+  void handleInvalidPayloads();
   bool m_debug;
   // starting date for sampling
   boost::posix_time::ptime m_startTime;
@@ -46,6 +81,11 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
   std::string m_connectionString, m_ecalConnectionString;
   std::string m_authpath;
   std::string m_omsBaseUrl;
+
+  float m_minEnergy;  // [GeV], applicable in duringFill mode only
+  float m_maxEnergy;  // [GeV], applicable in duringFill mode only
+  bool m_throwOnInvalid = true;
+
   std::unique_ptr<LHCInfoPerFill> m_fillPayload;
   std::shared_ptr<LHCInfoPerFill> m_prevPayload;
   std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerFill>>> m_tmpBuffer;

CondTools/RunInfo/interface/LHCInfoPerLSPopConSourceHandler.h

@@ -20,7 +20,7 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
 
 private:
   void populateIovs();
-  void filterInvalidPayloads();
+  void handleInvalidPayloads();
   bool isPayloadValid(const LHCInfoPerLS& payload) const;
   void addEmptyPayload(cond::Time_t iov);
   void addDefaultPayload(cond::Time_t iov, unsigned short fill, const cond::OMSService& oms);
@@ -62,15 +62,16 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
   float m_maxBetaStar;       // meters
   float m_minCrossingAngle;  // urad
   float m_maxCrossingAngle;  // urad
+  bool m_throwOnInvalid;     // duringFill: whether to throw exception or filter out invalid payloads
 
   std::unique_ptr<LHCInfoPerLS> m_fillPayload;
   std::shared_ptr<LHCInfoPerLS> m_prevPayload;
-  cond::Time_t m_startFillTime;
-  cond::Time_t m_endFillTime;
-  cond::Time_t m_prevEndFillTime = 0;
-  cond::Time_t m_prevStartFillTime;
-  cond::Time_t m_startStableBeamTime;
-  cond::Time_t m_endStableBeamTime;
+  cond::Time_t m_startFillTime{};
+  cond::Time_t m_endFillTime{};
+  cond::Time_t m_prevEndFillTime{};
+  cond::Time_t m_prevStartFillTime{};
+  cond::Time_t m_startStableBeamTime{};
+  cond::Time_t m_endStableBeamTime{};
   std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerLS>>> m_tmpBuffer;
   bool m_lastPayloadEmpty = false;
   // mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to

CondTools/RunInfo/interface/TestLHCInfoPerFillPopConSourceHandler.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
+#include "CondTools/RunInfo/interface/LHCInfoPerFillPopConSourceHandler.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <tuple>
+
+class TestLHCInfoPerFillPopConSourceHandler : public LHCInfoPerFillPopConSourceHandler {
+public:
+  TestLHCInfoPerFillPopConSourceHandler(edm::ParameterSet const& pset);
+  ~TestLHCInfoPerFillPopConSourceHandler() override = default;
+
+  std::vector<std::pair<cond::Time_t /*timestamp*/, std::shared_ptr<LHCInfoPerFill>>> mockOmsFills;
+  std::map<unsigned short /*fillNr*/, cond::OMSServiceResult> mockLumiData;
+  boost::posix_time::ptime mockExecutionTime;
+
+  Container& iovs() { return m_iovs; }
+
+protected:
+  std::unique_ptr<LHCInfoPerFill> findFillToProcess(cond::OMSService& oms,
+                                                    const boost::posix_time::ptime& nextFillSearchTime,
+                                                    bool inclusiveSearchTime) override;
+
+  cond::Time_t handleIfNewTagAndGetLastSince() override;
+
+  void fetchLastPayload() override {};
+
+  boost::posix_time::ptime getExecutionTime() const override;
+
+  std::tuple<cond::persistency::Session, cond::persistency::Session> createSubsystemDbSessions() const override;
+
+  void getDipData(const cond::OMSService& oms,
+                  const boost::posix_time::ptime& beginFillTime,
+                  const boost::posix_time::ptime& endFillTime) override {};
+
+  bool getCTPPSData(cond::persistency::Session& session,
+                    const boost::posix_time::ptime& beginFillTime,
+                    const boost::posix_time::ptime& endFillTime) override {
+    return true;
+  };
+
+  bool getEcalData(cond::persistency::Session& session,
+                   const boost::posix_time::ptime& lowerTime,
+                   const boost::posix_time::ptime& upperTime) override {
+    return true;
+  };
+
+  std::tuple<cond::OMSServiceResult, bool, std::unique_ptr<cond::OMSServiceQuery>> executeLumiQuery(
+      const cond::OMSService& oms,
+      unsigned short fillId,
+      const boost::posix_time::ptime& beginFillTime,
+      const boost::posix_time::ptime& endFillTime) const override;
+};

CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer_cfg.py

@@ -93,6 +93,19 @@
                   )
 
 
+options.register('minEnergy',  450  
+                , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float
+                , """duringFill only: [GeV] min value of the range of valid values (inclusive).
+                     If the value is outside of this range the payload is not uploaded""")
+options.register('maxEnergy',  8000.
+                , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float
+                , """duringFill only: [GeV] max value of the range of valid values (inclusive).
+                     If the value is outside of this range the payload is not uploaded""")
+options.register('throwOnInvalid', False,
+                VarParsing.VarParsing.multiplicity.singleton,
+                VarParsing.VarParsing.varType.bool,
+                "duringFill only: If true, throw on invalid payloads; if false, filter them out.")
+
 # so far there was no need to use option, added just in case
 options.register( 'authenticationPath'
                 , ""
@@ -173,7 +186,10 @@
                                    ecalConnectionString = cms.untracked.string(options.ecalConnection),
                                    omsBaseUrl = cms.untracked.string(options.oms),
                                    authenticationPath = cms.untracked.string(options.authenticationPath),
-                                   debug=cms.untracked.bool(False)
+                                   debug=cms.untracked.bool(False),
+                                   minEnergy = cms.untracked.double(options.minEnergy),
+                                   maxEnergy = cms.untracked.double(options.maxEnergy),
+                                   throwOnInvalid = cms.untracked.bool(options.throwOnInvalid)
                                ),
                                loggingOn = cms.untracked.bool(True),
                                IsDestDbCheckedInQueryLog = cms.untracked.bool(False)

CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer_cfg.py

@@ -136,20 +136,24 @@
 # it's unlikely to ever use values different from the defaults, added as a parameter just in case
 options.register('minBetaStar',  0.1 
                 , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float
-                , """duringFill only: [meters] min value of the range of valid values.
+                , """duringFill only: [meters] min value of the range of valid values (inclusive).
                      If the value is outside of this range the payload is not uploaded""")
 options.register('maxBetaStar',  100.
                 , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float
-                , """duringFill only: [meters] min value of the range of valid values.
+                , """duringFill only: [meters] max value of the range of valid values (inclusive).
                      If the value is outside of this range the payload is not uploaded""")
 options.register('minCrossingAngle',  10.
                 , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float
-                , """duringFill only: [urad] min value of the range of valid values.
+                , """duringFill only: [urad] min value of the range of valid values (inclusive).
                      If the value is outside of this range the payload is not uploaded""")
 options.register('maxCrossingAngle',  500.
                 , VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.float
-                , """duringFill only: [urad] min value of the range of valid values.
+                , """duringFill only: [urad] max value of the range of valid values (inclusive).
                      If the value is outside of this range the payload is not uploaded""")
+options.register('throwOnInvalid', False,
+                VarParsing.VarParsing.multiplicity.singleton,
+                VarParsing.VarParsing.varType.bool,
+                "duringFill only: If true, throw on invalid payloads; if false, filter them out.")
 
 # as the previous options, so far there was no need to use option, added just in case
 options.register( 'authenticationPath'
@@ -164,6 +168,8 @@
   raise ValueError("mode argument not provided. Supported modes are: duringFill endFill")
 if options.mode not in ("duringFill", "endFill"):
   raise ValueError("Wrong mode argument. Supported modes are: duringFill endFill")
+if options.throwOnInvalid and options.mode != "duringFill":
+  raise ValueError("throwOnInvalid option can be True only in duringFill mode")
 
 CondDBConnection = CondDB.clone( connect = cms.string( options.destinationConnection ) )
 CondDBConnection.DBParameters.messageLevel = cms.untracked.int32( options.messageLevel )
@@ -241,6 +247,7 @@
                                    maxBetaStar = cms.untracked.double(options.maxBetaStar),
                                    minCrossingAngle = cms.untracked.double(options.minCrossingAngle),
                                    maxCrossingAngle = cms.untracked.double(options.maxCrossingAngle),
+                                   throwOnInvalid = cms.untracked.bool(options.throwOnInvalid)
                                ),
                                loggingOn = cms.untracked.bool(True),
                                IsDestDbCheckedInQueryLog = cms.untracked.bool(False)