Merge pull request cms-sw#31544 from wddgit/localConnectInSiteLocalConfig

cmsbuild · web-flow · commit c06e9221654d · 2020-10-31T06:15:13.000+05:00
Allow setting connect string from SiteLocalConfig
diff --git a/CondCore/ESSources/plugins/BuildFile.xml b/CondCore/ESSources/plugins/BuildFile.xml
@@ -1,3 +1,4 @@
+<use name="FWCore/Catalog"/>
 <use name="FWCore/Framework"/>
 <use name="CondCore/ESSources"/>
 <library file="*.cc" name="CondCoreESSourcesPlugins">
diff --git a/CondCore/ESSources/plugins/CondDBESSource.cc b/CondCore/ESSources/plugins/CondDBESSource.cc
@@ -22,8 +22,10 @@
 #include "CondCore/ESSources/interface/DataProxy.h"
 
 #include "CondCore/CondDB/interface/PayloadProxy.h"
+#include "FWCore/Catalog/interface/SiteLocalConfig.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
+#include "FWCore/ServiceRegistry/interface/Service.h"
 #include <exception>
 
 #include <iomanip>
@@ -123,6 +125,17 @@ CondDBESSource::CondDBESSource(const edm::ParameterSet& iConfig)
     globaltag = iConfig.getParameter<std::string>("globaltag");
     // the global tag _requires_ a connection string
     m_connectionString = iConfig.getParameter<std::string>("connect");
+
+    if (!globaltag.empty()) {
+      edm::Service<edm::SiteLocalConfig> siteLocalConfig;
+      if (siteLocalConfig.isAvailable()) {
+        if (siteLocalConfig->useLocalConnectString()) {
+          std::string const& localConnectPrefix = siteLocalConfig->localConnectPrefix();
+          std::string const& localConnectSuffix = siteLocalConfig->localConnectSuffix();
+          m_connectionString = localConnectPrefix + globaltag + localConnectSuffix;
+        }
+      }
+    }
   } else if (iConfig.exists("connect"))  // default connection string
     m_connectionString = iConfig.getParameter<std::string>("connect");
 
diff --git a/FWCore/Catalog/interface/SiteLocalConfig.h b/FWCore/Catalog/interface/SiteLocalConfig.h
@@ -47,6 +47,9 @@ namespace edm {
     virtual struct addrinfo const* statisticsDestination() const = 0;
     virtual std::set<std::string> const* statisticsInfo() const = 0;
     virtual std::string const& siteName(void) const = 0;
+    virtual bool useLocalConnectString() const = 0;
+    virtual std::string const& localConnectPrefix() const = 0;
+    virtual std::string const& localConnectSuffix() const = 0;
 
     // implicit copy constructor
     // implicit assignment operator
diff --git a/FWCore/Catalog/test/FileLocator_t.cpp b/FWCore/Catalog/test/FileLocator_t.cpp
@@ -4,6 +4,8 @@
 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
 #include <boost/filesystem.hpp>
 
+#include <string>
+
 #define CATCH_CONFIG_MAIN
 #include "catch.hpp"
 
@@ -29,13 +31,14 @@ namespace {
       return nullptr;
     }
     std::set<std::string> const* statisticsInfo() const final { return nullptr; }
-    std::string const& siteName(void) const final {
-      static const std::string s_value = "TEST";
-      return s_value;
-    }
+    std::string const& siteName(void) const final { return m_emptyString; }
+    bool useLocalConnectString() const final { return false; }
+    std::string const& localConnectPrefix() const final { return m_emptyString; }
+    std::string const& localConnectSuffix() const final { return m_emptyString; }
 
   private:
     std::vector<std::string> m_catalogs;
+    std::string m_emptyString;
   };
 }  // namespace
 
diff --git a/FWCore/Services/bin/cmsGetFnConnect.cc b/FWCore/Services/bin/cmsGetFnConnect.cc
@@ -18,6 +18,7 @@
 #include "FWCore/Utilities/interface/Exception.h"
 #include <iostream>
 #include <cstring>
+#include <memory>
 
 int main(int argc, char* argv[]) {
   if ((argc != 2) || (strncmp(argv[1], "frontier://", 11) != 0)) {
diff --git a/FWCore/Services/src/SiteLocalConfigService.cc b/FWCore/Services/src/SiteLocalConfigService.cc
@@ -132,6 +132,15 @@ namespace edm {
       if (pset.exists("debugLevel")) {
         m_debugLevel = pset.getUntrackedParameter<unsigned int>("debugLevel");
       }
+      if (pset.exists("overrideUseLocalConnectString")) {
+        m_useLocalConnectString = pset.getUntrackedParameter<bool>("overrideUseLocalConnectString");
+      }
+      if (pset.exists("overrideLocalConnectPrefix")) {
+        m_localConnectPrefix = pset.getUntrackedParameter<std::string>("overrideLocalConnectPrefix");
+      }
+      if (pset.exists("overrideLocalConnectSuffix")) {
+        m_localConnectSuffix = pset.getUntrackedParameter<std::string>("overrideLocalConnectSuffix");
+      }
     }
 
     SiteLocalConfigService::~SiteLocalConfigService() {
@@ -259,6 +268,9 @@ namespace edm {
     }
 
     std::string const &SiteLocalConfigService::siteName() const { return m_siteName; }
+    bool SiteLocalConfigService::useLocalConnectString() const { return m_useLocalConnectString; }
+    std::string const &SiteLocalConfigService::localConnectPrefix() const { return m_localConnectPrefix; }
+    std::string const &SiteLocalConfigService::localConnectSuffix() const { return m_localConnectSuffix; }
 
     void SiteLocalConfigService::parse(std::string const &url) {
       tinyxml2::XMLDocument doc;
@@ -279,6 +291,9 @@ namespace edm {
       //     <frontier-connect>
       //       ... frontier-interpreted server/proxy xml ...
       //     </frontier-connect>
+      //     <local-connect>
+      //       <connectString prefix="anything1" suffix="anything2"/>
+      //     </local-connect>
       //   </calib-data>
       //   <source-config>
       //     <cache-temp-dir name="/a/b/c"/>
@@ -329,8 +344,22 @@ namespace edm {
             if (frontierConnect) {
               m_frontierConnect = _toParenString(*frontierConnect);
             }
+            auto localConnect = calibData->FirstChildElement("local-connect");
+            if (localConnect) {
+              if (frontierConnect) {
+                throw cms::Exception("Illegal site local configuration")
+                    << "It is illegal to include both frontier-connect and local-connect in the same XML file";
+              }
+              m_useLocalConnectString = true;
+              auto connectString = localConnect->FirstChildElement("connectString");
+              if (connectString) {
+                m_localConnectPrefix = safe(connectString->Attribute("prefix"));
+                m_localConnectSuffix = safe(connectString->Attribute("suffix"));
+              }
+            }
           }
         }
+
         // Parsing of the source config section
         {
           auto sourceConfig = site->FirstChildElement("source-config");
@@ -465,7 +494,9 @@ namespace edm {
           ->setComment(
               "Provide an alternate listing of statistics to send (comma separated list; current options are 'dn' or "
               "'nodn').  If left blank, all information is snet (including DNs).");
-
+      desc.addOptionalUntracked<bool>("overrideUseLocalConnectString");
+      desc.addOptionalUntracked<std::string>("overrideLocalConnectPrefix");
+      desc.addOptionalUntracked<std::string>("overrideLocalConnectSuffix");
       descriptions.add("SiteLocalConfigService", desc);
     }
   }  // namespace service
diff --git a/FWCore/Services/src/SiteLocalConfigService.h b/FWCore/Services/src/SiteLocalConfigService.h
@@ -47,6 +47,9 @@ namespace edm {
       struct addrinfo const* statisticsDestination() const override;
       std::set<std::string> const* statisticsInfo() const override;
       std::string const& siteName() const override;
+      bool useLocalConnectString() const override;
+      std::string const& localConnectPrefix() const override;
+      std::string const& localConnectSuffix() const override;
 
       // implicit copy constructor
       // implicit assignment operator
@@ -88,6 +91,9 @@ namespace edm {
       std::set<std::string> m_statisticsInfo;
       bool m_statisticsInfoAvail;
       std::string m_siteName;
+      bool m_useLocalConnectString = false;
+      std::string m_localConnectPrefix;
+      std::string m_localConnectSuffix;
     };
 
     inline bool isProcessWideService(SiteLocalConfigService const*) { return true; }
diff --git a/FWCore/Services/test/SiteLocalConfigServiceTester.cc b/FWCore/Services/test/SiteLocalConfigServiceTester.cc
@@ -10,14 +10,14 @@
 //         Created:  Tue Apr 20 16:51:38 CDT 2010
 //
 
-// system include files
 #include "FWCore/Framework/interface/EDAnalyzer.h"
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "FWCore/ServiceRegistry/interface/Service.h"
 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
 #include "FWCore/Utilities/interface/Exception.h"
 
-// user include files
+#include <string>
 
 namespace edmtest {
   class SiteLocalConfigServiceTester : public edm::EDAnalyzer {
@@ -33,54 +33,25 @@ namespace edmtest {
     unsigned int m_ttreeCacheSize;
     std::vector<std::string> m_nativeProtocols;
     bool m_valuesSet;
+    bool m_expectedUseLocalConnectString;
+    std::string m_expectedLocalConnectPrefix;
+    std::string m_expectedLocalConnectSuffix;
   };
 }  // namespace edmtest
 
 using namespace edmtest;
 
-//
-// constants, enums and typedefs
-//
-
-//
-// static data member definitions
-//
-
-//
-// constructors and destructor
-//
 SiteLocalConfigServiceTester::SiteLocalConfigServiceTester(const edm::ParameterSet& iPSet)
     : m_cacheHint(iPSet.getUntrackedParameter<std::string>("sourceCacheHint")),
       m_readHint(iPSet.getUntrackedParameter<std::string>("sourceReadHint")),
       m_tempDir(iPSet.getUntrackedParameter<std::string>("sourceTempDir")),
       m_ttreeCacheSize(iPSet.getUntrackedParameter<unsigned int>("sourceTTreeCacheSize")),
       m_nativeProtocols(iPSet.getUntrackedParameter<std::vector<std::string> >("sourceNativeProtocols")),
-      m_valuesSet(iPSet.getUntrackedParameter<bool>("sourceValuesSet", true)) {}
-
-// SiteLocalConfigServiceTester::SiteLocalConfigServiceTester(const SiteLocalConfigServiceTester& rhs)
-// {
-//    // do actual copying here;
-// }
-
-//SiteLocalConfigServiceTester::~SiteLocalConfigServiceTester()
-//{
-//}
-
-//
-// assignment operators
-//
-// const SiteLocalConfigServiceTester& SiteLocalConfigServiceTester::operator=(const SiteLocalConfigServiceTester& rhs)
-// {
-//   //An exception safe implementation is
-//   SiteLocalConfigServiceTester temp(rhs);
-//   swap(rhs);
-//
-//   return *this;
-// }
+      m_valuesSet(iPSet.getUntrackedParameter<bool>("sourceValuesSet", true)),
+      m_expectedUseLocalConnectString(iPSet.getUntrackedParameter<bool>("expectedUseLocalConnectString")),
+      m_expectedLocalConnectPrefix(iPSet.getUntrackedParameter<std::string>("expectedLocalConnectPrefix")),
+      m_expectedLocalConnectSuffix(iPSet.getUntrackedParameter<std::string>("expectedLocalConnectSuffix")) {}
 
-//
-// member functions
-//
 static void throwNotSet(const char* iName) {
   throw cms::Exception("TestFailure") << "The value " << iName << " should have been set but was not";
 }
@@ -116,9 +87,7 @@ namespace {
   }
 
 }  // namespace
-//
-// const member functions
-//
+
 void SiteLocalConfigServiceTester::analyze(const edm::Event&, const edm::EventSetup&) {
   edm::Service<edm::SiteLocalConfig> pConfig;
   if (m_valuesSet) {
@@ -148,10 +117,24 @@ void SiteLocalConfigServiceTester::analyze(const edm::Event&, const edm::EventSe
     checkNotSet("sourceTTreeCacheSize", pConfig->sourceTTreeCacheSize());
     checkNotSet("sourceNativeProtocols", pConfig->sourceNativeProtocols());
   }
-}
 
-//
-// static member functions
-//
+  if (pConfig->useLocalConnectString() != m_expectedUseLocalConnectString) {
+    throw cms::Exception("TestFailure") << "The value of useLocalConnectString is \""
+                                        << (pConfig->useLocalConnectString() ? std::string("true")
+                                                                             : std::string("false"))
+                                        << "\" but we expected the value \""
+                                        << (m_expectedUseLocalConnectString ? std::string("true")
+                                                                            : std::string("false"))
+                                        << "\"";
+  }
+  if (pConfig->localConnectPrefix() != m_expectedLocalConnectPrefix) {
+    throw cms::Exception("TestFailure") << "The value of localConnectPrefix is \"" << pConfig->localConnectPrefix()
+                                        << "\" but we expected the value \"" << m_expectedLocalConnectPrefix << "\"";
+  }
+  if (pConfig->localConnectSuffix() != m_expectedLocalConnectSuffix) {
+    throw cms::Exception("TestFailure") << "The value of localConnectSuffix is \"" << pConfig->localConnectSuffix()
+                                        << "\" but we expected the value \"" << m_expectedLocalConnectSuffix << "\"";
+  }
+}
 
 DEFINE_FWK_MODULE(SiteLocalConfigServiceTester);
diff --git a/FWCore/Services/test/full-site-local-config.testfile b/FWCore/Services/test/full-site-local-config.testfile
@@ -9,10 +9,9 @@
     <catalog url="trivialcatalog_file:/dummy/storage.xml?protocol=srm"/>
   </local-stage-out>
   <calib-data>
-    <frontier-connect>
-      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
-      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
-    </frontier-connect>
+    <local-connect>
+      <connectString prefix="Test:Prefix" suffix="Test.Suffix"/>
+    </local-connect>
   </calib-data>
   <source-config>
     <cache-temp-dir name="/a/b/c"/>
diff --git a/FWCore/Services/test/source-site-local-config.testfile b/FWCore/Services/test/source-site-local-config.testfile
@@ -9,10 +9,9 @@
     <catalog url="trivialcatalog_file:/dummy/storage.xml?protocol=srm"/>
   </local-stage-out>
   <calib-data>
-    <frontier-connect>
-      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
-      <proxy url="http://cmsfrontier.dummy.foo:3128"/>
-    </frontier-connect>
+    <local-connect>
+      <connectString prefix="Test:Prefix" suffix="Test.Suffix"/>
+    </local-connect>
   </calib-data>
   <source-config>
     <cache-temp-dir name="/a/b/c"/>
diff --git a/FWCore/Services/test/test_catch2_SiteLocalConfigService.cc b/FWCore/Services/test/test_catch2_SiteLocalConfigService.cc
@@ -1,5 +1,6 @@
 #include "FWCore/Services/src/SiteLocalConfigService.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/Utilities/interface/Exception.h"
 
 #define CATCH_CONFIG_MAIN
 #include "catch.hpp"
@@ -46,6 +47,9 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") {
     CHECK(slc.statisticsInfo()->size() == 1);
     CHECK(slc.statisticsInfo()->find("dn") != slc.statisticsInfo()->end());
     CHECK(slc.siteName() == "DUMMY");
+    REQUIRE(slc.useLocalConnectString() == true);
+    REQUIRE(slc.localConnectPrefix() == "Test:Prefix");
+    REQUIRE(slc.localConnectSuffix() == "Test.Suffix");
   }
 
   SECTION("overrides") {
@@ -63,6 +67,9 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") {
     pset.addUntrackedParameter<bool>("overridePrefetching", false);
     pset.addUntrackedParameter<std::string>("overrideStatisticsDestination", "");
     pset.addUntrackedParameter<std::vector<std::string> >("overrideStatisticsInfo", {{"nodn"}});
+    pset.addUntrackedParameter<bool>("overrideUseLocalConnectString", false);
+    pset.addUntrackedParameter<std::string>("overrideLocalConnectPrefix", "OverridePrefix");
+    pset.addUntrackedParameter<std::string>("overrideLocalConnectSuffix", "OverrideSuffix");
 
     edm::service::SiteLocalConfigService slc(pset);
 
@@ -90,5 +97,16 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") {
     CHECK(slc.statisticsInfo()->size() == 1);
     CHECK(slc.statisticsInfo()->find("nodn") != slc.statisticsInfo()->end());
     CHECK(slc.siteName() == "DUMMY");
+    REQUIRE(slc.useLocalConnectString() == false);
+    REQUIRE(slc.localConnectPrefix() == "OverridePrefix");
+    REQUIRE(slc.localConnectSuffix() == "OverrideSuffix");
+  }
+
+  SECTION("throwtest-site-local-config.testfile") {
+    edm::ParameterSet pset;
+    pset.addUntrackedParameter<std::string>("siteLocalConfigFileUrl",
+                                            dirString + "/throwtest-site-local-config.testfile");
+
+    REQUIRE_THROWS_AS(edm::service::SiteLocalConfigService(pset), cms::Exception);
   }
 }
diff --git a/FWCore/Services/test/test_sitelocalconfig_no_source_cfg.py b/FWCore/Services/test/test_sitelocalconfig_no_source_cfg.py
@@ -10,7 +10,10 @@
                             sourceReadHint=cms.untracked.string(""),
                             sourceTTreeCacheSize=cms.untracked.uint32(0),
                             sourceNativeProtocols=cms.untracked.vstring(),
-                            sourceValuesSet=cms.untracked.bool(False)
+                            sourceValuesSet=cms.untracked.bool(False),
+                            expectedUseLocalConnectString = cms.untracked.bool(False),
+                            expectedLocalConnectPrefix = cms.untracked.string(""),
+                            expectedLocalConnectSuffix = cms.untracked.string("")
 )
 
 process.o = cms.EndPath(process.tester)
diff --git a/FWCore/Services/test/test_sitelocalconfig_override_cfg.py b/FWCore/Services/test/test_sitelocalconfig_override_cfg.py
@@ -10,7 +10,11 @@
                             sourceReadHint=cms.untracked.string("direct-unbuffered"),
                             sourceTTreeCacheSize=cms.untracked.uint32(0),
                             sourceNativeProtocols=cms.untracked.vstring("rfio"),
-                            sourceValuesSet=cms.untracked.bool(True)
+                            sourceValuesSet=cms.untracked.bool(True),
+                            expectedUseLocalConnectString = cms.untracked.bool(True),
+                            expectedLocalConnectPrefix = cms.untracked.string("TestOverride:Prefix"),
+                            expectedLocalConnectSuffix = cms.untracked.string("TestOverride.Suffix")
+
 )
 
 process.o = cms.EndPath(process.tester)
@@ -20,4 +24,8 @@
                          overrideSourceCacheHintDir=cms.untracked.string("storage-only"),
                          overrideSourceReadHint=cms.untracked.string("direct-unbuffered"),
                          overrideSourceNativeProtocols=cms.untracked.vstring("rfio"),
-                         overrideSourceTTreeCacheSize=cms.untracked.uint32(0)))
+                         overrideSourceTTreeCacheSize=cms.untracked.uint32(0),
+                         overrideUseLocalConnectString = cms.untracked.bool(True),
+                         overrideLocalConnectPrefix = cms.untracked.string("TestOverride:Prefix"),
+                         overrideLocalConnectSuffix = cms.untracked.string("TestOverride.Suffix")
+))
diff --git a/FWCore/Services/test/test_sitelocalconfig_source_cfg.py b/FWCore/Services/test/test_sitelocalconfig_source_cfg.py
diff --git a/FWCore/Services/test/throwtest-site-local-config.testfile b/FWCore/Services/test/throwtest-site-local-config.testfile

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+<use name="FWCore/Catalog"/>`
`1`	`2`	`<use name="FWCore/Framework"/>`
`2`	`3`	`<use name="CondCore/ESSources"/>`
`3`	`4`	`<library file="*.cc" name="CondCoreESSourcesPlugins">`