Skip to content

Commit 54d7488

Browse files
ktfktf
authored
Merge de9dbc8 into sapling-pr-archive-ktf
2 parents 7a6e2a5 + de9dbc8 commit 54d7488

File tree

8 files changed

+78
-108
lines changed

8 files changed

+78
-108
lines changed

‎Framework/CCDBSupport/src/CCDBHelpers.cxx‎

Lines changed: 65 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,8 @@
2020
#include "CCDB/CcdbApi.h"
2121
#include "CommonConstants/LHCConstants.h"
2222
#include "Framework/Signpost.h"
23-
#include <typeinfo>
2423
#include <TError.h>
2524
#include <TMemFile.h>
26-
#include <functional>
2725

2826
O2_DECLARE_DYNAMIC_LOG(ccdb);
2927

@@ -159,6 +157,55 @@ CCDBHelpers::ParserResult CCDBHelpers::parseRemappings(char const* str)
159157
}
160158
}
161159

160+
void initialiseHelper(CCDBFetcherHelper& helper, ConfigParamRegistry const& options, std::vector<o2::framework::OutputRoute> const& outputRoutes)
161+
{
162+
std::unordered_map<std::string, bool> accountedSpecs;
163+
auto defHost = options.get<std::string>("condition-backend");
164+
auto checkRate = options.get<int>("condition-tf-per-query");
165+
auto checkMult = options.get<int>("condition-tf-per-query-multiplier");
166+
helper.timeToleranceMS = options.get<int64_t>("condition-time-tolerance");
167+
helper.queryPeriodGlo = checkRate > 0 ? checkRate : std::numeric_limits<int>::max();
168+
helper.queryPeriodFactor = checkMult > 0 ? checkMult : 1;
169+
LOGP(info, "CCDB Backend at: {}, validity check for every {} TF{}", defHost, helper.queryPeriodGlo, helper.queryPeriodFactor == 1 ? std::string{} : fmt::format(", (query for high-rate objects downscaled by {})", helper.queryPeriodFactor));
170+
LOGP(info, "Hook to enable signposts for CCDB messages at {}", (void*)&private_o2_log_ccdb->stacktrace);
171+
auto remapString = options.get<std::string>("condition-remap");
172+
CCDBHelpers::ParserResult result = CCDBHelpers::parseRemappings(remapString.c_str());
173+
if (!result.error.empty()) {
174+
throw runtime_error_f("Error while parsing remapping string %s", result.error.c_str());
175+
}
176+
helper.remappings = result.remappings;
177+
helper.apis[""].init(defHost); // default backend
178+
LOGP(info, "Initialised default CCDB host {}", defHost);
179+
//
180+
for (auto& entry : helper.remappings) { // init api instances for every host seen in the remapping
181+
if (helper.apis.find(entry.second) == helper.apis.end()) {
182+
helper.apis[entry.second].init(entry.second);
183+
LOGP(info, "Initialised custom CCDB host {}", entry.second);
184+
}
185+
LOGP(info, "{} is remapped to {}", entry.first, entry.second);
186+
}
187+
helper.createdNotBefore = std::to_string(options.get<int64_t>("condition-not-before"));
188+
helper.createdNotAfter = std::to_string(options.get<int64_t>("condition-not-after"));
189+
190+
for (auto& route : outputRoutes) {
191+
if (route.matcher.lifetime != Lifetime::Condition) {
192+
continue;
193+
}
194+
auto specStr = DataSpecUtils::describe(route.matcher);
195+
if (accountedSpecs.find(specStr) != accountedSpecs.end()) {
196+
continue;
197+
}
198+
accountedSpecs[specStr] = true;
199+
helper.routes.push_back(route);
200+
LOGP(info, "The following route is a condition {}", DataSpecUtils::describe(route.matcher));
201+
for (auto& metadata : route.matcher.metadata) {
202+
if (metadata.type == VariantType::String) {
203+
LOGP(info, "- {}: {}", metadata.name, metadata.defaultValue.asString());
204+
}
205+
}
206+
}
207+
}
208+
162209
auto getOrbitResetTime(std::pmr::vector<char> const& v) -> Long64_t
163210
{
164211
Int_t previousErrorLevel = gErrorIgnoreLevel;
@@ -261,19 +308,7 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
261308
LOGP(detail, "******** Default entry used for {} ********", path);
262309
}
263310
helper->mapURL2UUID[path].lastCheckedTF = timingInfo.tfCounter;
264-
if (etag.empty()) {
265-
helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
266-
helper->mapURL2UUID[path].cachePopulatedAt = timestampToUse;
267-
helper->mapURL2UUID[path].cacheMiss++;
268-
helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
269-
helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
270-
api.appendFlatHeader(v, headers);
271-
auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
272-
helper->mapURL2DPLCache[path] = cacheId;
273-
O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
274-
continue;
275-
}
276-
if (v.size()) { // but should be overridden by fresh object
311+
if (etag.empty() || v.size()) { // but should be overridden by fresh object
277312
// somewhere here pruneFromCache should be called
278313
helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
279314
helper->mapURL2UUID[path].cachePopulatedAt = timestampToUse;
@@ -283,15 +318,16 @@ auto populateCacheWith(std::shared_ptr<CCDBFetcherHelper> const& helper,
283318
helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
284319
api.appendFlatHeader(v, headers);
285320
auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodCCDB);
321+
if (v.size()) {
322+
// one could modify the adoptContainer to take optional old cacheID to clean:
323+
// mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
324+
}
286325
helper->mapURL2DPLCache[path] = cacheId;
287326
O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "populateCacheWith", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
288-
// one could modify the adoptContainer to take optional old cacheID to clean:
289-
// mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
290327
continue;
291-
} else {
292-
// Only once the etag is actually used, we get the information on how long the object is valid
293-
helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
294328
}
329+
// Only once the etag is actually used, we get the information on how long the object is valid
330+
helper->mapURL2UUID[path].cacheValidUntil = headers["Cache-Valid-Until"].empty() ? 0 : std::stoul(headers["Cache-Valid-Until"]);
295331
}
296332
// cached object is fine
297333
auto cacheId = helper->mapURL2DPLCache[path];
@@ -307,51 +343,7 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
307343
{
308344
return adaptStateful([](CallbackService& callbacks, ConfigParamRegistry const& options, DeviceSpec const& spec) {
309345
std::shared_ptr<CCDBFetcherHelper> helper = std::make_shared<CCDBFetcherHelper>();
310-
std::unordered_map<std::string, bool> accountedSpecs;
311-
auto defHost = options.get<std::string>("condition-backend");
312-
auto checkRate = options.get<int>("condition-tf-per-query");
313-
auto checkMult = options.get<int>("condition-tf-per-query-multiplier");
314-
helper->timeToleranceMS = options.get<int64_t>("condition-time-tolerance");
315-
helper->queryPeriodGlo = checkRate > 0 ? checkRate : std::numeric_limits<int>::max();
316-
helper->queryPeriodFactor = checkMult > 0 ? checkMult : 1;
317-
LOGP(info, "CCDB Backend at: {}, validity check for every {} TF{}", defHost, helper->queryPeriodGlo, helper->queryPeriodFactor == 1 ? std::string{} : fmt::format(", (query for high-rate objects downscaled by {})", helper->queryPeriodFactor));
318-
LOGP(info, "Hook to enable signposts for CCDB messages at {}", (void*)&private_o2_log_ccdb->stacktrace);
319-
auto remapString = options.get<std::string>("condition-remap");
320-
ParserResult result = CCDBHelpers::parseRemappings(remapString.c_str());
321-
if (!result.error.empty()) {
322-
throw runtime_error_f("Error while parsing remapping string %s", result.error.c_str());
323-
}
324-
helper->remappings = result.remappings;
325-
helper->apis[""].init(defHost); // default backend
326-
LOGP(info, "Initialised default CCDB host {}", defHost);
327-
//
328-
for (auto& entry : helper->remappings) { // init api instances for every host seen in the remapping
329-
if (helper->apis.find(entry.second) == helper->apis.end()) {
330-
helper->apis[entry.second].init(entry.second);
331-
LOGP(info, "Initialised custom CCDB host {}", entry.second);
332-
}
333-
LOGP(info, "{} is remapped to {}", entry.first, entry.second);
334-
}
335-
helper->createdNotBefore = std::to_string(options.get<int64_t>("condition-not-before"));
336-
helper->createdNotAfter = std::to_string(options.get<int64_t>("condition-not-after"));
337-
338-
for (auto &route : spec.outputs) {
339-
if (route.matcher.lifetime != Lifetime::Condition) {
340-
continue;
341-
}
342-
auto specStr = DataSpecUtils::describe(route.matcher);
343-
if (accountedSpecs.find(specStr) != accountedSpecs.end()) {
344-
continue;
345-
}
346-
accountedSpecs[specStr] = true;
347-
helper->routes.push_back(route);
348-
LOGP(info, "The following route is a condition {}", DataSpecUtils::describe(route.matcher));
349-
for (auto& metadata : route.matcher.metadata) {
350-
if (metadata.type == VariantType::String) {
351-
LOGP(info, "- {}: {}", metadata.name, metadata.defaultValue.asString());
352-
}
353-
}
354-
}
346+
initialiseHelper(*helper, options, spec.outputs);
355347
/// Add a callback on stop which dumps the statistics for the caching per
356348
/// path
357349
callbacks.set<CallbackService::Id::Stop>([helper]() {
@@ -398,18 +390,9 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
398390
// FIXME: I should send a dummy message.
399391
return;
400392
}
401-
if (etag.empty()) {
402-
helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
403-
helper->mapURL2UUID[path].cacheMiss++;
404-
helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
405-
helper->mapURL2UUID[path].maxSize = std::max(v.size(), helper->mapURL2UUID[path].maxSize);
406-
newOrbitResetTime = getOrbitResetTime(v);
407-
api.appendFlatHeader(v, headers);
408-
auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
409-
helper->mapURL2DPLCache[path] = cacheId;
410-
O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
411-
} else if (v.size()) { // but should be overridden by fresh object
412-
// somewhere here pruneFromCache should be called
393+
394+
if (etag.empty() || v.size()) {
395+
// Overwrite on cache miss
413396
helper->mapURL2UUID[path].etag = headers["ETag"]; // update uuid
414397
helper->mapURL2UUID[path].cacheMiss++;
415398
helper->mapURL2UUID[path].minSize = std::min(v.size(), helper->mapURL2UUID[path].minSize);
@@ -418,9 +401,12 @@ AlgorithmSpec CCDBHelpers::fetchFromCCDB()
418401
api.appendFlatHeader(v, headers);
419402
auto cacheId = allocator.adoptContainer(output, std::move(v), DataAllocator::CacheStrategy::Always, header::gSerializationMethodNone);
420403
helper->mapURL2DPLCache[path] = cacheId;
404+
if (v.size()) { // but should be overridden by fresh object
405+
// somewhere here pruneFromCache should be called
406+
// one could modify the adoptContainer to take optional old cacheID to clean:
407+
// mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
408+
}
421409
O2_SIGNPOST_EVENT_EMIT(ccdb, sid, "fetchFromCCDB", "Caching %{public}s for %{public}s (DPL id %" PRIu64 ")", path.data(), headers["ETag"].data(), cacheId.value);
422-
// one could modify the adoptContainer to take optional old cacheID to clean:
423-
// mapURL2DPLCache[URL] = ctx.outputs().adoptContainer(output, std::move(outputBuffer), DataAllocator::CacheStrategy::Always, mapURL2DPLCache[URL]);
424410
}
425411
// cached object is fine
426412
}

‎Framework/Core/CMakeLists.txt‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -315,11 +315,14 @@ set_property(TARGET o2-test-framework-root PROPERTY RUNTIME_OUTPUT_DIRECTORY ${o
315315
add_test(NAME framework:root COMMAND o2-test-framework-root --skip-benchmarks)
316316
add_test(NAME framework:crash COMMAND sh -e -c "PATH=${CMAKE_RUNTIME_OUTPUT_DIRECTORY}:$PATH ${CMAKE_CURRENT_LIST_DIR}/test/test_AllCrashTypes.sh")
317317

318-
o2_add_test(InfoLogger NAME test_Framework_test_InfoLogger
319-
SOURCES test/test_InfoLogger.cxx
320-
COMPONENT_NAME Framework
321-
LABELS framework
322-
PUBLIC_LINK_LIBRARIES O2::Framework AliceO2::InfoLogger)
318+
add_executable(o2-test-framework-infologger
319+
test/test_InfoLogger.cxx)
320+
target_link_libraries(o2-test-framework-infologger PRIVATE O2::Framework)
321+
target_link_libraries(o2-test-framework-infologger PRIVATE AliceO2::InfoLogger)
322+
target_link_libraries(o2-test-framework-infologger PRIVATE O2::Catch2)
323+
set_property(TARGET o2-test-framework-infologger
324+
PROPERTY RUNTIME_OUTPUT_DIRECTORY ${outdir})
325+
add_test(NAME framework:infologger COMMAND o2-test-framework-infologger)
323326

324327
o2_add_executable(dpl-null-sink
325328
SOURCES src/o2NullSink.cxx

‎Framework/Core/test/test_HTTPParser.cxx‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
1111

12-
#include <boost/test/tools/old/interface.hpp>
13-
1412
#include "../src/HTTPParser.h"
1513
#include <catch_amalgamated.hpp>
1614

‎Framework/Core/test/test_InfoLogger.cxx‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,12 @@
88
// In applying this license CERN does not waive the privileges and immunities
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
11-
#define BOOST_TEST_MODULE Test Framework InfoLoggerTest
12-
#define BOOST_TEST_MAIN
13-
#define BOOST_TEST_DYN_LINK
14-
15-
#include <boost/test/unit_test.hpp>
11+
#include <catch_amalgamated.hpp>
1612

1713
#include <InfoLogger/InfoLogger.hxx>
1814
using namespace AliceO2::InfoLogger;
1915

20-
BOOST_AUTO_TEST_CASE(InfoLoggerTest)
16+
TEST_CASE("InfoLoggerTest")
2117
{
2218

2319
// define infologger output to stdout, as we don't want to use the default infoLoggerD pipe which might not be running here
@@ -27,5 +23,5 @@ BOOST_AUTO_TEST_CASE(InfoLoggerTest)
2723
InfoLogger theLog;
2824

2925
// log a test message
30-
BOOST_CHECK(theLog.log("This is a log message test to stdout") == 0);
26+
CHECK(theLog.log("This is a log message test to stdout") == 0);
3127
}

‎Framework/Core/test/test_Parallel.cxx‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717
#include "Framework/ParallelContext.h"
1818
#include "Framework/runDataProcessing.h"
1919

20-
#include <boost/algorithm/string.hpp>
21-
2220
using namespace o2::framework;
2321

2422
struct FakeCluster {

‎Framework/Core/test/test_RegionInfoCallbackService.cxx‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,12 @@
88
// In applying this license CERN does not waive the privileges and immunities
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
11-
#include <boost/algorithm/string.hpp>
12-
13-
#include "Framework/InputSpec.h"
1411
#include "Framework/CallbackService.h"
1512
#include "Framework/ControlService.h"
1613
#include "Framework/DataProcessorSpec.h"
17-
#include "Framework/ParallelContext.h"
1814
#include "Framework/runDataProcessing.h"
1915
#include "Framework/Logger.h"
2016

21-
#include <chrono>
22-
#include <thread>
23-
2417
using namespace o2::framework;
2518
using DataHeader = o2::header::DataHeader;
2619

‎Framework/Core/test/test_Services.cxx‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88
// In applying this license CERN does not waive the privileges and immunities
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
11-
#include <boost/test/tools/old/interface.hpp>
12-
1311
#include "Framework/ServiceHandle.h"
1412
#include "Framework/ServiceRegistry.h"
1513
#include "Framework/CallbackService.h"

‎Framework/Core/test/test_TimePipeline.cxx‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,11 @@
88
// In applying this license CERN does not waive the privileges and immunities
99
// granted to it by virtue of its status as an Intergovernmental Organization
1010
// or submit itself to any jurisdiction.
11-
#include "Framework/InputSpec.h"
11+
1212
#include "Framework/DataProcessorSpec.h"
1313
#include "Framework/ParallelContext.h"
1414
#include "Framework/runDataProcessing.h"
15-
16-
#include <boost/algorithm/string.hpp>
17-
15+
#include <thread>
1816
#include <chrono>
1917

2018
using namespace o2::framework;

0 commit comments

Comments
 (0)