Skip to content

Commit c027e3c

Browse files
Net-burstNet-burst
authored
Core: Improve traceability for Prebid Cache stores (#3757)
1 parent b5a9b28 commit c027e3c

File tree

9 files changed

+788
-60
lines changed

9 files changed

+788
-60
lines changed

‎src/main/java/org/prebid/server/cache/CoreCacheService.java‎

Lines changed: 82 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import io.vertx.core.Future;
88
import io.vertx.core.MultiMap;
99
import org.apache.commons.collections4.CollectionUtils;
10+
import org.apache.commons.lang3.ObjectUtils;
1011
import org.apache.commons.lang3.StringUtils;
1112
import org.prebid.server.auction.model.AuctionContext;
1213
import org.prebid.server.auction.model.BidInfo;
@@ -63,6 +64,8 @@ public class CoreCacheService {
6364
private static final Logger logger = LoggerFactory.getLogger(CoreCacheService.class);
6465

6566
private static final String BID_WURL_ATTRIBUTE = "wurl";
67+
private static final String TRACE_INFO_SEPARATOR = "-";
68+
private static final int MAX_DATACENTER_REGION_LENGTH = 4;
6669

6770
private final HttpClient httpClient;
6871
private final URL endpointUrl;
@@ -78,13 +81,18 @@ public class CoreCacheService {
7881
private final MultiMap cacheHeaders;
7982
private final Map<String, List<String>> debugHeaders;
8083

84+
private final boolean appendTraceInfoToCacheId;
85+
private final String datacenterRegion;
86+
8187
public CoreCacheService(
8288
HttpClient httpClient,
8389
URL endpointUrl,
8490
String cachedAssetUrlTemplate,
8591
long expectedCacheTimeMs,
8692
String apiKey,
8793
boolean isApiKeySecured,
94+
boolean appendTraceInfoToCacheId,
95+
String datacenterRegion,
8896
VastModifier vastModifier,
8997
EventsService eventsService,
9098
Metrics metrics,
@@ -107,6 +115,9 @@ public CoreCacheService(
107115
? HttpUtil.headers().add(HttpUtil.X_PBC_API_KEY_HEADER, Objects.requireNonNull(apiKey))
108116
: HttpUtil.headers();
109117
debugHeaders = HttpUtil.toDebugHeaders(cacheHeaders);
118+
119+
this.appendTraceInfoToCacheId = appendTraceInfoToCacheId;
120+
this.datacenterRegion = normalizeDatacenterRegion(datacenterRegion);
110121
}
111122

112123
public String getEndpointHost() {
@@ -138,8 +149,10 @@ public String cacheVideoDebugLog(CachedDebugLog cachedDebugLog, Integer videoCac
138149
return cacheKey;
139150
}
140151

141-
private CachedCreative makeDebugCacheCreative(CachedDebugLog videoCacheDebugLog, String hbCacheId,
152+
private CachedCreative makeDebugCacheCreative(CachedDebugLog videoCacheDebugLog,
153+
String hbCacheId,
142154
Integer videoCacheTtl) {
155+
143156
final JsonNode value = mapper.mapper().valueToTree(videoCacheDebugLog.buildCacheBody());
144157
videoCacheDebugLog.setCacheKey(hbCacheId);
145158
return CachedCreative.of(BidPutObject.builder()
@@ -211,6 +224,7 @@ private List<CachedCreative> updatePutObjects(List<BidPutObject> bidPutObjects,
211224
.bidid(null)
212225
.bidder(null)
213226
.timestamp(null)
227+
.key(resolveCacheKey(accountId, putObject.getKey()))
214228
.value(vastModifier.modifyVastXml(isEventsEnabled,
215229
allowedBidders,
216230
putObject,
@@ -268,7 +282,8 @@ private Future<CacheServiceResult> doCacheOpenrtb(List<CacheBid> bids,
268282
final List<CachedCreative> cachedCreatives = Stream.concat(
269283
bids.stream().map(cacheBid ->
270284
createJsonPutObjectOpenrtb(cacheBid, accountId, eventsContext)),
271-
videoBids.stream().map(videoBid -> createXmlPutObjectOpenrtb(videoBid, requestId, hbCacheId)))
285+
videoBids.stream().map(videoBid ->
286+
createXmlPutObjectOpenrtb(videoBid, requestId, hbCacheId, accountId)))
272287
.collect(Collectors.toCollection(ArrayList::new));
273288

274289
if (cachedCreatives.isEmpty()) {
@@ -385,38 +400,43 @@ private CachedCreative createJsonPutObjectOpenrtb(CacheBid cacheBid,
385400
bidObjectNode.put(BID_WURL_ATTRIBUTE, eventUrl);
386401
}
387402

403+
final String resolvedCacheKey = resolveCacheKey(accountId);
404+
388405
final BidPutObject payload = BidPutObject.builder()
389406
.aid(eventsContext.getAuctionId())
390407
.type("json")
408+
.key(resolvedCacheKey)
391409
.value(bidObjectNode)
392410
.ttlseconds(cacheBid.getTtl())
393411
.build();
394412

395413
return CachedCreative.of(payload, creativeSizeFromAdm(bid.getAdm()));
396414
}
397415

398-
private CachedCreative createXmlPutObjectOpenrtb(CacheBid cacheBid, String requestId, String hbCacheId) {
416+
private CachedCreative createXmlPutObjectOpenrtb(CacheBid cacheBid,
417+
String requestId,
418+
String hbCacheId,
419+
String accountId) {
420+
399421
final BidInfo bidInfo = cacheBid.getBidInfo();
400422
final Bid bid = bidInfo.getBid();
401423
final String vastXml = bid.getAdm();
402424

403-
final String customCacheKey = resolveCustomCacheKey(hbCacheId, bidInfo.getCategory());
404-
405425
final BidPutObject payload = BidPutObject.builder()
406426
.aid(requestId)
407427
.type("xml")
428+
.key(resolveCacheKey(accountId, hbCacheId, bidInfo.getCategory()))
408429
.value(vastXml != null ? new TextNode(vastXml) : null)
409430
.ttlseconds(cacheBid.getTtl())
410-
.key(customCacheKey)
411431
.build();
412432

413433
return CachedCreative.of(payload, creativeSizeFromTextNode(payload.getValue()));
414434
}
415435

416-
private static String resolveCustomCacheKey(String hbCacheId, String category) {
436+
private static String formatCategoryMappedCacheKey(String hbCacheId, String category) {
417437
return StringUtils.isNoneEmpty(category, hbCacheId)
418438
? "%s_%s".formatted(category, hbCacheId)
419-
: null;
439+
: hbCacheId;
420440
}
421441

422442
private String generateWinUrl(String bidId,
@@ -515,10 +535,16 @@ private static String resolveVideoBidUuid(String uuid, String hbCacheId) {
515535
}
516536

517537
private void updateCreativeMetrics(String accountId, List<CachedCreative> cachedCreatives) {
518-
for (final CachedCreative cachedCreative : cachedCreatives) {
519-
metrics.updateCacheCreativeSize(accountId,
520-
cachedCreative.getSize(),
521-
resolveCreativeTypeName(cachedCreative.getPayload()));
538+
for (CachedCreative cachedCreative : cachedCreatives) {
539+
final BidPutObject payload = cachedCreative.getPayload();
540+
final MetricName creativeType = resolveCreativeTypeName(payload);
541+
final Integer creativeTtl = ObjectUtils.defaultIfNull(payload.getTtlseconds(), payload.getExpiry());
542+
543+
if (creativeTtl != null) {
544+
metrics.updateCacheCreativeTtl(accountId, creativeTtl, creativeType);
545+
}
546+
547+
metrics.updateCacheCreativeSize(accountId, cachedCreative.getSize(), creativeType);
522548
}
523549
}
524550

@@ -553,4 +579,48 @@ private BidCacheRequest toBidCacheRequest(List<CachedCreative> cachedCreatives)
553579
.map(CachedCreative::getPayload)
554580
.toList());
555581
}
582+
583+
private String resolveCacheKey(String accountId, String existingKey, String category) {
584+
final String resolvedCacheKey = resolveCacheKey(accountId, existingKey);
585+
return formatCategoryMappedCacheKey(resolvedCacheKey, category);
586+
587+
}
588+
589+
private String resolveCacheKey(String accountId) {
590+
return resolveCacheKey(accountId, null);
591+
}
592+
593+
private String resolveCacheKey(String accountId, String existingCacheKey) {
594+
if (!appendTraceInfoToCacheId || existingCacheKey != null) {
595+
return existingCacheKey;
596+
}
597+
598+
final boolean isDatacenterNamePopulated = StringUtils.isNotBlank(datacenterRegion);
599+
final int separatorCount = isDatacenterNamePopulated ? 2 : 1;
600+
final int accountIdLength = accountId.length();
601+
final int traceInfoLength = isDatacenterNamePopulated
602+
? accountIdLength + datacenterRegion.length() + separatorCount
603+
: accountIdLength + separatorCount;
604+
605+
final String cacheKey = idGenerator.generateId();
606+
if (cacheKey == null || traceInfoLength >= (cacheKey.length() / 2)) {
607+
return null;
608+
}
609+
610+
final String substring = cacheKey.substring(0, cacheKey.length() - traceInfoLength);
611+
return isDatacenterNamePopulated
612+
? accountId + TRACE_INFO_SEPARATOR + datacenterRegion + TRACE_INFO_SEPARATOR + substring
613+
: accountId + TRACE_INFO_SEPARATOR + substring;
614+
}
615+
616+
private static String normalizeDatacenterRegion(String datacenterRegion) {
617+
if (datacenterRegion == null) {
618+
return null;
619+
}
620+
621+
final String trimmedDatacenterRegion = datacenterRegion.trim();
622+
return trimmedDatacenterRegion.length() > MAX_DATACENTER_REGION_LENGTH
623+
? trimmedDatacenterRegion.substring(0, MAX_DATACENTER_REGION_LENGTH)
624+
: trimmedDatacenterRegion;
625+
}
556626
}

‎src/main/java/org/prebid/server/metric/CacheCreativeTtlMetrics.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package org.prebid.server.metric;
2+
3+
import com.codahale.metrics.MetricRegistry;
4+
5+
import java.util.Objects;
6+
import java.util.function.Function;
7+
8+
public class CacheCreativeTtlMetrics extends UpdatableMetrics {
9+
10+
CacheCreativeTtlMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
11+
super(Objects.requireNonNull(metricRegistry),
12+
Objects.requireNonNull(counterType),
13+
nameCreator(Objects.requireNonNull(prefix)));
14+
}
15+
16+
private static Function<MetricName, String> nameCreator(String prefix) {
17+
return metricName -> "%s.creative_ttl.%s".formatted(prefix, metricName);
18+
}
19+
}

‎src/main/java/org/prebid/server/metric/CacheMetrics.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ class CacheMetrics extends UpdatableMetrics {
1212

1313
private final RequestMetrics requestsMetrics;
1414
private final CacheCreativeSizeMetrics cacheCreativeSizeMetrics;
15+
private final CacheCreativeTtlMetrics cacheCreativeTtlMetrics;
1516

1617
CacheMetrics(MetricRegistry metricRegistry, CounterType counterType) {
1718
super(
@@ -21,6 +22,7 @@ class CacheMetrics extends UpdatableMetrics {
2122

2223
requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix());
2324
cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix());
25+
cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix());
2426
}
2527

2628
CacheMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
@@ -31,6 +33,7 @@ class CacheMetrics extends UpdatableMetrics {
3133

3234
requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix(prefix));
3335
cacheCreativeSizeMetrics = new CacheCreativeSizeMetrics(metricRegistry, counterType, createPrefix(prefix));
36+
cacheCreativeTtlMetrics = new CacheCreativeTtlMetrics(metricRegistry, counterType, createPrefix(prefix));
3437
}
3538

3639
private static String createPrefix(String prefix) {
@@ -52,4 +55,8 @@ RequestMetrics requests() {
5255
CacheCreativeSizeMetrics creativeSize() {
5356
return cacheCreativeSizeMetrics;
5457
}
58+
59+
CacheCreativeTtlMetrics creativeTtl() {
60+
return cacheCreativeTtlMetrics;
61+
}
5562
}

‎src/main/java/org/prebid/server/metric/Metrics.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,11 @@ public void updateAccountModuleDurationMetric(Account account, String moduleCode
694694
}
695695
}
696696

697+
public void updateCacheCreativeTtl(String accountId, Integer creativeTtl, MetricName creativeType) {
698+
cache().creativeTtl().updateHistogram(creativeType, creativeTtl);
699+
forAccount(accountId).cache().creativeTtl().updateHistogram(creativeType, creativeTtl);
700+
}
701+
697702
private static class HookMetricMapper {
698703

699704
private static final EnumMap<ExecutionStatus, MetricName> STATUS_TO_METRIC =

‎src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,8 @@ CoreCacheService cacheService(
164164
@Value("${auction.cache.expected-request-time-ms}") long expectedCacheTimeMs,
165165
@Value("${pbc.api.key:#{null}}") String apiKey,
166166
@Value("${cache.api-key-secured:false}") boolean apiKeySecured,
167+
@Value("${cache.append-trace-info-to-cache-id:false}") boolean appendTraceInfoToCacheId,
168+
@Value("${datacenter-region:#{null}}") String datacenterRegion,
167169
VastModifier vastModifier,
168170
EventsService eventsService,
169171
HttpClient httpClient,
@@ -178,6 +180,8 @@ CoreCacheService cacheService(
178180
expectedCacheTimeMs,
179181
apiKey,
180182
apiKeySecured,
183+
appendTraceInfoToCacheId,
184+
datacenterRegion,
181185
vastModifier,
182186
eventsService,
183187
metrics,

‎src/test/groovy/org/prebid/server/functional/model/request/cache/BidCachePut.groovy‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,5 @@ class BidCachePut implements ObjectMapperWrapper {
1414
String bidder
1515
Long timestamp
1616
String aid
17+
String key
1718
}

0 commit comments

Comments
 (0)