From fa305b8152ef46a32a6c4b71392e3c0bb5292090 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 2 Dec 2025 02:08:55 -0700 Subject: [PATCH 01/52] fix test --- src/main/java/com/uid2/optout/Main.java | 16 +- .../optout/vertx/DeltaProductionResult.java | 47 +- .../optout/vertx/OptOutSqsLogProducer.java | 239 +++- .../vertx/OptOutSqsLogProducerTest.java | 1084 ++++++++++++++++- 4 files changed, 1327 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/uid2/optout/Main.java b/src/main/java/com/uid2/optout/Main.java index dbccd32e..f6bf5cad 100644 --- a/src/main/java/com/uid2/optout/Main.java +++ b/src/main/java/com/uid2/optout/Main.java @@ -1,6 +1,8 @@ package com.uid2.optout; import com.uid2.optout.vertx.*; +import com.uid2.optout.vertx.OptOutTrafficFilter.MalformedTrafficFilterConfigException; +import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import com.uid2.shared.ApplicationVersion; import com.uid2.shared.Utils; import com.uid2.shared.attest.AttestationResponseHandler; @@ -27,7 +29,6 @@ import io.vertx.config.ConfigRetriever; import io.vertx.core.*; import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.http.impl.HttpUtils; import io.vertx.core.json.JsonObject; import io.vertx.micrometer.MetricsDomain; import org.slf4j.Logger; @@ -296,14 +297,23 @@ public void run(String[] args) throws IOException { fsSqs = CloudUtils.createStorage(optoutBucket, sqsConfig); } - // Deploy SQS log producer with its own storage instance - OptOutSqsLogProducer sqsLogProducer = new OptOutSqsLogProducer(this.config, fsSqs, sqsCs); + // Create SQS-specific cloud storage instance for dropped requests (different bucket) + String optoutBucketDroppedRequests = this.config.getString(Const.Config.OptOutS3BucketDroppedRequestsProp); + ICloudStorage fsSqsDroppedRequests = CloudUtils.createStorage(optoutBucketDroppedRequests, config); + + // Deploy SQS log producer with its own storage instance + OptOutSqsLogProducer sqsLogProducer = new OptOutSqsLogProducer(this.config, fsSqs, fsSqsDroppedRequests, sqsCs, Const.Event.DeltaProduce, null); futs.add(this.deploySingleInstance(sqsLogProducer)); LOGGER.info("SQS log producer deployed - bucket: {}, folder: {}", this.config.getString(Const.Config.OptOutS3BucketProp), sqsFolder); } catch (IOException e) { LOGGER.error("Failed to initialize SQS log producer: " + e.getMessage(), e); + LOGGER.error("Failed to initialize SQS log producer, delta production will be disabled: " + e.getMessage(), e); + } catch (MalformedTrafficFilterConfigException e) { + LOGGER.error("The traffic filter config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); + } catch (MalformedTrafficCalcConfigException e) { + LOGGER.error("The traffic calc config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); } } diff --git a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java b/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java index fa18556f..790b1277 100644 --- a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java +++ b/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java @@ -1,16 +1,25 @@ package com.uid2.optout.vertx; +import io.vertx.core.json.JsonObject; + /** - * Result object containing statistics from delta production. + * Data class containing statistics from delta production. + * + * This class holds the counts and provides JSON encoding methods. + * API response status is determined by the caller based on these statistics. */ public class DeltaProductionResult { private final int deltasProduced; private final int entriesProcessed; + private final int droppedRequestFilesProduced; + private final int droppedRequestsProcessed; private final boolean stoppedDueToRecentMessages; - public DeltaProductionResult(int deltasProduced, int entriesProcessed, boolean stoppedDueToRecentMessages) { + public DeltaProductionResult(int deltasProduced, int entriesProcessed, int droppedRequestFilesProduced, int droppedRequestsProcessed, boolean stoppedDueToRecentMessages) { this.deltasProduced = deltasProduced; this.entriesProcessed = entriesProcessed; + this.droppedRequestFilesProduced = droppedRequestFilesProduced; + this.droppedRequestsProcessed = droppedRequestsProcessed; this.stoppedDueToRecentMessages = stoppedDueToRecentMessages; } @@ -25,5 +34,37 @@ public int getEntriesProcessed() { public boolean stoppedDueToRecentMessages() { return stoppedDueToRecentMessages; } -} + public int getDroppedRequestFilesProduced() { + return droppedRequestFilesProduced; + } + + public int getDroppedRequestsProcessed() { + return droppedRequestsProcessed; + } + + /** + * Convert to JSON with just the production counts. + */ + public JsonObject toJson() { + return new JsonObject() + .put("deltas_produced", deltasProduced) + .put("entries_processed", entriesProcessed) + .put("dropped_request_files_produced", droppedRequestFilesProduced) + .put("dropped_requests_processed", droppedRequestsProcessed); + } + + /** + * Convert to JSON with status and counts. + */ + public JsonObject toJsonWithStatus(String status) { + return toJson().put("status", status); + } + + /** + * Convert to JSON with status, reason/error, and counts. + */ + public JsonObject toJsonWithStatus(String status, String reasonKey, String reasonValue) { + return toJsonWithStatus(status).put(reasonKey, reasonValue); + } +} diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 8ac1d1bf..ae2619bd 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -2,6 +2,8 @@ import com.uid2.optout.Const; import com.uid2.optout.auth.InternalAuthMiddleware; +import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import com.uid2.optout.vertx.OptOutTrafficFilter.MalformedTrafficFilterConfigException; import com.uid2.shared.Utils; import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.health.HealthComponent; @@ -14,6 +16,9 @@ import io.vertx.core.json.JsonObject; import io.vertx.ext.web.Router; import io.vertx.ext.web.RoutingContext; +import io.vertx.core.json.JsonArray; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; @@ -22,6 +27,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; @@ -80,6 +86,7 @@ public class OptOutSqsLogProducer extends AbstractVerticle { private final String eventDeltaProduced; private final int replicaId; private final ICloudStorage cloudStorage; + private final ICloudStorage cloudStorageDroppedRequests; private final OptOutCloudSync cloudSync; private final int maxMessagesPerPoll; private final int visibilityTimeout; @@ -88,6 +95,9 @@ public class OptOutSqsLogProducer extends AbstractVerticle { private final int listenPort; private final String internalApiKey; private final InternalAuthMiddleware internalAuth; + private final OptOutTrafficFilter trafficFilter; + private final OptOutTrafficCalculator trafficCalculator; + private final String manualOverrideS3Path; private Counter counterDeltaProduced = Counter .builder("uid2_optout_sqs_delta_produced_total") @@ -99,6 +109,16 @@ public class OptOutSqsLogProducer extends AbstractVerticle { .description("counter for how many optout entries are processed from SQS") .register(Metrics.globalRegistry); + private Counter counterDroppedRequestFilesProduced = Counter + .builder("uid2_optout_sqs_dropped_request_files_produced_total") + .description("counter for how many optout dropped request files are produced from SQS") + .register(Metrics.globalRegistry); + + private Counter counterDroppedRequestsProcessed = Counter + .builder("uid2_optout_sqs_dropped_requests_processed_total") + .description("counter for how many optout dropped requests are processed from SQS") + .register(Metrics.globalRegistry); + private ByteBuffer buffer; private boolean shutdownInProgress = false; @@ -108,19 +128,20 @@ public class OptOutSqsLogProducer extends AbstractVerticle { // Helper for reading complete 5-minute windows from SQS private final SqsWindowReader windowReader; - public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync) throws IOException { + public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { this(jsonConfig, cloudStorage, cloudSync, Const.Event.DeltaProduce); } - public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced) throws IOException { - this(jsonConfig, cloudStorage, cloudSync, eventDeltaProduced, null); + public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { + this(jsonConfig, cloudStorage, null, cloudSync, eventDeltaProduced, null); } // Constructor for testing - allows injecting mock SqsClient - public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced, SqsClient sqsClient) throws IOException { + public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, ICloudStorage cloudStorageDroppedRequests, OptOutCloudSync cloudSync, String eventDeltaProduced, SqsClient sqsClient) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { this.eventDeltaProduced = eventDeltaProduced; this.replicaId = OptOutUtils.getReplicaId(jsonConfig); this.cloudStorage = cloudStorage; + this.cloudStorageDroppedRequests = cloudStorageDroppedRequests; this.cloudSync = cloudSync; // Initialize SQS client @@ -148,6 +169,10 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, O int bufferSize = jsonConfig.getInteger(Const.Config.OptOutProducerBufferSizeProp); this.buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN); + + this.trafficFilter = new OptOutTrafficFilter(jsonConfig.getString(Const.Config.TrafficFilterConfigPathProp)); + this.trafficCalculator = new OptOutTrafficCalculator(cloudStorage, jsonConfig.getString(Const.Config.OptOutSqsS3FolderProp), jsonConfig.getString(Const.Config.TrafficCalcConfigPathProp)); + this.manualOverrideS3Path = jsonConfig.getString(Const.Config.ManualOverrideS3PathProp); // Initialize window reader this.windowReader = new SqsWindowReader( @@ -228,10 +253,7 @@ private void handleDeltaProduceStatus(RoutingContext routingContext) { if (job == null) { resp.setStatusCode(200) .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject() - .put("state", "idle") - .put("message", "No job running on this pod") - .encode()); + .end(new JsonObject().put("status", "idle").put("message", "No job running on this pod").encode()); return; } @@ -256,6 +278,26 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { LOGGER.info("Delta production job requested via /deltaproduce endpoint"); + try { + this.trafficFilter.reloadTrafficFilterConfig(); + } catch (MalformedTrafficFilterConfigException e) { + LOGGER.error("Error reloading traffic filter config: " + e.getMessage(), e); + resp.setStatusCode(500) + .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(new JsonObject().put("status", "failed").put("error", e.getMessage()).encode()); + return; + } + + try { + this.trafficCalculator.reloadTrafficCalcConfig(); + } catch (MalformedTrafficCalcConfigException e) { + LOGGER.error("Error reloading traffic calculator config: " + e.getMessage(), e); + resp.setStatusCode(500) + .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(new JsonObject().put("status", "failed").put("error", e.getMessage()).encode()); + return; + } + DeltaProduceJobStatus existingJob = currentJob.get(); @@ -266,11 +308,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { LOGGER.warn("Delta production job already running on this pod"); resp.setStatusCode(409) .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject() - .put("status", "conflict") - .put("message", "A delta production job is already running on this pod") - .put("current_job", existingJob.toJson()) - .encode()); + .end(new JsonObject().put("status", "conflict").put("reason", "A delta production job is already running on this pod").encode()); return; } @@ -283,10 +321,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { if (!currentJob.compareAndSet(existingJob, newJob)) { resp.setStatusCode(409) .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject() - .put("status", "conflict") - .put("message", "Job state changed, please retry") - .encode()); + .end(new JsonObject().put("status", "conflict").put("reason", "Job state changed, please retry").encode()); return; } @@ -297,10 +332,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { // Return immediately with 202 Accepted resp.setStatusCode(202) .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject() - .put("status", "accepted") - .put("message", "Delta production job started on this pod") - .encode()); + .end(new JsonObject().put("status", "accepted").put("message", "Delta production job started on this pod").encode()); } /** @@ -336,8 +368,6 @@ private JsonObject produceDeltasBlocking() throws Exception { if (this.shutdownInProgress) { throw new Exception("Producer is shutting down"); } - - JsonObject result = new JsonObject(); LOGGER.info("Starting delta production from SQS queue"); // Process messages until queue is empty or messages are too recent @@ -345,20 +375,14 @@ private JsonObject produceDeltasBlocking() throws Exception { // Determine status based on results if (deltaResult.getDeltasProduced() == 0 && deltaResult.stoppedDueToRecentMessages()) { - // No deltas produced because all messages were too recent - result.put("status", "skipped"); - result.put("reason", "All messages too recent"); - LOGGER.info("Delta production skipped: all messages too recent"); + // No deltas produced - either messages too recent or traffic spike detected + LOGGER.info("Delta production skipped: {} entries processed", deltaResult.getEntriesProcessed()); + return deltaResult.toJsonWithStatus("skipped", "reason", "No deltas produced"); } else { - result.put("status", "success"); LOGGER.info("Delta production complete: {} deltas, {} entries", deltaResult.getDeltasProduced(), deltaResult.getEntriesProcessed()); + return deltaResult.toJsonWithStatus("success"); } - - result.put("deltas_produced", deltaResult.getDeltasProduced()); - result.put("entries_processed", deltaResult.getEntriesProcessed()); - - return result; } @@ -370,8 +394,18 @@ private JsonObject produceDeltasBlocking() throws Exception { * @throws IOException if delta production fails */ private DeltaProductionResult produceBatchedDeltas() throws IOException { + // Check for manual override at the start of the batch (and then between each delta window) + if (getManualOverride().equals("DELAYED_PROCESSING")) { + LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); + return new DeltaProductionResult(0, 0, 0, 0, false); + } + int deltasProduced = 0; int totalEntriesProcessed = 0; + + int droppedRequestFilesProduced = 0; + int droppedRequestsProcessed = 0; + boolean stoppedDueToRecentMessages = false; long jobStartTime = OptOutUtils.nowEpochSeconds(); @@ -393,36 +427,73 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { break; } + // Create delta file buffer + String deltaName = OptOutUtils.newDeltaFileName(this.replicaId); + ByteArrayOutputStream deltaStream = new ByteArrayOutputStream(); + + // Create dropped request file buffer + JsonArray droppedRequestStream = new JsonArray(); + String currentDroppedRequestName = String.format("%s%03d_%s_%08x.json", "optout-dropped-", replicaId, Instant.now().truncatedTo(ChronoUnit.SECONDS).toString().replace(':', '.'), OptOutUtils.rand.nextInt()); + // Produce delta for this window long windowStart = windowResult.getWindowStart(); List messages = windowResult.getMessages(); - - // Create delta file - String deltaName = OptOutUtils.newDeltaFileName(this.replicaId); - ByteArrayOutputStream deltaStream = new ByteArrayOutputStream(); + writeStartOfDelta(deltaStream, windowStart); // Write all messages - List sqsMessages = new ArrayList<>(); + List currentDeltaMessages = new ArrayList<>(); + List droppedRequestMessages = new ArrayList<>(); for (SqsParsedMessage msg : messages) { - writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp()); - sqsMessages.add(msg.getOriginalMessage()); + if (trafficFilter.isBlacklisted(msg)) { + this.writeDroppedRequestEntry(droppedRequestStream, msg); + droppedRequestMessages.add(msg.getOriginalMessage()); + droppedRequestsProcessed++; + } else { + writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp()); + currentDeltaMessages.add(msg.getOriginalMessage()); + totalEntriesProcessed++; + } + } + + // check for manual override + if (getManualOverride().equals("DELAYED_PROCESSING")) { + LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); + return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); + } else if (getManualOverride().equals("DEFAULT")) { + LOGGER.info("Manual override set to DEFAULT, skipping traffic calculation"); + } else { + // check traffic calculator status + OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(currentDeltaMessages); + if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { + LOGGER.error("OptOut Delta Production has hit DELAYED_PROCESSING status, stopping production"); + this.setDelayedProcessingOverride(); + return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); + } } - // Upload and delete - uploadDeltaAndDeleteMessages(deltaStream, deltaName, windowStart, sqsMessages); - deltasProduced++; - totalEntriesProcessed += messages.size(); + // Upload delta file if there are non-blacklisted messages + if (!currentDeltaMessages.isEmpty()) { + uploadDeltaAndDeleteMessages(deltaStream, deltaName, windowStart, currentDeltaMessages); + deltasProduced++; + } + + // Upload dropped request file if there are blacklisted messages + if (!droppedRequestMessages.isEmpty()) { + this.uploadDroppedRequestsAndDeleteMessages(droppedRequestStream, currentDroppedRequestName, windowStart, droppedRequestMessages); + droppedRequestFilesProduced++; + droppedRequestMessages.clear(); + } LOGGER.info("Produced delta for window [{}, {}] with {} messages", windowStart, windowStart + this.deltaWindowSeconds, messages.size()); } long totalDuration = OptOutUtils.nowEpochSeconds() - jobStartTime; - LOGGER.info("Delta production complete: took {}s, produced {} deltas, processed {} entries", + LOGGER.info("Delta production complete: took {}s, produced {} deltas, processed {} entries, produced {} dropped request files, processed {} dropped requests", totalDuration, deltasProduced, totalEntriesProcessed); - return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, stoppedDueToRecentMessages); + return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, stoppedDueToRecentMessages); } /** @@ -537,6 +608,82 @@ private void uploadDeltaAndDeleteMessages(ByteArrayOutputStream deltaStream, Str } } + /** + * Writes a dropped request entry to the dropped request stream. + */ + private void writeDroppedRequestEntry(JsonArray droppedRequestArray, SqsParsedMessage parsed) throws IOException { + String messageBody = parsed.getOriginalMessage().body(); + JsonObject messageJson = new JsonObject(messageBody); + droppedRequestArray.add(messageJson); + } + + // Upload a dropped request file to S3 and delete messages from SQS after successful upload + private void uploadDroppedRequestsAndDeleteMessages(JsonArray droppedRequestStream, String droppedRequestName, Long windowStart, List messages) throws IOException { + try { + // upload + byte[] droppedRequestData = droppedRequestStream.encode().getBytes(); + + LOGGER.info("SQS Dropped Requests Upload - fileName: {}, s3Path: {}, size: {} bytes, messages: {}, window: [{}, {})", + droppedRequestName, droppedRequestData.length, messages.size(), windowStart, windowStart + this.deltaWindowSeconds); + + boolean uploadSucceeded = false; + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(droppedRequestData)) { + this.cloudStorageDroppedRequests.upload(inputStream, droppedRequestName); + LOGGER.info("Successfully uploaded dropped requests to S3: {}", droppedRequestName); + uploadSucceeded = true; + + // publish event + this.counterDroppedRequestFilesProduced.increment(); + this.counterDroppedRequestsProcessed.increment(messages.size()); + } catch (Exception uploadEx) { + LOGGER.error("Failed to upload dropped requests to S3: " + uploadEx.getMessage(), uploadEx); + throw new IOException("S3 upload failed", uploadEx); + } + + // CRITICAL: Only delete messages from SQS after successful S3 upload + if (uploadSucceeded && !messages.isEmpty()) { + LOGGER.info("Deleting {} messages from SQS after successful S3 upload", messages.size()); + SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, messages); + } + + // Clear the array + droppedRequestStream.clear(); + + } catch (Exception ex) { + LOGGER.error("Error uploading dropped requests: " + ex.getMessage(), ex); + throw new IOException("Dropped requests upload failed", ex); + } + } + + /** + * Upload a JSON config file to S3 containing the following: + * {"manual_override": "DELAYED_PROCESSING"} + * Manual override file is at the root of the S3 bucket + */ + private void setDelayedProcessingOverride() { + try { + JsonObject config = new JsonObject().put("manual_override", "DELAYED_PROCESSING"); + this.cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), this.manualOverrideS3Path); + } catch (Exception e) { + LOGGER.error("Error setting delayed processing override", e); + } + } + + /** + * Check if there is a manual override set in S3 for DEFAULT or DELAYED_PROCESSING status + * Manual override file is at the root of the S3 bucket + */ + private String getManualOverride() { + try { + InputStream inputStream = this.cloudStorage.download(this.manualOverrideS3Path); + JsonObject configJson = Utils.toJsonObject(inputStream); + return configJson.getString("manual_override", ""); + } catch (Exception e) { + LOGGER.error("Error checking for manual override in S3: " + e.getMessage(), e); + return ""; + } + } + private void publishDeltaProducedEvent(String newDelta) { vertx.eventBus().publish(this.eventDeltaProduced, newDelta); LOGGER.info("Published delta.produced event for: {}", newDelta); diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index a46023f6..414db4bd 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -4,6 +4,8 @@ import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.optout.OptOutCloudSync; import com.uid2.shared.vertx.VertxUtils; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.shared.optout.OptOutCollection; import io.vertx.core.Vertx; import io.vertx.core.json.JsonObject; import io.vertx.ext.unit.Async; @@ -14,12 +16,19 @@ import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.InputStream; +import java.io.ByteArrayInputStream; import java.util.*; +import java.nio.file.Files; +import java.nio.file.Path; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.*; +import static org.mockito.Mockito.doAnswer; +import org.mockito.ArgumentCaptor; /** * Integration tests for OptOutSqsLogProducer deltaproduce endpoint. @@ -33,12 +42,18 @@ public class OptOutSqsLogProducerTest { private SqsClient sqsClient; private ICloudStorage cloudStorage; + private ICloudStorage cloudStorageDroppedRequests; private OptOutCloudSync cloudSync; private static final String TEST_QUEUE_URL = "https://sqs.test.amazonaws.com/123456789/test"; private static final String TEST_API_KEY = "test-api-key"; private static final String VALID_HASH_BASE64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; private static final String VALID_ID_BASE64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE="; + private static final String TRAFFIC_FILTER_CONFIG_PATH = "./traffic-filter.json"; + private static final String TRAFFIC_CALC_CONFIG_PATH = "./traffic-calc.json"; + private static final String MANUAL_OVERRIDE_S3_PATH = "manual-override.json"; + private static final String S3_DELTA_PREFIX = "sqs-delta"; + private static final String TEST_BUCKET_DROPPED_REQUESTS = "test-bucket-dropped-requests"; @Before public void setup(TestContext context) throws Exception { @@ -47,6 +62,7 @@ public void setup(TestContext context) throws Exception { // Create mocks sqsClient = mock(SqsClient.class); cloudStorage = mock(ICloudStorage.class); + cloudStorageDroppedRequests = mock(ICloudStorage.class); cloudSync = mock(OptOutCloudSync.class); JsonObject config = VertxUtils.getJsonConfig(vertx); @@ -54,7 +70,13 @@ public void setup(TestContext context) throws Exception { .put(Const.Config.OptOutSqsVisibilityTimeoutProp, 240) .put(Const.Config.OptOutProducerBufferSizeProp, 65536) .put(Const.Config.OptOutProducerReplicaIdProp, 1) - .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY); + .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY) + .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY) + .put(Const.Config.TrafficFilterConfigPathProp, TRAFFIC_FILTER_CONFIG_PATH) + .put(Const.Config.TrafficCalcConfigPathProp, TRAFFIC_CALC_CONFIG_PATH) + .put(Const.Config.ManualOverrideS3PathProp, MANUAL_OVERRIDE_S3_PATH) + .put(Const.Config.OptOutS3BucketDroppedRequestsProp, TEST_BUCKET_DROPPED_REQUESTS) + .put(Const.Config.OptOutSqsS3FolderProp, S3_DELTA_PREFIX); // Mock cloud sync to return proper S3 paths when(cloudSync.toCloudPath(anyString())) @@ -62,9 +84,36 @@ public void setup(TestContext context) throws Exception { // Mock S3 upload to succeed by default doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); + doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString()); + + // Don't mock download with anyString() - let tests mock specific paths as needed + // Unmocked downloads will return null by default + + + try { + String traficFilterConfig = """ + { + "blacklist_requests": [ + ] + } + """; + createTrafficConfigFile(traficFilterConfig); + + String trafficCalcConfig = """ + { + "traffic_calc_evaluation_window_seconds": 86400, + "traffic_calc_baseline_traffic": 100, + "traffic_calc_threshold_multiplier": 5, + "traffic_calc_allowlist_ranges": [] + } + """; + createTrafficCalcConfigFile(trafficCalcConfig); + } catch (Exception e) { + throw new RuntimeException(e); + } // Create producer with mock SqsClient - producer = new OptOutSqsLogProducer(config, cloudStorage, cloudSync, Const.Event.DeltaProduce, sqsClient); + producer = new OptOutSqsLogProducer(config, cloudStorage, cloudStorageDroppedRequests, cloudSync, Const.Event.DeltaProduce, sqsClient); // Deploy verticle Async async = context.async(); @@ -76,12 +125,53 @@ public void tearDown(TestContext context) { if (vertx != null) { vertx.close(context.asyncAssertSuccess()); } + if (Files.exists(Path.of(TRAFFIC_FILTER_CONFIG_PATH))) { + try { + Files.delete(Path.of(TRAFFIC_FILTER_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + if (Files.exists(Path.of(TRAFFIC_CALC_CONFIG_PATH))) { + try { + Files.delete(Path.of(TRAFFIC_CALC_CONFIG_PATH)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } - + + private void createTrafficConfigFile(String content) { + try { + Path configPath = Path.of(TRAFFIC_FILTER_CONFIG_PATH); + Files.writeString(configPath, content); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void createTrafficCalcConfigFile(String content) { + try { + Path configPath = Path.of(TRAFFIC_CALC_CONFIG_PATH); + Files.writeString(configPath, content); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private Message createMessage(String hash, String id, long timestampMs) { + return createMessage(hash, id, timestampMs, null, null, null, null); + } + + private Message createMessage(String hash, String id, long timestampMs, String email, String phone, String clientIp, String traceId) { JsonObject body = new JsonObject() .put("identity_hash", hash) .put("advertising_id", id); + + if (email != null) body.put("email", email); + if (phone != null) body.put("phone", phone); + if (clientIp != null) body.put("client_ip", clientIp); + if (traceId != null) body.put("trace_id", traceId); Map attrs = new HashMap<>(); attrs.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampMs)); @@ -287,7 +377,7 @@ public void testDeltaProduceEndpoint_allMessagesTooRecent(TestContext context) { JsonObject result = finalStatus.getJsonObject("result"); context.assertNotNull(result); context.assertEquals("skipped", result.getString("status")); - context.assertEquals("All messages too recent", result.getString("reason")); + context.assertEquals("No deltas produced", result.getString("reason")); // No processing should occur try { @@ -331,9 +421,15 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime) ); - // Mock SQS to return messages + // Use CountDownLatch to control when the mock returns - ensures job stays running + java.util.concurrent.CountDownLatch processingLatch = new java.util.concurrent.CountDownLatch(1); + + // Mock SQS to wait on latch before returning - keeps job in RUNNING state when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) - .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenAnswer(inv -> { + processingLatch.await(); // Wait until test releases + return ReceiveMessageResponse.builder().messages(messages).build(); + }) .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) @@ -373,8 +469,10 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context .onComplete(context.asyncAssertSuccess(body -> { JsonObject response = new JsonObject(body.toString()); context.assertEquals("conflict", response.getString("status")); - context.assertTrue(response.getString("message").contains("already running")); + context.assertTrue(response.getString("reason").contains("already running")); + // Release the latch so first job can complete + processingLatch.countDown(); async.complete(); })); } @@ -460,5 +558,977 @@ public void testDeltaProduceEndpoint_autoClearCompletedJob(TestContext context) async.complete(); })); } + + @Test + public void testTrafficFilter_blacklistedMessagesAreDropped(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - update traffic filter config to blacklist specific IP and time range + long baseTime = System.currentTimeMillis() / 1000 - 400; // 400 seconds ago + String filterConfig = String.format(""" + { + "blacklist_requests": [ + { + "range": [%d, %d], + "IPs": ["192.168.1.100"] + } + ] + } + """, baseTime - 100, baseTime + 100); + createTrafficConfigFile(filterConfig); + + // Setup - create messages: some blacklisted, some not + long blacklistedTime = (baseTime) * 1000; // Within blacklist range + long normalTime = (baseTime - 200) * 1000; // Outside blacklist range + List messages = Arrays.asList( + // These should be dropped (blacklisted) + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime, null, null, "192.168.1.100", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime + 1000, null, null, "192.168.1.100", null), + // These should be processed normally + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime + 1000, null, null, "10.0.0.2", null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // Should process 2 normal entries + context.assertEquals(2, result.getInteger("entries_processed")); + + // Should have 2 dropped requests + context.assertEquals(2, result.getInteger("dropped_requests_processed")); + + // Verify both delta and dropped request files were uploaded + try { + verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), anyString()); + verify(cloudStorageDroppedRequests, atLeastOnce()).upload(any(InputStream.class), anyString()); + } catch (Exception e) { + context.fail(e); + } + + async.complete(); + })); + } + + @Test + public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - traffic filter with a blacklisted IP + long baseTime = System.currentTimeMillis() / 1000 - 400; + String filterConfig = String.format(""" + { + "blacklist_requests": [ + { + "range": [%d, %d], + "IPs": ["192.168.1.100"] + } + ] + } + """, baseTime - 100, baseTime + 100); + createTrafficConfigFile(filterConfig); + + // Setup - create messages that don't match blacklist + long normalTime = (baseTime - 200) * 1000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime + 1000, null, null, "10.0.0.2", null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + context.assertEquals(2, result.getInteger("entries_processed")); + context.assertEquals(0, result.getInteger("dropped_requests_processed")); + + // Should not upload dropped request file + try { + verify(cloudStorageDroppedRequests, never()).upload(any(InputStream.class), anyString()); + } catch (Exception e) { + context.fail(e); + } + + async.complete(); + })); + } + + @Test + public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - traffic filter with a blacklisted IP + long baseTime = System.currentTimeMillis() / 1000 - 400; + String filterConfig = String.format(""" + { + "blacklist_requests": [ + { + "range": [%d, %d], + "IPs": ["192.168.1.100"] + } + ] + } + """, baseTime - 100, baseTime + 100); + createTrafficConfigFile(filterConfig); + + // Setup - create messages that are blacklisted + long blacklistedTime = baseTime * 1000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime, null, null, "192.168.1.100", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime + 1000, null, null, "192.168.1.100", null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // No entries processed (all blacklisted) + context.assertEquals(0, result.getInteger("entries_processed")); + + // All messages dropped + context.assertEquals(2, result.getInteger("dropped_requests_processed")); + + // Should upload dropped request file but not delta file + try { + verify(cloudStorageDroppedRequests, atLeastOnce()).upload(any(InputStream.class), anyString()); + verify(cloudStorage, never()).upload(any(InputStream.class), anyString()); + } catch (Exception e) { + context.fail(e); + } + + async.complete(); + })); + } + + @Test + public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - traffic filter with a blacklisted IP + long baseTime = System.currentTimeMillis() / 1000 - 400; + String filterConfig = String.format(""" + { + "blacklist_requests": [ + { + "range": [%d, %d], + "IPs": ["192.168.1.100"] + } + ] + } + """, baseTime - 100, baseTime + 100); + createTrafficConfigFile(filterConfig); + + // Create messages without client IP (should not be blacklisted) + long time = baseTime * 1000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, time, null, null, null, null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // Message should be processed (not blacklisted due to missing IP) + context.assertEquals(1, result.getInteger("entries_processed")); + context.assertEquals(0, result.getInteger("dropped_requests_processed")); + + async.complete(); + })); + } + + @Test + public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - initial config with no blacklist + String initialConfig = """ + { + "blacklist_requests": [] + } + """; + createTrafficConfigFile(initialConfig); + + // Setup - create messages + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "192.168.1.100", null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + // Act & Assert - first request - should process normally + int port = Const.Port.ServicePortForOptOut + 1; + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals(1, result.getInteger("entries_processed")); + context.assertEquals(0, result.getInteger("dropped_requests_processed")); + + // Update config to blacklist the IP + try { + long baseTime = System.currentTimeMillis() / 1000 - 400; + String updatedConfig = String.format(""" + { + "blacklist_requests": [ + { + "range": [%d, %d], + "IPs": ["192.168.1.100"] + } + ] + } + """, baseTime - 100, baseTime + 100); + createTrafficConfigFile(updatedConfig); + + // Reset mocks for second request + reset(sqsClient, cloudStorage, cloudStorageDroppedRequests); + + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); + doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString()); + + // Act & Assert - second request - should now be blacklisted + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body2 -> { + JsonObject response2 = new JsonObject(body2.toString()); + context.assertEquals("accepted", response2.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus2 -> { + context.assertEquals("completed", finalStatus2.getString("state")); + JsonObject result2 = finalStatus2.getJsonObject("result"); + // Now should be blacklisted + context.assertEquals(0, result2.getInteger("entries_processed")); + context.assertEquals(1, result2.getInteger("dropped_requests_processed")); + async.complete(); + })); + } catch (Exception e) { + context.fail(e); + } + })); + } + + @Test + public void testTrafficCalculator_defaultStatus(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - traffic calc config with required fields + String trafficCalcConfig = """ + { + "traffic_calc_evaluation_window_seconds": 86400, + "traffic_calc_baseline_traffic": 100, + "traffic_calc_threshold_multiplier": 5, + "traffic_calc_allowlist_ranges": [] + } + """; + createTrafficCalcConfigFile(trafficCalcConfig); + + // Setup - no manual override + when(cloudStorage.download(anyString())) + .thenReturn(null); + + // Setup - create messages that will result in DEFAULT status + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime + 1000, null, null, "10.0.0.2", null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // Should process messages normally (DEFAULT status) + context.assertEquals(2, result.getInteger("entries_processed")); + context.assertTrue(result.getInteger("deltas_produced") >= 1); + + // Verify upload happened + try { + verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), anyString()); + } catch (Exception e) { + context.fail(e); + } + + async.complete(); + })); + } + + @Test + public void testManualOverride_delayedProcessing(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - mock manual override set to DELAYED_PROCESSING + JsonObject manualOverride = new JsonObject().put("manual_override", "DELAYED_PROCESSING"); + doReturn(new java.io.ByteArrayInputStream(manualOverride.encode().getBytes())) + .when(cloudStorage).download(MANUAL_OVERRIDE_S3_PATH); // At root of bucket + + // Setup - create messages (won't be processed due to override) + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime + 1000, null, null, "10.0.0.2", null) + ); + + List allMessages = new ArrayList<>(messages); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(allMessages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // Should not process anything - manual override checked at start + context.assertEquals(0, result.getInteger("entries_processed")); + context.assertEquals(0, result.getInteger("deltas_produced")); + + // No SQS deletions should occur (messages not processed) + verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + + async.complete(); + })); + } + + @Test + public void testManualOverride_default_bypassesTrafficCalculation(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - setup time: current time + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create delta files with timestamps distributed over 48 hours + List timestamps = new ArrayList<>(); + + // Past window: t-47h to t-25h (add 10 entries) + for (int i = 0; i < 10; i++) { + timestamps.add(t - 47*3600 + i * 1000); + } + + // Current window: t-23h to t-1h (add 100 entries - 10x past) + for (int i = 0; i < 100; i++) { + timestamps.add(t - 23*3600 + i * 1000); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + // Setup - mock manual override set to DEFAULT + JsonObject manualOverride = new JsonObject().put("manual_override", "DEFAULT"); + + // Mock S3 operations for this test + // Use doAnswer to create fresh streams on each call (streams are consumed on read) + doReturn(Arrays.asList("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .when(cloudStorage).list("sqs-delta"); + doAnswer(inv -> new ByteArrayInputStream(deltaFileBytes)) + .when(cloudStorage).download("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); + doAnswer(inv -> new java.io.ByteArrayInputStream(manualOverride.encode().getBytes())) + .when(cloudStorage).download("manual-override.json"); + + // Setup - SQS messages, 10 messages in same window + long oldTime = (t - 600) * 1000; + List messages = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime - (i * 1000), null, null, "10.0.0." + i, null)); + } + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // Should process 10 messages and produce 1 delta (all in same window) + context.assertEquals(10, result.getInteger("entries_processed")); + context.assertEquals(1, result.getInteger("deltas_produced")); + + verify(sqsClient, atLeastOnce()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + + async.complete(); + })); + } + + @Test + public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext context) throws Exception { + Async async = context.async(); + + // Setup time + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create historical delta files showing low baseline traffic (2 records from 24-48h ago) + List timestamps = new ArrayList<>(); + timestamps.add(t - 36*3600); // 36 hours ago + timestamps.add(t - 36*3600 + 1000); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + // Reset cloudStorage mock to ensure clean state + reset(cloudStorage); + + // Re-mock S3 upload (needed after reset) + doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); + + // Mock S3 operations for historical data + // Use doAnswer to create fresh streams on each call + doReturn(Arrays.asList("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_baseline.dat")) + .when(cloudStorage).list("sqs-delta"); + doAnswer(inv -> new ByteArrayInputStream(deltaFileBytes)) + .when(cloudStorage).download("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_baseline.dat"); + + // No manual override set (returns null) + doReturn(null).when(cloudStorage).download("manual-override.json"); + + // Setup SQS messages in 2 different 5-minute windows + long oldTime = (t - 600) * 1000; // 10 minutes ago + + // Window 1 (oldest): Low traffic (10 messages) - within baseline, should process + List window1Messages = new ArrayList<>(); + long window1Time = oldTime - 600*1000; // 20 minutes ago + for (int i = 0; i < 10; i++) { + window1Messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, + window1Time - (i * 1000), null, null, "10.0.0." + i, null)); + } + + // Window 2: High traffic spike (600 messages) - exceeds threshold (baseline=100 * multiplier=5 = 500) + // NOTE: The traffic calculator is called with currentDeltaMessages (window 2's messages), + // so it calculates the spike AFTER processing all of window 2's messages (but the delta is not uploaded). + List window2Messages = new ArrayList<>(); + long window2Time = oldTime - 300*1000; // 15 minutes ago (5 min after window1) + for (int i = 0; i < 600; i++) { + window2Messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, + window2Time - (i * 100), null, null, "10.0.1." + (i % 256), null)); + } + + // Combine all messages (oldest first for SQS ordering) + List allMessages = new ArrayList<>(); + allMessages.addAll(window1Messages); + allMessages.addAll(window2Messages); + + // Mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(allMessages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + + // Act & Assert + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("skipped", result.getString("status")); + + // Expected behavior: + // SqsWindowReader groups all 610 messages together (window 1 + window 2) + // Traffic calculator detects spike (>=500 threshold) → DELAYED_PROCESSING + // encodeSkippedResult preserves the actual counts + context.assertEquals(610, result.getInteger("entries_processed")); + context.assertEquals(0, result.getInteger("deltas_produced")); + + // Verify manual override was set to DELAYED_PROCESSING on S3 + try { + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor streamCaptor = ArgumentCaptor.forClass(InputStream.class); + verify(cloudStorage, atLeastOnce()).upload(streamCaptor.capture(), pathCaptor.capture()); + + // Check if manual-override.json was uploaded + boolean overrideSet = false; + for (int i = 0; i < pathCaptor.getAllValues().size(); i++) { + if (pathCaptor.getAllValues().get(i).equals("manual-override.json")) { + overrideSet = true; + break; + } + } + context.assertTrue(overrideSet, "Manual override should be set to DELAYED_PROCESSING after detecting spike"); + } catch (Exception e) { + context.fail(e); + } + + async.complete(); + })); + } + + @Test + public void testManualOverride_notSet(TestContext context) throws Exception { + Async async = context.async(); + + // Setup - mock no manual override file + when(cloudStorage.download(anyString())) + .thenReturn(null); + + // Setup - create messages + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null) + ); + + // Setup - mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + // Act & Assert - call endpoint via HTTP + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("success", result.getString("status")); + + // Should process normally with traffic calc (no override) + context.assertEquals(1, result.getInteger("entries_processed")); + + async.complete(); + })); + } + + @Test + public void testS3UploadFailure_messagesNotDeletedFromSqs(TestContext context) throws Exception { + Async async = context.async(); + + // Create messages to process + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime + 1000, null, null, "10.0.0.2", null) + ); + + // Mock SQS to return messages + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + // Mock S3 upload to FAIL + doThrow(new RuntimeException("S3 upload failed - simulated error")) + .when(cloudStorage).upload(any(InputStream.class), anyString()); + + int port = Const.Port.ServicePortForOptOut + 1; + + // Start job + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + // Job should fail due to S3 error + context.assertEquals("failed", finalStatus.getString("state")); + context.assertTrue(finalStatus.getString("error").contains("S3") || + finalStatus.getString("error").contains("upload")); + + // CRITICAL: Messages should NOT be deleted from SQS when upload fails + verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + + async.complete(); + })); + } + + @Test + public void testStatusEndpoint_showsRunningJob(TestContext context) throws Exception { + Async async = context.async(); + + // Create messages + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime) + ); + + // Use CountDownLatch to keep job running + java.util.concurrent.CountDownLatch processingLatch = new java.util.concurrent.CountDownLatch(1); + + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenAnswer(inv -> { + processingLatch.await(); + return ReceiveMessageResponse.builder().messages(messages).build(); + }) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + + // Start job + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + // Immediately check status - should show "running" + return vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.GET, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString() + "/status") + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()); + }) + .compose(resp -> { + context.assertEquals(200, resp.statusCode()); + return resp.body(); + }) + .onComplete(context.asyncAssertSuccess(body -> { + JsonObject status = new JsonObject(body.toString()); + context.assertEquals("running", status.getString("state")); + context.assertNotNull(status.getString("start_time")); + + // Release latch so job can complete + processingLatch.countDown(); + async.complete(); + })); + } + + @Test + public void testStatusEndpoint_showsFailedJob(TestContext context) throws Exception { + Async async = context.async(); + + // Create messages + long oldTime = System.currentTimeMillis() - 400_000; + List messages = Arrays.asList( + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime) + ); + + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + + // Make S3 upload fail + doThrow(new RuntimeException("Simulated S3 failure")) + .when(cloudStorage).upload(any(InputStream.class), anyString()); + + int port = Const.Port.ServicePortForOptOut + 1; + + // Start job and wait for it to fail + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> pollForCompletion(context, port, 100, 50)) + .compose(finalStatus -> { + context.assertEquals("failed", finalStatus.getString("state")); + + // Now call status endpoint directly to verify failed state is persisted + return vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.GET, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString() + "/status") + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()); + }) + .compose(resp -> { + context.assertEquals(200, resp.statusCode()); + return resp.body(); + }) + .onComplete(context.asyncAssertSuccess(body -> { + JsonObject status = new JsonObject(body.toString()); + context.assertEquals("failed", status.getString("state")); + context.assertNotNull(status.getString("error")); + context.assertNotNull(status.getString("start_time")); + context.assertNotNull(status.getString("end_time")); + context.assertNotNull(status.getInteger("duration_seconds")); + + async.complete(); + })); + } + + @Test + public void testDeltaProduceEndpoint_invalidApiKey(TestContext context) { + Async async = context.async(); + + int port = Const.Port.ServicePortForOptOut + 1; + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer wrong-api-key") + .send()) + .compose(resp -> { + context.assertEquals(401, resp.statusCode()); + return resp.body(); + }) + .onComplete(context.asyncAssertSuccess(body -> { + // Should not call SQS when unauthorized + verify(sqsClient, never()).receiveMessage(any(ReceiveMessageRequest.class)); + async.complete(); + })); + } + + /** + * Create delta file bytes with specified timestamps + */ + private byte[] createDeltaFileBytes(List timestamps) throws Exception { + // Create OptOutEntry objects using newTestEntry + List entries = new ArrayList<>(); + + long idCounter = 1000; // Use incrementing IDs for test entries + for (long timestamp : timestamps) { + entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp)); + } + + // Create OptOutCollection + OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); + return collection.getStore(); + } } From 695f6b4602a9af0f6654bf7488e49a4a0d674532 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 2 Dec 2025 02:41:53 -0700 Subject: [PATCH 02/52] threshold in reading --- .../optout/vertx/OptOutSqsLogProducer.java | 15 ++- .../optout/vertx/OptOutTrafficCalculator.java | 8 ++ .../uid2/optout/vertx/SqsWindowReader.java | 44 +++++++- .../vertx/OptOutSqsLogProducerTest.java | 105 ++++++++++++++++++ 4 files changed, 163 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index ae2619bd..af1d9372 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -174,10 +174,10 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I this.trafficCalculator = new OptOutTrafficCalculator(cloudStorage, jsonConfig.getString(Const.Config.OptOutSqsS3FolderProp), jsonConfig.getString(Const.Config.TrafficCalcConfigPathProp)); this.manualOverrideS3Path = jsonConfig.getString(Const.Config.ManualOverrideS3PathProp); - // Initialize window reader + // Initialize window reader with traffic threshold this.windowReader = new SqsWindowReader( this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, - this.visibilityTimeout, this.deltaWindowSeconds + this.visibilityTimeout, this.deltaWindowSeconds, this.trafficCalculator.getThreshold() ); } @@ -290,6 +290,8 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { try { this.trafficCalculator.reloadTrafficCalcConfig(); + // Update window reader's message limit to match new threshold + this.windowReader.setMaxMessagesPerWindow(this.trafficCalculator.getThreshold()); } catch (MalformedTrafficCalcConfigException e) { LOGGER.error("Error reloading traffic calculator config: " + e.getMessage(), e); resp.setStatusCode(500) @@ -298,7 +300,6 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { return; } - DeltaProduceJobStatus existingJob = currentJob.get(); // If there's an existing job, check if it's still running @@ -427,6 +428,14 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { break; } + // if message limit exceeded, treat as traffic spike + if (windowResult.exceededMessageLimit()) { + LOGGER.error("Message limit exceeded ({} messages) - triggering DELAYED_PROCESSING to prevent memory exhaustion", + windowResult.getMessages().size()); + this.setDelayedProcessingOverride(); + return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); + } + // Create delta file buffer String deltaName = OptOutUtils.newDeltaFileName(this.replicaId); ByteArrayOutputStream deltaStream = new ByteArrayOutputStream(); diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 7795021c..a7d73d31 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -515,6 +515,14 @@ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { return TrafficStatus.DEFAULT; } + /** + * Get the traffic threshold (baseline × multiplier). + * Used for early termination + */ + public int getThreshold() { + return this.baselineTraffic * this.thresholdMultiplier; + } + /** * Get cache statistics for monitoring */ diff --git a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java b/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java index 782bb93f..6aa18a0d 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java @@ -1,5 +1,7 @@ package com.uid2.optout.vertx; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.Message; @@ -11,23 +13,35 @@ * Handles accumulation of all messages for a window before returning. */ public class SqsWindowReader { + private static final Logger LOGGER = LoggerFactory.getLogger(SqsWindowReader.class); + private final SqsClient sqsClient; private final String queueUrl; private final int maxMessagesPerPoll; private final int visibilityTimeout; private final int deltaWindowSeconds; private final SqsBatchProcessor batchProcessor; + private int maxMessagesPerWindow; public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerPoll, - int visibilityTimeout, int deltaWindowSeconds) { + int visibilityTimeout, int deltaWindowSeconds, int maxMessagesPerWindow) { this.sqsClient = sqsClient; this.queueUrl = queueUrl; this.maxMessagesPerPoll = maxMessagesPerPoll; this.visibilityTimeout = visibilityTimeout; this.deltaWindowSeconds = deltaWindowSeconds; + this.maxMessagesPerWindow = maxMessagesPerWindow; this.batchProcessor = new SqsBatchProcessor(sqsClient, queueUrl, deltaWindowSeconds); } + /** + * Update the max messages limit (e.g., after config reload). + */ + public void setMaxMessagesPerWindow(int maxMessagesPerWindow) { + this.maxMessagesPerWindow = maxMessagesPerWindow; + LOGGER.info("Updated maxMessagesPerWindow to {}", maxMessagesPerWindow); + } + /** * Result of reading messages for a 5-minute window. */ @@ -35,17 +49,21 @@ public static class WindowReadResult { private final List messages; private final long windowStart; private final boolean stoppedDueToRecentMessages; + private final boolean exceededMessageLimit; - public WindowReadResult(List messages, long windowStart, boolean stoppedDueToRecentMessages) { + public WindowReadResult(List messages, long windowStart, + boolean stoppedDueToRecentMessages, boolean exceededMessageLimit) { this.messages = messages; this.windowStart = windowStart; this.stoppedDueToRecentMessages = stoppedDueToRecentMessages; + this.exceededMessageLimit = exceededMessageLimit; } public List getMessages() { return messages; } public long getWindowStart() { return windowStart; } public boolean isEmpty() { return messages.isEmpty(); } public boolean stoppedDueToRecentMessages() { return stoppedDueToRecentMessages; } + public boolean exceededMessageLimit() { return exceededMessageLimit; } } /** @@ -54,6 +72,7 @@ public WindowReadResult(List messages, long windowStart, boole * - We discover the next window * - Queue is empty (no more messages) * - Messages are too recent (all messages younger than 5 minutes) + * - Message count exceeds maxMessagesPerWindow * * @return WindowReadResult with messages for the window, or empty if done */ @@ -62,13 +81,20 @@ public WindowReadResult readWindow() { long currentWindowStart = 0; while (true) { + + if (windowMessages.size() >= maxMessagesPerWindow) { + LOGGER.warn("Message limit exceeded: {} messages >= limit {}. Stopping to prevent memory exhaustion.", + windowMessages.size(), maxMessagesPerWindow); + return new WindowReadResult(windowMessages, currentWindowStart, false, true); + } + // Read one batch from SQS (up to 10 messages) List rawBatch = SqsMessageOperations.receiveMessagesFromSqs( this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout); if (rawBatch.isEmpty()) { // Queue empty - return what we have - return new WindowReadResult(windowMessages, currentWindowStart, false); + return new WindowReadResult(windowMessages, currentWindowStart, false, false); } // Process batch: parse, validate, filter @@ -77,7 +103,7 @@ public WindowReadResult readWindow() { if (batchResult.isEmpty()) { if (batchResult.shouldStopProcessing()) { // Messages too recent - return what we have - return new WindowReadResult(windowMessages, currentWindowStart, true); + return new WindowReadResult(windowMessages, currentWindowStart, true, false); } // corrupt messages deleted, read next messages continue; @@ -99,13 +125,19 @@ public WindowReadResult readWindow() { } windowMessages.add(msg); + + // Check limit after each message addition + if (windowMessages.size() >= maxMessagesPerWindow) { + LOGGER.warn("Message limit exceeded during batch: {} messages >= limit {}", + windowMessages.size(), maxMessagesPerWindow); + return new WindowReadResult(windowMessages, currentWindowStart, false, true); + } } if (newWindow) { // close current window and return - return new WindowReadResult(windowMessages, currentWindowStart, false); + return new WindowReadResult(windowMessages, currentWindowStart, false, false); } } } } - diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 414db4bd..bfada923 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -1159,6 +1159,18 @@ public void testManualOverride_default_bypassesTrafficCalculation(TestContext co public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext context) throws Exception { Async async = context.async(); + // Use higher threshold (1000) so circuit breaker doesn't trigger before traffic calculator + // This tests the traffic calculator spike detection, not the circuit breaker + String trafficCalcConfig = """ + { + "traffic_calc_evaluation_window_seconds": 86400, + "traffic_calc_baseline_traffic": 200, + "traffic_calc_threshold_multiplier": 5, + "traffic_calc_allowlist_ranges": [] + } + """; + createTrafficCalcConfigFile(trafficCalcConfig); + // Setup time long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1271,6 +1283,99 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex })); } + @Test + public void testCircuitBreaker_stopsProcessingWhenMessageLimitExceeded(TestContext context) throws Exception { + Async async = context.async(); + + // Use low threshold (100) so circuit breaker triggers before traffic calculator + String trafficCalcConfig = """ + { + "traffic_calc_evaluation_window_seconds": 86400, + "traffic_calc_baseline_traffic": 20, + "traffic_calc_threshold_multiplier": 5, + "traffic_calc_allowlist_ranges": [] + } + """; + createTrafficCalcConfigFile(trafficCalcConfig); + + // Reset cloudStorage mock to ensure clean state + reset(cloudStorage); + + // Re-mock S3 upload (needed after reset) + doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); + + // No manual override set (returns null) + doReturn(null).when(cloudStorage).download("manual-override.json"); + + // Setup time + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create 200 messages - exceeds threshold (20 * 5 = 100) + long oldTime = (t - 600) * 1000; // 10 minutes ago + List messages = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, + oldTime - (i * 1000), null, null, "10.0.0." + (i % 256), null)); + } + + // Mock SQS operations + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) + .thenReturn(DeleteMessageBatchResponse.builder().build()); + + int port = Const.Port.ServicePortForOptOut + 1; + + // Act & Assert + vertx.createHttpClient() + .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", + Endpoints.OPTOUT_DELTA_PRODUCE.toString()) + .compose(req -> req + .putHeader("Authorization", "Bearer " + TEST_API_KEY) + .send()) + .compose(resp -> { + context.assertEquals(202, resp.statusCode()); + return resp.body(); + }) + .compose(body -> { + JsonObject response = new JsonObject(body.toString()); + context.assertEquals("accepted", response.getString("status")); + return pollForCompletion(context, port, 100, 50); + }) + .onComplete(context.asyncAssertSuccess(finalStatus -> { + context.assertEquals("completed", finalStatus.getString("state")); + JsonObject result = finalStatus.getJsonObject("result"); + context.assertEquals("skipped", result.getString("status")); + + // Expected behavior: + // SqsWindowReader hits maxMessagesPerWindow limit (100) during reading + // Circuit breaker triggers DELAYED_PROCESSING immediately + // Processing stops before any messages are counted as processed + context.assertEquals(0, result.getInteger("entries_processed")); + context.assertEquals(0, result.getInteger("deltas_produced")); + + // Verify manual override was set to DELAYED_PROCESSING on S3 + try { + ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); + verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), pathCaptor.capture()); + + // Check if manual-override.json was uploaded + boolean overrideSet = pathCaptor.getAllValues().stream() + .anyMatch(path -> path.equals("manual-override.json")); + context.assertTrue(overrideSet, "Circuit breaker should set DELAYED_PROCESSING override"); + } catch (Exception e) { + context.fail(e); + } + + // Verify NO messages were deleted from SQS (processing stopped before completion) + verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + + async.complete(); + })); + } + @Test public void testManualOverride_notSet(TestContext context) throws Exception { Async async = context.async(); From 9dab244c2f096e9a54613842025b48ab15e85a35 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 2 Dec 2025 18:49:31 -0700 Subject: [PATCH 03/52] factgoring in the queue attributes adjustment --- .../optout/vertx/OptOutSqsLogProducer.java | 7 +- .../optout/vertx/OptOutTrafficCalculator.java | 32 ++- .../optout/vertx/SqsMessageOperations.java | 91 ++++++++ .../vertx/OptOutSqsLogProducerTest.java | 72 ++++--- .../vertx/OptOutTrafficCalculatorTest.java | 197 ++++++++++++++++++ 5 files changed, 361 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index af1d9372..575371e1 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -472,8 +472,11 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { } else if (getManualOverride().equals("DEFAULT")) { LOGGER.info("Manual override set to DEFAULT, skipping traffic calculation"); } else { - // check traffic calculator status - OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(currentDeltaMessages); + // Get queue attributes (including invisible messages) for traffic calculation + SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); + + // check traffic calculator status (including invisible messages in case of multiple consumers) + OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(currentDeltaMessages, queueAttributes); if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { LOGGER.error("OptOut Delta Production has hit DELAYED_PROCESSING status, stopping production"); this.setDelayedProcessingOverride(); diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index a7d73d31..53b7695e 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -193,10 +193,26 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, * and the oldest queue timestamp to anchor the 5-minute queue window. * - * @param sqsMessages List of SQS messages + * @param sqsMessages List of SQS messages in the current batch * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ public TrafficStatus calculateStatus(List sqsMessages) { + return calculateStatus(sqsMessages, null); + } + + /** + * Calculate traffic status based on delta files, SQS queue messages, and queue attributes. + * + * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, + * and the oldest queue timestamp to anchor the 5-minute queue window. + * + * The invisible message count from queue attributes is added in case of multiple consumers. + * + * @param sqsMessages List of SQS messages in the current batch + * @param queueAttributes SQS queue attributes including invisible message count (may be null) + * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) + */ + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes) { try { // Get list of delta files from S3 (sorted newest to oldest) @@ -258,11 +274,21 @@ public TrafficStatus calculateStatus(List sqsMessages) { sum += sqsCount; } + // Add invisible messages from queue attributes (messages being processed by other consumers) + // These represent in-flight work that will soon become delta records + int invisibleMessages = 0; + if (queueAttributes != null) { + invisibleMessages = queueAttributes.getApproximateNumberOfMessagesNotVisible(); + sum += invisibleMessages; + LOGGER.info("Traffic calculation: adding {} invisible SQS messages to sum (queue: {})", + invisibleMessages, queueAttributes); + } + // Determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("Traffic calculation complete: sum={}, baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, this.baselineTraffic, this.thresholdMultiplier, status); + LOGGER.info("Traffic calculation complete: sum={} (including {} invisible), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, invisibleMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java b/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java index 6c2715b1..104215c6 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java @@ -7,6 +7,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Utility class for SQS message operations. @@ -15,6 +16,96 @@ public class SqsMessageOperations { private static final Logger LOGGER = LoggerFactory.getLogger(SqsMessageOperations.class); private static final int SQS_MAX_DELETE_BATCH_SIZE = 10; + /** + * Result of getting queue attributes from SQS. + */ + public static class QueueAttributes { + private final int approximateNumberOfMessages; + private final int approximateNumberOfMessagesNotVisible; + private final int approximateNumberOfMessagesDelayed; + + public QueueAttributes(int approximateNumberOfMessages, + int approximateNumberOfMessagesNotVisible, + int approximateNumberOfMessagesDelayed) { + this.approximateNumberOfMessages = approximateNumberOfMessages; + this.approximateNumberOfMessagesNotVisible = approximateNumberOfMessagesNotVisible; + this.approximateNumberOfMessagesDelayed = approximateNumberOfMessagesDelayed; + } + + /** Number of messages available for retrieval from the queue (visible messages) */ + public int getApproximateNumberOfMessages() { + return approximateNumberOfMessages; + } + + /** Number of messages that are in flight (being processed by consumers, invisible) */ + public int getApproximateNumberOfMessagesNotVisible() { + return approximateNumberOfMessagesNotVisible; + } + + /** Number of messages in the queue that are delayed and not available yet */ + public int getApproximateNumberOfMessagesDelayed() { + return approximateNumberOfMessagesDelayed; + } + + /** Total messages in queue = visible + invisible + delayed */ + public int getTotalMessages() { + return approximateNumberOfMessages + approximateNumberOfMessagesNotVisible + approximateNumberOfMessagesDelayed; + } + + @Override + public String toString() { + return String.format("QueueAttributes{visible=%d, invisible=%d, delayed=%d, total=%d}", + approximateNumberOfMessages, approximateNumberOfMessagesNotVisible, + approximateNumberOfMessagesDelayed, getTotalMessages()); + } + } + + /** + * Gets queue attributes from SQS including message counts. + * + * @param sqsClient The SQS client + * @param queueUrl The queue URL + * @return QueueAttributes with message counts, or null if failed + */ + public static QueueAttributes getQueueAttributes(SqsClient sqsClient, String queueUrl) { + try { + GetQueueAttributesRequest request = GetQueueAttributesRequest.builder() + .queueUrl(queueUrl) + .attributeNames( + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED + ) + .build(); + + GetQueueAttributesResponse response = sqsClient.getQueueAttributes(request); + Map attrs = response.attributes(); + + int visible = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES), 0); + int invisible = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE), 0); + int delayed = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED), 0); + + QueueAttributes queueAttributes = new QueueAttributes(visible, invisible, delayed); + LOGGER.debug("Queue attributes: {}", queueAttributes); + return queueAttributes; + + } catch (Exception e) { + LOGGER.error("Error getting queue attributes from SQS", e); + return null; + } + } + + private static int parseIntOrDefault(String value, int defaultValue) { + if (value == null) { + return defaultValue; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + /** * Receives all available messages from an SQS queue up to a maximum number of batches. * diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index bfada923..5942bbe2 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -16,8 +16,6 @@ import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.InputStream; import java.io.ByteArrayInputStream; import java.util.*; @@ -86,6 +84,16 @@ public void setup(TestContext context) throws Exception { doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString()); + // Mock getQueueAttributes by default (returns zero messages) + Map defaultQueueAttrs = new HashMap<>(); + defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "0"); + defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, "0"); + defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, "0"); + doReturn(GetQueueAttributesResponse.builder() + .attributes(defaultQueueAttrs) + .build()) + .when(sqsClient).getQueueAttributes(any(GetQueueAttributesRequest.class)); + // Don't mock download with anyString() - let tests mock specific paths as needed // Unmocked downloads will return null by default @@ -1159,12 +1167,12 @@ public void testManualOverride_default_bypassesTrafficCalculation(TestContext co public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext context) throws Exception { Async async = context.async(); - // Use higher threshold (1000) so circuit breaker doesn't trigger before traffic calculator - // This tests the traffic calculator spike detection, not the circuit breaker + // Threshold = baseline * multiplier = 100 * 5 = 500 + // We have 610 messages, which exceeds 500, so spike should be detected String trafficCalcConfig = """ { "traffic_calc_evaluation_window_seconds": 86400, - "traffic_calc_baseline_traffic": 200, + "traffic_calc_baseline_traffic": 100, "traffic_calc_threshold_multiplier": 5, "traffic_calc_allowlist_ranges": [] } @@ -1197,31 +1205,18 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex // No manual override set (returns null) doReturn(null).when(cloudStorage).download("manual-override.json"); - // Setup SQS messages in 2 different 5-minute windows - long oldTime = (t - 600) * 1000; // 10 minutes ago - - // Window 1 (oldest): Low traffic (10 messages) - within baseline, should process - List window1Messages = new ArrayList<>(); - long window1Time = oldTime - 600*1000; // 20 minutes ago - for (int i = 0; i < 10; i++) { - window1Messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, - window1Time - (i * 1000), null, null, "10.0.0." + i, null)); - } - - // Window 2: High traffic spike (600 messages) - exceeds threshold (baseline=100 * multiplier=5 = 500) - // NOTE: The traffic calculator is called with currentDeltaMessages (window 2's messages), - // so it calculates the spike AFTER processing all of window 2's messages (but the delta is not uploaded). - List window2Messages = new ArrayList<>(); - long window2Time = oldTime - 300*1000; // 15 minutes ago (5 min after window1) - for (int i = 0; i < 600; i++) { - window2Messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, - window2Time - (i * 100), null, null, "10.0.1." + (i % 256), null)); - } - - // Combine all messages (oldest first for SQS ordering) + // Setup SQS messages + long baseTime = (t - 600) * 1000; + List allMessages = new ArrayList<>(); - allMessages.addAll(window1Messages); - allMessages.addAll(window2Messages); + // Create 610 messages with timestamps spread over ~4 minutes (within the 5-minute window) + for (int i = 0; i < 610; i++) { + // Timestamps range from (t-600) to (t-600-240) seconds = t-600 to t-840 + // All within a single 5-minute window for traffic calculation, and all > 5 min old + long timestampMs = baseTime - (i * 400); // ~400ms apart going backwards, total span ~244 seconds + allMessages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, + timestampMs, null, null, "10.0.0." + (i % 256), null)); + } // Mock SQS operations when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) @@ -1229,6 +1224,16 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) .thenReturn(DeleteMessageBatchResponse.builder().build()); + + // Mock getQueueAttributes to return zero invisible messages (doesn't affect the spike detection) + Map queueAttrs = new HashMap<>(); + queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "0"); + queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, "0"); + queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, "0"); + doReturn(GetQueueAttributesResponse.builder() + .attributes(queueAttrs) + .build()) + .when(sqsClient).getQueueAttributes(any(GetQueueAttributesRequest.class)); int port = Const.Port.ServicePortForOptOut + 1; @@ -1254,10 +1259,11 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex context.assertEquals("skipped", result.getString("status")); // Expected behavior: - // SqsWindowReader groups all 610 messages together (window 1 + window 2) - // Traffic calculator detects spike (>=500 threshold) → DELAYED_PROCESSING - // encodeSkippedResult preserves the actual counts - context.assertEquals(610, result.getInteger("entries_processed")); + // All 610 messages are within a single 5-minute window + // Traffic calculator counts them all and detects spike (>=500 threshold) + // DELAYED_PROCESSING is triggered, no delta uploaded + // The entries_processed count reflects how many were read before spike detection + context.assertTrue(result.getInteger("entries_processed") <= 610); context.assertEquals(0, result.getInteger("deltas_produced")); // Verify manual override was set to DELAYED_PROCESSING on S3 diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 0824c3b2..9e3a3510 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -1357,4 +1357,201 @@ void testCalculateStatus_timestampsCached() throws Exception { assertEquals(2, stats.get("total_cached_timestamps")); } + // ============================================================================ + // SECTION 10: Tests for calculateStatus() with QueueAttributes + // ============================================================================ + + /** + * Create a QueueAttributes object for testing + */ + private SqsMessageOperations.QueueAttributes createQueueAttributes(int visible, int invisible, int delayed) { + return new SqsMessageOperations.QueueAttributes(visible, invisible, delayed); + } + + @Test + void testCalculateStatus_withQueueAttributes_nullAttributes() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - null queue attributes should behave same as single-parameter version + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + + // Assert - DEFAULT (same as without queue attributes) + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_withQueueAttributes_invisibleMessagesAddedToSum() throws Exception { + // Setup - create delta files with entries just under threshold + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create 490 entries (just under threshold of 500 = 5 * 100) + List timestamps = new ArrayList<>(); + for (int i = 0; i < 490; i++) { + timestamps.add(t - 23*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - without invisible messages, should be DEFAULT + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus statusWithoutInvisible = calculator.calculateStatus(sqsMessages, null); + + // With invisible messages that push over threshold, should be DELAYED_PROCESSING + // 490 (delta) + 1 (sqs) + 10 (invisible) = 501 >= 500 + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(5, 10, 0); + OptOutTrafficCalculator.TrafficStatus statusWithInvisible = calculator.calculateStatus(sqsMessages, queueAttributes); + + // Assert + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, statusWithoutInvisible); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, statusWithInvisible); + } + + @Test + void testCalculateStatus_withQueueAttributes_zeroInvisibleMessages() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600, t - 7200); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - queue attributes with zero invisible messages + List sqsMessages = Arrays.asList(createSqsMessage(t)); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(100, 0, 50); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + + // Assert - same result as without queue attributes (only invisible is added to sum) + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + @Test + void testCalculateStatus_withQueueAttributes_largeInvisibleCount() throws Exception { + // Setup - create delta files with minimal entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - large number of invisible messages alone should trigger DELAYED_PROCESSING + // threshold = 5 * 100 = 500 + // 1 (delta) + 1 (sqs) + 500 (invisible) = 502 >= 500 + List sqsMessages = Arrays.asList(createSqsMessage(t)); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(0, 500, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + + // Assert - DELAYED_PROCESSING due to invisible messages + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); + } + + @Test + void testCalculateStatus_withQueueAttributes_delayedMessagesNotAdded() throws Exception { + // Setup - create delta files with entries near threshold + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + // Create 495 entries + List timestamps = new ArrayList<>(); + for (int i = 0; i < 495; i++) { + timestamps.add(t - 23*3600 + i * 60); + } + + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - delayed messages should NOT be added to sum (only invisible) + // 495 (delta) + 1 (sqs) + 0 (invisible) = 496 < 500 + // If delayed was added: 496 + 1000 = 1496 >= 500 (would fail) + List sqsMessages = Arrays.asList(createSqsMessage(t)); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(100, 0, 1000); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + + // Assert - DEFAULT (delayed messages not counted) + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + + // ============================================================================ + // SECTION 11: Tests for QueueAttributes class + // ============================================================================ + + @Test + void testQueueAttributes_getters() { + // Setup + SqsMessageOperations.QueueAttributes attrs = createQueueAttributes(10, 20, 5); + + // Assert + assertEquals(10, attrs.getApproximateNumberOfMessages()); + assertEquals(20, attrs.getApproximateNumberOfMessagesNotVisible()); + assertEquals(5, attrs.getApproximateNumberOfMessagesDelayed()); + assertEquals(35, attrs.getTotalMessages()); + } + + @Test + void testQueueAttributes_toString() { + // Setup + SqsMessageOperations.QueueAttributes attrs = createQueueAttributes(100, 50, 25); + + // Act + String str = attrs.toString(); + + // Assert - should contain all values + assertTrue(str.contains("visible=100")); + assertTrue(str.contains("invisible=50")); + assertTrue(str.contains("delayed=25")); + assertTrue(str.contains("total=175")); + } + + @Test + void testQueueAttributes_zeroValues() { + // Setup + SqsMessageOperations.QueueAttributes attrs = createQueueAttributes(0, 0, 0); + + // Assert + assertEquals(0, attrs.getApproximateNumberOfMessages()); + assertEquals(0, attrs.getApproximateNumberOfMessagesNotVisible()); + assertEquals(0, attrs.getApproximateNumberOfMessagesDelayed()); + assertEquals(0, attrs.getTotalMessages()); + } + } From 0fbbc6f245f253cde04ccc2ab2aa76d5c99080ab Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 4 Dec 2025 19:37:34 +0000 Subject: [PATCH 04/52] [CI Pipeline] Released Snapshot version: 4.5.1-alpha-114-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2d85c1fd..58442310 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.0 + 4.5.1-alpha-114-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 18262c365f2f92564a7e8445662ede8ed33060ce Mon Sep 17 00:00:00 2001 From: Ian Nara <135270994+Ian-Nara@users.noreply.github.com> Date: Thu, 4 Dec 2025 12:39:30 -0700 Subject: [PATCH 05/52] Update .trivyignore --- .trivyignore | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index 223846fb..374b2307 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,5 +5,8 @@ # UID2-6097 CVE-2025-59375 exp:2025-12-15 -# UID2-6128 -CVE-2025-55163 exp:2025-11-30 +# UID2-6340 +CVE-2025-64720 exp:2025-12-16 + +# UID2-6340 +CVE-2025-65018 exp:2025-12-16 From f7cecdee81ae5b43e1902a13dfcd4521aae0b66f Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 4 Dec 2025 19:41:54 +0000 Subject: [PATCH 06/52] [CI Pipeline] Released Snapshot version: 4.5.2-alpha-115-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 58442310..48e55529 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.1-alpha-114-SNAPSHOT + 4.5.2-alpha-115-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From d23c792bf56f65e6c259ad5f25b8c16d7f0dff1b Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 4 Dec 2025 14:05:40 -0700 Subject: [PATCH 07/52] update naming --- .../vertx/OptOutSqsLogProducerTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 5942bbe2..172c71c8 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -101,7 +101,7 @@ public void setup(TestContext context) throws Exception { try { String traficFilterConfig = """ { - "blacklist_requests": [ + "denylist_requests": [ ] } """; @@ -571,11 +571,11 @@ public void testDeltaProduceEndpoint_autoClearCompletedJob(TestContext context) public void testTrafficFilter_blacklistedMessagesAreDropped(TestContext context) throws Exception { Async async = context.async(); - // Setup - update traffic filter config to blacklist specific IP and time range + // Setup - update traffic filter config to denyhlist specific IP and time range long baseTime = System.currentTimeMillis() / 1000 - 400; // 400 seconds ago String filterConfig = String.format(""" { - "blacklist_requests": [ + "denylist_requests": [ { "range": [%d, %d], "IPs": ["192.168.1.100"] @@ -586,8 +586,8 @@ public void testTrafficFilter_blacklistedMessagesAreDropped(TestContext context) createTrafficConfigFile(filterConfig); // Setup - create messages: some blacklisted, some not - long blacklistedTime = (baseTime) * 1000; // Within blacklist range - long normalTime = (baseTime - 200) * 1000; // Outside blacklist range + long blacklistedTime = (baseTime) * 1000; // Within denyhlist range + long normalTime = (baseTime - 200) * 1000; // Outside denyhlist range List messages = Arrays.asList( // These should be dropped (blacklisted) createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime, null, null, "192.168.1.100", null), @@ -653,7 +653,7 @@ public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws long baseTime = System.currentTimeMillis() / 1000 - 400; String filterConfig = String.format(""" { - "blacklist_requests": [ + "denylist_requests": [ { "range": [%d, %d], "IPs": ["192.168.1.100"] @@ -663,7 +663,7 @@ public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws """, baseTime - 100, baseTime + 100); createTrafficConfigFile(filterConfig); - // Setup - create messages that don't match blacklist + // Setup - create messages that don't match denyhlist long normalTime = (baseTime - 200) * 1000; List messages = Arrays.asList( createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null), @@ -721,7 +721,7 @@ public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws long baseTime = System.currentTimeMillis() / 1000 - 400; String filterConfig = String.format(""" { - "blacklist_requests": [ + "denylist_requests": [ { "range": [%d, %d], "IPs": ["192.168.1.100"] @@ -793,7 +793,7 @@ public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throw long baseTime = System.currentTimeMillis() / 1000 - 400; String filterConfig = String.format(""" { - "blacklist_requests": [ + "denylist_requests": [ { "range": [%d, %d], "IPs": ["192.168.1.100"] @@ -851,10 +851,10 @@ public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throw public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throws Exception { Async async = context.async(); - // Setup - initial config with no blacklist + // Setup - initial config with no denyhlist String initialConfig = """ { - "blacklist_requests": [] + "denylist_requests": [] } """; createTrafficConfigFile(initialConfig); @@ -895,12 +895,12 @@ public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throw context.assertEquals(1, result.getInteger("entries_processed")); context.assertEquals(0, result.getInteger("dropped_requests_processed")); - // Update config to blacklist the IP + // Update config to denyhlist the IP try { long baseTime = System.currentTimeMillis() / 1000 - 400; String updatedConfig = String.format(""" { - "blacklist_requests": [ + "denylist_requests": [ { "range": [%d, %d], "IPs": ["192.168.1.100"] From d03eb732946f3c64d4d6714fbf6d60a2f545ed15 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 4 Dec 2025 21:08:40 +0000 Subject: [PATCH 08/52] [CI Pipeline] Released Snapshot version: 4.5.3-alpha-117-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 48e55529..6b11c199 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.2-alpha-115-SNAPSHOT + 4.5.3-alpha-117-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From b09ac28fc31addb40d82c351bc6353232820795a Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Thu, 4 Dec 2025 15:11:07 -0700 Subject: [PATCH 09/52] update logging --- .../com/uid2/optout/vertx/OptOutServiceVerticle.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java b/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java index 75073239..e01c49e1 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java @@ -365,21 +365,23 @@ private void handleQueue(RoutingContext routingContext) { String email = body != null ? body.getString(EMAIL) : null; String phone = body != null ? body.getString(PHONE) : null; - HttpServerResponse resp = routingContext.response(); - // while old delta production is enabled, response is handled by replicate logic // Validate parameters - same as replicate if (identityHash == null || params.getAll(IDENTITY_HASH).size() != 1) { + LOGGER.warn("handleQueue: Invalid identity_hash parameter"); // this.sendBadRequestError(resp); return; } if (advertisingId == null || params.getAll(ADVERTISING_ID).size() != 1) { + LOGGER.warn("handleQueue: Invalid advertising_id parameter"); + // this.sendBadRequestError(resp); return; } if (!this.isGetOrPost(req)) { + LOGGER.warn("handleQueue: Invalid HTTP method: {}", req.method()); // this.sendBadRequestError(resp); return; } @@ -408,8 +410,6 @@ private void handleQueue(RoutingContext routingContext) { } }, res -> { if (res.failed()) { - // this.sendInternalServerError(resp, "Failed to queue message: " + res.cause().getMessage()); - LOGGER.error("Failed to queue message: " + res.cause().getMessage()); } else { String messageId = (String) res.result(); @@ -426,8 +426,7 @@ private void handleQueue(RoutingContext routingContext) { } }); } catch (Exception ex) { - // this.sendInternalServerError(resp, ex.getMessage()); - LOGGER.error("Error processing queue request: " + ex.getMessage(), ex); + LOGGER.error("handleQueue: Error processing queue request"); } } From de8ae4a999ab38dcf2114156ef245bdb3cb19a13 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Thu, 4 Dec 2025 22:13:57 +0000 Subject: [PATCH 10/52] [CI Pipeline] Released Snapshot version: 4.5.4-alpha-118-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b11c199..4bab0733 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.3-alpha-117-SNAPSHOT + 4.5.4-alpha-118-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 7603858972a3809e67607571f8903bb430416b49 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 5 Dec 2025 14:19:16 -0700 Subject: [PATCH 11/52] Update name --- .../optout/vertx/OptOutSqsLogProducer.java | 4 +-- .../vertx/OptOutSqsLogProducerTest.java | 36 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 24d5ab76..b1fc864e 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -488,13 +488,13 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { } } - // Upload delta file if there are non-blacklisted messages + // Upload delta file if there are non-denylisted messages if (!currentDeltaMessages.isEmpty()) { uploadDeltaAndDeleteMessages(deltaStream, deltaName, windowStart, currentDeltaMessages); deltasProduced++; } - // Upload dropped request file if there are blacklisted messages + // Upload dropped request file if there are denylisted messages if (!droppedRequestMessages.isEmpty()) { this.uploadDroppedRequestsAndDeleteMessages(droppedRequestStream, currentDroppedRequestName, windowStart, droppedRequestMessages); droppedRequestFilesProduced++; diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 172c71c8..7b5900f1 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -568,7 +568,7 @@ public void testDeltaProduceEndpoint_autoClearCompletedJob(TestContext context) } @Test - public void testTrafficFilter_blacklistedMessagesAreDropped(TestContext context) throws Exception { + public void testTrafficFilter_denylistedMessagesAreDropped(TestContext context) throws Exception { Async async = context.async(); // Setup - update traffic filter config to denyhlist specific IP and time range @@ -585,13 +585,13 @@ public void testTrafficFilter_blacklistedMessagesAreDropped(TestContext context) """, baseTime - 100, baseTime + 100); createTrafficConfigFile(filterConfig); - // Setup - create messages: some blacklisted, some not - long blacklistedTime = (baseTime) * 1000; // Within denyhlist range + // Setup - create messages: some denylisted, some not + long denylistedTime = (baseTime) * 1000; // Within denyhlist range long normalTime = (baseTime - 200) * 1000; // Outside denyhlist range List messages = Arrays.asList( - // These should be dropped (blacklisted) - createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime, null, null, "192.168.1.100", null), - createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime + 1000, null, null, "192.168.1.100", null), + // These should be dropped (denylisted) + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime, null, null, "192.168.1.100", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime + 1000, null, null, "192.168.1.100", null), // These should be processed normally createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null), createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime + 1000, null, null, "10.0.0.2", null) @@ -649,7 +649,7 @@ public void testTrafficFilter_blacklistedMessagesAreDropped(TestContext context) public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws Exception { Async async = context.async(); - // Setup - traffic filter with a blacklisted IP + // Setup - traffic filter with a denylisted IP long baseTime = System.currentTimeMillis() / 1000 - 400; String filterConfig = String.format(""" { @@ -717,7 +717,7 @@ public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws Exception { Async async = context.async(); - // Setup - traffic filter with a blacklisted IP + // Setup - traffic filter with a denylisted IP long baseTime = System.currentTimeMillis() / 1000 - 400; String filterConfig = String.format(""" { @@ -731,11 +731,11 @@ public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws """, baseTime - 100, baseTime + 100); createTrafficConfigFile(filterConfig); - // Setup - create messages that are blacklisted - long blacklistedTime = baseTime * 1000; + // Setup - create messages that are denylisted + long denylistedTime = baseTime * 1000; List messages = Arrays.asList( - createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime, null, null, "192.168.1.100", null), - createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, blacklistedTime + 1000, null, null, "192.168.1.100", null) + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime, null, null, "192.168.1.100", null), + createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime + 1000, null, null, "192.168.1.100", null) ); // Setup - mock SQS operations @@ -767,7 +767,7 @@ public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws JsonObject result = finalStatus.getJsonObject("result"); context.assertEquals("success", result.getString("status")); - // No entries processed (all blacklisted) + // No entries processed (all denylisted) context.assertEquals(0, result.getInteger("entries_processed")); // All messages dropped @@ -789,7 +789,7 @@ public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throws Exception { Async async = context.async(); - // Setup - traffic filter with a blacklisted IP + // Setup - traffic filter with a denylisted IP long baseTime = System.currentTimeMillis() / 1000 - 400; String filterConfig = String.format(""" { @@ -803,7 +803,7 @@ public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throw """, baseTime - 100, baseTime + 100); createTrafficConfigFile(filterConfig); - // Create messages without client IP (should not be blacklisted) + // Create messages without client IP (should not be denylisted) long time = baseTime * 1000; List messages = Arrays.asList( createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, time, null, null, null, null) @@ -839,7 +839,7 @@ public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throw JsonObject result = finalStatus.getJsonObject("result"); context.assertEquals("success", result.getString("status")); - // Message should be processed (not blacklisted due to missing IP) + // Message should be processed (not denylisted due to missing IP) context.assertEquals(1, result.getInteger("entries_processed")); context.assertEquals(0, result.getInteger("dropped_requests_processed")); @@ -923,7 +923,7 @@ public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throw doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString()); - // Act & Assert - second request - should now be blacklisted + // Act & Assert - second request - should now be denylisted vertx.createHttpClient() .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", Endpoints.OPTOUT_DELTA_PRODUCE.toString()) @@ -942,7 +942,7 @@ public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throw .onComplete(context.asyncAssertSuccess(finalStatus2 -> { context.assertEquals("completed", finalStatus2.getString("state")); JsonObject result2 = finalStatus2.getJsonObject("result"); - // Now should be blacklisted + // Now should be denylisted context.assertEquals(0, result2.getInteger("entries_processed")); context.assertEquals(1, result2.getInteger("dropped_requests_processed")); async.complete(); From 29a16acbe2d952d8d1de9b401216af82e7af9b10 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 5 Dec 2025 14:46:50 -0700 Subject: [PATCH 12/52] remove if block for DEFAULT --- src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index b1fc864e..2228e8b7 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -473,8 +473,6 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { if (getManualOverride().equals("DELAYED_PROCESSING")) { LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); - } else if (getManualOverride().equals("DEFAULT")) { - LOGGER.info("Manual override set to DEFAULT, skipping traffic calculation"); } else { // Get queue attributes (including invisible messages) for traffic calculation SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); From a200825452de669a77ba150cf2c882b72988733e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 5 Dec 2025 21:51:04 +0000 Subject: [PATCH 13/52] [CI Pipeline] Released Snapshot version: 4.5.5-alpha-119-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f049a390..2c9c8a38 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.4-alpha-118-SNAPSHOT + 4.5.5-alpha-119-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From bcf170952af7dfe094f4b5b2818e02de02df810e Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 5 Dec 2025 23:47:58 +0000 Subject: [PATCH 14/52] [CI Pipeline] Released Snapshot version: 4.5.6-alpha-120-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2c9c8a38..1392438c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.5-alpha-119-SNAPSHOT + 4.5.6-alpha-120-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 5b7124ee674c365f0291dcc8ad400ae6a147412f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 5 Dec 2025 16:51:49 -0700 Subject: [PATCH 15/52] commons logging missing? --- pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2c9c8a38..a75be3cf 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.5-alpha-119-SNAPSHOT + 4.5.4-alpha-118-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout @@ -160,6 +160,11 @@ software.amazon.awssdk sqs + + commons-logging + commons-logging + 1.2 + From 1e631350e432264b209dc3dd8bea1cfecf9d9db9 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Fri, 5 Dec 2025 23:56:13 +0000 Subject: [PATCH 16/52] [CI Pipeline] Released Snapshot version: 4.5.7-alpha-121-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c8d70f53..b9ea09ad 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.6-alpha-120-SNAPSHOT + 4.5.7-alpha-121-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 5442674473e7ce469c18f53bbac058d934c04951 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 5 Dec 2025 17:40:41 -0700 Subject: [PATCH 17/52] whitespace --- .trivyignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.trivyignore b/.trivyignore index 11f08b2b..82ae41f4 100644 --- a/.trivyignore +++ b/.trivyignore @@ -10,3 +10,4 @@ CVE-2025-64720 exp:2026-06-05 # UID2-6340 CVE-2025-65018 exp:2026-06-05 + From d2364e0424837e204603ec6a4a713ba060f59e37 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Fri, 5 Dec 2025 18:40:08 -0700 Subject: [PATCH 18/52] improve config error logging details --- .../com/uid2/optout/vertx/OptOutTrafficFilter.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index e8bd04b8..f9ba8165 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -123,6 +123,12 @@ List parseFilterRules(JsonObject config) throws MalformedTraf range.add(end); } + // log error and throw exception if range is not 2 elements + if (range.size() != 2) { + LOGGER.error("Invalid traffic filter rule: range is not 2 elements: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range is not 2 elements"); + } + // parse IPs var ipAddressesJson = ruleJson.getJsonArray("IPs"); List ipAddresses = new ArrayList<>(); @@ -132,8 +138,14 @@ List parseFilterRules(JsonObject config) throws MalformedTraf } } + // log error and throw exception if IPs is empty + if (ipAddresses.size() == 0) { + LOGGER.error("Invalid traffic filter rule: IPs is empty: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: IPs is empty"); + } + // log error and throw exception if rule is invalid - if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less + if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less LOGGER.error("Invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule, range must be 24 hours or less"); } From 03fe8a9dc3a7d2726ff0f152a231f243604d0a70 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sat, 6 Dec 2025 01:43:40 +0000 Subject: [PATCH 19/52] [CI Pipeline] Released Snapshot version: 4.5.8-alpha-122-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b9ea09ad..21b5420b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.7-alpha-121-SNAPSHOT + 4.5.8-alpha-122-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 1bc84d5fef6926919e3b3e3df5c2217dca26d74f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sat, 6 Dec 2025 16:15:42 -0700 Subject: [PATCH 20/52] remove unneeded variable --- .../java/com/uid2/optout/vertx/OptOutSqsLogProducer.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 2228e8b7..e2afddf7 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -92,7 +92,6 @@ public class OptOutSqsLogProducer extends AbstractVerticle { private final int visibilityTimeout; private final int deltaWindowSeconds; // Time window for each delta file (5 minutes = 300 seconds) private final int jobTimeoutSeconds; - private final int maxMessagesPerFile; // Memory protection: max messages per delta file private final int listenPort; private final String internalApiKey; private final InternalAuthMiddleware internalAuth; @@ -129,10 +128,6 @@ public class OptOutSqsLogProducer extends AbstractVerticle { // Helper for reading complete 5-minute windows from SQS private final SqsWindowReader windowReader; - public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { - this(jsonConfig, cloudStorage, cloudSync, Const.Event.DeltaProduce); - } - public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { this(jsonConfig, cloudStorage, null, cloudSync, eventDeltaProduced, null); } @@ -160,7 +155,6 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I this.visibilityTimeout = jsonConfig.getInteger(Const.Config.OptOutSqsVisibilityTimeoutProp, 240); // 4 minutes default this.deltaWindowSeconds = 300; // Fixed 5 minutes for all deltas this.jobTimeoutSeconds = jsonConfig.getInteger(Const.Config.OptOutDeltaJobTimeoutSecondsProp, 10800); // 3 hours default - this.maxMessagesPerFile = jsonConfig.getInteger(Const.Config.OptOutMaxMessagesPerFileProp, 10000); // Memory protection limit // HTTP server configuration - use port offset + 1 to avoid conflicts this.listenPort = Const.Port.ServicePortForOptOut + Utils.getPortOffset() + 1; @@ -181,7 +175,6 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout, this.deltaWindowSeconds, this.trafficCalculator.getThreshold() ); - LOGGER.info("OptOutSqsLogProducer initialized with maxMessagesPerFile: {}", this.maxMessagesPerFile); } @Override @@ -414,7 +407,7 @@ private DeltaProductionResult produceBatchedDeltas() throws IOException { boolean stoppedDueToRecentMessages = false; long jobStartTime = OptOutUtils.nowEpochSeconds(); - LOGGER.info("Starting delta production from SQS queue (maxMessagesPerFile: {})", this.maxMessagesPerFile); + LOGGER.info("Starting delta production from SQS queue"); // Read and process windows until done while (true) { From b85a0171bed394b406998ae9d90724b087609fa4 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sat, 6 Dec 2025 19:58:18 -0700 Subject: [PATCH 21/52] refactoring circuit breaker and delta producer --- .../uid2/optout/delta/DeltaFileWriter.java | 114 ++++ .../delta/DeltaManualOverrideService.java | 84 +++ .../DeltaProductionJobStatus.java} | 6 +- .../optout/delta/DeltaProductionMetrics.java | 60 ++ .../delta/DeltaProductionOrchestrator.java | 332 +++++++++++ .../DeltaProductionResult.java | 30 +- .../uid2/optout/delta/DeltaUploadService.java | 89 +++ .../com/uid2/optout/delta/StopReason.java | 38 ++ .../{vertx => sqs}/SqsBatchProcessor.java | 110 ++-- .../{vertx => sqs}/SqsMessageOperations.java | 40 +- .../{vertx => sqs}/SqsMessageParser.java | 2 +- .../{vertx => sqs}/SqsParsedMessage.java | 2 +- .../{vertx => sqs}/SqsWindowReader.java | 78 ++- .../uid2/optout/util/HttpResponseHelper.java | 70 +++ .../optout/vertx/OptOutSqsLogProducer.java | 551 ++++-------------- .../optout/vertx/OptOutTrafficCalculator.java | 126 +--- .../optout/vertx/OptOutTrafficFilter.java | 1 + .../{vertx => sqs}/SqsBatchProcessorTest.java | 3 +- .../{vertx => sqs}/SqsMessageParserTest.java | 4 +- .../vertx/OptOutSqsLogProducerTest.java | 8 +- .../vertx/OptOutTrafficCalculatorTest.java | 250 +++----- .../optout/vertx/OptOutTrafficFilterTest.java | 3 + 22 files changed, 1107 insertions(+), 894 deletions(-) create mode 100644 src/main/java/com/uid2/optout/delta/DeltaFileWriter.java create mode 100644 src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java rename src/main/java/com/uid2/optout/{vertx/DeltaProduceJobStatus.java => delta/DeltaProductionJobStatus.java} (95%) create mode 100644 src/main/java/com/uid2/optout/delta/DeltaProductionMetrics.java create mode 100644 src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java rename src/main/java/com/uid2/optout/{vertx => delta}/DeltaProductionResult.java (75%) create mode 100644 src/main/java/com/uid2/optout/delta/DeltaUploadService.java create mode 100644 src/main/java/com/uid2/optout/delta/StopReason.java rename src/main/java/com/uid2/optout/{vertx => sqs}/SqsBatchProcessor.java (54%) rename src/main/java/com/uid2/optout/{vertx => sqs}/SqsMessageOperations.java (84%) rename src/main/java/com/uid2/optout/{vertx => sqs}/SqsMessageParser.java (99%) rename src/main/java/com/uid2/optout/{vertx => sqs}/SqsParsedMessage.java (97%) rename src/main/java/com/uid2/optout/{vertx => sqs}/SqsWindowReader.java (62%) create mode 100644 src/main/java/com/uid2/optout/util/HttpResponseHelper.java rename src/test/java/com/uid2/optout/{vertx => sqs}/SqsBatchProcessorTest.java (99%) rename src/test/java/com/uid2/optout/{vertx => sqs}/SqsMessageParserTest.java (99%) diff --git a/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java b/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java new file mode 100644 index 00000000..d31bcccf --- /dev/null +++ b/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java @@ -0,0 +1,114 @@ +package com.uid2.optout.delta; + +import com.uid2.shared.optout.OptOutConst; +import com.uid2.shared.optout.OptOutEntry; +import com.uid2.shared.optout.OptOutUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * Handles binary writing of delta file entries. + * + * Delta files have the following format: + * - Start entry: null hash (32 bytes) + null hash (32 bytes) + timestamp (8 bytes) + * - Opt-out entries: hash (32 bytes) + id (32 bytes) + timestamp (8 bytes) + * - End entry: ones hash (32 bytes) + ones hash (32 bytes) + timestamp (8 bytes) + * + * Each entry is 72 bytes (OptOutConst.EntrySize). + * This format is consistent with OptOutLogProducer.java. + */ +public class DeltaFileWriter { + private static final Logger LOGGER = LoggerFactory.getLogger(DeltaFileWriter.class); + + private ByteBuffer buffer; + + /** + * Create a DeltaFileWriter with the specified initial buffer size. + * + * @param bufferSize Initial buffer size in bytes + */ + public DeltaFileWriter(int bufferSize) { + this.buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN); + } + + /** + * Write the start-of-delta sentinel entry. + * Uses null hash bytes and the window start timestamp. + * + * @param stream Output stream to write to + * @param windowStart Window start timestamp (epoch seconds) + * @throws IOException if write fails + */ + public void writeStartOfDelta(ByteArrayOutputStream stream, long windowStart) throws IOException { + ensureCapacity(OptOutConst.EntrySize); + + buffer.put(OptOutUtils.nullHashBytes); + buffer.put(OptOutUtils.nullHashBytes); + buffer.putLong(windowStart); + + flushToStream(stream); + } + + /** + * Write a single opt-out entry. + * + * @param stream Output stream to write to + * @param hashBytes Hash bytes (32 bytes) + * @param idBytes ID bytes (32 bytes) + * @param timestamp Entry timestamp (epoch seconds) + * @throws IOException if write fails + */ + public void writeOptOutEntry(ByteArrayOutputStream stream, byte[] hashBytes, byte[] idBytes, long timestamp) throws IOException { + ensureCapacity(OptOutConst.EntrySize); + + OptOutEntry.writeTo(buffer, hashBytes, idBytes, timestamp); + + flushToStream(stream); + } + + /** + * Write the end-of-delta sentinel entry. + * Uses ones hash bytes and the window end timestamp. + * + * @param stream Output stream to write to + * @param windowEnd Window end timestamp (epoch seconds) + * @throws IOException if write fails + */ + public void writeEndOfDelta(ByteArrayOutputStream stream, long windowEnd) throws IOException { + ensureCapacity(OptOutConst.EntrySize); + + buffer.put(OptOutUtils.onesHashBytes); + buffer.put(OptOutUtils.onesHashBytes); + buffer.putLong(windowEnd); + + flushToStream(stream); + } + + /** + * Flush the buffer contents to the output stream and clear the buffer. + */ + private void flushToStream(ByteArrayOutputStream stream) throws IOException { + buffer.flip(); + byte[] entry = new byte[buffer.remaining()]; + buffer.get(entry); + stream.write(entry); + buffer.clear(); + } + + /** + * Ensure buffer has sufficient capacity, expanding if necessary. + */ + private void ensureCapacity(int dataSize) { + if (buffer.capacity() < dataSize) { + int newCapacity = Integer.highestOneBit(dataSize) << 1; + LOGGER.warn("Expanding buffer size: current {}, need {}, new {}", buffer.capacity(), dataSize, newCapacity); + this.buffer = ByteBuffer.allocate(newCapacity).order(ByteOrder.LITTLE_ENDIAN); + } + } +} + diff --git a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java new file mode 100644 index 00000000..02a581bc --- /dev/null +++ b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java @@ -0,0 +1,84 @@ +package com.uid2.optout.delta; + +import com.uid2.shared.Utils; +import com.uid2.shared.cloud.ICloudStorage; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; + +/** + * Service for managing manual override status in S3. + * + * The manual override allows operators to force DELAYED_PROCESSING status, + * which stops delta production until manually cleared. + * + * Override file format: + * 
+ * {"manual_override": "DELAYED_PROCESSING"}
+ *
+ */ +public class DeltaManualOverrideService { + private static final Logger LOGGER = LoggerFactory.getLogger(DeltaManualOverrideService.class); + + private static final String OVERRIDE_KEY = "manual_override"; + private static final String DELAYED_PROCESSING_VALUE = "DELAYED_PROCESSING"; + + private final ICloudStorage cloudStorage; + private final String overrideS3Path; + + /** + * Create a ManualOverrideService. + * + * @param cloudStorage Cloud storage client for reading/writing override file + * @param overrideS3Path S3 path where the override file is stored + */ + public DeltaManualOverrideService(ICloudStorage cloudStorage, String overrideS3Path) { + this.cloudStorage = cloudStorage; + this.overrideS3Path = overrideS3Path; + } + + /** + * Check if DELAYED_PROCESSING override is currently set. + * + * @return true if manual override is set to DELAYED_PROCESSING + */ + public boolean isDelayedProcessing() { + return DELAYED_PROCESSING_VALUE.equals(getOverrideValue()); + } + + /** + * Set the manual override to DELAYED_PROCESSING. + * This will stop delta production until manually cleared. + * + * @return true if override was set successfully + */ + public boolean setDelayedProcessing() { + try { + JsonObject config = new JsonObject().put(OVERRIDE_KEY, DELAYED_PROCESSING_VALUE); + cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), overrideS3Path); + LOGGER.info("Set manual override to DELAYED_PROCESSING at {}", overrideS3Path); + return true; + } catch (Exception e) { + LOGGER.error("Error setting delayed processing override at {}: {}", overrideS3Path, e.getMessage(), e); + return false; + } + } + + /** + * Get the current manual override value + */ + private String getOverrideValue() { + try { + InputStream inputStream = cloudStorage.download(overrideS3Path); + JsonObject configJson = Utils.toJsonObject(inputStream); + return configJson.getString(OVERRIDE_KEY, ""); + } catch (Exception e) { + LOGGER.debug("No manual override found at {}: {}", overrideS3Path, e.getMessage()); + return ""; + } + } +} + diff --git a/src/main/java/com/uid2/optout/vertx/DeltaProduceJobStatus.java b/src/main/java/com/uid2/optout/delta/DeltaProductionJobStatus.java similarity index 95% rename from src/main/java/com/uid2/optout/vertx/DeltaProduceJobStatus.java rename to src/main/java/com/uid2/optout/delta/DeltaProductionJobStatus.java index e503e200..d649eaad 100644 --- a/src/main/java/com/uid2/optout/vertx/DeltaProduceJobStatus.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionJobStatus.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.delta; import io.vertx.core.json.JsonObject; import java.time.Instant; @@ -10,7 +10,7 @@ * (running, completed, failed), timing information, and result or error details. * */ -public class DeltaProduceJobStatus { +public class DeltaProductionJobStatus { private final Instant startTime; private volatile JobState state; private volatile JsonObject result; @@ -23,7 +23,7 @@ public enum JobState { FAILED } - public DeltaProduceJobStatus() { + public DeltaProductionJobStatus() { this.startTime = Instant.now(); this.state = JobState.RUNNING; } diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionMetrics.java b/src/main/java/com/uid2/optout/delta/DeltaProductionMetrics.java new file mode 100644 index 00000000..eca36d20 --- /dev/null +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionMetrics.java @@ -0,0 +1,60 @@ +package com.uid2.optout.delta; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Metrics; + +/** + * Metrics counters for delta production operations. + * + * Tracks: + * - Number of delta files produced + * - Number of opt-out entries processed + * - Number of dropped request files produced + * - Number of dropped requests processed + */ +public class DeltaProductionMetrics { + + private final Counter deltasProduced; + private final Counter entriesProcessed; + private final Counter droppedRequestFilesProduced; + private final Counter droppedRequestsProcessed; + + public DeltaProductionMetrics() { + this.deltasProduced = Counter + .builder("uid2_optout_sqs_delta_produced_total") + .description("counter for how many optout delta files are produced from SQS") + .register(Metrics.globalRegistry); + + this.entriesProcessed = Counter + .builder("uid2_optout_sqs_entries_processed_total") + .description("counter for how many optout entries are processed from SQS") + .register(Metrics.globalRegistry); + + this.droppedRequestFilesProduced = Counter + .builder("uid2_optout_sqs_dropped_request_files_produced_total") + .description("counter for how many optout dropped request files are produced from SQS") + .register(Metrics.globalRegistry); + + this.droppedRequestsProcessed = Counter + .builder("uid2_optout_sqs_dropped_requests_processed_total") + .description("counter for how many optout dropped requests are processed from SQS") + .register(Metrics.globalRegistry); + } + + /** + * Record that a delta file was produced with the given number of entries. + */ + public void recordDeltaProduced(int entryCount) { + deltasProduced.increment(); + entriesProcessed.increment(entryCount); + } + + /** + * Record that a dropped requests file was produced with the given number of entries. + */ + public void recordDroppedRequestsProduced(int requestCount) { + droppedRequestFilesProduced.increment(); + droppedRequestsProcessed.increment(requestCount); + } +} + diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java new file mode 100644 index 00000000..dc86724d --- /dev/null +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -0,0 +1,332 @@ +package com.uid2.optout.delta; + +import com.uid2.optout.sqs.SqsMessageOperations; +import com.uid2.optout.sqs.SqsParsedMessage; +import com.uid2.optout.sqs.SqsWindowReader; +import com.uid2.optout.vertx.OptOutTrafficCalculator; +import com.uid2.optout.vertx.OptOutTrafficFilter; +import com.uid2.shared.optout.OptOutCloudSync; +import com.uid2.shared.optout.OptOutUtils; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +/** + * Orchestrates the delta production workflow. + * + *

This class encapsulates the core delta production logic:

+ *
    + *
  • Reading messages from SQS in 5-minute windows
    • + *
  • Filtering denylisted messages
    • + *
  • Checking circuit breakers (manual override, traffic calculator)
    • + *
  • Constructingdelta files and dropped request files
    • + *
  • Uploading to S3 and deleting processed messages
    • + *
+ * + *

The orchestrator is stateless and thread-safe.

+ */ +public class DeltaProductionOrchestrator { + private static final Logger LOGGER = LoggerFactory.getLogger(DeltaProductionOrchestrator.class); + + private final SqsClient sqsClient; + private final String queueUrl; + private final int replicaId; + private final int deltaWindowSeconds; + private final int jobTimeoutSeconds; + + private final SqsWindowReader windowReader; + private final DeltaFileWriter deltaFileWriter; + private final DeltaUploadService deltaUploadService; + private final DeltaUploadService droppedRequestUploadService; + private final DeltaManualOverrideService manualOverrideService; + private final OptOutTrafficFilter trafficFilter; + private final OptOutTrafficCalculator trafficCalculator; + private final OptOutCloudSync cloudSync; + private final DeltaProductionMetrics metrics; + + public DeltaProductionOrchestrator( + SqsClient sqsClient, + String queueUrl, + int replicaId, + int deltaWindowSeconds, + int jobTimeoutSeconds, + SqsWindowReader windowReader, + DeltaFileWriter deltaFileWriter, + DeltaUploadService deltaUploadService, + DeltaUploadService droppedRequestUploadService, + DeltaManualOverrideService manualOverrideService, + OptOutTrafficFilter trafficFilter, + OptOutTrafficCalculator trafficCalculator, + OptOutCloudSync cloudSync, + DeltaProductionMetrics metrics) { + this.sqsClient = sqsClient; + this.queueUrl = queueUrl; + this.replicaId = replicaId; + this.deltaWindowSeconds = deltaWindowSeconds; + this.jobTimeoutSeconds = jobTimeoutSeconds; + this.windowReader = windowReader; + this.deltaFileWriter = deltaFileWriter; + this.deltaUploadService = deltaUploadService; + this.droppedRequestUploadService = droppedRequestUploadService; + this.manualOverrideService = manualOverrideService; + this.trafficFilter = trafficFilter; + this.trafficCalculator = trafficCalculator; + this.cloudSync = cloudSync; + this.metrics = metrics; + } + + /** + * Produces delta files from SQS queue in batched 5-minute windows. + * + *

Continues until queue is empty, messages are too recent, or a circuit breaker triggers.

+ * + * @param onDeltaProduced Called with delta filename after each successful delta upload (for event publishing) + * @return DeltaProduceResult with production statistics + * @throws IOException if delta production fails + */ + public DeltaProductionResult produceBatchedDeltas(Consumer onDeltaProduced) throws IOException { + // Check for manual override at the start + if (manualOverrideService.isDelayedProcessing()) { + LOGGER.info("Manual override set to DELAYED_PROCESSING, skipping production"); + return DeltaProductionResult.empty(StopReason.MANUAL_OVERRIDE_ACTIVE); + } + + ProductionStats stats = new ProductionStats(); + long jobStartTime = OptOutUtils.nowEpochSeconds(); + + LOGGER.info("Starting delta production from SQS queue (replicaId: {}, deltaWindowSeconds: {}, jobTimeoutSeconds: {})", + this.replicaId, this.deltaWindowSeconds, this.jobTimeoutSeconds); + + // Read and process windows until done + while (!isJobTimedOut(jobStartTime)) { + + // Read one complete 5-minute window + SqsWindowReader.WindowReadResult windowResult = windowReader.readWindow(); + + // If no messages, we're done (queue empty or messages too recent) + if (windowResult.isEmpty()) { + stats.stopReason = windowResult.getStopReason(); + LOGGER.info("Delta production complete - no more eligible messages (reason: {})", stats.stopReason); + break; + } + + // Process this window + WindowProcessingResult windowProcessingResult = processWindow(windowResult, onDeltaProduced); + + // Check if we should stop (circuit breaker triggered) + if (windowProcessingResult.shouldStop) { + stats.merge(windowProcessingResult); + stats.stopReason = StopReason.CIRCUIT_BREAKER_TRIGGERED; + return stats.toResult(); + } + + stats.merge(windowProcessingResult); + + LOGGER.info("Processed window [{}, {}]: {} entries, {} dropped requests", + windowResult.getWindowStart(), + windowResult.getWindowStart() + this.deltaWindowSeconds, + windowProcessingResult.entriesProcessed, + windowProcessingResult.droppedRequestsProcessed); + } + + long totalDuration = OptOutUtils.nowEpochSeconds() - jobStartTime; + LOGGER.info("Delta production complete: took {}s, produced {} deltas, processed {} entries, " + + "produced {} dropped request files, processed {} dropped requests, stop reason: {}", + totalDuration, stats.deltasProduced, stats.entriesProcessed, + stats.droppedRequestFilesProduced, stats.droppedRequestsProcessed, stats.stopReason); + + return stats.toResult(); + } + + /** + * Processes a single 5-minute window of messages. + */ + private WindowProcessingResult processWindow(SqsWindowReader.WindowReadResult windowResult, + Consumer onDeltaProduced) throws IOException { + WindowProcessingResult result = new WindowProcessingResult(); + + long windowStart = windowResult.getWindowStart(); + List messages = windowResult.getMessages(); + + // Create buffers + ByteArrayOutputStream deltaStream = new ByteArrayOutputStream(); + JsonArray droppedRequestStream = new JsonArray(); + + String deltaName = OptOutUtils.newDeltaFileName(this.replicaId); + String droppedRequestName = generateDroppedRequestFileName(); + + // Write start of delta + deltaFileWriter.writeStartOfDelta(deltaStream, windowStart); + + // Separate messages into delta entries and dropped requests + List deltaMessages = new ArrayList<>(); + List droppedMessages = new ArrayList<>(); + + for (SqsParsedMessage msg : messages) { + if (trafficFilter.isDenylisted(msg)) { + writeDroppedRequestEntry(droppedRequestStream, msg); + droppedMessages.add(msg.getOriginalMessage()); + result.droppedRequestsProcessed++; + } else { + deltaFileWriter.writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp()); + deltaMessages.add(msg.getOriginalMessage()); + result.entriesProcessed++; + } + } + + // Check for manual override + if (manualOverrideService.isDelayedProcessing()) { + LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); + result.shouldStop = true; + return result; + } + + // Check traffic calculator + SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); + OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(queueAttributes); + + if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { + LOGGER.error("OptOut Delta Production has hit DELAYED_PROCESSING status, stopping production"); + manualOverrideService.setDelayedProcessing(); + result.shouldStop = true; + return result; + } + + // Upload delta file if there are non-denylisted messages + if (!deltaMessages.isEmpty()) { + uploadDelta(deltaStream, deltaName, windowStart, deltaMessages, onDeltaProduced); + result.deltasProduced++; + } + + // Upload dropped request file if there are denylisted messages + if (!droppedMessages.isEmpty() && droppedRequestUploadService != null) { + uploadDroppedRequests(droppedRequestStream, droppedRequestName, windowStart, droppedMessages); + result.droppedRequestFilesProduced++; + } + + deltaStream.close(); + return result; + } + + /** + * Uploads a delta file to S3 and deletes processed messages from SQS. + */ + private void uploadDelta(ByteArrayOutputStream deltaStream, String deltaName, + long windowStart, List messages, + Consumer onDeltaProduced) throws IOException { + // Add end-of-delta entry + long endTimestamp = windowStart + this.deltaWindowSeconds; + deltaFileWriter.writeEndOfDelta(deltaStream, endTimestamp); + + byte[] deltaData = deltaStream.toByteArray(); + String s3Path = this.cloudSync.toCloudPath(deltaName); + + LOGGER.info("SQS Delta Upload - fileName: {}, s3Path: {}, size: {} bytes, messages: {}, window: [{}, {})", + deltaName, s3Path, deltaData.length, messages.size(), windowStart, endTimestamp); + + deltaUploadService.uploadAndDeleteMessages(deltaData, s3Path, messages, (count) -> { + metrics.recordDeltaProduced(count); + onDeltaProduced.accept(deltaName); + }); + } + + /** + * Uploads dropped requests to S3 and deletes processed messages from SQS. + */ + private void uploadDroppedRequests(JsonArray droppedRequestStream, String droppedRequestName, + long windowStart, List messages) throws IOException { + byte[] droppedRequestData = droppedRequestStream.encode().getBytes(); + + LOGGER.info("SQS Dropped Requests Upload - fileName: {}, size: {} bytes, messages: {}, window: [{}, {})", + droppedRequestName, droppedRequestData.length, messages.size(), + windowStart, windowStart + this.deltaWindowSeconds); + + droppedRequestUploadService.uploadAndDeleteMessages(droppedRequestData, droppedRequestName, messages, + metrics::recordDroppedRequestsProduced); + } + + /** + * Writes a dropped request entry to the JSON array. + */ + private void writeDroppedRequestEntry(JsonArray droppedRequestArray, SqsParsedMessage parsed) { + String messageBody = parsed.getOriginalMessage().body(); + JsonObject messageJson = new JsonObject(messageBody); + droppedRequestArray.add(messageJson); + } + + /** + * Generates a unique filename for dropped requests. + */ + private String generateDroppedRequestFileName() { + return String.format("%s%03d_%s_%08x.json", + "optout-dropped-", + replicaId, + Instant.now().truncatedTo(ChronoUnit.SECONDS).toString().replace(':', '.'), + OptOutUtils.rand.nextInt()); + } + + /** + * Checks if the job has exceeded its timeout. + */ + private boolean isJobTimedOut(long jobStartTime) { + long elapsedTime = OptOutUtils.nowEpochSeconds() - jobStartTime; + + if (elapsedTime > 3600) { // 1 hour - log warning + LOGGER.error("Delta production job has been running for {} seconds", elapsedTime); + } + + if (elapsedTime > this.jobTimeoutSeconds) { + LOGGER.error("Delta production job has been running for {} seconds (exceeds timeout of {}s)", + elapsedTime, this.jobTimeoutSeconds); + return true; + } + return false; + } + + /** + * Mutable class for tracking production statistics during a job. + */ + private static class ProductionStats { + int deltasProduced = 0; + int entriesProcessed = 0; + int droppedRequestFilesProduced = 0; + int droppedRequestsProcessed = 0; + StopReason stopReason = StopReason.NONE; + + ProductionStats merge(WindowProcessingResult windowResult) { + this.deltasProduced += windowResult.deltasProduced; + this.entriesProcessed += windowResult.entriesProcessed; + this.droppedRequestFilesProduced += windowResult.droppedRequestFilesProduced; + this.droppedRequestsProcessed += windowResult.droppedRequestsProcessed; + return this; + } + + DeltaProductionResult toResult() { + return new DeltaProductionResult(deltasProduced, entriesProcessed, + droppedRequestFilesProduced, droppedRequestsProcessed, stopReason); + } + } + + /** + * Result of processing a single window. + */ + private static class WindowProcessingResult { + int deltasProduced = 0; + int entriesProcessed = 0; + int droppedRequestFilesProduced = 0; + int droppedRequestsProcessed = 0; + boolean shouldStop = false; + } +} + diff --git a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java similarity index 75% rename from src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java rename to src/main/java/com/uid2/optout/delta/DeltaProductionResult.java index 47af49ac..dfcf1f71 100644 --- a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.delta; import io.vertx.core.json.JsonObject; @@ -13,20 +13,23 @@ public class DeltaProductionResult { private final int entriesProcessed; private final int droppedRequestFilesProduced; private final int droppedRequestsProcessed; - - /* - * indicates that there are still messages in the queue, however, - * not enough time has elapsed to produce a delta file. - * We produce in batches of (5 minutes) - */ - private final boolean stoppedDueToRecentMessages; + private final StopReason stopReason; - public DeltaProductionResult(int deltasProduced, int entriesProcessed, int droppedRequestFilesProduced, int droppedRequestsProcessed, boolean stoppedDueToRecentMessages) { + public DeltaProductionResult(int deltasProduced, int entriesProcessed, + int droppedRequestFilesProduced, int droppedRequestsProcessed, + StopReason stopReason) { this.deltasProduced = deltasProduced; this.entriesProcessed = entriesProcessed; this.droppedRequestFilesProduced = droppedRequestFilesProduced; this.droppedRequestsProcessed = droppedRequestsProcessed; - this.stoppedDueToRecentMessages = stoppedDueToRecentMessages; + this.stopReason = stopReason; + } + + /** + * Factory method for an empty result (no production occurred). + */ + public static DeltaProductionResult empty(StopReason stopReason) { + return new DeltaProductionResult(0, 0, 0, 0, stopReason); } public int getDeltasProduced() { @@ -37,8 +40,8 @@ public int getEntriesProcessed() { return entriesProcessed; } - public boolean stoppedDueToRecentMessages() { - return stoppedDueToRecentMessages; + public StopReason getStopReason() { + return stopReason; } public int getDroppedRequestFilesProduced() { @@ -57,7 +60,8 @@ public JsonObject toJson() { .put("deltas_produced", deltasProduced) .put("entries_processed", entriesProcessed) .put("dropped_request_files_produced", droppedRequestFilesProduced) - .put("dropped_requests_processed", droppedRequestsProcessed); + .put("dropped_requests_processed", droppedRequestsProcessed) + .put("stop_reason", stopReason.name()); } /** diff --git a/src/main/java/com/uid2/optout/delta/DeltaUploadService.java b/src/main/java/com/uid2/optout/delta/DeltaUploadService.java new file mode 100644 index 00000000..a165929c --- /dev/null +++ b/src/main/java/com/uid2/optout/delta/DeltaUploadService.java @@ -0,0 +1,89 @@ +package com.uid2.optout.delta; + +import com.uid2.optout.sqs.SqsMessageOperations; +import com.uid2.shared.cloud.ICloudStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.List; + +/** + * Service for uploading data to S3 and deleting messages from SQS after successful upload. + * + * This class encapsulates the critical "upload then delete" pattern that ensures + * data is persisted to S3 before messages are removed from the queue. + */ +public class DeltaUploadService { + private static final Logger LOGGER = LoggerFactory.getLogger(DeltaUploadService.class); + + private final ICloudStorage cloudStorage; + private final SqsClient sqsClient; + private final String queueUrl; + + /** + * Callback interface for metrics updates after successful upload. + */ + @FunctionalInterface + public interface UploadSuccessCallback { + /** + * Called after successful S3 upload, before SQS message deletion. + * + * @param messageCount Number of messages in the uploaded batch + */ + void onSuccess(int messageCount); + } + + /** + * Create an S3UploadService. + * + * @param cloudStorage Cloud storage client for S3 operations + * @param sqsClient SQS client for message deletion + * @param queueUrl SQS queue URL + */ + public DeltaUploadService(ICloudStorage cloudStorage, SqsClient sqsClient, String queueUrl) { + this.cloudStorage = cloudStorage; + this.sqsClient = sqsClient; + this.queueUrl = queueUrl; + } + + /** + * Upload data to S3 and delete associated messages from SQS after successful upload. + * + *

Critical behavior: Messages are ONLY deleted from SQS after + * the S3 upload succeeds. This ensures no data loss if upload fails.

+ * + * @param data Data to upload + * @param s3Path S3 path (key) for the upload + * @param messages SQS messages to delete after successful upload + * @param onSuccess Callback invoked after successful upload (before message deletion) + * @throws IOException if upload fails + */ + public void uploadAndDeleteMessages(byte[] data, String s3Path, List messages, UploadSuccessCallback onSuccess) throws IOException { + LOGGER.info("Uploading to S3: path={}, size={} bytes, messages={}", s3Path, data.length, messages.size()); + + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(data)) { + cloudStorage.upload(inputStream, s3Path); + LOGGER.info("Successfully uploaded to S3: {}", s3Path); + + // Invoke callback for metrics/events + if (onSuccess != null) { + onSuccess.onSuccess(messages.size()); + } + + } catch (Exception e) { + LOGGER.error("Failed to upload to S3: path={}, error={}", s3Path, e.getMessage(), e); + throw new IOException("S3 upload failed: " + s3Path, e); + } + + // CRITICAL: Only delete messages after successful S3 upload + if (!messages.isEmpty()) { + LOGGER.info("Deleting {} messages from SQS after successful S3 upload", messages.size()); + SqsMessageOperations.deleteMessagesFromSqs(sqsClient, queueUrl, messages); + } + } +} + diff --git a/src/main/java/com/uid2/optout/delta/StopReason.java b/src/main/java/com/uid2/optout/delta/StopReason.java new file mode 100644 index 00000000..40239e16 --- /dev/null +++ b/src/main/java/com/uid2/optout/delta/StopReason.java @@ -0,0 +1,38 @@ +package com.uid2.optout.delta; + +/** + * Represents why delta production stopped. + * Used across all layers (batch, window, orchestrator) for consistent stop reason tracking. + */ +public enum StopReason { + /** + * Processing completed normally with work done, or still in progress. + */ + NONE, + + /** + * No messages available in the SQS queue. + */ + QUEUE_EMPTY, + + /** + * Messages exist in the queue but are too recent (less than deltaWindowSeconds old). + */ + MESSAGES_TOO_RECENT, + + /** + * Hit the maximum messages per window limit. + */ + MESSAGE_LIMIT_EXCEEDED, + + /** + * Pre-existing manual override was set (checked at job start). + */ + MANUAL_OVERRIDE_ACTIVE, + + /** + * Circuit breaker triggered during processing (traffic spike detected). + */ + CIRCUIT_BREAKER_TRIGGERED +} + diff --git a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java similarity index 54% rename from src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java rename to src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java index 0a656e98..14eccf76 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java +++ b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java @@ -1,15 +1,15 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; +import com.uid2.optout.delta.StopReason; import com.uid2.shared.optout.OptOutUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.Message; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * Applies parsing, validation, filtering, and deletion of corrupted SQS messages. @@ -29,46 +29,46 @@ public SqsBatchProcessor(SqsClient sqsClient, String queueUrl, int deltaWindowSe } /** - * Result of processing a batch of messages from SQS. - * Encapsulates eligible messages and metadata about the processing. + * Result of processing a batch of (10) messages from SQS. + * Encapsulates eligible messages and the reason for stopping (if any). */ public static class BatchProcessingResult { private final List eligibleMessages; - private final boolean shouldStopProcessing; + private final StopReason stopReason; - private BatchProcessingResult(List eligibleMessages, boolean shouldStopProcessing) { + private BatchProcessingResult(List eligibleMessages, StopReason stopReason) { this.eligibleMessages = eligibleMessages; - this.shouldStopProcessing = shouldStopProcessing; + this.stopReason = stopReason; } - public static BatchProcessingResult withEligibleMessages(List messages) { - return new BatchProcessingResult(messages, false); + public static BatchProcessingResult withMessages(List messages) { + return new BatchProcessingResult(messages, StopReason.NONE); } - public static BatchProcessingResult stopProcessing() { - return new BatchProcessingResult(new ArrayList<>(), true); + public static BatchProcessingResult messagesTooRecent() { + return new BatchProcessingResult(List.of(), StopReason.MESSAGES_TOO_RECENT); } - public static BatchProcessingResult empty() { - return new BatchProcessingResult(new ArrayList<>(), false); + public static BatchProcessingResult corruptMessagesDeleted() { + return new BatchProcessingResult(List.of(), StopReason.NONE); } - public boolean isEmpty() { - return eligibleMessages.isEmpty(); + public boolean hasMessages() { + return !eligibleMessages.isEmpty(); } - public boolean shouldStopProcessing() { - return shouldStopProcessing; + public StopReason getStopReason() { + return stopReason; } - public List getEligibleMessages() { + public List getMessages() { return eligibleMessages; } } /** * Processes a batch of messages: parses, validates, cleans up invalid messages, - * and filters for eligible messages based on age threshold (message is less than 5 minutes old) + * and filters for eligible messages based on age threshold (message is older than deltaWindowSeconds) * * @param messageBatch Raw messages from SQS * @param batchNumber The batch number (for logging) @@ -88,53 +88,48 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN } } - // If no valid messages, return empty result + // No valid messages after deleting corrupt ones, continue reading if (parsedBatch.isEmpty()) { - LOGGER.warn("No valid messages in batch {} (all failed parsing)", batchNumber); - return BatchProcessingResult.empty(); + LOGGER.error("No valid messages in batch {} (all failed parsing)", batchNumber); + return BatchProcessingResult.corruptMessagesDeleted(); } // Check if the oldest message in this batch is too recent long currentTime = OptOutUtils.nowEpochSeconds(); SqsParsedMessage oldestMessage = parsedBatch.get(0); - long messageAge = currentTime - oldestMessage.getTimestamp(); - if (messageAge < this.deltaWindowSeconds) { - // Signal to stop processing - messages are too recent - return BatchProcessingResult.stopProcessing(); + if (!isMessageEligible(oldestMessage, currentTime)) { + return BatchProcessingResult.messagesTooRecent(); } - // Filter for eligible messages (>= 5 minutes old) + // Filter for eligible messages (>= deltaWindowSeconds old) List eligibleMessages = filterEligibleMessages(parsedBatch, currentTime); - if (eligibleMessages.isEmpty()) { - LOGGER.debug("No eligible messages in batch {} (all too recent)", batchNumber); - return BatchProcessingResult.empty(); - } + return BatchProcessingResult.withMessages(eligibleMessages); + } - return BatchProcessingResult.withEligibleMessages(eligibleMessages); + /** + * Checks if a message is old enough to be processed. + * + * @param message The parsed message to check + * @param currentTime Current time in epoch seconds + * @return true if the message is at least deltaWindowSeconds old + */ + private boolean isMessageEligible(SqsParsedMessage message, long currentTime) { + return currentTime - message.getTimestamp() >= this.deltaWindowSeconds; } /** * Filters messages to only include those where sufficient time has elapsed. -. * + * * @param messages List of parsed messages * @param currentTime Current time in seconds * @return List of messages that meet the time threshold */ - public List filterEligibleMessages( - List messages, - long currentTime) { - - List eligibleMessages = new ArrayList<>(); - - for (SqsParsedMessage pm : messages) { - if (currentTime - pm.getTimestamp() >= this.deltaWindowSeconds) { - eligibleMessages.add(pm); - } - } - - return eligibleMessages; + List filterEligibleMessages(List messages, long currentTime) { + return messages.stream() + .filter(msg -> isMessageEligible(msg, currentTime)) + .collect(Collectors.toList()); } /** @@ -145,21 +140,12 @@ public List filterEligibleMessages( * @return List of messages that failed to parse */ private List identifyInvalidMessages(List originalBatch, List parsedBatch) { - // Create a set of message IDs from successfully parsed messages - Set validMessageIds = new HashSet<>(); - for (SqsParsedMessage parsed : parsedBatch) { - validMessageIds.add(parsed.getOriginalMessage().messageId()); - } + Set validIds = parsedBatch.stream() + .map(p -> p.getOriginalMessage().messageId()) + .collect(Collectors.toSet()); - // Find messages that were not successfully parsed - List invalidMessages = new ArrayList<>(); - for (Message msg : originalBatch) { - if (!validMessageIds.contains(msg.messageId())) { - invalidMessages.add(msg); - } - } - - return invalidMessages; + return originalBatch.stream() + .filter(msg -> !validIds.contains(msg.messageId())) + .collect(Collectors.toList()); } } - diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java similarity index 84% rename from src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java rename to src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java index 104215c6..87d13ace 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,44 +106,6 @@ private static int parseIntOrDefault(String value, int defaultValue) { } } - /** - * Receives all available messages from an SQS queue up to a maximum number of batches. - * - * @param sqsClient The SQS client - * @param queueUrl The queue URL - * @param maxMessagesPerPoll Maximum messages to receive per poll (max 10) - * @param visibilityTimeout Visibility timeout in seconds - * @param maxBatches Maximum number of receive batches - * @return List of all received messages - */ - public static List receiveAllAvailableMessages( - SqsClient sqsClient, - String queueUrl, - int maxMessagesPerPoll, - int visibilityTimeout, - int maxBatches) { - - List allMessages = new ArrayList<>(); - int batchCount = 0; - - // Keep receiving messages until we get an empty batch or hit the limit - while (batchCount < maxBatches) { - List batch = receiveMessagesFromSqs(sqsClient, queueUrl, maxMessagesPerPoll, visibilityTimeout); - if (batch.isEmpty()) { - break; - } - allMessages.addAll(batch); - batchCount++; - - // If we got fewer messages than the max (of 10), the queue is likely empty - if (batch.size() < maxMessagesPerPoll) { - break; - } - } - - return allMessages; - } - /** * Receives a batch of messages from SQS. * diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java similarity index 99% rename from src/main/java/com/uid2/optout/vertx/SqsMessageParser.java rename to src/main/java/com/uid2/optout/sqs/SqsMessageParser.java index 44a6c5e9..3d4c534b 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; import com.uid2.shared.optout.OptOutUtils; import io.vertx.core.json.JsonObject; diff --git a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java b/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java similarity index 97% rename from src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java rename to src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java index 1ad8ba77..850cd7a1 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java +++ b/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; import software.amazon.awssdk.services.sqs.model.Message; diff --git a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java similarity index 62% rename from src/main/java/com/uid2/optout/vertx/SqsWindowReader.java rename to src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index 2149baa3..444b23eb 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -1,5 +1,6 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; +import com.uid2.optout.delta.StopReason; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; @@ -37,36 +38,40 @@ public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerP maxMessagesPerWindow, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds); } - /** - * Update the max messages limit (e.g., after config reload). - */ - public void setMaxMessagesPerWindow(int maxMessagesPerWindow) { - this.maxMessagesPerWindow = maxMessagesPerWindow; - LOGGER.info("Updated maxMessagesPerWindow to {}", maxMessagesPerWindow); - } - /** * Result of reading messages for a 5-minute window. */ public static class WindowReadResult { private final List messages; private final long windowStart; - private final boolean stoppedDueToRecentMessages; - private final boolean exceededMessageLimit; + private final StopReason stopReason; - public WindowReadResult(List messages, long windowStart, - boolean stoppedDueToRecentMessages, boolean exceededMessageLimit) { + private WindowReadResult(List messages, long windowStart, StopReason stopReason) { this.messages = messages; this.windowStart = windowStart; - this.stoppedDueToRecentMessages = stoppedDueToRecentMessages; - this.exceededMessageLimit = exceededMessageLimit; + this.stopReason = stopReason; + } + + public static WindowReadResult withMessages(List messages, long windowStart) { + return new WindowReadResult(messages, windowStart, StopReason.NONE); + } + + public static WindowReadResult queueEmpty(List messages, long windowStart) { + return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY); + } + + public static WindowReadResult messagesTooRecent(List messages, long windowStart) { + return new WindowReadResult(messages, windowStart, StopReason.MESSAGES_TOO_RECENT); + } + + public static WindowReadResult messageLimitExceeded(List messages, long windowStart) { + return new WindowReadResult(messages, windowStart, StopReason.MESSAGE_LIMIT_EXCEEDED); } public List getMessages() { return messages; } public long getWindowStart() { return windowStart; } public boolean isEmpty() { return messages.isEmpty(); } - public boolean stoppedDueToRecentMessages() { return stoppedDueToRecentMessages; } - public boolean exceededMessageLimit() { return exceededMessageLimit; } + public StopReason getStopReason() { return stopReason; } } /** @@ -74,7 +79,7 @@ public WindowReadResult(List messages, long windowStart, * Keeps reading batches and accumulating messages until: * - We discover the next window * - Queue is empty (no more messages) - * - Messages are too recent (all messages younger than 5 minutes) + * - Messages are too recent (all messages younger than deltaWindowSeconds) * - Message count exceeds maxMessagesPerWindow * * @return WindowReadResult with messages for the window, or empty if done @@ -82,12 +87,13 @@ public WindowReadResult(List messages, long windowStart, public WindowReadResult readWindow() { List windowMessages = new ArrayList<>(); long currentWindowStart = 0; + int batchNumber = 0; while (true) { if (windowMessages.size() >= maxMessagesPerWindow) { - LOGGER.warn("Message limit exceeded: {} messages >= limit {}. Stopping to prevent memory exhaustion.", + LOGGER.warn("Message limit exceeded: {} messages >= limit {}. Closing window.", windowMessages.size(), maxMessagesPerWindow); - return new WindowReadResult(windowMessages, currentWindowStart, false, true); + return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart); } // Read one batch from SQS (up to 10 messages) @@ -95,50 +101,40 @@ public WindowReadResult readWindow() { this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout); if (rawBatch.isEmpty()) { - // Queue empty - return what we have - return new WindowReadResult(windowMessages, currentWindowStart, false, false); + return WindowReadResult.queueEmpty(windowMessages, currentWindowStart); } - // Process batch: parse, validate, filter - SqsBatchProcessor.BatchProcessingResult batchResult = batchProcessor.processBatch(rawBatch, 0); + // parse, validate, filter + SqsBatchProcessor.BatchProcessingResult batchResult = batchProcessor.processBatch(rawBatch, batchNumber++); - if (batchResult.isEmpty()) { - if (batchResult.shouldStopProcessing()) { - // Messages too recent - return what we have - return new WindowReadResult(windowMessages, currentWindowStart, true, false); + if (!batchResult.hasMessages()) { + if (batchResult.getStopReason() == StopReason.MESSAGES_TOO_RECENT) { + return WindowReadResult.messagesTooRecent(windowMessages, currentWindowStart); } - // corrupt messages deleted, read next messages + // Corrupt messages were deleted, continue reading continue; } // Add eligible messages to current window boolean newWindow = false; - for (SqsParsedMessage msg : batchResult.getEligibleMessages()) { + for (SqsParsedMessage msg : batchResult.getMessages()) { long msgWindowStart = (msg.getTimestamp() / this.deltaWindowSeconds) * this.deltaWindowSeconds; - // discover start of window + // Discover start of window if (currentWindowStart == 0) { currentWindowStart = msgWindowStart; } - // discover new window + // Discover next window if (msgWindowStart > currentWindowStart + this.deltaWindowSeconds) { newWindow = true; } windowMessages.add(msg); - - // Check limit after each message addition - if (windowMessages.size() >= maxMessagesPerWindow) { - LOGGER.warn("Message limit exceeded during batch: {} messages >= limit {}", - windowMessages.size(), maxMessagesPerWindow); - return new WindowReadResult(windowMessages, currentWindowStart, false, true); - } } if (newWindow) { - // close current window and return - return new WindowReadResult(windowMessages, currentWindowStart, false, false); + return WindowReadResult.withMessages(windowMessages, currentWindowStart); } } } diff --git a/src/main/java/com/uid2/optout/util/HttpResponseHelper.java b/src/main/java/com/uid2/optout/util/HttpResponseHelper.java new file mode 100644 index 00000000..6d82f568 --- /dev/null +++ b/src/main/java/com/uid2/optout/util/HttpResponseHelper.java @@ -0,0 +1,70 @@ +package com.uid2.optout.util; + +import io.vertx.core.http.HttpHeaders; +import io.vertx.core.http.HttpServerResponse; +import io.vertx.core.json.JsonObject; + +/** + * Utility class for HTTP JSON response handling. + * Ensures consistent response format across handlers. + */ +public class HttpResponseHelper { + + /** + * Send a JSON response with the specified status code. + */ + public static void sendJson(HttpServerResponse resp, int statusCode, JsonObject body) { + resp.setStatusCode(statusCode) + .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") + .end(body.encode()); + } + + /** + * Send a 200 OK response with JSON body. + */ + public static void sendSuccess(HttpServerResponse resp, JsonObject body) { + sendJson(resp, 200, body); + } + + /** + * Send a 200 OK response with status and message. + */ + public static void sendSuccess(HttpServerResponse resp, String status, String message) { + sendJson(resp, 200, new JsonObject().put("status", status).put("message", message)); + } + + /** + * Send a 200 OK response with idle status and message. + */ + public static void sendIdle(HttpServerResponse resp, String message) { + sendJson(resp, 200, new JsonObject().put("status", "idle").put("message", message)); + } + /** + * Send a 202 Accepted response indicating async job started. + */ + public static void sendAccepted(HttpServerResponse resp, String message) { + sendJson(resp, 202, new JsonObject().put("status", "accepted").put("message", message)); + } + + /** + * Send a 409 Conflict response. + */ + public static void sendConflict(HttpServerResponse resp, String reason) { + sendJson(resp, 409, new JsonObject().put("status", "conflict").put("reason", reason)); + } + + /** + * Send a 500 Internal Server Error response. + */ + public static void sendError(HttpServerResponse resp, String error) { + sendJson(resp, 500, new JsonObject().put("status", "failed").put("error", error)); + } + + /** + * Send a 500 Internal Server Error response from an exception. + */ + public static void sendError(HttpServerResponse resp, Exception e) { + sendError(resp, e.getMessage()); + } +} + diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index e2afddf7..5f882028 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -2,36 +2,37 @@ import com.uid2.optout.Const; import com.uid2.optout.auth.InternalAuthMiddleware; +import com.uid2.optout.delta.DeltaFileWriter; +import com.uid2.optout.delta.DeltaManualOverrideService; +import com.uid2.optout.delta.DeltaProductionJobStatus; +import com.uid2.optout.delta.DeltaProductionResult; +import com.uid2.optout.delta.DeltaProductionMetrics; +import com.uid2.optout.delta.DeltaProductionOrchestrator; +import com.uid2.optout.delta.DeltaUploadService; +import com.uid2.optout.delta.StopReason; +import com.uid2.optout.sqs.SqsWindowReader; import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import com.uid2.optout.vertx.OptOutTrafficFilter.MalformedTrafficFilterConfigException; import com.uid2.shared.Utils; import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.health.HealthComponent; import com.uid2.shared.health.HealthManager; -import com.uid2.shared.optout.*; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Metrics; +import com.uid2.shared.optout.OptOutCloudSync; +import com.uid2.shared.optout.OptOutUtils; + import io.vertx.core.*; import io.vertx.core.http.*; import io.vertx.core.json.JsonObject; import io.vertx.ext.web.Router; import io.vertx.ext.web.RoutingContext; -import io.vertx.core.json.JsonArray; -import java.time.Instant; -import java.time.temporal.ChronoUnit; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.*; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; +import static com.uid2.optout.util.HttpResponseHelper.*; + import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.atomic.AtomicReference; /** @@ -82,104 +83,83 @@ public class OptOutSqsLogProducer extends AbstractVerticle { private final HealthComponent healthComponent = HealthManager.instance.registerComponent("sqs-log-producer"); private final SqsClient sqsClient; - private final String queueUrl; private final String eventDeltaProduced; - private final int replicaId; - private final ICloudStorage cloudStorage; - private final ICloudStorage cloudStorageDroppedRequests; - private final OptOutCloudSync cloudSync; - private final int maxMessagesPerPoll; - private final int visibilityTimeout; - private final int deltaWindowSeconds; // Time window for each delta file (5 minutes = 300 seconds) - private final int jobTimeoutSeconds; private final int listenPort; - private final String internalApiKey; private final InternalAuthMiddleware internalAuth; private final OptOutTrafficFilter trafficFilter; private final OptOutTrafficCalculator trafficCalculator; - private final String manualOverrideS3Path; - - private Counter counterDeltaProduced = Counter - .builder("uid2_optout_sqs_delta_produced_total") - .description("counter for how many optout delta files are produced from SQS") - .register(Metrics.globalRegistry); - - private Counter counterEntriesProcessed = Counter - .builder("uid2_optout_sqs_entries_processed_total") - .description("counter for how many optout entries are processed from SQS") - .register(Metrics.globalRegistry); - - private Counter counterDroppedRequestFilesProduced = Counter - .builder("uid2_optout_sqs_dropped_request_files_produced_total") - .description("counter for how many optout dropped request files are produced from SQS") - .register(Metrics.globalRegistry); - - private Counter counterDroppedRequestsProcessed = Counter - .builder("uid2_optout_sqs_dropped_requests_processed_total") - .description("counter for how many optout dropped requests are processed from SQS") - .register(Metrics.globalRegistry); - - private ByteBuffer buffer; - private boolean shutdownInProgress = false; + private final DeltaProductionOrchestrator orchestrator; - //Tracks the current delta production job status for this pod. - private final AtomicReference currentJob = new AtomicReference<>(null); + // Tracks the current delta production job status for this pod + private final AtomicReference currentJob = new AtomicReference<>(null); - // Helper for reading complete 5-minute windows from SQS - private final SqsWindowReader windowReader; - - public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { - this(jsonConfig, cloudStorage, null, cloudSync, eventDeltaProduced, null); - } + private volatile boolean shutdownInProgress = false; - // Constructor for testing - allows injecting mock SqsClient public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, ICloudStorage cloudStorageDroppedRequests, OptOutCloudSync cloudSync, String eventDeltaProduced, SqsClient sqsClient) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException { this.eventDeltaProduced = eventDeltaProduced; - this.replicaId = OptOutUtils.getReplicaId(jsonConfig); - this.cloudStorage = cloudStorage; - this.cloudStorageDroppedRequests = cloudStorageDroppedRequests; - this.cloudSync = cloudSync; - + // Initialize SQS client - this.queueUrl = jsonConfig.getString(Const.Config.OptOutSqsQueueUrlProp); - if (this.queueUrl == null || this.queueUrl.isEmpty()) { + String queueUrl = jsonConfig.getString(Const.Config.OptOutSqsQueueUrlProp); + if (queueUrl == null || queueUrl.isEmpty()) { throw new IOException("SQS queue URL not configured"); } - - // Use injected client for testing, or create new one this.sqsClient = sqsClient != null ? sqsClient : SqsClient.builder().build(); - LOGGER.info("SQS client initialized for queue: " + this.queueUrl); + LOGGER.info("SQS client initialized for queue: {}", queueUrl); - // SQS Configuration - this.maxMessagesPerPoll = 10; // SQS max is 10 - this.visibilityTimeout = jsonConfig.getInteger(Const.Config.OptOutSqsVisibilityTimeoutProp, 240); // 4 minutes default - this.deltaWindowSeconds = 300; // Fixed 5 minutes for all deltas - this.jobTimeoutSeconds = jsonConfig.getInteger(Const.Config.OptOutDeltaJobTimeoutSecondsProp, 10800); // 3 hours default - - // HTTP server configuration - use port offset + 1 to avoid conflicts + // HTTP server configuration this.listenPort = Const.Port.ServicePortForOptOut + Utils.getPortOffset() + 1; // Authentication - this.internalApiKey = jsonConfig.getString(Const.Config.OptOutInternalApiTokenProp); - this.internalAuth = new InternalAuthMiddleware(this.internalApiKey, "optout-sqs"); - - int bufferSize = jsonConfig.getInteger(Const.Config.OptOutProducerBufferSizeProp); - this.buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN); + String internalApiKey = jsonConfig.getString(Const.Config.OptOutInternalApiTokenProp); + this.internalAuth = new InternalAuthMiddleware(internalApiKey, "optout-sqs"); + // Circuit breaker tools this.trafficFilter = new OptOutTrafficFilter(jsonConfig.getString(Const.Config.TrafficFilterConfigPathProp)); this.trafficCalculator = new OptOutTrafficCalculator(cloudStorage, jsonConfig.getString(Const.Config.OptOutSqsS3FolderProp), jsonConfig.getString(Const.Config.TrafficCalcConfigPathProp)); - this.manualOverrideS3Path = jsonConfig.getString(Const.Config.ManualOverrideS3PathProp); - - // Initialize window reader with traffic threshold - this.windowReader = new SqsWindowReader( - this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, - this.visibilityTimeout, this.deltaWindowSeconds, this.trafficCalculator.getThreshold() + + // Configuration values for orchestrator setup + int replicaId = OptOutUtils.getReplicaId(jsonConfig); + int maxMessagesPerPoll = 10; // SQS max is 10 + int deltaWindowSeconds = 300; // Fixed 5 minutes for all deltas + int visibilityTimeout = jsonConfig.getInteger(Const.Config.OptOutSqsVisibilityTimeoutProp, 240); + int jobTimeoutSeconds = jsonConfig.getInteger(Const.Config.OptOutDeltaJobTimeoutSecondsProp, 10800); + int maxMessagesPerFile = jsonConfig.getInteger(Const.Config.OptOutMaxMessagesPerFileProp, 10000); + int bufferSize = jsonConfig.getInteger(Const.Config.OptOutProducerBufferSizeProp); + + // Orchestrator setup + DeltaFileWriter deltaFileWriter = new DeltaFileWriter(bufferSize); + DeltaUploadService deltaUploadService = new DeltaUploadService(cloudStorage, this.sqsClient, queueUrl); + DeltaUploadService droppedRequestUploadService = new DeltaUploadService(cloudStorageDroppedRequests, this.sqsClient, queueUrl) ; + DeltaManualOverrideService manualOverrideService = new DeltaManualOverrideService(cloudStorage, jsonConfig.getString(Const.Config.ManualOverrideS3PathProp)); + SqsWindowReader windowReader = new SqsWindowReader( + this.sqsClient, queueUrl, maxMessagesPerPoll, + visibilityTimeout, deltaWindowSeconds, maxMessagesPerFile + ); + + this.orchestrator = new DeltaProductionOrchestrator( + this.sqsClient, + queueUrl, + replicaId, + deltaWindowSeconds, + jobTimeoutSeconds, + windowReader, + deltaFileWriter, + deltaUploadService, + droppedRequestUploadService, + manualOverrideService, + this.trafficFilter, + this.trafficCalculator, + cloudSync, + new DeltaProductionMetrics() ); + + LOGGER.info("OptOutSqsLogProducer initialized with maxMessagesPerFile: {}, maxMessagesPerPoll: {}, visibilityTimeout: {}, deltaWindowSeconds: {}, jobTimeoutSeconds: {}", + maxMessagesPerFile, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds, jobTimeoutSeconds); } @Override public void start(Promise startPromise) { - LOGGER.info("Starting SQS Log Producer with HTTP endpoint..."); + LOGGER.info("Attempting to start SQS Log Producer HTTP server on port: {}", listenPort); try { vertx.createHttpServer() @@ -187,8 +167,7 @@ public void start(Promise startPromise) { .listen(listenPort, result -> { if (result.succeeded()) { this.healthComponent.setHealthStatus(true); - LOGGER.info("SQS Log Producer HTTP server started on port: {} (delta window: {}s)", - listenPort, this.deltaWindowSeconds); + LOGGER.info("SQS Log Producer HTTP server started on port: {}", listenPort); startPromise.complete(); } else { LOGGER.error("Failed to start SQS Log Producer HTTP server", result.cause()); @@ -198,7 +177,7 @@ public void start(Promise startPromise) { }); } catch (Exception e) { - LOGGER.error("Failed to start SQS Log Producer", e); + LOGGER.error("Failed to start SQS Log Producer HTTP server", e); this.healthComponent.setHealthStatus(false, e.getMessage()); startPromise.fail(e); } @@ -244,18 +223,14 @@ private Router createRouter() { private void handleDeltaProduceStatus(RoutingContext routingContext) { HttpServerResponse resp = routingContext.response(); - DeltaProduceJobStatus job = currentJob.get(); + DeltaProductionJobStatus job = currentJob.get(); if (job == null) { - resp.setStatusCode(200) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject().put("status", "idle").put("message", "No job running on this pod").encode()); + sendIdle(resp, "No job running on this pod"); return; } - resp.setStatusCode(200) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(job.toJson().encode()); + sendSuccess(resp, job.toJson()); } /** @@ -278,67 +253,55 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { this.trafficFilter.reloadTrafficFilterConfig(); } catch (MalformedTrafficFilterConfigException e) { LOGGER.error("Error reloading traffic filter config: " + e.getMessage(), e); - resp.setStatusCode(500) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject().put("status", "failed").put("error", e.getMessage()).encode()); + sendError(resp, e); return; } try { this.trafficCalculator.reloadTrafficCalcConfig(); - // Update window reader's message limit to match new threshold - this.windowReader.setMaxMessagesPerWindow(this.trafficCalculator.getThreshold()); } catch (MalformedTrafficCalcConfigException e) { LOGGER.error("Error reloading traffic calculator config: " + e.getMessage(), e); - resp.setStatusCode(500) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject().put("status", "failed").put("error", e.getMessage()).encode()); + sendError(resp, e); return; } - DeltaProduceJobStatus existingJob = currentJob.get(); + DeltaProductionJobStatus existingJob = currentJob.get(); // If there's an existing job, check if it's still running if (existingJob != null) { - if (existingJob.getState() == DeltaProduceJobStatus.JobState.RUNNING) { + if (existingJob.getState() == DeltaProductionJobStatus.JobState.RUNNING) { // Cannot replace a running job - 409 Conflict LOGGER.warn("Delta production job already running on this pod"); - resp.setStatusCode(409) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject().put("status", "conflict").put("reason", "A delta production job is already running on this pod").encode()); + sendConflict(resp, "A delta production job is already running on this pod"); return; } LOGGER.info("Auto-clearing previous {} job to start new one", existingJob.getState()); } - DeltaProduceJobStatus newJob = new DeltaProduceJobStatus(); + DeltaProductionJobStatus newJob = new DeltaProductionJobStatus(); // Try to set the new job if (!currentJob.compareAndSet(existingJob, newJob)) { - resp.setStatusCode(409) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject().put("status", "conflict").put("reason", "Job state changed, please retry").encode()); + sendConflict(resp, "Job state changed, please retry"); return; } // Start the job asynchronously - LOGGER.info("Starting delta production job"); + LOGGER.info("New delta production job initialized"); this.startDeltaProductionJob(newJob); // Return immediately with 202 Accepted - resp.setStatusCode(202) - .putHeader(HttpHeaders.CONTENT_TYPE, "application/json") - .end(new JsonObject().put("status", "accepted").put("message", "Delta production job started on this pod").encode()); + sendAccepted(resp, "Delta production job started on this pod"); } /** * Starts the delta production job asynchronously * The job runs on a worker thread and updates the DeltaProduceJobStatus when complete */ - private void startDeltaProductionJob(DeltaProduceJobStatus job) { + private void startDeltaProductionJob(DeltaProductionJobStatus job) { vertx.executeBlocking(() -> { - LOGGER.info("Executing delta production job"); + LOGGER.info("Delta production job starting on worker thread"); return produceDeltasBlocking(); }).onComplete(ar -> { if (ar.succeeded()) { @@ -370,338 +333,46 @@ private JsonObject produceDeltasBlocking() throws Exception { // Process messages until queue is empty or messages are too recent DeltaProductionResult deltaResult = this.produceBatchedDeltas(); - // Determine status based on results - if (deltaResult.getDeltasProduced() == 0 && deltaResult.stoppedDueToRecentMessages()) { - // No deltas produced - either messages too recent or traffic spike detected - LOGGER.info("Delta production skipped: {} entries processed", deltaResult.getEntriesProcessed()); - return deltaResult.toJsonWithStatus("skipped", "reason", "No deltas produced"); - } else { - LOGGER.info("Delta production complete: {} deltas, {} entries", - deltaResult.getDeltasProduced(), deltaResult.getEntriesProcessed()); + StopReason stopReason = deltaResult.getStopReason(); + boolean producedDeltas = deltaResult.getDeltasProduced() > 0; + boolean producedDroppedRequests = deltaResult.getDroppedRequestFilesProduced() > 0; + + // Determine status based on results: + // "success" = produced work OR completed normally + // "skipped" = stopped early due to abnormal conditions (circuit breaker, override, too recent) + boolean isSkipped = stopReason == StopReason.CIRCUIT_BREAKER_TRIGGERED + || stopReason == StopReason.MESSAGES_TOO_RECENT + || stopReason == StopReason.MANUAL_OVERRIDE_ACTIVE; + + boolean isSuccess = !isSkipped && (producedDeltas + || producedDroppedRequests + || stopReason == StopReason.QUEUE_EMPTY + || stopReason == StopReason.NONE); + + if (isSuccess) { + LOGGER.info("Delta production complete: {} deltas, {} entries, dropped request files: {}, dropped requests: {}, stop reason: {}", + deltaResult.getDeltasProduced(), deltaResult.getEntriesProcessed(), deltaResult.getDroppedRequestFilesProduced(), deltaResult.getDroppedRequestsProcessed(), stopReason); return deltaResult.toJsonWithStatus("success"); + } else { + LOGGER.info("Delta production skipped: {}, {} entries processed, dropped request files: {}, dropped requests: {}", + stopReason, deltaResult.getEntriesProcessed(), deltaResult.getDroppedRequestFilesProduced(), deltaResult.getDroppedRequestsProcessed()); + return deltaResult.toJsonWithStatus("skipped", "reason", stopReason.name()); } } /** - * Reads messages from SQS and produces delta files in 5 minute batches. - * Continues until queue is empty or messages are too recent. - * Windows are limited to maxMessagesPerFile for memory protection. + * Delegates to the orchestrator to produce delta files. * - * @return DeltaProductionResult with counts and stop reason + * @return DeltaProduceResult with counts and stop reason * @throws IOException if delta production fails */ private DeltaProductionResult produceBatchedDeltas() throws IOException { - // Check for manual override at the start of the batch (and then between each delta window) - if (getManualOverride().equals("DELAYED_PROCESSING")) { - LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); - return new DeltaProductionResult(0, 0, 0, 0, false); - } - - int deltasProduced = 0; - int totalEntriesProcessed = 0; - - int droppedRequestFilesProduced = 0; - int droppedRequestsProcessed = 0; - - boolean stoppedDueToRecentMessages = false; - - long jobStartTime = OptOutUtils.nowEpochSeconds(); - LOGGER.info("Starting delta production from SQS queue"); - - // Read and process windows until done - while (true) { - if(checkJobTimeout(jobStartTime)){ - break; - } - - // Read one complete 5-minute window (limited to maxMessagesPerFile) - SqsWindowReader.WindowReadResult windowResult = windowReader.readWindow(); - - // If no messages, we're done (queue empty or messages too recent) - if (windowResult.isEmpty()) { - stoppedDueToRecentMessages = windowResult.stoppedDueToRecentMessages(); - LOGGER.info("Delta production complete - no more eligible messages"); - break; - } - - // if message limit exceeded, treat as traffic spike - if (windowResult.exceededMessageLimit()) { - LOGGER.error("Message limit exceeded ({} messages) - triggering DELAYED_PROCESSING to prevent memory exhaustion", - windowResult.getMessages().size()); - this.setDelayedProcessingOverride(); - return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); - } - - // Create delta file buffer - String deltaName = OptOutUtils.newDeltaFileName(this.replicaId); - ByteArrayOutputStream deltaStream = new ByteArrayOutputStream(); - - // Create dropped request file buffer - JsonArray droppedRequestStream = new JsonArray(); - String currentDroppedRequestName = String.format("%s%03d_%s_%08x.json", "optout-dropped-", replicaId, Instant.now().truncatedTo(ChronoUnit.SECONDS).toString().replace(':', '.'), OptOutUtils.rand.nextInt()); - - // Produce delta for this window - long windowStart = windowResult.getWindowStart(); - List messages = windowResult.getMessages(); - - writeStartOfDelta(deltaStream, windowStart); - - // Write all messages - List currentDeltaMessages = new ArrayList<>(); - List droppedRequestMessages = new ArrayList<>(); - for (SqsParsedMessage msg : messages) { - if (trafficFilter.isDenylisted(msg)) { - this.writeDroppedRequestEntry(droppedRequestStream, msg); - droppedRequestMessages.add(msg.getOriginalMessage()); - droppedRequestsProcessed++; - } else { - writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp()); - currentDeltaMessages.add(msg.getOriginalMessage()); - totalEntriesProcessed++; - } - } - - // check for manual override - if (getManualOverride().equals("DELAYED_PROCESSING")) { - LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); - return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); - } else { - // Get queue attributes (including invisible messages) for traffic calculation - SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); - - // check traffic calculator status (including invisible messages in case of multiple consumers) - OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(currentDeltaMessages, queueAttributes); - if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { - LOGGER.error("OptOut Delta Production has hit DELAYED_PROCESSING status, stopping production"); - this.setDelayedProcessingOverride(); - return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, true); - } - } - - // Upload delta file if there are non-denylisted messages - if (!currentDeltaMessages.isEmpty()) { - uploadDeltaAndDeleteMessages(deltaStream, deltaName, windowStart, currentDeltaMessages); - deltasProduced++; - } - - // Upload dropped request file if there are denylisted messages - if (!droppedRequestMessages.isEmpty()) { - this.uploadDroppedRequestsAndDeleteMessages(droppedRequestStream, currentDroppedRequestName, windowStart, droppedRequestMessages); - droppedRequestFilesProduced++; - droppedRequestMessages.clear(); - } - - LOGGER.info("Produced delta for window [{}, {}] with {} messages", - windowStart, windowStart + this.deltaWindowSeconds, messages.size()); - } - - long totalDuration = OptOutUtils.nowEpochSeconds() - jobStartTime; - LOGGER.info("Delta production complete: took {}s, produced {} deltas, processed {} entries, produced {} dropped request files, processed {} dropped requests", - totalDuration, deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed); - - return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, droppedRequestFilesProduced, droppedRequestsProcessed, stoppedDueToRecentMessages); - } - - /** - * Checks if job has exceeded timeout - */ - private boolean checkJobTimeout(long jobStartTime) { - long elapsedTime = OptOutUtils.nowEpochSeconds() - jobStartTime; - if (elapsedTime > 3600) { // 1 hour - log warning message - LOGGER.error("Delta production job has been running for {} seconds", - elapsedTime); - return false; - } - if (elapsedTime > this.jobTimeoutSeconds) { - LOGGER.error("Delta production job has been running for {} seconds (exceeds timeout of {}s)", - elapsedTime, this.jobTimeoutSeconds); - return true; // deadline exceeded - } - return false; - } - - /** - * Writes the start-of-delta entry with null hash and window start timestamp. - */ - private void writeStartOfDelta(ByteArrayOutputStream stream, long windowStart) throws IOException { - - this.checkBufferSize(OptOutConst.EntrySize); - - buffer.put(OptOutUtils.nullHashBytes); - buffer.put(OptOutUtils.nullHashBytes); - buffer.putLong(windowStart); - - buffer.flip(); - byte[] entry = new byte[buffer.remaining()]; - buffer.get(entry); - - stream.write(entry); - buffer.clear(); - } - - /** - * Writes a single opt-out entry to the delta stream. - */ - private void writeOptOutEntry(ByteArrayOutputStream stream, byte[] hashBytes, byte[] idBytes, long timestamp) throws IOException { - this.checkBufferSize(OptOutConst.EntrySize); - OptOutEntry.writeTo(buffer, hashBytes, idBytes, timestamp); - buffer.flip(); - byte[] entry = new byte[buffer.remaining()]; - buffer.get(entry); - stream.write(entry); - buffer.clear(); - } - - /** - * Writes the end-of-delta sentinel entry with ones hash and window end timestamp. - */ - private void writeEndOfDelta(ByteArrayOutputStream stream, long windowEnd) throws IOException { - this.checkBufferSize(OptOutConst.EntrySize); - buffer.put(OptOutUtils.onesHashBytes); - buffer.put(OptOutUtils.onesHashBytes); - buffer.putLong(windowEnd); - buffer.flip(); - byte[] entry = new byte[buffer.remaining()]; - buffer.get(entry); - stream.write(entry); - buffer.clear(); + return orchestrator.produceBatchedDeltas(this::publishDeltaProducedEvent); } - - - // Upload a delta to S3 and delete messages from SQS after successful upload - private void uploadDeltaAndDeleteMessages(ByteArrayOutputStream deltaStream, String deltaName, Long windowStart, List messages) throws IOException { - try { - // Add end-of-delta entry - long endTimestamp = windowStart + this.deltaWindowSeconds; - this.writeEndOfDelta(deltaStream, endTimestamp); - - // upload - byte[] deltaData = deltaStream.toByteArray(); - String s3Path = this.cloudSync.toCloudPath(deltaName); - - LOGGER.info("SQS Delta Upload - fileName: {}, s3Path: {}, size: {} bytes, messages: {}, window: [{}, {})", - deltaName, s3Path, deltaData.length, messages.size(), windowStart, endTimestamp); - - boolean uploadSucceeded = false; - try (ByteArrayInputStream inputStream = new ByteArrayInputStream(deltaData)) { - this.cloudStorage.upload(inputStream, s3Path); - LOGGER.info("Successfully uploaded delta to S3: {}", s3Path); - uploadSucceeded = true; - - // publish event - this.publishDeltaProducedEvent(deltaName); - this.counterDeltaProduced.increment(); - this.counterEntriesProcessed.increment(messages.size()); - - } catch (Exception uploadEx) { - LOGGER.error("Failed to upload delta to S3: " + uploadEx.getMessage(), uploadEx); - throw new IOException("S3 upload failed", uploadEx); - } - - // CRITICAL: Only delete messages from SQS after successful S3 upload - if (uploadSucceeded && !messages.isEmpty()) { - LOGGER.info("Deleting {} messages from SQS after successful S3 upload", messages.size()); - SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, messages); - } - - // Close the stream - deltaStream.close(); - - } catch (Exception ex) { - LOGGER.error("Error uploading delta: " + ex.getMessage(), ex); - throw new IOException("Delta upload failed", ex); - } - } - - /** - * Writes a dropped request entry to the dropped request stream. - */ - private void writeDroppedRequestEntry(JsonArray droppedRequestArray, SqsParsedMessage parsed) throws IOException { - String messageBody = parsed.getOriginalMessage().body(); - JsonObject messageJson = new JsonObject(messageBody); - droppedRequestArray.add(messageJson); - } - - // Upload a dropped request file to S3 and delete messages from SQS after successful upload - private void uploadDroppedRequestsAndDeleteMessages(JsonArray droppedRequestStream, String droppedRequestName, Long windowStart, List messages) throws IOException { - try { - // upload - byte[] droppedRequestData = droppedRequestStream.encode().getBytes(); - - LOGGER.info("SQS Dropped Requests Upload - fileName: {}, s3Path: {}, size: {} bytes, messages: {}, window: [{}, {})", - droppedRequestName, droppedRequestData.length, messages.size(), windowStart, windowStart + this.deltaWindowSeconds); - - boolean uploadSucceeded = false; - try (ByteArrayInputStream inputStream = new ByteArrayInputStream(droppedRequestData)) { - this.cloudStorageDroppedRequests.upload(inputStream, droppedRequestName); - LOGGER.info("Successfully uploaded dropped requests to S3: {}", droppedRequestName); - uploadSucceeded = true; - - // publish event - this.counterDroppedRequestFilesProduced.increment(); - this.counterDroppedRequestsProcessed.increment(messages.size()); - } catch (Exception uploadEx) { - LOGGER.error("Failed to upload dropped requests to S3: " + uploadEx.getMessage(), uploadEx); - throw new IOException("S3 upload failed", uploadEx); - } - - // CRITICAL: Only delete messages from SQS after successful S3 upload - if (uploadSucceeded && !messages.isEmpty()) { - LOGGER.info("Deleting {} messages from SQS after successful S3 upload", messages.size()); - SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, messages); - } - - // Clear the array - droppedRequestStream.clear(); - - } catch (Exception ex) { - LOGGER.error("Error uploading dropped requests: " + ex.getMessage(), ex); - throw new IOException("Dropped requests upload failed", ex); - } - } - - /** - * Upload a JSON config file to S3 containing the following: - * {"manual_override": "DELAYED_PROCESSING"} - * Manual override file is at the root of the S3 bucket - */ - private void setDelayedProcessingOverride() { - try { - JsonObject config = new JsonObject().put("manual_override", "DELAYED_PROCESSING"); - this.cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), this.manualOverrideS3Path); - } catch (Exception e) { - LOGGER.error("Error setting delayed processing override", e); - } - } - - /** - * Check if there is a manual override set in S3 for DEFAULT or DELAYED_PROCESSING status - * Manual override file is at the root of the S3 bucket - */ - private String getManualOverride() { - try { - InputStream inputStream = this.cloudStorage.download(this.manualOverrideS3Path); - JsonObject configJson = Utils.toJsonObject(inputStream); - return configJson.getString("manual_override", ""); - } catch (Exception e) { - LOGGER.error("Error checking for manual override in S3: " + e.getMessage(), e); - return ""; - } - } - - private void publishDeltaProducedEvent(String newDelta) { - vertx.eventBus().publish(this.eventDeltaProduced, newDelta); - LOGGER.info("Published delta.produced event for: {}", newDelta); - } - - private void checkBufferSize(int dataSize) { - ByteBuffer b = this.buffer; - if (b.capacity() < dataSize) { - int newCapacity = Integer.highestOneBit(dataSize) << 1; - LOGGER.warn("Expanding buffer size: current {}, need {}, new {}", b.capacity(), dataSize, newCapacity); - this.buffer = ByteBuffer.allocate(newCapacity).order(ByteOrder.LITTLE_ENDIAN); - } + private void publishDeltaProducedEvent(String deltaName) { + vertx.eventBus().publish(this.eventDeltaProduced, deltaName); + LOGGER.info("Published delta.produced event for: {}", deltaName); } } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 650126a3..7215514d 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -5,11 +5,11 @@ import com.uid2.shared.optout.OptOutEntry; import com.uid2.shared.optout.OptOutUtils; import com.uid2.optout.Const; +import com.uid2.optout.sqs.SqsMessageOperations; + import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; import java.nio.charset.StandardCharsets; @@ -202,31 +202,15 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic } /** - * Calculate traffic status based on delta files and SQS queue messages. - * - * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, - * and the oldest queue timestamp to anchor the 5-minute queue window. - * - * @param sqsMessages List of SQS messages in the current batch - * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) - */ - public TrafficStatus calculateStatus(List sqsMessages) { - return calculateStatus(sqsMessages, null); - } - - /** - * Calculate traffic status based on delta files, SQS queue messages, and queue attributes. - * - * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, - * and the oldest queue timestamp to anchor the 5-minute queue window. + * Calculate traffic status based on delta files and SQS queue attributes. * - * The invisible message count from queue attributes is added in case of multiple consumers. + * Uses the newest delta file timestamp to anchor the evaluation window, + * and adds the total queue depth from queue attributes. * - * @param sqsMessages List of SQS messages in the current batch - * @param queueAttributes SQS queue attributes including invisible message count (may be null) + * @param queueAttributes SQS queue attributes including message counts * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ - public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes) { + public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueAttributes) { try { // Get list of delta files from S3 (sorted newest to oldest) @@ -241,10 +225,6 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs); - // Find oldest SQS queue message timestamp for queue window - long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); - LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs); - // Define start time of the delta evaluation window // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend // the window to account for any allowlist ranges in the extended portion @@ -259,7 +239,6 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat for (String s3Path : deltaS3Paths) { List timestamps = getTimestampsFromFile(s3Path); - boolean shouldStop = false; for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { @@ -276,35 +255,24 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat if (ts >= deltaWindowStart) { sum++; } - } - - if (shouldStop) { - break; - } - } - - // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] - if (sqsMessages != null && !sqsMessages.isEmpty()) { - int sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); - sum += sqsCount; } - // Add invisible messages from queue attributes (messages being processed by other consumers) - // These represent in-flight work that will soon become delta records - int invisibleMessages = 0; + // Add total queue depth from queue attributes + // This includes visible, invisible (in-flight), and delayed messages + int queueMessages = 0; if (queueAttributes != null) { - invisibleMessages = queueAttributes.getApproximateNumberOfMessagesNotVisible(); - sum += invisibleMessages; - LOGGER.info("Traffic calculation: adding {} invisible SQS messages to sum (queue: {})", - invisibleMessages, queueAttributes); + queueMessages = queueAttributes.getTotalMessages(); + sum += queueMessages; + LOGGER.info("Traffic calculation: adding {} SQS queue messages to sum ({})", + queueMessages, queueAttributes); } // Determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("Traffic calculation complete: sum={} (including {} invisible), baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, invisibleMessages, this.baselineTraffic, this.thresholdMultiplier, status); + LOGGER.info("Traffic calculation complete: sum={} (deltaRecords={}, queueMessages={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, sum - queueMessages, queueMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; @@ -458,68 +426,6 @@ long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowS return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; } - /** - * Find the oldest SQS queue message timestamp - */ - private long findOldestQueueTimestamp(List sqsMessages) throws IOException { - long oldest = System.currentTimeMillis() / 1000; - - if (sqsMessages != null && !sqsMessages.isEmpty()) { - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - if (ts != null && ts < oldest) { - oldest = ts; - } - } - } - - return oldest; - } - - /** - * Extract timestamp from SQS message (from SentTimestamp attribute) - */ - private Long extractTimestampFromMessage(Message msg) { - // Get SentTimestamp attribute (milliseconds) - String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); - if (sentTimestamp != null) { - try { - return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds - } catch (NumberFormatException e) { - LOGGER.debug("Invalid SentTimestamp: {}", sentTimestamp); - } - } - - // Fallback: use current time - return System.currentTimeMillis() / 1000; - } - - /** - * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes - */ - private int countSqsMessages(List sqsMessages, long oldestQueueTs) { - - int count = 0; - long windowEnd = oldestQueueTs + 5 * 60; - - for (Message msg : sqsMessages) { - Long ts = extractTimestampFromMessage(msg); - - if (ts < oldestQueueTs || ts > windowEnd) { - continue; - } - - if (isInAllowlist(ts)) { - continue; - } - count++; - - } - - LOGGER.info("SQS messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); - return count; - } - /** * Check if a timestamp falls within any allowlist range */ diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index f9ba8165..a15c5f89 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -1,5 +1,6 @@ package com.uid2.optout.vertx; +import com.uid2.optout.sqs.SqsParsedMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java b/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java similarity index 99% rename from src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java rename to src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java index 6fc1c865..778f858c 100644 --- a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java +++ b/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java @@ -1,8 +1,9 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; + import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; diff --git a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java similarity index 99% rename from src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java rename to src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java index 810a7a41..5d6810db 100644 --- a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java +++ b/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java @@ -1,7 +1,8 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; import io.vertx.core.json.JsonObject; import org.junit.jupiter.api.Test; + import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; import java.util.*; @@ -269,3 +270,4 @@ public void testParseAndSortMessages_multipleValidMessages() { } } + diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 7b5900f1..de1eef4c 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -74,7 +74,8 @@ public void setup(TestContext context) throws Exception { .put(Const.Config.TrafficCalcConfigPathProp, TRAFFIC_CALC_CONFIG_PATH) .put(Const.Config.ManualOverrideS3PathProp, MANUAL_OVERRIDE_S3_PATH) .put(Const.Config.OptOutS3BucketDroppedRequestsProp, TEST_BUCKET_DROPPED_REQUESTS) - .put(Const.Config.OptOutSqsS3FolderProp, S3_DELTA_PREFIX); + .put(Const.Config.OptOutSqsS3FolderProp, S3_DELTA_PREFIX) + .put(Const.Config.OptOutMaxMessagesPerFileProp, 100); // Low limit for circuit breaker tests // Mock cloud sync to return proper S3 paths when(cloudSync.toCloudPath(anyString())) @@ -385,7 +386,7 @@ public void testDeltaProduceEndpoint_allMessagesTooRecent(TestContext context) { JsonObject result = finalStatus.getJsonObject("result"); context.assertNotNull(result); context.assertEquals("skipped", result.getString("status")); - context.assertEquals("No deltas produced", result.getString("reason")); + context.assertEquals("MESSAGES_TOO_RECENT", result.getString("reason")); // No processing should occur try { @@ -1069,7 +1070,8 @@ public void testManualOverride_delayedProcessing(TestContext context) throws Exc .onComplete(context.asyncAssertSuccess(finalStatus -> { context.assertEquals("completed", finalStatus.getString("state")); JsonObject result = finalStatus.getJsonObject("result"); - context.assertEquals("success", result.getString("status")); + context.assertEquals("skipped", result.getString("status")); + context.assertEquals("MANUAL_OVERRIDE_ACTIVE", result.getString("reason")); // Should not process anything - manual override checked at start context.assertEquals(0, result.getInteger("entries_processed")); diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 99dc9506..e073fd2e 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -5,6 +5,8 @@ import com.uid2.shared.optout.OptOutCollection; import com.uid2.shared.optout.OptOutEntry; import com.uid2.optout.Const; +import com.uid2.optout.sqs.SqsMessageOperations; + import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import java.nio.file.Files; @@ -20,8 +22,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import java.io.ByteArrayInputStream; @@ -1065,31 +1065,6 @@ void testGetCacheStats_emptyCache() throws Exception { // SECTION 8: Helper Methods for Test Data Creation // ============================================================================ - /** - * Create a mock SQS message with specified timestamp - */ - private Message createSqsMessage(long timestampSeconds) { - Map attributes = new HashMap<>(); - attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); - - return Message.builder() - .messageId("test-msg-" + timestampSeconds) - .body("{\"test\": \"data\"}") - .attributes(attributes) - .build(); - } - - /** - * Create a mock SQS message without timestamp - */ - private Message createSqsMessageWithoutTimestamp() { - return Message.builder() - .messageId("test-msg-no-timestamp") - .body("{\"test\": \"data\"}") - .attributes(new HashMap<>()) - .build(); - } - /** * Create delta file bytes with specified timestamps */ @@ -1121,7 +1096,7 @@ void testCalculateStatus_noDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); // Assert - should return DEFAULT when no delta files assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1133,10 +1108,10 @@ void testCalculateStatus_normalTraffic() throws Exception { long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - // Create delta files with timestamps distributed over 48 hours + // Create delta files with timestamps distributed over 24 hours List timestamps = new ArrayList<>(); - // add 499 entries in current window + // add 49 entries in current window for (int i = 0; i < 49; i++) { timestamps.add(t - 23*3600 + i * 60); } @@ -1150,11 +1125,11 @@ void testCalculateStatus_normalTraffic() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + // Act - pass queue attributes with 1 message + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT + // Assert - 49 + 1 = 50 < 5 * 100 = 500, so should be DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @@ -1181,16 +1156,16 @@ void testCalculateStatus_delayedProcessing() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + // Act - 500 delta + 1 queue = 501 >= 500 + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING + // Assert - 500 + 1 >= 5 * 100 = 500, DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test - void testCalculateStatus_noSqsMessages() throws Exception { + void testCalculateStatus_nullQueueAttributes() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1205,7 +1180,7 @@ void testCalculateStatus_noSqsMessages() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - null SQS messages + // Act - null queue attributes OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); // Assert - should still calculate based on delta files, DEFAULT @@ -1213,7 +1188,7 @@ void testCalculateStatus_noSqsMessages() throws Exception { } @Test - void testCalculateStatus_emptySqsMessages() throws Exception { + void testCalculateStatus_emptyQueue() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1228,21 +1203,22 @@ void testCalculateStatus_emptySqsMessages() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + // Act - empty queue + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(0, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @Test - void testCalculateStatus_multipleSqsMessages() throws Exception { - // Setup - create delta files with some entries + void testCalculateStatus_queueMessagesPushOverThreshold() throws Exception { + // Setup - create delta files with entries just under threshold long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; List timestamps = new ArrayList<>(); - // add 470 entries in window + // add 470 entries in window (under threshold of 500) for (int i = 0; i < 470; i++) { timestamps.add(t - 24*3600 + i * 60); } @@ -1256,12 +1232,10 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Add 30 SQS entries in [t, t+5min] - List sqsMessages = new ArrayList<>(); - for (int i = 0; i < 30; i++) { - sqsMessages.add(createSqsMessage(t - i * 10)); - } - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + // Add 30 messages in queue to push over threshold + // 470 + 30 = 500 >= 500 + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(30, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); // Assert - DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1303,11 +1277,11 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - // Assert - should filter out entries in traffic calc config ranges - // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 + // Assert - should filter out entries in allowlist ranges + // Only 300 from window count (not in allowlist ranges) + 1 queue = 301 // 301 < 5*100, so DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @@ -1329,14 +1303,14 @@ void testCalculateStatus_cacheUtilization() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - first call should populate cache - List sqsMessages = Arrays.asList(createSqsMessage(t)); - calculator.calculateStatus(sqsMessages); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + calculator.calculateStatus(queueAttributes); Map stats = calculator.getCacheStats(); int cachedFiles = (Integer) stats.get("cached_files"); // Second call should use cache (no additional S3 download) - calculator.calculateStatus(sqsMessages); + calculator.calculateStatus(queueAttributes); Map stats2 = calculator.getCacheStats(); int cachedFiles2 = (Integer) stats2.get("cached_files"); @@ -1358,7 +1332,7 @@ void testCalculateStatus_s3Exception() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should not throw exception - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1374,37 +1348,13 @@ void testCalculateStatus_deltaFileReadException() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList()); + // Act - empty queue + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } - @Test - void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - SQS message without timestamp (should use current time) - List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); - - // Assert - DEFAULT - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - @Test void testCalculateStatus_multipleDeltaFiles() throws Exception { // Setup - create delta files with some entries @@ -1438,8 +1388,8 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1472,10 +1422,10 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - // Assert - DEFAULT + // Assert - 500 + 1 >= 500, DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @@ -1496,8 +1446,8 @@ void testCalculateStatus_timestampsCached() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages); + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); // Assert assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1511,39 +1461,8 @@ void testCalculateStatus_timestampsCached() throws Exception { // SECTION 10: Tests for calculateStatus() with QueueAttributes // ============================================================================ - /** - * Create a QueueAttributes object for testing - */ - private SqsMessageOperations.QueueAttributes createQueueAttributes(int visible, int invisible, int delayed) { - return new SqsMessageOperations.QueueAttributes(visible, invisible, delayed); - } - @Test - void testCalculateStatus_withQueueAttributes_nullAttributes() throws Exception { - // Setup - create delta files with some entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - null queue attributes should behave same as single-parameter version - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); - - // Assert - DEFAULT (same as without queue attributes) - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_withQueueAttributes_invisibleMessagesAddedToSum() throws Exception { + void testCalculateStatus_withQueueAttributes_allTypesAddedToSum() throws Exception { // Setup - create delta files with entries just under threshold long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1563,22 +1482,22 @@ void testCalculateStatus_withQueueAttributes_invisibleMessagesAddedToSum() throw OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - without invisible messages, should be DEFAULT - List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus statusWithoutInvisible = calculator.calculateStatus(sqsMessages, null); + // Act - with only 1 queue message, should be DEFAULT: 490 + 1 = 491 < 500 + SqsMessageOperations.QueueAttributes queueAttributesSmall = createQueueAttributes(1, 0, 0); + OptOutTrafficCalculator.TrafficStatus statusDefault = calculator.calculateStatus(queueAttributesSmall); - // With invisible messages that push over threshold, should be DELAYED_PROCESSING - // 490 (delta) + 1 (sqs) + 10 (invisible) = 501 >= 500 - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(5, 10, 0); - OptOutTrafficCalculator.TrafficStatus statusWithInvisible = calculator.calculateStatus(sqsMessages, queueAttributes); + // With more queue messages that push over threshold, should be DELAYED_PROCESSING + // 490 (delta) + 5 (visible) + 10 (invisible) = 505 >= 500 + SqsMessageOperations.QueueAttributes queueAttributesLarge = createQueueAttributes(5, 5, 5); + OptOutTrafficCalculator.TrafficStatus statusDelayed = calculator.calculateStatus(queueAttributesLarge); // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, statusWithoutInvisible); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, statusWithInvisible); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, statusDefault); + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, statusDelayed); } @Test - void testCalculateStatus_withQueueAttributes_zeroInvisibleMessages() throws Exception { + void testCalculateStatus_withQueueAttributes_zeroMessages() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1593,17 +1512,16 @@ void testCalculateStatus_withQueueAttributes_zeroInvisibleMessages() throws Exce OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - queue attributes with zero invisible messages - List sqsMessages = Arrays.asList(createSqsMessage(t)); - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(100, 0, 50); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + // Act - queue attributes with zero messages + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(0, 0, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - // Assert - same result as without queue attributes (only invisible is added to sum) + // Assert - DEFAULT (only delta records count) assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @Test - void testCalculateStatus_withQueueAttributes_largeInvisibleCount() throws Exception { + void testCalculateStatus_withQueueAttributes_largeQueueCount() throws Exception { // Setup - create delta files with minimal entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1618,53 +1536,27 @@ void testCalculateStatus_withQueueAttributes_largeInvisibleCount() throws Except OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - large number of invisible messages alone should trigger DELAYED_PROCESSING + // Act - large queue count alone should trigger DELAYED_PROCESSING // threshold = 5 * 100 = 500 - // 1 (delta) + 1 (sqs) + 500 (invisible) = 502 >= 500 - List sqsMessages = Arrays.asList(createSqsMessage(t)); - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(0, 500, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + // 1 (delta) + 500 (total queue) = 501 >= 500 + SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(200, 200, 100); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - // Assert - DELAYED_PROCESSING due to invisible messages + // Assert - DELAYED_PROCESSING due to queue messages assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } - @Test - void testCalculateStatus_withQueueAttributes_delayedMessagesNotAdded() throws Exception { - // Setup - create delta files with entries near threshold - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create 495 entries - List timestamps = new ArrayList<>(); - for (int i = 0; i < 495; i++) { - timestamps.add(t - 23*3600 + i * 60); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - - // Act - delayed messages should NOT be added to sum (only invisible) - // 495 (delta) + 1 (sqs) + 0 (invisible) = 496 < 500 - // If delayed was added: 496 + 1000 = 1496 >= 500 (would fail) - List sqsMessages = Arrays.asList(createSqsMessage(t)); - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(100, 0, 1000); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); - - // Assert - DEFAULT (delayed messages not counted) - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - // ============================================================================ // SECTION 11: Tests for QueueAttributes class // ============================================================================ + /** + * Create a QueueAttributes object for testing + */ + private SqsMessageOperations.QueueAttributes createQueueAttributes(int visible, int invisible, int delayed) { + return new SqsMessageOperations.QueueAttributes(visible, invisible, delayed); + } + @Test void testQueueAttributes_getters() { // Setup diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java index 63f6807c..ccf425b5 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java @@ -3,6 +3,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; + +import com.uid2.optout.sqs.SqsParsedMessage; + import software.amazon.awssdk.services.sqs.model.Message; import java.nio.file.Files; From a2b34396d7fe9a24f2793e841fe84bf38cd7c5f3 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sat, 6 Dec 2025 22:17:23 -0700 Subject: [PATCH 22/52] update comments --- .../uid2/optout/delta/DeltaFileWriter.java | 2 +- .../delta/DeltaManualOverrideService.java | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java b/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java index d31bcccf..092df828 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java +++ b/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java @@ -16,7 +16,7 @@ * * Delta files have the following format: * - Start entry: null hash (32 bytes) + null hash (32 bytes) + timestamp (8 bytes) - * - Opt-out entries: hash (32 bytes) + id (32 bytes) + timestamp (8 bytes) + * - Opt-out entries: hash (32 bytes) + id (32 bytes) + timestamp (7 bytes) + metadata (1 byte) * - End entry: ones hash (32 bytes) + ones hash (32 bytes) + timestamp (8 bytes) * * Each entry is 72 bytes (OptOutConst.EntrySize). diff --git a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java index 02a581bc..d84e2f1e 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java +++ b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java @@ -49,23 +49,23 @@ public boolean isDelayedProcessing() { return DELAYED_PROCESSING_VALUE.equals(getOverrideValue()); } - /** + /** * Set the manual override to DELAYED_PROCESSING. * This will stop delta production until manually cleared. * * @return true if override was set successfully */ - public boolean setDelayedProcessing() { - try { - JsonObject config = new JsonObject().put(OVERRIDE_KEY, DELAYED_PROCESSING_VALUE); - cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), overrideS3Path); - LOGGER.info("Set manual override to DELAYED_PROCESSING at {}", overrideS3Path); - return true; - } catch (Exception e) { - LOGGER.error("Error setting delayed processing override at {}: {}", overrideS3Path, e.getMessage(), e); - return false; - } + public boolean setDelayedProcessing() { + try { + JsonObject config = new JsonObject().put(OVERRIDE_KEY, DELAYED_PROCESSING_VALUE); + cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), overrideS3Path); + LOGGER.info("Set manual override to DELAYED_PROCESSING at {}", overrideS3Path); + return true; + } catch (Exception e) { + LOGGER.error("Error setting delayed processing override at {}: {}", overrideS3Path, e.getMessage(), e); + return false; } + } /** * Get the current manual override value From fa1bd62b9f5d84926c2422b577241cdc60106b1f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 13:44:16 -0700 Subject: [PATCH 23/52] more refactoring --- .../uid2/optout/delta/DeltaFileWriter.java | 5 +- .../delta/DeltaManualOverrideService.java | 6 +- .../delta/DeltaProductionOrchestrator.java | 169 ++++++------------ .../optout/delta/DeltaProductionResult.java | 99 +++++++--- .../uid2/optout/delta/DeltaUploadService.java | 13 +- .../com/uid2/optout/delta/StopReason.java | 1 + .../uid2/optout/sqs/SqsBatchProcessor.java | 5 +- .../uid2/optout/sqs/SqsMessageOperations.java | 14 +- .../com/uid2/optout/sqs/SqsMessageParser.java | 8 +- .../com/uid2/optout/sqs/SqsWindowReader.java | 7 +- .../optout/vertx/OptOutSqsLogProducer.java | 105 ++++------- .../optout/vertx/OptOutTrafficCalculator.java | 8 - .../optout/vertx/OptOutTrafficFilter.java | 3 +- 13 files changed, 197 insertions(+), 246 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java b/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java index 092df828..b9d54bcc 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java +++ b/src/main/java/com/uid2/optout/delta/DeltaFileWriter.java @@ -19,8 +19,7 @@ * - Opt-out entries: hash (32 bytes) + id (32 bytes) + timestamp (7 bytes) + metadata (1 byte) * - End entry: ones hash (32 bytes) + ones hash (32 bytes) + timestamp (8 bytes) * - * Each entry is 72 bytes (OptOutConst.EntrySize). - * This format is consistent with OptOutLogProducer.java. + * Each entry is 72 bytes (OptOutConst.EntrySize) */ public class DeltaFileWriter { private static final Logger LOGGER = LoggerFactory.getLogger(DeltaFileWriter.class); @@ -106,7 +105,7 @@ private void flushToStream(ByteArrayOutputStream stream) throws IOException { private void ensureCapacity(int dataSize) { if (buffer.capacity() < dataSize) { int newCapacity = Integer.highestOneBit(dataSize) << 1; - LOGGER.warn("Expanding buffer size: current {}, need {}, new {}", buffer.capacity(), dataSize, newCapacity); + LOGGER.info("expanding buffer size: current {}, need {}, new {}", buffer.capacity(), dataSize, newCapacity); this.buffer = ByteBuffer.allocate(newCapacity).order(ByteOrder.LITTLE_ENDIAN); } } diff --git a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java index d84e2f1e..fede0297 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java +++ b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java @@ -59,10 +59,10 @@ public boolean setDelayedProcessing() { try { JsonObject config = new JsonObject().put(OVERRIDE_KEY, DELAYED_PROCESSING_VALUE); cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), overrideS3Path); - LOGGER.info("Set manual override to DELAYED_PROCESSING at {}", overrideS3Path); + LOGGER.info("set manual override to DELAYED_PROCESSING: {}", overrideS3Path); return true; } catch (Exception e) { - LOGGER.error("Error setting delayed processing override at {}: {}", overrideS3Path, e.getMessage(), e); + LOGGER.error("error setting manual override: {}", overrideS3Path, e); return false; } } @@ -76,7 +76,7 @@ private String getOverrideValue() { JsonObject configJson = Utils.toJsonObject(inputStream); return configJson.getString(OVERRIDE_KEY, ""); } catch (Exception e) { - LOGGER.debug("No manual override found at {}: {}", overrideS3Path, e.getMessage()); + LOGGER.debug("no manual override found: {}", overrideS3Path); return ""; } } diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index dc86724d..18d1d172 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -30,11 +30,10 @@ *
  • Reading messages from SQS in 5-minute windows
    • *
  • Filtering denylisted messages
    • *
  • Checking circuit breakers (manual override, traffic calculator)
    • - *
  • Constructingdelta files and dropped request files
    • + *
  • Constructing delta files and dropped request files
    • *
  • Uploading to S3 and deleting processed messages
    • * * - *

    The orchestrator is stateless and thread-safe.

    */ public class DeltaProductionOrchestrator { private static final Logger LOGGER = LoggerFactory.getLogger(DeltaProductionOrchestrator.class); @@ -89,87 +88,84 @@ public DeltaProductionOrchestrator( /** * Produces delta files from SQS queue in batched 5-minute windows. * - *

    Continues until queue is empty, messages are too recent, or a circuit breaker triggers.

    + * Continues until queue is empty, messages are too recent, circuit breaker is triggered, or job timeout is reached. * * @param onDeltaProduced Called with delta filename after each successful delta upload (for event publishing) - * @return DeltaProduceResult with production statistics + * @return DeltaProductionResult with production statistics * @throws IOException if delta production fails */ public DeltaProductionResult produceBatchedDeltas(Consumer onDeltaProduced) throws IOException { - // Check for manual override at the start + + // check for manual override if (manualOverrideService.isDelayedProcessing()) { - LOGGER.info("Manual override set to DELAYED_PROCESSING, skipping production"); - return DeltaProductionResult.empty(StopReason.MANUAL_OVERRIDE_ACTIVE); + LOGGER.info("manual override set to DELAYED_PROCESSING, skipping production"); + return DeltaProductionResult.builder().stopReason(StopReason.MANUAL_OVERRIDE_ACTIVE).build(); } - ProductionStats stats = new ProductionStats(); + DeltaProductionResult.Builder result = DeltaProductionResult.builder(); long jobStartTime = OptOutUtils.nowEpochSeconds(); - LOGGER.info("Starting delta production from SQS queue (replicaId: {}, deltaWindowSeconds: {}, jobTimeoutSeconds: {})", + LOGGER.info("starting delta production from SQS queue (replicaId: {}, deltaWindowSeconds: {}, jobTimeoutSeconds: {})", this.replicaId, this.deltaWindowSeconds, this.jobTimeoutSeconds); - // Read and process windows until done + // read and process windows until done while (!isJobTimedOut(jobStartTime)) { - // Read one complete 5-minute window + // read one complete 5-minute window SqsWindowReader.WindowReadResult windowResult = windowReader.readWindow(); - // If no messages, we're done (queue empty or messages too recent) + // if no messages, we're done (queue empty or messages too recent) if (windowResult.isEmpty()) { - stats.stopReason = windowResult.getStopReason(); - LOGGER.info("Delta production complete - no more eligible messages (reason: {})", stats.stopReason); + result.stopReason(windowResult.getStopReason()); + LOGGER.info("delta production complete - no more eligible messages (reason: {})", windowResult.getStopReason().name()); break; } - // Process this window - WindowProcessingResult windowProcessingResult = processWindow(windowResult, onDeltaProduced); + // process this window + boolean isDelayedProcessing = processWindow(windowResult, result, onDeltaProduced); - // Check if we should stop (circuit breaker triggered) - if (windowProcessingResult.shouldStop) { - stats.merge(windowProcessingResult); - stats.stopReason = StopReason.CIRCUIT_BREAKER_TRIGGERED; - return stats.toResult(); + // circuit breaker triggered + if (isDelayedProcessing) { + result.stopReason(StopReason.CIRCUIT_BREAKER_TRIGGERED); + return result.build(); } - - stats.merge(windowProcessingResult); - - LOGGER.info("Processed window [{}, {}]: {} entries, {} dropped requests", - windowResult.getWindowStart(), - windowResult.getWindowStart() + this.deltaWindowSeconds, - windowProcessingResult.entriesProcessed, - windowProcessingResult.droppedRequestsProcessed); } - long totalDuration = OptOutUtils.nowEpochSeconds() - jobStartTime; - LOGGER.info("Delta production complete: took {}s, produced {} deltas, processed {} entries, " + - "produced {} dropped request files, processed {} dropped requests, stop reason: {}", - totalDuration, stats.deltasProduced, stats.entriesProcessed, - stats.droppedRequestFilesProduced, stats.droppedRequestsProcessed, stats.stopReason); - - return stats.toResult(); + return result.build(); } /** * Processes a single 5-minute window of messages. + * + * @param windowResult The window data to process + * @param result The builder to accumulate statistics into + * @param onDeltaProduced Callback for when a delta is produced + * @return true if the circuit breaker triggered */ - private WindowProcessingResult processWindow(SqsWindowReader.WindowReadResult windowResult, - Consumer onDeltaProduced) throws IOException { - WindowProcessingResult result = new WindowProcessingResult(); - + private boolean processWindow(SqsWindowReader.WindowReadResult windowResult, + DeltaProductionResult.Builder result, + Consumer onDeltaProduced) throws IOException { long windowStart = windowResult.getWindowStart(); List messages = windowResult.getMessages(); - // Create buffers + // check for manual override + if (manualOverrideService.isDelayedProcessing()) { + LOGGER.info("manual override set to DELAYED_PROCESSING, stopping production"); + return true; + } + + // create buffers for current window ByteArrayOutputStream deltaStream = new ByteArrayOutputStream(); JsonArray droppedRequestStream = new JsonArray(); + // get file names for current window String deltaName = OptOutUtils.newDeltaFileName(this.replicaId); String droppedRequestName = generateDroppedRequestFileName(); - // Write start of delta + // write start of delta deltaFileWriter.writeStartOfDelta(deltaStream, windowStart); - // Separate messages into delta entries and dropped requests + // separate messages into delta entries and dropped requests List deltaMessages = new ArrayList<>(); List droppedMessages = new ArrayList<>(); @@ -177,81 +173,63 @@ private WindowProcessingResult processWindow(SqsWindowReader.WindowReadResult wi if (trafficFilter.isDenylisted(msg)) { writeDroppedRequestEntry(droppedRequestStream, msg); droppedMessages.add(msg.getOriginalMessage()); - result.droppedRequestsProcessed++; } else { deltaFileWriter.writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp()); deltaMessages.add(msg.getOriginalMessage()); - result.entriesProcessed++; } } - // Check for manual override - if (manualOverrideService.isDelayedProcessing()) { - LOGGER.info("Manual override set to DELAYED_PROCESSING, stopping production"); - result.shouldStop = true; - return result; - } - - // Check traffic calculator + // check traffic calculator SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(queueAttributes); if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { - LOGGER.error("OptOut Delta Production has hit DELAYED_PROCESSING status, stopping production"); + LOGGER.error("optout delta production has hit DELAYED_PROCESSING status, stopping production and setting manual override"); manualOverrideService.setDelayedProcessing(); - result.shouldStop = true; - return result; + return true; } - // Upload delta file if there are non-denylisted messages + // upload delta file if there are non-denylisted messages if (!deltaMessages.isEmpty()) { uploadDelta(deltaStream, deltaName, windowStart, deltaMessages, onDeltaProduced); - result.deltasProduced++; + result.incrementDeltasProduced(); + result.incrementEntriesProcessed(deltaMessages.size()); } - // Upload dropped request file if there are denylisted messages + // upload dropped request file if there are denylisted messages if (!droppedMessages.isEmpty() && droppedRequestUploadService != null) { uploadDroppedRequests(droppedRequestStream, droppedRequestName, windowStart, droppedMessages); - result.droppedRequestFilesProduced++; + result.incrementDroppedRequestFilesProduced(); + result.incrementDroppedRequestsProcessed(droppedMessages.size()); } - deltaStream.close(); - return result; + LOGGER.info("processed window [{}, {}]: {} entries, {} dropped requests", + windowStart, windowStart + this.deltaWindowSeconds, + deltaMessages.size(), droppedMessages.size()); + + return false; } - /** - * Uploads a delta file to S3 and deletes processed messages from SQS. - */ private void uploadDelta(ByteArrayOutputStream deltaStream, String deltaName, long windowStart, List messages, Consumer onDeltaProduced) throws IOException { - // Add end-of-delta entry + // add end-of-delta entry long endTimestamp = windowStart + this.deltaWindowSeconds; deltaFileWriter.writeEndOfDelta(deltaStream, endTimestamp); byte[] deltaData = deltaStream.toByteArray(); String s3Path = this.cloudSync.toCloudPath(deltaName); - LOGGER.info("SQS Delta Upload - fileName: {}, s3Path: {}, size: {} bytes, messages: {}, window: [{}, {})", - deltaName, s3Path, deltaData.length, messages.size(), windowStart, endTimestamp); - deltaUploadService.uploadAndDeleteMessages(deltaData, s3Path, messages, (count) -> { metrics.recordDeltaProduced(count); onDeltaProduced.accept(deltaName); }); } - /** - * Uploads dropped requests to S3 and deletes processed messages from SQS. - */ private void uploadDroppedRequests(JsonArray droppedRequestStream, String droppedRequestName, long windowStart, List messages) throws IOException { byte[] droppedRequestData = droppedRequestStream.encode().getBytes(); - LOGGER.info("SQS Dropped Requests Upload - fileName: {}, size: {} bytes, messages: {}, window: [{}, {})", - droppedRequestName, droppedRequestData.length, messages.size(), - windowStart, windowStart + this.deltaWindowSeconds); - droppedRequestUploadService.uploadAndDeleteMessages(droppedRequestData, droppedRequestName, messages, metrics::recordDroppedRequestsProduced); } @@ -283,50 +261,15 @@ private boolean isJobTimedOut(long jobStartTime) { long elapsedTime = OptOutUtils.nowEpochSeconds() - jobStartTime; if (elapsedTime > 3600) { // 1 hour - log warning - LOGGER.error("Delta production job has been running for {} seconds", elapsedTime); + LOGGER.error("delta production job has been running for {} seconds", elapsedTime); } if (elapsedTime > this.jobTimeoutSeconds) { - LOGGER.error("Delta production job has been running for {} seconds (exceeds timeout of {}s)", + LOGGER.error("delta production job has been running for {} seconds (exceeds timeout of {}s)", elapsedTime, this.jobTimeoutSeconds); return true; } return false; } - - /** - * Mutable class for tracking production statistics during a job. - */ - private static class ProductionStats { - int deltasProduced = 0; - int entriesProcessed = 0; - int droppedRequestFilesProduced = 0; - int droppedRequestsProcessed = 0; - StopReason stopReason = StopReason.NONE; - - ProductionStats merge(WindowProcessingResult windowResult) { - this.deltasProduced += windowResult.deltasProduced; - this.entriesProcessed += windowResult.entriesProcessed; - this.droppedRequestFilesProduced += windowResult.droppedRequestFilesProduced; - this.droppedRequestsProcessed += windowResult.droppedRequestsProcessed; - return this; - } - - DeltaProductionResult toResult() { - return new DeltaProductionResult(deltasProduced, entriesProcessed, - droppedRequestFilesProduced, droppedRequestsProcessed, stopReason); - } - } - - /** - * Result of processing a single window. - */ - private static class WindowProcessingResult { - int deltasProduced = 0; - int entriesProcessed = 0; - int droppedRequestFilesProduced = 0; - int droppedRequestsProcessed = 0; - boolean shouldStop = false; - } } diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java index dfcf1f71..d88ef128 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java @@ -3,10 +3,13 @@ import io.vertx.core.json.JsonObject; /** - * Data class containing statistics from delta production. + * Immutable result containing statistics from a delta production job. * - * This class holds the counts and provides JSON encoding methods. - * API response status is determined by the caller based on these statistics. + *

    This class holds production counts and the stop reason, with methods for JSON serialization. + * Use {@link Builder} to accumulate statistics during production, then call {@link Builder#build()} + * to create the immutable result.

    + * + *

    Note: Job duration is tracked by {@link DeltaProductionJobStatus}, not this class.

    */ public class DeltaProductionResult { private final int deltasProduced; @@ -15,7 +18,10 @@ public class DeltaProductionResult { private final int droppedRequestsProcessed; private final StopReason stopReason; - public DeltaProductionResult(int deltasProduced, int entriesProcessed, + /** + * Private constructor. Use {@link #builder()} to create instances. + */ + private DeltaProductionResult(int deltasProduced, int entriesProcessed, int droppedRequestFilesProduced, int droppedRequestsProcessed, StopReason stopReason) { this.deltasProduced = deltasProduced; @@ -26,12 +32,14 @@ public DeltaProductionResult(int deltasProduced, int entriesProcessed, } /** - * Factory method for an empty result (no production occurred). + * Creates a new Builder for accumulating production statistics. */ - public static DeltaProductionResult empty(StopReason stopReason) { - return new DeltaProductionResult(0, 0, 0, 0, stopReason); + public static Builder builder() { + return new Builder(); } + // ==================== Getters ==================== + public int getDeltasProduced() { return deltasProduced; } @@ -40,10 +48,6 @@ public int getEntriesProcessed() { return entriesProcessed; } - public StopReason getStopReason() { - return stopReason; - } - public int getDroppedRequestFilesProduced() { return droppedRequestFilesProduced; } @@ -52,9 +56,12 @@ public int getDroppedRequestsProcessed() { return droppedRequestsProcessed; } - /** - * Convert to JSON with just the production counts. - */ + public StopReason getStopReason() { + return stopReason; + } + + // ==================== JSON Serialization ==================== + public JsonObject toJson() { return new JsonObject() .put("deltas_produced", deltasProduced) @@ -64,17 +71,69 @@ public JsonObject toJson() { .put("stop_reason", stopReason.name()); } - /** - * Convert to JSON with status and counts. - */ public JsonObject toJsonWithStatus(String status) { return toJson().put("status", status); } + @Override + public String toString() { + return String.format( + "DeltaProductionResult{deltasProduced=%d, entriesProcessed=%d, " + + "droppedRequestFilesProduced=%d, droppedRequestsProcessed=%d, stopReason=%s}", + deltasProduced, entriesProcessed, droppedRequestFilesProduced, + droppedRequestsProcessed, stopReason); + } + + // ==================== Builder ==================== + /** - * Convert to JSON with status, reason/error, and counts. + * Mutable builder for accumulating production statistics. + * + *

    Use this builder to track stats during delta production jobs, + * then call {@link #build()} to create the immutable result.

    */ - public JsonObject toJsonWithStatus(String status, String reasonKey, String reasonValue) { - return toJsonWithStatus(status).put(reasonKey, reasonValue); + public static class Builder { + private int deltasProduced; + private int entriesProcessed; + private int droppedRequestFilesProduced; + private int droppedRequestsProcessed; + private StopReason stopReason = StopReason.NONE; + + public Builder incrementDeltasProduced() { + deltasProduced++; + return this; + } + + public Builder incrementEntriesProcessed(int count) { + entriesProcessed += count; + return this; + } + + public Builder incrementDroppedRequestFilesProduced() { + droppedRequestFilesProduced++; + return this; + } + + public Builder incrementDroppedRequestsProcessed(int count) { + droppedRequestsProcessed += count; + return this; + } + + public Builder stopReason(StopReason reason) { + this.stopReason = reason; + return this; + } + + /** + * Builds the DeltaProductionResult with the accumulated statistics. + */ + public DeltaProductionResult build() { + return new DeltaProductionResult( + deltasProduced, + entriesProcessed, + droppedRequestFilesProduced, + droppedRequestsProcessed, + stopReason); + } } } diff --git a/src/main/java/com/uid2/optout/delta/DeltaUploadService.java b/src/main/java/com/uid2/optout/delta/DeltaUploadService.java index a165929c..6929e333 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaUploadService.java +++ b/src/main/java/com/uid2/optout/delta/DeltaUploadService.java @@ -25,7 +25,7 @@ public class DeltaUploadService { private final String queueUrl; /** - * Callback interface for metrics updates after successful upload. + * Callback interface for after successful upload. */ @FunctionalInterface public interface UploadSuccessCallback { @@ -63,25 +63,20 @@ public DeltaUploadService(ICloudStorage cloudStorage, SqsClient sqsClient, Strin * @throws IOException if upload fails */ public void uploadAndDeleteMessages(byte[] data, String s3Path, List messages, UploadSuccessCallback onSuccess) throws IOException { - LOGGER.info("Uploading to S3: path={}, size={} bytes, messages={}", s3Path, data.length, messages.size()); + LOGGER.info("uploading to s3: path={}, size={} bytes, messages={}", s3Path, data.length, messages.size()); try (ByteArrayInputStream inputStream = new ByteArrayInputStream(data)) { cloudStorage.upload(inputStream, s3Path); - LOGGER.info("Successfully uploaded to S3: {}", s3Path); - // Invoke callback for metrics/events if (onSuccess != null) { onSuccess.onSuccess(messages.size()); } - } catch (Exception e) { - LOGGER.error("Failed to upload to S3: path={}, error={}", s3Path, e.getMessage(), e); - throw new IOException("S3 upload failed: " + s3Path, e); + LOGGER.error("failed to upload to s3: path={}", s3Path, e); + throw new IOException("s3 upload failed: " + s3Path, e); } - // CRITICAL: Only delete messages after successful S3 upload if (!messages.isEmpty()) { - LOGGER.info("Deleting {} messages from SQS after successful S3 upload", messages.size()); SqsMessageOperations.deleteMessagesFromSqs(sqsClient, queueUrl, messages); } } diff --git a/src/main/java/com/uid2/optout/delta/StopReason.java b/src/main/java/com/uid2/optout/delta/StopReason.java index 40239e16..0f61d7d5 100644 --- a/src/main/java/com/uid2/optout/delta/StopReason.java +++ b/src/main/java/com/uid2/optout/delta/StopReason.java @@ -5,6 +5,7 @@ * Used across all layers (batch, window, orchestrator) for consistent stop reason tracking. */ public enum StopReason { + /** * Processing completed normally with work done, or still in progress. */ diff --git a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java index 14eccf76..5db15890 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java +++ b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java @@ -82,15 +82,14 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN if (parsedBatch.size() < messageBatch.size()) { List invalidMessages = identifyInvalidMessages(messageBatch, parsedBatch); if (!invalidMessages.isEmpty()) { - LOGGER.error("Found {} invalid messages in batch {} (failed parsing). Deleting from queue.", - invalidMessages.size(), batchNumber); + LOGGER.error("found {} invalid messages in batch {}, deleting", invalidMessages.size(), batchNumber); SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages); } } // No valid messages after deleting corrupt ones, continue reading if (parsedBatch.isEmpty()) { - LOGGER.error("No valid messages in batch {} (all failed parsing)", batchNumber); + LOGGER.error("no valid messages in batch {}", batchNumber); return BatchProcessingResult.corruptMessagesDeleted(); } diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java index 87d13ace..43607d08 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java @@ -86,11 +86,11 @@ public static QueueAttributes getQueueAttributes(SqsClient sqsClient, String que int delayed = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED), 0); QueueAttributes queueAttributes = new QueueAttributes(visible, invisible, delayed); - LOGGER.debug("Queue attributes: {}", queueAttributes); + LOGGER.debug("queue attributes: {}", queueAttributes); return queueAttributes; } catch (Exception e) { - LOGGER.error("Error getting queue attributes from SQS", e); + LOGGER.error("error getting queue attributes", e); return null; } } @@ -132,11 +132,11 @@ public static List receiveMessagesFromSqs( ReceiveMessageResponse response = sqsClient.receiveMessage(receiveRequest); - LOGGER.debug("Received {} messages from SQS", response.messages().size()); + LOGGER.debug("received {} messages", response.messages().size()); return response.messages(); } catch (Exception e) { - LOGGER.error("Error receiving messages from SQS", e); + LOGGER.error("error receiving messages", e); return new ArrayList<>(); } } @@ -174,7 +174,7 @@ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, L DeleteMessageBatchResponse deleteResponse = sqsClient.deleteMessageBatch(deleteRequest); if (!deleteResponse.failed().isEmpty()) { - LOGGER.error("Failed to delete {} messages from SQS", deleteResponse.failed().size()); + LOGGER.error("failed to delete {} messages", deleteResponse.failed().size()); } else { totalDeleted += entries.size(); } @@ -183,10 +183,10 @@ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, L } } - LOGGER.info("Deleted {} messages from SQS", totalDeleted); + LOGGER.debug("deleted {} messages", totalDeleted); } catch (Exception e) { - LOGGER.error("Error deleting messages from SQS", e); + LOGGER.error("error deleting messages", e); } } } diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java index 3d4c534b..1bcf6751 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java @@ -40,7 +40,7 @@ public static List parseAndSortMessages(List messages String phone = body.getString("phone"); if (identityHash == null || advertisingId == null) { - LOGGER.error("Invalid message format, skipping: {}", message.body()); + LOGGER.error("invalid message format: {}", message.body()); continue; } @@ -48,13 +48,13 @@ public static List parseAndSortMessages(List messages byte[] idBytes = OptOutUtils.base64StringTobyteArray(advertisingId); if (hashBytes == null || idBytes == null) { - LOGGER.error("Invalid base64 encoding, skipping message"); + LOGGER.error("invalid base64 encoding"); continue; } parsedMessages.add(new SqsParsedMessage(message, hashBytes, idBytes, timestampSeconds, email, phone, clientIp, traceId)); } catch (Exception e) { - LOGGER.error("Error parsing SQS message", e); + LOGGER.error("error parsing message", e); } } @@ -73,7 +73,7 @@ public static List parseAndSortMessages(List messages private static long extractTimestamp(Message message) { String sentTimestampStr = message.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); if (sentTimestampStr == null) { - LOGGER.warn("Message missing SentTimestamp attribute, using current time"); + LOGGER.warn("message missing SentTimestamp, using current time"); return OptOutUtils.nowEpochSeconds(); } return Long.parseLong(sentTimestampStr) / 1000; // ms to seconds diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index 444b23eb..ff41ab27 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -34,8 +34,8 @@ public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerP this.deltaWindowSeconds = deltaWindowSeconds; this.maxMessagesPerWindow = maxMessagesPerWindow; this.batchProcessor = new SqsBatchProcessor(sqsClient, queueUrl, deltaWindowSeconds); - LOGGER.info("SqsWindowReader initialized with: maxMessagesPerWindow: {}, maxMessagesPerPoll: {}, visibilityTimeout: {}, deltaWindowSeconds: {}", - maxMessagesPerWindow, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds); + LOGGER.info("initialized: maxMessagesPerWindow={}, maxMessagesPerPoll={}, visibilityTimeout={}, deltaWindowSeconds={}", + maxMessagesPerWindow, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds); } /** @@ -91,8 +91,7 @@ public WindowReadResult readWindow() { while (true) { if (windowMessages.size() >= maxMessagesPerWindow) { - LOGGER.warn("Message limit exceeded: {} messages >= limit {}. Closing window.", - windowMessages.size(), maxMessagesPerWindow); + LOGGER.warn("message limit exceeded: {} messages >= limit {}", windowMessages.size(), maxMessagesPerWindow); return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart); } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 5f882028..cfc66af1 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -101,10 +101,10 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I // Initialize SQS client String queueUrl = jsonConfig.getString(Const.Config.OptOutSqsQueueUrlProp); if (queueUrl == null || queueUrl.isEmpty()) { - throw new IOException("SQS queue URL not configured"); + throw new IOException("sqs queue url not configured"); } this.sqsClient = sqsClient != null ? sqsClient : SqsClient.builder().build(); - LOGGER.info("SQS client initialized for queue: {}", queueUrl); + LOGGER.info("sqs client initialized for queue: {}", queueUrl); // HTTP server configuration this.listenPort = Const.Port.ServicePortForOptOut + Utils.getPortOffset() + 1; @@ -153,13 +153,13 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I new DeltaProductionMetrics() ); - LOGGER.info("OptOutSqsLogProducer initialized with maxMessagesPerFile: {}, maxMessagesPerPoll: {}, visibilityTimeout: {}, deltaWindowSeconds: {}, jobTimeoutSeconds: {}", + LOGGER.info("initialized with maxMessagesPerFile={}, maxMessagesPerPoll={}, visibilityTimeout={}, deltaWindowSeconds={}, jobTimeoutSeconds={}", maxMessagesPerFile, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds, jobTimeoutSeconds); } @Override public void start(Promise startPromise) { - LOGGER.info("Attempting to start SQS Log Producer HTTP server on port: {}", listenPort); + LOGGER.info("starting http server on port {}", listenPort); try { vertx.createHttpServer() @@ -167,17 +167,17 @@ public void start(Promise startPromise) { .listen(listenPort, result -> { if (result.succeeded()) { this.healthComponent.setHealthStatus(true); - LOGGER.info("SQS Log Producer HTTP server started on port: {}", listenPort); + LOGGER.info("http server started on port {}", listenPort); startPromise.complete(); } else { - LOGGER.error("Failed to start SQS Log Producer HTTP server", result.cause()); + LOGGER.error("failed to start http server", result.cause()); this.healthComponent.setHealthStatus(false, result.cause().getMessage()); startPromise.fail(result.cause()); } }); } catch (Exception e) { - LOGGER.error("Failed to start SQS Log Producer HTTP server", e); + LOGGER.error("failed to start http server", e); this.healthComponent.setHealthStatus(false, e.getMessage()); startPromise.fail(e); } @@ -185,20 +185,19 @@ public void start(Promise startPromise) { @Override public void stop(Promise stopPromise) { - LOGGER.info("Stopping SQS Log Producer..."); + LOGGER.info("stopping"); this.shutdownInProgress = true; if (this.sqsClient != null) { try { this.sqsClient.close(); - LOGGER.info("SQS client closed"); } catch (Exception e) { - LOGGER.error("Error closing SQS client", e); + LOGGER.error("error closing sqs client", e); } } stopPromise.complete(); - LOGGER.info("SQS Log Producer stopped"); + LOGGER.info("stopped"); } private Router createRouter() { @@ -226,7 +225,7 @@ private void handleDeltaProduceStatus(RoutingContext routingContext) { DeltaProductionJobStatus job = currentJob.get(); if (job == null) { - sendIdle(resp, "No job running on this pod"); + sendIdle(resp, "no job running on this pod"); return; } @@ -247,12 +246,12 @@ private void handleDeltaProduceStatus(RoutingContext routingContext) { private void handleDeltaProduceStart(RoutingContext routingContext) { HttpServerResponse resp = routingContext.response(); - LOGGER.info("Delta production job requested via /deltaproduce endpoint"); + LOGGER.info("delta production job requested"); try { this.trafficFilter.reloadTrafficFilterConfig(); } catch (MalformedTrafficFilterConfigException e) { - LOGGER.error("Error reloading traffic filter config: " + e.getMessage(), e); + LOGGER.error("error reloading traffic filter config", e); sendError(resp, e); return; } @@ -260,7 +259,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { try { this.trafficCalculator.reloadTrafficCalcConfig(); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.error("Error reloading traffic calculator config: " + e.getMessage(), e); + LOGGER.error("error reloading traffic calculator config", e); sendError(resp, e); return; } @@ -270,29 +269,27 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { // If there's an existing job, check if it's still running if (existingJob != null) { if (existingJob.getState() == DeltaProductionJobStatus.JobState.RUNNING) { - // Cannot replace a running job - 409 Conflict - LOGGER.warn("Delta production job already running on this pod"); - sendConflict(resp, "A delta production job is already running on this pod"); + LOGGER.warn("job already running"); + sendConflict(resp, "job already running on this pod"); return; } - LOGGER.info("Auto-clearing previous {} job to start new one", existingJob.getState()); + LOGGER.info("clearing previous {} job", existingJob.getState()); } DeltaProductionJobStatus newJob = new DeltaProductionJobStatus(); // Try to set the new job if (!currentJob.compareAndSet(existingJob, newJob)) { - sendConflict(resp, "Job state changed, please retry"); + sendConflict(resp, "job state changed, please retry"); return; } - // Start the job asynchronously - LOGGER.info("New delta production job initialized"); + LOGGER.info("starting new job"); this.startDeltaProductionJob(newJob); // Return immediately with 202 Accepted - sendAccepted(resp, "Delta production job started on this pod"); + sendAccepted(resp, "job started"); } /** @@ -300,18 +297,12 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { * The job runs on a worker thread and updates the DeltaProduceJobStatus when complete */ private void startDeltaProductionJob(DeltaProductionJobStatus job) { - vertx.executeBlocking(() -> { - LOGGER.info("Delta production job starting on worker thread"); - return produceDeltasBlocking(); - }).onComplete(ar -> { + vertx.executeBlocking(() -> produceDeltasBlocking()).onComplete(ar -> { if (ar.succeeded()) { - JsonObject result = ar.result(); - job.complete(result); - LOGGER.info("Delta production job succeeded: {}", result.encode()); + job.complete(ar.result()); } else { - String errorMsg = ar.cause().getMessage(); - job.fail(errorMsg); - LOGGER.error("Delta production job failed: {}", errorMsg, ar.cause()); + job.fail(ar.cause().getMessage()); + LOGGER.error("job failed", ar.cause()); } }); } @@ -326,53 +317,25 @@ private void startDeltaProductionJob(DeltaProductionJobStatus job) { */ private JsonObject produceDeltasBlocking() throws Exception { if (this.shutdownInProgress) { - throw new Exception("Producer is shutting down"); + throw new Exception("producer is shutting down"); } - LOGGER.info("Starting delta production from SQS queue"); - // Process messages until queue is empty or messages are too recent - DeltaProductionResult deltaResult = this.produceBatchedDeltas(); + DeltaProductionResult result = orchestrator.produceBatchedDeltas(this::publishDeltaProducedEvent); - StopReason stopReason = deltaResult.getStopReason(); - boolean producedDeltas = deltaResult.getDeltasProduced() > 0; - boolean producedDroppedRequests = deltaResult.getDroppedRequestFilesProduced() > 0; + StopReason stopReason = result.getStopReason(); + boolean producedWork = result.getDeltasProduced() > 0 || result.getDroppedRequestFilesProduced() > 0; + boolean halted = stopReason == StopReason.CIRCUIT_BREAKER_TRIGGERED || stopReason == StopReason.MANUAL_OVERRIDE_ACTIVE; - // Determine status based on results: - // "success" = produced work OR completed normally - // "skipped" = stopped early due to abnormal conditions (circuit breaker, override, too recent) - boolean isSkipped = stopReason == StopReason.CIRCUIT_BREAKER_TRIGGERED - || stopReason == StopReason.MESSAGES_TOO_RECENT - || stopReason == StopReason.MANUAL_OVERRIDE_ACTIVE; + String status = halted ? "halted" : (producedWork ? "success" : "skipped"); - boolean isSuccess = !isSkipped && (producedDeltas - || producedDroppedRequests - || stopReason == StopReason.QUEUE_EMPTY - || stopReason == StopReason.NONE); - - if (isSuccess) { - LOGGER.info("Delta production complete: {} deltas, {} entries, dropped request files: {}, dropped requests: {}, stop reason: {}", - deltaResult.getDeltasProduced(), deltaResult.getEntriesProcessed(), deltaResult.getDroppedRequestFilesProduced(), deltaResult.getDroppedRequestsProcessed(), stopReason); - return deltaResult.toJsonWithStatus("success"); - } else { - LOGGER.info("Delta production skipped: {}, {} entries processed, dropped request files: {}, dropped requests: {}", - stopReason, deltaResult.getEntriesProcessed(), deltaResult.getDroppedRequestFilesProduced(), deltaResult.getDroppedRequestsProcessed()); - return deltaResult.toJsonWithStatus("skipped", "reason", stopReason.name()); - } - } - + LOGGER.info("delta production {}: {} deltas, {} entries, {} dropped files, {} dropped requests, reason={}", + status, result.getDeltasProduced(), result.getEntriesProcessed(), + result.getDroppedRequestFilesProduced(), result.getDroppedRequestsProcessed(), stopReason); - /** - * Delegates to the orchestrator to produce delta files. - * - * @return DeltaProduceResult with counts and stop reason - * @throws IOException if delta production fails - */ - private DeltaProductionResult produceBatchedDeltas() throws IOException { - return orchestrator.produceBatchedDeltas(this::publishDeltaProducedEvent); + return result.toJsonWithStatus(status); } private void publishDeltaProducedEvent(String deltaName) { vertx.eventBus().publish(this.eventDeltaProduced, deltaName); - LOGGER.info("Published delta.produced event for: {}", deltaName); } } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 7215514d..1b1d4d8e 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -487,14 +487,6 @@ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DEFAULT; } - - /** - * Get the traffic threshold (baseline × multiplier). - * Used for early termination - */ - public int getThreshold() { - return this.baselineTraffic * this.thresholdMultiplier; - } /** * Get cache statistics for monitoring diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index a15c5f89..dc06a238 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -1,9 +1,10 @@ package com.uid2.optout.vertx; -import com.uid2.optout.sqs.SqsParsedMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.uid2.optout.sqs.SqsParsedMessage; + import java.util.ArrayList; import java.util.List; import java.util.Collections; From 6dbccecb4341e0b39be2967e79c86706f0a59cfd Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 13:48:20 -0700 Subject: [PATCH 24/52] update logging --- .../delta/DeltaManualOverrideService.java | 2 +- .../uid2/optout/sqs/SqsMessageOperations.java | 8 +-- .../optout/vertx/OptOutTrafficCalculator.java | 72 +++++++++---------- .../optout/vertx/OptOutTrafficFilter.java | 36 +++++----- 4 files changed, 58 insertions(+), 60 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java index fede0297..f6f038ea 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java +++ b/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java @@ -76,7 +76,7 @@ private String getOverrideValue() { JsonObject configJson = Utils.toJsonObject(inputStream); return configJson.getString(OVERRIDE_KEY, ""); } catch (Exception e) { - LOGGER.debug("no manual override found: {}", overrideS3Path); + LOGGER.error("no manual override found: {}", overrideS3Path); return ""; } } diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java index 43607d08..5db890d4 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java @@ -86,11 +86,11 @@ public static QueueAttributes getQueueAttributes(SqsClient sqsClient, String que int delayed = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED), 0); QueueAttributes queueAttributes = new QueueAttributes(visible, invisible, delayed); - LOGGER.debug("queue attributes: {}", queueAttributes); + LOGGER.info("queue attributes: {}", queueAttributes); return queueAttributes; } catch (Exception e) { - LOGGER.error("error getting queue attributes", e); + LOGGER.info("error getting queue attributes", e); return null; } } @@ -132,7 +132,7 @@ public static List receiveMessagesFromSqs( ReceiveMessageResponse response = sqsClient.receiveMessage(receiveRequest); - LOGGER.debug("received {} messages", response.messages().size()); + LOGGER.info("received {} messages", response.messages().size()); return response.messages(); } catch (Exception e) { @@ -183,7 +183,7 @@ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, L } } - LOGGER.debug("deleted {} messages", totalDeleted); + LOGGER.info("deleted {} messages", totalDeleted); } catch (Exception e) { LOGGER.error("error deleting messages", e); diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 1b1d4d8e..267c798f 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -87,7 +87,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, this.trafficCalcConfigPath = trafficCalcConfigPath; reloadTrafficCalcConfig(); // Load ConfigMap - LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x", + LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", s3DeltaPrefix, thresholdMultiplier); } @@ -107,23 +107,23 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, * Can be called periodically to pick up config changes without restarting. */ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { - LOGGER.info("Loading traffic calc config from ConfigMap"); + LOGGER.info("loading traffic calc config"); try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject trafficCalcConfig = new JsonObject(content); // Validate required fields exist if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_allowlist_ranges"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); } this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); @@ -133,15 +133,15 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException List> ranges = parseAllowlistRanges(trafficCalcConfig); this.allowlistRanges = ranges; - LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", + LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.warn("Failed to load traffic calc config. Config is malformed: {}", trafficCalcConfigPath, e); + LOGGER.warn("failed to load traffic calc config, malformed: {}", trafficCalcConfigPath, e); throw e; } catch (Exception e) { - LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e); - throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); + LOGGER.warn("failed to load traffic calc config: {}", trafficCalcConfigPath, e); + throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); } } @@ -161,18 +161,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long end = rangeArray.getLong(1); if(start >= end) { - LOGGER.error("Invalid allowlist range: start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": start must be less than end"); + LOGGER.error("invalid allowlist range, start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); } if (end - start > 86400) { - LOGGER.error("Invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": range must be less than 24 hours"); + LOGGER.error("invalid allowlist range, must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); } List range = Arrays.asList(start, end); ranges.add(range); - LOGGER.info("Loaded allowlist range: [{}, {}]", start, end); + LOGGER.info("loaded allowlist range: [{}, {}]", start, end); } } } @@ -184,18 +184,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long currentEnd = ranges.get(i).get(1); long nextStart = ranges.get(i + 1).get(0); if (currentEnd >= nextStart) { - LOGGER.error("Overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", + LOGGER.error("overlapping allowlist ranges: [{}, {}] overlaps with [{}, {}]", ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); throw new MalformedTrafficCalcConfigException( - "Overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); + "overlapping allowlist ranges at indices " + i + " and " + (i + 1)); } } } catch (MalformedTrafficCalcConfigException e) { throw e; } catch (Exception e) { - LOGGER.error("Failed to parse allowlist ranges", e); - throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage()); + LOGGER.error("failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); } return ranges; @@ -217,13 +217,13 @@ public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueA List deltaS3Paths = listDeltaFiles(); if (deltaS3Paths.isEmpty()) { - LOGGER.warn("No delta files found in S3 with prefix: {}", s3DeltaPrefix); + LOGGER.warn("no delta files found in s3 with prefix: {}", s3DeltaPrefix); return TrafficStatus.DEFAULT; } // Find newest delta file timestamp for delta traffic window long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); - LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs); + LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); // Define start time of the delta evaluation window // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend @@ -242,7 +242,7 @@ public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueA for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { - LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); + LOGGER.info("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); break; } @@ -264,20 +264,20 @@ public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueA if (queueAttributes != null) { queueMessages = queueAttributes.getTotalMessages(); sum += queueMessages; - LOGGER.info("Traffic calculation: adding {} SQS queue messages to sum ({})", + LOGGER.info("traffic calculation: adding {} sqs queue messages to sum ({})", queueMessages, queueAttributes); } // Determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("Traffic calculation complete: sum={} (deltaRecords={}, queueMessages={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + LOGGER.info("traffic calculation complete: sum={} (deltaRecords={}, queueMessages={}), baselineTraffic={}, thresholdMultiplier={}, status={}", sum, sum - queueMessages, queueMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; } catch (Exception e) { - LOGGER.error("Error calculating traffic status", e); + LOGGER.error("error calculating traffic status", e); return TrafficStatus.DEFAULT; } } @@ -296,12 +296,12 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept List timestamps = getTimestampsFromFile(newestDeltaPath); if (timestamps.isEmpty()) { - LOGGER.warn("Newest delta file has no timestamps: {}", newestDeltaPath); + LOGGER.warn("newest delta file has no timestamps: {}", newestDeltaPath); return System.currentTimeMillis() / 1000; } long newestTs = Collections.max(timestamps); - LOGGER.debug("Found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); return newestTs; } @@ -320,7 +320,7 @@ private List listDeltaFiles() { .collect(Collectors.toList()); } catch (Exception e) { - LOGGER.error("Failed to list delta files from S3 with prefix: {}", s3DeltaPrefix, e); + LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); return Collections.emptyList(); } } @@ -335,12 +335,12 @@ private List getTimestampsFromFile(String s3Path) throws IOException { // Check cache first FileRecordCache cached = deltaFileCache.get(filename); if (cached != null) { - LOGGER.debug("Using cached timestamps for file: {}", filename); + LOGGER.info("using cached timestamps for file: {}", filename); return cached.timestamps; } // Cache miss - download from S3 - LOGGER.debug("Downloading and reading timestamps from S3: {}", s3Path); + LOGGER.info("downloading and reading timestamps from s3: {}", s3Path); List timestamps = readTimestampsFromS3(s3Path); // Store in cache @@ -371,8 +371,8 @@ private List readTimestampsFromS3(String s3Path) throws IOException { return timestamps; } catch (Exception e) { - LOGGER.error("Failed to read delta file from S3: {}", s3Path, e); - throw new IOException("Failed to read delta file from S3: " + s3Path, e); + LOGGER.error("failed to read delta file from s3: {}", s3Path, e); + throw new IOException("failed to read delta file from s3: " + s3Path, e); } } @@ -462,7 +462,7 @@ private void evictOldCacheEntries(long cutoffTimestamp) { int afterSize = deltaFileCache.size(); if (beforeSize != afterSize) { - LOGGER.info("Evicted {} old cache entries (before={}, after={})", + LOGGER.info("evicted {} old cache entries (before={}, after={})", beforeSize - afterSize, beforeSize, afterSize); } } @@ -473,17 +473,17 @@ private void evictOldCacheEntries(long cutoffTimestamp) { TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { if (baselineTraffic == 0 || thresholdMultiplier == 0) { // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0 returning DEFAULT status."); + LOGGER.warn("baselineTraffic or thresholdMultiplier is 0, returning default status"); return TrafficStatus.DEFAULT; } if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×baselineTraffic={}", + LOGGER.warn("delayed_processing threshold breached: sumCurrent={} >= {}x baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } - LOGGER.info("Traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", + LOGGER.info("traffic within normal range: sumCurrent={} < {}x baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DEFAULT; } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java index dc06a238..b566c09e 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java @@ -62,8 +62,7 @@ public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTraff this.filterRules = Collections.emptyList(); // start empty reloadTrafficFilterConfig(); // load ConfigMap - LOGGER.info("OptOutTrafficFilter initialized: filterRules={}", - filterRules.size()); + LOGGER.info("initialized: filterRules={}", filterRules.size()); } /** @@ -80,18 +79,17 @@ public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTraff * Can be called periodically to pick up config changes without restarting. */ public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException { - LOGGER.info("Loading traffic filter config from ConfigMap"); + LOGGER.info("loading traffic filter config"); try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject filterConfigJson = new JsonObject(content); this.filterRules = parseFilterRules(filterConfigJson); - LOGGER.info("Successfully loaded traffic filter config from ConfigMap: filterRules={}", - filterRules.size()); + LOGGER.info("loaded traffic filter config: filterRules={}", filterRules.size()); } catch (Exception e) { - LOGGER.warn("No traffic filter config found at: {}", trafficFilterConfigPath, e); + LOGGER.warn("no traffic filter config found at: {}", trafficFilterConfigPath, e); throw new MalformedTrafficFilterConfigException(e.getMessage()); } } @@ -104,8 +102,8 @@ List parseFilterRules(JsonObject config) throws MalformedTraf try { JsonArray denylistRequests = config.getJsonArray("denylist_requests"); if (denylistRequests == null) { - LOGGER.error("Invalid traffic filter config: denylist_requests is null"); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: denylist_requests is null"); + LOGGER.error("invalid traffic filter config: denylist_requests is null"); + throw new MalformedTrafficFilterConfigException("invalid traffic filter config: denylist_requests is null"); } for (int i = 0; i < denylistRequests.size(); i++) { JsonObject ruleJson = denylistRequests.getJsonObject(i); @@ -118,8 +116,8 @@ List parseFilterRules(JsonObject config) throws MalformedTraf long end = rangeJson.getLong(1); if (start >= end) { - LOGGER.error("Invalid traffic filter rule: range start must be less than end: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range start must be less than end"); + LOGGER.error("invalid traffic filter rule, range start must be less than end: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); } range.add(start); range.add(end); @@ -127,8 +125,8 @@ List parseFilterRules(JsonObject config) throws MalformedTraf // log error and throw exception if range is not 2 elements if (range.size() != 2) { - LOGGER.error("Invalid traffic filter rule: range is not 2 elements: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range is not 2 elements"); + LOGGER.error("invalid traffic filter rule, range is not 2 elements: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements"); } // parse IPs @@ -142,24 +140,24 @@ List parseFilterRules(JsonObject config) throws MalformedTraf // log error and throw exception if IPs is empty if (ipAddresses.size() == 0) { - LOGGER.error("Invalid traffic filter rule: IPs is empty: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: IPs is empty"); + LOGGER.error("invalid traffic filter rule, IPs is empty: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); } // log error and throw exception if rule is invalid if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less - LOGGER.error("Invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode()); - throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule, range must be 24 hours or less"); + LOGGER.error("invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode()); + throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less"); } TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses); - LOGGER.info("Loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); + LOGGER.info("loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses()); rules.add(rule); } return rules; } catch (Exception e) { - LOGGER.error("Failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); + LOGGER.error("failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); throw new MalformedTrafficFilterConfigException(e.getMessage()); } } @@ -169,7 +167,7 @@ public boolean isDenylisted(SqsParsedMessage message) { String clientIp = message.getClientIp(); if (clientIp == null || clientIp.isEmpty()) { - LOGGER.error("Request does not contain client IP, timestamp={}", timestamp); + LOGGER.error("request does not contain client ip, timestamp={}", timestamp); return false; } From da7ac07fa8bedb4d3668a8704f34c3379130ee5c Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 13:51:31 -0700 Subject: [PATCH 25/52] update logging and tests --- src/main/java/com/uid2/optout/Main.java | 8 ++++---- .../uid2/optout/vertx/OptOutSqsLogProducerTest.java | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/uid2/optout/Main.java b/src/main/java/com/uid2/optout/Main.java index f6bf5cad..cbdda97f 100644 --- a/src/main/java/com/uid2/optout/Main.java +++ b/src/main/java/com/uid2/optout/Main.java @@ -308,12 +308,12 @@ public void run(String[] args) throws IOException { LOGGER.info("SQS log producer deployed - bucket: {}, folder: {}", this.config.getString(Const.Config.OptOutS3BucketProp), sqsFolder); } catch (IOException e) { - LOGGER.error("Failed to initialize SQS log producer: " + e.getMessage(), e); - LOGGER.error("Failed to initialize SQS log producer, delta production will be disabled: " + e.getMessage(), e); + LOGGER.error("failed to initialize SQS log producer: " + e.getMessage(), e); + LOGGER.error("failed to initialize SQS log producer, delta production will be disabled: " + e.getMessage(), e); } catch (MalformedTrafficFilterConfigException e) { - LOGGER.error("The traffic filter config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); + LOGGER.error("the traffic filter config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.error("The traffic calc config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); + LOGGER.error("the traffic calc config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); } } diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index de1eef4c..9351c7ca 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -331,7 +331,7 @@ public void testDeltaProduceEndpoint_noMessages(TestContext context) { context.assertEquals("completed", finalStatus.getString("state")); JsonObject result = finalStatus.getJsonObject("result"); context.assertNotNull(result); - context.assertEquals("success", result.getString("status")); + context.assertEquals("skipped", result.getString("status")); context.assertEquals(0, result.getInteger("deltas_produced")); context.assertEquals(0, result.getInteger("entries_processed")); @@ -386,7 +386,7 @@ public void testDeltaProduceEndpoint_allMessagesTooRecent(TestContext context) { JsonObject result = finalStatus.getJsonObject("result"); context.assertNotNull(result); context.assertEquals("skipped", result.getString("status")); - context.assertEquals("MESSAGES_TOO_RECENT", result.getString("reason")); + context.assertEquals("MESSAGES_TOO_RECENT", result.getString("stop_reason")); // No processing should occur try { @@ -1070,8 +1070,8 @@ public void testManualOverride_delayedProcessing(TestContext context) throws Exc .onComplete(context.asyncAssertSuccess(finalStatus -> { context.assertEquals("completed", finalStatus.getString("state")); JsonObject result = finalStatus.getJsonObject("result"); - context.assertEquals("skipped", result.getString("status")); - context.assertEquals("MANUAL_OVERRIDE_ACTIVE", result.getString("reason")); + context.assertEquals("halted", result.getString("status")); + context.assertEquals("MANUAL_OVERRIDE_ACTIVE", result.getString("stop_reason")); // Should not process anything - manual override checked at start context.assertEquals(0, result.getInteger("entries_processed")); @@ -1258,7 +1258,7 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex .onComplete(context.asyncAssertSuccess(finalStatus -> { context.assertEquals("completed", finalStatus.getString("state")); JsonObject result = finalStatus.getJsonObject("result"); - context.assertEquals("skipped", result.getString("status")); + context.assertEquals("halted", result.getString("status")); // Expected behavior: // All 610 messages are within a single 5-minute window @@ -1355,7 +1355,7 @@ public void testCircuitBreaker_stopsProcessingWhenMessageLimitExceeded(TestConte .onComplete(context.asyncAssertSuccess(finalStatus -> { context.assertEquals("completed", finalStatus.getString("state")); JsonObject result = finalStatus.getJsonObject("result"); - context.assertEquals("skipped", result.getString("status")); + context.assertEquals("halted", result.getString("status")); // Expected behavior: // SqsWindowReader hits maxMessagesPerWindow limit (100) during reading From c6be5a1bf4465b77209ff236a1f3ae012cab2e17 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 13:57:54 -0700 Subject: [PATCH 26/52] update test --- .../vertx/OptOutSqsLogProducerTest.java | 136 +++--------------- 1 file changed, 16 insertions(+), 120 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 9351c7ca..548fff64 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -1170,7 +1170,8 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex Async async = context.async(); // Threshold = baseline * multiplier = 100 * 5 = 500 - // We have 610 messages, which exceeds 500, so spike should be detected + // Traffic calculator counts: delta file records + queue depth + // We'll set queue depth to 600 to exceed threshold String trafficCalcConfig = """ { "traffic_calc_evaluation_window_seconds": 86400, @@ -1185,10 +1186,10 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - // Create historical delta files showing low baseline traffic (2 records from 24-48h ago) + // Create historical delta file with recent timestamps (within 24h evaluation window) List timestamps = new ArrayList<>(); - timestamps.add(t - 36*3600); // 36 hours ago - timestamps.add(t - 36*3600 + 1000); + timestamps.add(t - 3600); // 1 hour ago + timestamps.add(t - 3600 + 1000); byte[] deltaFileBytes = createDeltaFileBytes(timestamps); // Reset cloudStorage mock to ensure clean state @@ -1198,7 +1199,6 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); // Mock S3 operations for historical data - // Use doAnswer to create fresh streams on each call doReturn(Arrays.asList("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_baseline.dat")) .when(cloudStorage).list("sqs-delta"); doAnswer(inv -> new ByteArrayInputStream(deltaFileBytes)) @@ -1207,17 +1207,13 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex // No manual override set (returns null) doReturn(null).when(cloudStorage).download("manual-override.json"); - // Setup SQS messages - long baseTime = (t - 600) * 1000; - + // Setup SQS messages - just a few to trigger processing + long baseTime = (t - 600) * 1000; // 10 minutes ago List allMessages = new ArrayList<>(); - // Create 610 messages with timestamps spread over ~4 minutes (within the 5-minute window) - for (int i = 0; i < 610; i++) { - // Timestamps range from (t-600) to (t-600-240) seconds = t-600 to t-840 - // All within a single 5-minute window for traffic calculation, and all > 5 min old - long timestampMs = baseTime - (i * 400); // ~400ms apart going backwards, total span ~244 seconds + for (int i = 0; i < 10; i++) { + long timestampMs = baseTime - (i * 1000); allMessages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, - timestampMs, null, null, "10.0.0." + (i % 256), null)); + timestampMs, null, null, "10.0.0." + i, null)); } // Mock SQS operations @@ -1227,9 +1223,10 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) .thenReturn(DeleteMessageBatchResponse.builder().build()); - // Mock getQueueAttributes to return zero invisible messages (doesn't affect the spike detection) + // Mock getQueueAttributes to return 600 messages - exceeds threshold (500) + // Traffic calculator adds queue depth to delta record count Map queueAttrs = new HashMap<>(); - queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "0"); + queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "600"); queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, "0"); queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, "0"); doReturn(GetQueueAttributesResponse.builder() @@ -1259,108 +1256,11 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex context.assertEquals("completed", finalStatus.getString("state")); JsonObject result = finalStatus.getJsonObject("result"); context.assertEquals("halted", result.getString("status")); + context.assertEquals("CIRCUIT_BREAKER_TRIGGERED", result.getString("stop_reason")); // Expected behavior: - // All 610 messages are within a single 5-minute window - // Traffic calculator counts them all and detects spike (>=500 threshold) + // Traffic calculator detects spike (queue depth 600 >= threshold 500) // DELAYED_PROCESSING is triggered, no delta uploaded - // The entries_processed count reflects how many were read before spike detection - context.assertTrue(result.getInteger("entries_processed") <= 610); - context.assertEquals(0, result.getInteger("deltas_produced")); - - // Verify manual override was set to DELAYED_PROCESSING on S3 - try { - ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); - ArgumentCaptor streamCaptor = ArgumentCaptor.forClass(InputStream.class); - verify(cloudStorage, atLeastOnce()).upload(streamCaptor.capture(), pathCaptor.capture()); - - // Check if manual-override.json was uploaded - boolean overrideSet = false; - for (int i = 0; i < pathCaptor.getAllValues().size(); i++) { - if (pathCaptor.getAllValues().get(i).equals("manual-override.json")) { - overrideSet = true; - break; - } - } - context.assertTrue(overrideSet, "Manual override should be set to DELAYED_PROCESSING after detecting spike"); - } catch (Exception e) { - context.fail(e); - } - - async.complete(); - })); - } - - @Test - public void testCircuitBreaker_stopsProcessingWhenMessageLimitExceeded(TestContext context) throws Exception { - Async async = context.async(); - - // Use low threshold (100) so circuit breaker triggers before traffic calculator - String trafficCalcConfig = """ - { - "traffic_calc_evaluation_window_seconds": 86400, - "traffic_calc_baseline_traffic": 20, - "traffic_calc_threshold_multiplier": 5, - "traffic_calc_allowlist_ranges": [] - } - """; - createTrafficCalcConfigFile(trafficCalcConfig); - - // Reset cloudStorage mock to ensure clean state - reset(cloudStorage); - - // Re-mock S3 upload (needed after reset) - doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); - - // No manual override set (returns null) - doReturn(null).when(cloudStorage).download("manual-override.json"); - - // Setup time - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create 200 messages - exceeds threshold (20 * 5 = 100) - long oldTime = (t - 600) * 1000; // 10 minutes ago - List messages = new ArrayList<>(); - for (int i = 0; i < 200; i++) { - messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, - oldTime - (i * 1000), null, null, "10.0.0." + (i % 256), null)); - } - - // Mock SQS operations - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) - .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) - .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); - when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) - .thenReturn(DeleteMessageBatchResponse.builder().build()); - - int port = Const.Port.ServicePortForOptOut + 1; - - // Act & Assert - vertx.createHttpClient() - .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", - Endpoints.OPTOUT_DELTA_PRODUCE.toString()) - .compose(req -> req - .putHeader("Authorization", "Bearer " + TEST_API_KEY) - .send()) - .compose(resp -> { - context.assertEquals(202, resp.statusCode()); - return resp.body(); - }) - .compose(body -> { - JsonObject response = new JsonObject(body.toString()); - context.assertEquals("accepted", response.getString("status")); - return pollForCompletion(context, port, 100, 50); - }) - .onComplete(context.asyncAssertSuccess(finalStatus -> { - context.assertEquals("completed", finalStatus.getString("state")); - JsonObject result = finalStatus.getJsonObject("result"); - context.assertEquals("halted", result.getString("status")); - - // Expected behavior: - // SqsWindowReader hits maxMessagesPerWindow limit (100) during reading - // Circuit breaker triggers DELAYED_PROCESSING immediately - // Processing stops before any messages are counted as processed context.assertEquals(0, result.getInteger("entries_processed")); context.assertEquals(0, result.getInteger("deltas_produced")); @@ -1369,17 +1269,13 @@ public void testCircuitBreaker_stopsProcessingWhenMessageLimitExceeded(TestConte ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), pathCaptor.capture()); - // Check if manual-override.json was uploaded boolean overrideSet = pathCaptor.getAllValues().stream() .anyMatch(path -> path.equals("manual-override.json")); - context.assertTrue(overrideSet, "Circuit breaker should set DELAYED_PROCESSING override"); + context.assertTrue(overrideSet, "Manual override should be set to DELAYED_PROCESSING after detecting spike"); } catch (Exception e) { context.fail(e); } - // Verify NO messages were deleted from SQS (processing stopped before completion) - verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); - async.complete(); })); } From 521118d4aa1b119bf7125f7f9077e85092434bbc Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 13:59:25 -0700 Subject: [PATCH 27/52] update logging --- src/main/java/com/uid2/optout/Main.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/uid2/optout/Main.java b/src/main/java/com/uid2/optout/Main.java index cbdda97f..216cb391 100644 --- a/src/main/java/com/uid2/optout/Main.java +++ b/src/main/java/com/uid2/optout/Main.java @@ -308,12 +308,11 @@ public void run(String[] args) throws IOException { LOGGER.info("SQS log producer deployed - bucket: {}, folder: {}", this.config.getString(Const.Config.OptOutS3BucketProp), sqsFolder); } catch (IOException e) { - LOGGER.error("failed to initialize SQS log producer: " + e.getMessage(), e); - LOGGER.error("failed to initialize SQS log producer, delta production will be disabled: " + e.getMessage(), e); + LOGGER.error("failed to initialize SQS log producer, delta production will be disabled: {}", e.getMessage(), e); } catch (MalformedTrafficFilterConfigException e) { - LOGGER.error("the traffic filter config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); + LOGGER.error("the traffic filter config is malformed, refusing to process messages, delta production will be disabled: {}", e.getMessage(), e); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.error("the traffic calc config is malformed, refusing to process messages, delta production will be disabled: " + e.getMessage(), e); + LOGGER.error("the traffic calc config is malformed, refusing to process messages, delta production will be disabled: {}", e.getMessage(), e); } } From 51b59a4240c94703aa2fbeee4566afbe40b95c73 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:30:37 -0700 Subject: [PATCH 28/52] update traffic calculator --- .../delta/DeltaProductionOrchestrator.java | 2 +- .../optout/vertx/OptOutTrafficCalculator.java | 190 ++++++++---- .../vertx/OptOutSqsLogProducerTest.java | 26 +- .../vertx/OptOutTrafficCalculatorTest.java | 276 +++++++++--------- 4 files changed, 281 insertions(+), 213 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index 18d1d172..b1990083 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -181,7 +181,7 @@ private boolean processWindow(SqsWindowReader.WindowReadResult windowResult, // check traffic calculator SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); - OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(queueAttributes); + OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(deltaMessages, queueAttributes); if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { LOGGER.error("optout delta production has hit DELAYED_PROCESSING status, stopping production and setting manual override"); diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index 267c798f..e4dd2094 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -6,10 +6,11 @@ import com.uid2.shared.optout.OptOutUtils; import com.uid2.optout.Const; import com.uid2.optout.sqs.SqsMessageOperations; - import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; import java.nio.charset.StandardCharsets; @@ -87,7 +88,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, this.trafficCalcConfigPath = trafficCalcConfigPath; reloadTrafficCalcConfig(); // Load ConfigMap - LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", + LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x", s3DeltaPrefix, thresholdMultiplier); } @@ -107,23 +108,23 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, * Can be called periodically to pick up config changes without restarting. */ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { - LOGGER.info("loading traffic calc config"); + LOGGER.info("Loading traffic calc config from ConfigMap"); try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject trafficCalcConfig = new JsonObject(content); // Validate required fields exist if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); + throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_allowlist_ranges"); } this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); @@ -133,15 +134,15 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException List> ranges = parseAllowlistRanges(trafficCalcConfig); this.allowlistRanges = ranges; - LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", + LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.warn("failed to load traffic calc config, malformed: {}", trafficCalcConfigPath, e); + LOGGER.warn("Failed to load traffic calc config. Config is malformed: {}", trafficCalcConfigPath, e); throw e; } catch (Exception e) { - LOGGER.warn("failed to load traffic calc config: {}", trafficCalcConfigPath, e); - throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); + LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e); + throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); } } @@ -161,18 +162,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long end = rangeArray.getLong(1); if(start >= end) { - LOGGER.error("invalid allowlist range, start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); + LOGGER.error("Invalid allowlist range: start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": start must be less than end"); } if (end - start > 86400) { - LOGGER.error("invalid allowlist range, must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); + LOGGER.error("Invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": range must be less than 24 hours"); } List range = Arrays.asList(start, end); ranges.add(range); - LOGGER.info("loaded allowlist range: [{}, {}]", start, end); + LOGGER.info("Loaded allowlist range: [{}, {}]", start, end); } } } @@ -184,46 +185,56 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long currentEnd = ranges.get(i).get(1); long nextStart = ranges.get(i + 1).get(0); if (currentEnd >= nextStart) { - LOGGER.error("overlapping allowlist ranges: [{}, {}] overlaps with [{}, {}]", + LOGGER.error("Overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); throw new MalformedTrafficCalcConfigException( - "overlapping allowlist ranges at indices " + i + " and " + (i + 1)); + "Overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); } } } catch (MalformedTrafficCalcConfigException e) { throw e; } catch (Exception e) { - LOGGER.error("failed to parse allowlist ranges", e); - throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); + LOGGER.error("Failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage()); } return ranges; } /** - * Calculate traffic status based on delta files and SQS queue attributes. + * Calculate traffic status based on delta files and SQS queue messages. + * + * Uses the newest delta file timestamp to anchor the 24-hour delta traffic window, + * and the oldest queue timestamp to anchor the 5-minute queue window. * - * Uses the newest delta file timestamp to anchor the evaluation window, - * and adds the total queue depth from queue attributes. + * Counts: + * - Delta file records (with allowlist filtering) + * - SQS messages passed in (with allowlist filtering) + * - Invisible messages from other consumers (from queue attributes, avoiding double count) * - * @param queueAttributes SQS queue attributes including message counts + * @param sqsMessages List of SQS messages this consumer has read + * @param queueAttributes Queue attributes including invisible message count (can be null) * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ - public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueAttributes) { + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes) { try { // Get list of delta files from S3 (sorted newest to oldest) List deltaS3Paths = listDeltaFiles(); if (deltaS3Paths.isEmpty()) { - LOGGER.warn("no delta files found in s3 with prefix: {}", s3DeltaPrefix); + LOGGER.warn("No delta files found in S3 with prefix: {}", s3DeltaPrefix); return TrafficStatus.DEFAULT; } // Find newest delta file timestamp for delta traffic window long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); - LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); + LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs); + + // Find oldest SQS queue message timestamp for queue window + long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); + LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs); // Define start time of the delta evaluation window // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend @@ -239,10 +250,11 @@ public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueA for (String s3Path : deltaS3Paths) { List timestamps = getTimestampsFromFile(s3Path); + boolean shouldStop = false; for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { - LOGGER.info("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); + LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); break; } @@ -255,29 +267,43 @@ public TrafficStatus calculateStatus(SqsMessageOperations.QueueAttributes queueA if (ts >= deltaWindowStart) { sum++; } + + } + + if (shouldStop) { + break; } } - // Add total queue depth from queue attributes - // This includes visible, invisible (in-flight), and delayed messages - int queueMessages = 0; + // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering + int sqsCount = 0; + if (sqsMessages != null && !sqsMessages.isEmpty()) { + sqsCount = countSqsMessages(sqsMessages, oldestQueueTs); + sum += sqsCount; + } + + // Add invisible messages being processed by OTHER consumers + // (notVisible count includes our messages, so subtract what we've read to avoid double counting) + int otherConsumersMessages = 0; if (queueAttributes != null) { - queueMessages = queueAttributes.getTotalMessages(); - sum += queueMessages; - LOGGER.info("traffic calculation: adding {} sqs queue messages to sum ({})", - queueMessages, queueAttributes); + int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); + int ourMessages = sqsMessages != null ? sqsMessages.size() : 0; + otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); + sum += otherConsumersMessages; + LOGGER.info("Traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", + otherConsumersMessages, totalInvisible, ourMessages); } // Determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("traffic calculation complete: sum={} (deltaRecords={}, queueMessages={}), baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, sum - queueMessages, queueMessages, this.baselineTraffic, this.thresholdMultiplier, status); + LOGGER.info("Traffic calculation complete: sum={} (deltaRecords + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; } catch (Exception e) { - LOGGER.error("error calculating traffic status", e); + LOGGER.error("Error calculating traffic status", e); return TrafficStatus.DEFAULT; } } @@ -296,12 +322,12 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept List timestamps = getTimestampsFromFile(newestDeltaPath); if (timestamps.isEmpty()) { - LOGGER.warn("newest delta file has no timestamps: {}", newestDeltaPath); + LOGGER.warn("Newest delta file has no timestamps: {}", newestDeltaPath); return System.currentTimeMillis() / 1000; } long newestTs = Collections.max(timestamps); - LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + LOGGER.debug("Found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); return newestTs; } @@ -320,7 +346,7 @@ private List listDeltaFiles() { .collect(Collectors.toList()); } catch (Exception e) { - LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); + LOGGER.error("Failed to list delta files from S3 with prefix: {}", s3DeltaPrefix, e); return Collections.emptyList(); } } @@ -335,12 +361,12 @@ private List getTimestampsFromFile(String s3Path) throws IOException { // Check cache first FileRecordCache cached = deltaFileCache.get(filename); if (cached != null) { - LOGGER.info("using cached timestamps for file: {}", filename); + LOGGER.debug("Using cached timestamps for file: {}", filename); return cached.timestamps; } // Cache miss - download from S3 - LOGGER.info("downloading and reading timestamps from s3: {}", s3Path); + LOGGER.debug("Downloading and reading timestamps from S3: {}", s3Path); List timestamps = readTimestampsFromS3(s3Path); // Store in cache @@ -371,8 +397,8 @@ private List readTimestampsFromS3(String s3Path) throws IOException { return timestamps; } catch (Exception e) { - LOGGER.error("failed to read delta file from s3: {}", s3Path, e); - throw new IOException("failed to read delta file from s3: " + s3Path, e); + LOGGER.error("Failed to read delta file from S3: {}", s3Path, e); + throw new IOException("Failed to read delta file from S3: " + s3Path, e); } } @@ -426,6 +452,68 @@ long calculateWindowStartWithAllowlist(long newestDeltaTs, int evaluationWindowS return newestDeltaTs - evaluationWindowSeconds - allowlistDuration; } + /** + * Find the oldest SQS queue message timestamp + */ + private long findOldestQueueTimestamp(List sqsMessages) throws IOException { + long oldest = System.currentTimeMillis() / 1000; + + if (sqsMessages != null && !sqsMessages.isEmpty()) { + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + if (ts != null && ts < oldest) { + oldest = ts; + } + } + } + + return oldest; + } + + /** + * Extract timestamp from SQS message (from SentTimestamp attribute) + */ + private Long extractTimestampFromMessage(Message msg) { + // Get SentTimestamp attribute (milliseconds) + String sentTimestamp = msg.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); + if (sentTimestamp != null) { + try { + return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds + } catch (NumberFormatException e) { + LOGGER.debug("Invalid SentTimestamp: {}", sentTimestamp); + } + } + + // Fallback: use current time + return System.currentTimeMillis() / 1000; + } + + /** + * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes + */ + private int countSqsMessages(List sqsMessages, long oldestQueueTs) { + + int count = 0; + long windowEnd = oldestQueueTs + 5 * 60; + + for (Message msg : sqsMessages) { + Long ts = extractTimestampFromMessage(msg); + + if (ts < oldestQueueTs || ts > windowEnd) { + continue; + } + + if (isInAllowlist(ts)) { + continue; + } + count++; + + } + + LOGGER.info("SQS messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); + return count; + } + /** * Check if a timestamp falls within any allowlist range */ @@ -462,7 +550,7 @@ private void evictOldCacheEntries(long cutoffTimestamp) { int afterSize = deltaFileCache.size(); if (beforeSize != afterSize) { - LOGGER.info("evicted {} old cache entries (before={}, after={})", + LOGGER.info("Evicted {} old cache entries (before={}, after={})", beforeSize - afterSize, beforeSize, afterSize); } } @@ -473,21 +561,21 @@ private void evictOldCacheEntries(long cutoffTimestamp) { TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { if (baselineTraffic == 0 || thresholdMultiplier == 0) { // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic or thresholdMultiplier is 0, returning default status"); + LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0 returning DEFAULT status."); return TrafficStatus.DEFAULT; } if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.warn("delayed_processing threshold breached: sumCurrent={} >= {}x baselineTraffic={}", + LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } - LOGGER.info("traffic within normal range: sumCurrent={} < {}x baselineTraffic={}", + LOGGER.info("Traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DEFAULT; } - + /** * Get cache statistics for monitoring */ @@ -503,4 +591,4 @@ public Map getCacheStats() { return stats; } -} +} \ No newline at end of file diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 548fff64..0504fa67 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -1170,8 +1170,8 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex Async async = context.async(); // Threshold = baseline * multiplier = 100 * 5 = 500 - // Traffic calculator counts: delta file records + queue depth - // We'll set queue depth to 600 to exceed threshold + // Traffic calculator counts: delta file records + SQS messages (with allowlist filtering) + // We'll create 600 SQS messages to exceed threshold String trafficCalcConfig = """ { "traffic_calc_evaluation_window_seconds": 86400, @@ -1207,13 +1207,14 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex // No manual override set (returns null) doReturn(null).when(cloudStorage).download("manual-override.json"); - // Setup SQS messages - just a few to trigger processing + // Setup SQS messages - 600 messages to exceed threshold (500) + // Traffic calculator counts SQS messages and can apply allowlist filtering long baseTime = (t - 600) * 1000; // 10 minutes ago List allMessages = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - long timestampMs = baseTime - (i * 1000); + for (int i = 0; i < 600; i++) { + long timestampMs = baseTime - (i * 100); // spread over ~60 seconds allMessages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, - timestampMs, null, null, "10.0.0." + i, null)); + timestampMs, null, null, "10.0.0." + (i % 256), null)); } // Mock SQS operations @@ -1222,17 +1223,6 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) .thenReturn(DeleteMessageBatchResponse.builder().build()); - - // Mock getQueueAttributes to return 600 messages - exceeds threshold (500) - // Traffic calculator adds queue depth to delta record count - Map queueAttrs = new HashMap<>(); - queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "600"); - queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, "0"); - queueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, "0"); - doReturn(GetQueueAttributesResponse.builder() - .attributes(queueAttrs) - .build()) - .when(sqsClient).getQueueAttributes(any(GetQueueAttributesRequest.class)); int port = Const.Port.ServicePortForOptOut + 1; @@ -1259,7 +1249,7 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex context.assertEquals("CIRCUIT_BREAKER_TRIGGERED", result.getString("stop_reason")); // Expected behavior: - // Traffic calculator detects spike (queue depth 600 >= threshold 500) + // Traffic calculator detects spike (delta records + SQS messages >= threshold 500) // DELAYED_PROCESSING is triggered, no delta uploaded context.assertEquals(0, result.getInteger("entries_processed")); context.assertEquals(0, result.getInteger("deltas_produced")); diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index e073fd2e..0d0232b1 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -5,8 +5,6 @@ import com.uid2.shared.optout.OptOutCollection; import com.uid2.shared.optout.OptOutEntry; import com.uid2.optout.Const; -import com.uid2.optout.sqs.SqsMessageOperations; - import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import java.nio.file.Files; @@ -22,6 +20,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import java.io.ByteArrayInputStream; @@ -1065,6 +1065,31 @@ void testGetCacheStats_emptyCache() throws Exception { // SECTION 8: Helper Methods for Test Data Creation // ============================================================================ + /** + * Create a mock SQS message with specified timestamp + */ + private Message createSqsMessage(long timestampSeconds) { + Map attributes = new HashMap<>(); + attributes.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampSeconds * 1000)); + + return Message.builder() + .messageId("test-msg-" + timestampSeconds) + .body("{\"test\": \"data\"}") + .attributes(attributes) + .build(); + } + + /** + * Create a mock SQS message without timestamp + */ + private Message createSqsMessageWithoutTimestamp() { + return Message.builder() + .messageId("test-msg-no-timestamp") + .body("{\"test\": \"data\"}") + .attributes(new HashMap<>()) + .build(); + } + /** * Create delta file bytes with specified timestamps */ @@ -1096,7 +1121,7 @@ void testCalculateStatus_noDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); // Assert - should return DEFAULT when no delta files assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1108,10 +1133,10 @@ void testCalculateStatus_normalTraffic() throws Exception { long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - // Create delta files with timestamps distributed over 24 hours + // Create delta files with timestamps distributed over 48 hours List timestamps = new ArrayList<>(); - // add 49 entries in current window + // add 499 entries in current window for (int i = 0; i < 49; i++) { timestamps.add(t - 23*3600 + i * 60); } @@ -1125,11 +1150,11 @@ void testCalculateStatus_normalTraffic() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - pass queue attributes with 1 message - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); - // Assert - 49 + 1 = 50 < 5 * 100 = 500, so should be DEFAULT + // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @@ -1156,16 +1181,16 @@ void testCalculateStatus_delayedProcessing() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - 500 delta + 1 queue = 501 >= 500 - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + // Act + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); - // Assert - 500 + 1 >= 5 * 100 = 500, DELAYED_PROCESSING + // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test - void testCalculateStatus_nullQueueAttributes() throws Exception { + void testCalculateStatus_noSqsMessages() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1180,15 +1205,15 @@ void testCalculateStatus_nullQueueAttributes() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - null queue attributes - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); + // Act - null SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @Test - void testCalculateStatus_emptyQueue() throws Exception { + void testCalculateStatus_emptySqsMessages() throws Exception { // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; @@ -1203,22 +1228,21 @@ void testCalculateStatus_emptyQueue() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - empty queue - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(0, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + // Act - empty SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @Test - void testCalculateStatus_queueMessagesPushOverThreshold() throws Exception { - // Setup - create delta files with entries just under threshold + void testCalculateStatus_multipleSqsMessages() throws Exception { + // Setup - create delta files with some entries long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; List timestamps = new ArrayList<>(); - // add 470 entries in window (under threshold of 500) + // add 470 entries in window for (int i = 0; i < 470; i++) { timestamps.add(t - 24*3600 + i * 60); } @@ -1232,10 +1256,12 @@ void testCalculateStatus_queueMessagesPushOverThreshold() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Add 30 messages in queue to push over threshold - // 470 + 30 = 500 >= 500 - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(30, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + // Add 30 SQS entries in [t, t+5min] + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + sqsMessages.add(createSqsMessage(t - i * 10)); + } + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); // Assert - DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1277,11 +1303,11 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); - // Assert - should filter out entries in allowlist ranges - // Only 300 from window count (not in allowlist ranges) + 1 queue = 301 + // Assert - should filter out entries in traffic calc config ranges + // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 // 301 < 5*100, so DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } @@ -1303,14 +1329,14 @@ void testCalculateStatus_cacheUtilization() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - first call should populate cache - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - calculator.calculateStatus(queueAttributes); + List sqsMessages = Arrays.asList(createSqsMessage(t)); + calculator.calculateStatus(sqsMessages, null); Map stats = calculator.getCacheStats(); int cachedFiles = (Integer) stats.get("cached_files"); // Second call should use cache (no additional S3 download) - calculator.calculateStatus(queueAttributes); + calculator.calculateStatus(sqsMessages, null); Map stats2 = calculator.getCacheStats(); int cachedFiles2 = (Integer) stats2.get("cached_files"); @@ -1332,7 +1358,7 @@ void testCalculateStatus_s3Exception() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should not throw exception - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1348,13 +1374,37 @@ void testCalculateStatus_deltaFileReadException() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - empty queue - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null); + // Act - empty SQS messages + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); } + @Test + void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { + // Setup - create delta files with some entries + long currentTime = System.currentTimeMillis() / 1000; + long t = currentTime; + + List timestamps = Arrays.asList(t - 3600); + byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + .thenReturn(new ByteArrayInputStream(deltaFileBytes)); + + OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( + cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + + // Act - SQS message without timestamp (should use current time) + List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + + // Assert - DEFAULT + assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + } + @Test void testCalculateStatus_multipleDeltaFiles() throws Exception { // Setup - create delta files with some entries @@ -1388,8 +1438,8 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1422,10 +1472,10 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); - // Assert - 500 + 1 >= 500, DELAYED_PROCESSING + // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @@ -1446,8 +1496,8 @@ void testCalculateStatus_timestampsCached() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); + List sqsMessages = Arrays.asList(createSqsMessage(t)); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); // Assert assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1458,21 +1508,21 @@ void testCalculateStatus_timestampsCached() throws Exception { } // ============================================================================ - // SECTION 10: Tests for calculateStatus() with QueueAttributes + // SECTION 10: Tests for queue attributes (invisible messages from other consumers) // ============================================================================ @Test - void testCalculateStatus_withQueueAttributes_allTypesAddedToSum() throws Exception { - // Setup - create delta files with entries just under threshold + void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Exception { + // Setup - delta files with low traffic (10 records) + // Threshold = 100 * 5 = 500 + // Queue attributes will have 600 invisible messages (other consumers processing) long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - // Create 490 entries (just under threshold of 500 = 5 * 100) List timestamps = new ArrayList<>(); - for (int i = 0; i < 490; i++) { - timestamps.add(t - 23*3600 + i * 60); + for (int i = 0; i < 10; i++) { + timestamps.add(t - 3600 + i); // 10 entries from 1 hour ago } - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); @@ -1482,27 +1532,33 @@ void testCalculateStatus_withQueueAttributes_allTypesAddedToSum() throws Excepti OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - with only 1 queue message, should be DEFAULT: 490 + 1 = 491 < 500 - SqsMessageOperations.QueueAttributes queueAttributesSmall = createQueueAttributes(1, 0, 0); - OptOutTrafficCalculator.TrafficStatus statusDefault = calculator.calculateStatus(queueAttributesSmall); + // Act - 1 message read by us, but 600 invisible messages from other consumers + List sqsMessages = Arrays.asList(createSqsMessage(t)); + + // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed + // Since we read 1 message, otherConsumers = 600 - 1 = 599 + // Total = 10 (delta) + 1 (our message) + 599 (other consumers) = 610 >= 500 threshold + com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes queueAttributes = + new com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes(0, 600, 0); - // With more queue messages that push over threshold, should be DELAYED_PROCESSING - // 490 (delta) + 5 (visible) + 10 (invisible) = 505 >= 500 - SqsMessageOperations.QueueAttributes queueAttributesLarge = createQueueAttributes(5, 5, 5); - OptOutTrafficCalculator.TrafficStatus statusDelayed = calculator.calculateStatus(queueAttributesLarge); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, statusDefault); - assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, statusDelayed); + // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers + assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } @Test - void testCalculateStatus_withQueueAttributes_zeroMessages() throws Exception { - // Setup - create delta files with some entries + void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exception { + // Setup - delta files with moderate traffic (100 records) + // Threshold = 100 * 5 = 500 + // We'll have 200 messages + 250 invisible from other consumers = 550 > 500 long currentTime = System.currentTimeMillis() / 1000; long t = currentTime; - List timestamps = Arrays.asList(t - 3600, t - 7200); + List timestamps = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + timestamps.add(t - 3600 + i); // 100 entries from 1 hour ago + } byte[] deltaFileBytes = createDeltaFileBytes(timestamps); when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); @@ -1512,88 +1568,22 @@ void testCalculateStatus_withQueueAttributes_zeroMessages() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - queue attributes with zero messages - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(0, 0, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - - // Assert - DEFAULT (only delta records count) - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); - } - - @Test - void testCalculateStatus_withQueueAttributes_largeQueueCount() throws Exception { - // Setup - create delta files with minimal entries - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; + // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) + List sqsMessages = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + sqsMessages.add(createSqsMessage(t - i * 10)); + } - List timestamps = Arrays.asList(t - 3600); - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); + // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed + // otherConsumers = 450 - 200 = 250 + // Total = 100 (delta) + 200 (our messages) + 250 (other consumers) = 550 >= 500 threshold + com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes queueAttributes = + new com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes(0, 450, 0); - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .thenReturn(new ByteArrayInputStream(deltaFileBytes)); - - OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( - cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); - // Act - large queue count alone should trigger DELAYED_PROCESSING - // threshold = 5 * 100 = 500 - // 1 (delta) + 500 (total queue) = 501 >= 500 - SqsMessageOperations.QueueAttributes queueAttributes = createQueueAttributes(200, 200, 100); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(queueAttributes); - - // Assert - DELAYED_PROCESSING due to queue messages + // Assert - DELAYED_PROCESSING due to combined count exceeding threshold assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); } - // ============================================================================ - // SECTION 11: Tests for QueueAttributes class - // ============================================================================ - - /** - * Create a QueueAttributes object for testing - */ - private SqsMessageOperations.QueueAttributes createQueueAttributes(int visible, int invisible, int delayed) { - return new SqsMessageOperations.QueueAttributes(visible, invisible, delayed); - } - - @Test - void testQueueAttributes_getters() { - // Setup - SqsMessageOperations.QueueAttributes attrs = createQueueAttributes(10, 20, 5); - - // Assert - assertEquals(10, attrs.getApproximateNumberOfMessages()); - assertEquals(20, attrs.getApproximateNumberOfMessagesNotVisible()); - assertEquals(5, attrs.getApproximateNumberOfMessagesDelayed()); - assertEquals(35, attrs.getTotalMessages()); - } - - @Test - void testQueueAttributes_toString() { - // Setup - SqsMessageOperations.QueueAttributes attrs = createQueueAttributes(100, 50, 25); - - // Act - String str = attrs.toString(); - - // Assert - should contain all values - assertTrue(str.contains("visible=100")); - assertTrue(str.contains("invisible=50")); - assertTrue(str.contains("delayed=25")); - assertTrue(str.contains("total=175")); - } - - @Test - void testQueueAttributes_zeroValues() { - // Setup - SqsMessageOperations.QueueAttributes attrs = createQueueAttributes(0, 0, 0); - - // Assert - assertEquals(0, attrs.getApproximateNumberOfMessages()); - assertEquals(0, attrs.getApproximateNumberOfMessagesNotVisible()); - assertEquals(0, attrs.getApproximateNumberOfMessagesDelayed()); - assertEquals(0, attrs.getTotalMessages()); - } - -} +} \ No newline at end of file From daf4d84fa29cc6c48c5e6cd8b20c55375149068a Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:30:48 -0700 Subject: [PATCH 29/52] update test --- .../com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index 0d0232b1..d2ee3ea6 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -1569,9 +1569,10 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others) + // Messages must be within 5-minute window to be counted, so use 1-second spacing List sqsMessages = new ArrayList<>(); for (int i = 0; i < 200; i++) { - sqsMessages.add(createSqsMessage(t - i * 10)); + sqsMessages.add(createSqsMessage(t - i)); // 1 second apart, all within 5-minute window } // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed From a05eb9055811b22225460efbe8562a8746018b92 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:34:56 -0700 Subject: [PATCH 30/52] logging in lowercase --- .../optout/vertx/OptOutTrafficCalculator.java | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java index e4dd2094..2658f395 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java @@ -88,7 +88,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, this.trafficCalcConfigPath = trafficCalcConfigPath; reloadTrafficCalcConfig(); // Load ConfigMap - LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x", + LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x", s3DeltaPrefix, thresholdMultiplier); } @@ -108,23 +108,23 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix, * Can be called periodically to pick up config changes without restarting. */ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException { - LOGGER.info("Loading traffic calc config from ConfigMap"); + LOGGER.info("loading traffic calc config from configmap"); try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) { String content = new String(is.readAllBytes(), StandardCharsets.UTF_8); JsonObject trafficCalcConfig = new JsonObject(content); // Validate required fields exist if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier"); } if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) { - throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_allowlist_ranges"); + throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges"); } this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp); @@ -134,15 +134,15 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException List> ranges = parseAllowlistRanges(trafficCalcConfig); this.allowlistRanges = ranges; - LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", + LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}", this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.warn("Failed to load traffic calc config. Config is malformed: {}", trafficCalcConfigPath, e); + LOGGER.warn("failed to load traffic calc config, config is malformed: {}", trafficCalcConfigPath, e); throw e; } catch (Exception e) { - LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e); - throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage()); + LOGGER.warn("failed to load traffic calc config, config is malformed or missing: {}", trafficCalcConfigPath, e); + throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); } } @@ -162,18 +162,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long end = rangeArray.getLong(1); if(start >= end) { - LOGGER.error("Invalid allowlist range: start must be less than end: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": start must be less than end"); + LOGGER.error("invalid allowlist range: start must be less than end: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); } if (end - start > 86400) { - LOGGER.error("Invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); - throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": range must be less than 24 hours"); + LOGGER.error("invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); + throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); } List range = Arrays.asList(start, end); ranges.add(range); - LOGGER.info("Loaded allowlist range: [{}, {}]", start, end); + LOGGER.info("loaded allowlist range: [{}, {}]", start, end); } } } @@ -185,18 +185,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long currentEnd = ranges.get(i).get(1); long nextStart = ranges.get(i + 1).get(0); if (currentEnd >= nextStart) { - LOGGER.error("Overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", + LOGGER.error("overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); throw new MalformedTrafficCalcConfigException( - "Overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); + "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); } } } catch (MalformedTrafficCalcConfigException e) { throw e; } catch (Exception e) { - LOGGER.error("Failed to parse allowlist ranges", e); - throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage()); + LOGGER.error("failed to parse allowlist ranges", e); + throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); } return ranges; @@ -224,17 +224,17 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat List deltaS3Paths = listDeltaFiles(); if (deltaS3Paths.isEmpty()) { - LOGGER.warn("No delta files found in S3 with prefix: {}", s3DeltaPrefix); + LOGGER.warn("no delta files found in s3 with prefix: {}", s3DeltaPrefix); return TrafficStatus.DEFAULT; } // Find newest delta file timestamp for delta traffic window long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths); - LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs); + LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs); // Find oldest SQS queue message timestamp for queue window long oldestQueueTs = findOldestQueueTimestamp(sqsMessages); - LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs); + LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs); // Define start time of the delta evaluation window // We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend @@ -254,7 +254,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { - LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); + LOGGER.debug("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); break; } @@ -290,20 +290,20 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat int ourMessages = sqsMessages != null ? sqsMessages.size() : 0; otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); sum += otherConsumersMessages; - LOGGER.info("Traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", + LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", otherConsumersMessages, totalInvisible, ourMessages); } // Determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("Traffic calculation complete: sum={} (deltaRecords + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + LOGGER.info("traffic calculation complete: sum={} (deltaRecords + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", sum, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; } catch (Exception e) { - LOGGER.error("Error calculating traffic status", e); + LOGGER.error("error calculating traffic status", e); return TrafficStatus.DEFAULT; } } @@ -322,12 +322,12 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept List timestamps = getTimestampsFromFile(newestDeltaPath); if (timestamps.isEmpty()) { - LOGGER.warn("Newest delta file has no timestamps: {}", newestDeltaPath); + LOGGER.warn("newest delta file has no timestamps: {}", newestDeltaPath); return System.currentTimeMillis() / 1000; } long newestTs = Collections.max(timestamps); - LOGGER.debug("Found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + LOGGER.debug("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); return newestTs; } @@ -346,7 +346,7 @@ private List listDeltaFiles() { .collect(Collectors.toList()); } catch (Exception e) { - LOGGER.error("Failed to list delta files from S3 with prefix: {}", s3DeltaPrefix, e); + LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); return Collections.emptyList(); } } @@ -361,12 +361,12 @@ private List getTimestampsFromFile(String s3Path) throws IOException { // Check cache first FileRecordCache cached = deltaFileCache.get(filename); if (cached != null) { - LOGGER.debug("Using cached timestamps for file: {}", filename); + LOGGER.debug("using cached timestamps for file: {}", filename); return cached.timestamps; } // Cache miss - download from S3 - LOGGER.debug("Downloading and reading timestamps from S3: {}", s3Path); + LOGGER.debug("downloading and reading timestamps from s3: {}", s3Path); List timestamps = readTimestampsFromS3(s3Path); // Store in cache @@ -397,8 +397,8 @@ private List readTimestampsFromS3(String s3Path) throws IOException { return timestamps; } catch (Exception e) { - LOGGER.error("Failed to read delta file from S3: {}", s3Path, e); - throw new IOException("Failed to read delta file from S3: " + s3Path, e); + LOGGER.error("failed to read delta file from s3: {}", s3Path, e); + throw new IOException("failed to read delta file from s3: " + s3Path, e); } } @@ -480,7 +480,7 @@ private Long extractTimestampFromMessage(Message msg) { try { return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds } catch (NumberFormatException e) { - LOGGER.debug("Invalid SentTimestamp: {}", sentTimestamp); + LOGGER.debug("invalid sentTimestamp: {}", sentTimestamp); } } @@ -510,7 +510,7 @@ private int countSqsMessages(List sqsMessages, long oldestQueueTs) { } - LOGGER.info("SQS messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); + LOGGER.info("sqs messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); return count; } @@ -550,7 +550,7 @@ private void evictOldCacheEntries(long cutoffTimestamp) { int afterSize = deltaFileCache.size(); if (beforeSize != afterSize) { - LOGGER.info("Evicted {} old cache entries (before={}, after={})", + LOGGER.info("evicted {} old cache entries (before={}, after={})", beforeSize - afterSize, beforeSize, afterSize); } } @@ -561,17 +561,17 @@ private void evictOldCacheEntries(long cutoffTimestamp) { TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { if (baselineTraffic == 0 || thresholdMultiplier == 0) { // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0 returning DEFAULT status."); + LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0, returning default status"); return TrafficStatus.DEFAULT; } if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×baselineTraffic={}", + LOGGER.warn("delayed_processing threshold breached: sumCurrent={} >= {}×baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } - LOGGER.info("Traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", + LOGGER.info("traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DEFAULT; } From 0a0790af12ab5929c466c37ad2856ae5938e385c Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:45:39 -0700 Subject: [PATCH 31/52] test format --- .../com/uid2/optout/vertx/OptOutSqsLogProducerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 0504fa67..3395df85 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -21,6 +21,7 @@ import java.util.*; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -69,7 +70,6 @@ public void setup(TestContext context) throws Exception { .put(Const.Config.OptOutProducerBufferSizeProp, 65536) .put(Const.Config.OptOutProducerReplicaIdProp, 1) .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY) - .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY) .put(Const.Config.TrafficFilterConfigPathProp, TRAFFIC_FILTER_CONFIG_PATH) .put(Const.Config.TrafficCalcConfigPathProp, TRAFFIC_CALC_CONFIG_PATH) .put(Const.Config.ManualOverrideS3PathProp, MANUAL_OVERRIDE_S3_PATH) @@ -431,7 +431,7 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context ); // Use CountDownLatch to control when the mock returns - ensures job stays running - java.util.concurrent.CountDownLatch processingLatch = new java.util.concurrent.CountDownLatch(1); + CountDownLatch processingLatch = new CountDownLatch(1); // Mock SQS to wait on latch before returning - keeps job in RUNNING state when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) @@ -1383,7 +1383,7 @@ public void testStatusEndpoint_showsRunningJob(TestContext context) throws Excep ); // Use CountDownLatch to keep job running - java.util.concurrent.CountDownLatch processingLatch = new java.util.concurrent.CountDownLatch(1); + CountDownLatch processingLatch = new CountDownLatch(1); when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) .thenAnswer(inv -> { From 28d20d6c5315af39d8f8fdd29ac4a4e69fc5eac8 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:47:52 -0700 Subject: [PATCH 32/52] git diff detect file move --- .../uid2/optout/{delta => vertx}/DeltaProductionResult.java | 5 ++++- .../com/uid2/optout/{sqs => vertx}/SqsMessageOperations.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) rename src/main/java/com/uid2/optout/{delta => vertx}/DeltaProductionResult.java (97%) rename src/main/java/com/uid2/optout/{sqs => vertx}/SqsMessageOperations.java (99%) diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java b/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java similarity index 97% rename from src/main/java/com/uid2/optout/delta/DeltaProductionResult.java rename to src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java index d88ef128..66761048 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java +++ b/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java @@ -1,4 +1,7 @@ -package com.uid2.optout.delta; +package com.uid2.optout.vertx; + +import com.uid2.optout.delta.DeltaProductionJobStatus; +import com.uid2.optout.delta.StopReason; import io.vertx.core.json.JsonObject; diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java b/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java similarity index 99% rename from src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java rename to src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java index 5db890d4..02a22621 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java @@ -1,4 +1,4 @@ -package com.uid2.optout.sqs; +package com.uid2.optout.vertx; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From b5793e06e838416ba4eea3123e7d0da83675df26 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:48:41 -0700 Subject: [PATCH 33/52] git diff detect filel move --- .../uid2/optout/{vertx => delta}/DeltaProductionResult.java | 5 +---- .../com/uid2/optout/{vertx => sqs}/SqsMessageOperations.java | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) rename src/main/java/com/uid2/optout/{vertx => delta}/DeltaProductionResult.java (97%) rename src/main/java/com/uid2/optout/{vertx => sqs}/SqsMessageOperations.java (99%) diff --git a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java similarity index 97% rename from src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java rename to src/main/java/com/uid2/optout/delta/DeltaProductionResult.java index 66761048..d88ef128 100644 --- a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java @@ -1,7 +1,4 @@ -package com.uid2.optout.vertx; - -import com.uid2.optout.delta.DeltaProductionJobStatus; -import com.uid2.optout.delta.StopReason; +package com.uid2.optout.delta; import io.vertx.core.json.JsonObject; diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java similarity index 99% rename from src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java rename to src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java index 02a22621..5db890d4 100644 --- a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.sqs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 64c244cb3aa8df600d53828ab1c492ddb0e9aa43 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:57:28 -0700 Subject: [PATCH 34/52] update test --- .../vertx/OptOutSqsLogProducerTest.java | 96 ++----------------- 1 file changed, 6 insertions(+), 90 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 3395df85..3df0a96b 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -95,9 +95,6 @@ public void setup(TestContext context) throws Exception { .build()) .when(sqsClient).getQueueAttributes(any(GetQueueAttributesRequest.class)); - // Don't mock download with anyString() - let tests mock specific paths as needed - // Unmocked downloads will return null by default - try { String traficFilterConfig = """ @@ -572,7 +569,7 @@ public void testDeltaProduceEndpoint_autoClearCompletedJob(TestContext context) public void testTrafficFilter_denylistedMessagesAreDropped(TestContext context) throws Exception { Async async = context.async(); - // Setup - update traffic filter config to denyhlist specific IP and time range + // Setup - update traffic filter config to denylist specific IP and time range long baseTime = System.currentTimeMillis() / 1000 - 400; // 400 seconds ago String filterConfig = String.format(""" { @@ -587,8 +584,8 @@ public void testTrafficFilter_denylistedMessagesAreDropped(TestContext context) createTrafficConfigFile(filterConfig); // Setup - create messages: some denylisted, some not - long denylistedTime = (baseTime) * 1000; // Within denyhlist range - long normalTime = (baseTime - 200) * 1000; // Outside denyhlist range + long denylistedTime = (baseTime) * 1000; // Within denylist range + long normalTime = (baseTime - 200) * 1000; // Outside denylist range List messages = Arrays.asList( // These should be dropped (denylisted) createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime, null, null, "192.168.1.100", null), @@ -664,7 +661,7 @@ public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws """, baseTime - 100, baseTime + 100); createTrafficConfigFile(filterConfig); - // Setup - create messages that don't match denyhlist + // Setup - create messages that don't match denylist long normalTime = (baseTime - 200) * 1000; List messages = Arrays.asList( createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null), @@ -852,7 +849,7 @@ public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throw public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throws Exception { Async async = context.async(); - // Setup - initial config with no denyhlist + // Setup - initial config with no denylist String initialConfig = """ { "denylist_requests": [] @@ -896,7 +893,7 @@ public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throw context.assertEquals(1, result.getInteger("entries_processed")); context.assertEquals(0, result.getInteger("dropped_requests_processed")); - // Update config to denyhlist the IP + // Update config to denylist the IP try { long baseTime = System.currentTimeMillis() / 1000 - 400; String updatedConfig = String.format(""" @@ -1084,87 +1081,6 @@ public void testManualOverride_delayedProcessing(TestContext context) throws Exc })); } - @Test - public void testManualOverride_default_bypassesTrafficCalculation(TestContext context) throws Exception { - Async async = context.async(); - - // Setup - setup time: current time - long currentTime = System.currentTimeMillis() / 1000; - long t = currentTime; - - // Create delta files with timestamps distributed over 48 hours - List timestamps = new ArrayList<>(); - - // Past window: t-47h to t-25h (add 10 entries) - for (int i = 0; i < 10; i++) { - timestamps.add(t - 47*3600 + i * 1000); - } - - // Current window: t-23h to t-1h (add 100 entries - 10x past) - for (int i = 0; i < 100; i++) { - timestamps.add(t - 23*3600 + i * 1000); - } - - byte[] deltaFileBytes = createDeltaFileBytes(timestamps); - - // Setup - mock manual override set to DEFAULT - JsonObject manualOverride = new JsonObject().put("manual_override", "DEFAULT"); - - // Mock S3 operations for this test - // Use doAnswer to create fresh streams on each call (streams are consumed on read) - doReturn(Arrays.asList("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) - .when(cloudStorage).list("sqs-delta"); - doAnswer(inv -> new ByteArrayInputStream(deltaFileBytes)) - .when(cloudStorage).download("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"); - doAnswer(inv -> new java.io.ByteArrayInputStream(manualOverride.encode().getBytes())) - .when(cloudStorage).download("manual-override.json"); - - // Setup - SQS messages, 10 messages in same window - long oldTime = (t - 600) * 1000; - List messages = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - messages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime - (i * 1000), null, null, "10.0.0." + i, null)); - } - - // Setup - mock SQS operations - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) - .thenReturn(ReceiveMessageResponse.builder().messages(messages).build()) - .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); - when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))) - .thenReturn(DeleteMessageBatchResponse.builder().build()); - - int port = Const.Port.ServicePortForOptOut + 1; - // Act & Assert - call endpoint via HTTP - vertx.createHttpClient() - .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", - Endpoints.OPTOUT_DELTA_PRODUCE.toString()) - .compose(req -> req - .putHeader("Authorization", "Bearer " + TEST_API_KEY) - .send()) - .compose(resp -> { - context.assertEquals(202, resp.statusCode()); - return resp.body(); - }) - .compose(body -> { - JsonObject response = new JsonObject(body.toString()); - context.assertEquals("accepted", response.getString("status")); - return pollForCompletion(context, port, 100, 50); - }) - .onComplete(context.asyncAssertSuccess(finalStatus -> { - context.assertEquals("completed", finalStatus.getString("state")); - JsonObject result = finalStatus.getJsonObject("result"); - context.assertEquals("success", result.getString("status")); - - // Should process 10 messages and produce 1 delta (all in same window) - context.assertEquals(10, result.getInteger("entries_processed")); - context.assertEquals(1, result.getInteger("deltas_produced")); - - verify(sqsClient, atLeastOnce()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); - - async.complete(); - })); - } - @Test public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext context) throws Exception { Async async = context.async(); From 3a72d0f6b330c202aabef15088e00ed864062862 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 14:58:50 -0700 Subject: [PATCH 35/52] update test --- .../uid2/optout/vertx/OptOutTrafficCalculatorTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java index d2ee3ea6..611270ab 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java @@ -4,6 +4,7 @@ import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.optout.OptOutCollection; import com.uid2.shared.optout.OptOutEntry; +import com.uid2.optout.sqs.SqsMessageOperations; import com.uid2.optout.Const; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; @@ -1538,8 +1539,8 @@ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Excep // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed // Since we read 1 message, otherConsumers = 600 - 1 = 599 // Total = 10 (delta) + 1 (our message) + 599 (other consumers) = 610 >= 500 threshold - com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes queueAttributes = - new com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes(0, 600, 0); + SqsMessageOperations.QueueAttributes queueAttributes = + new SqsMessageOperations.QueueAttributes(0, 600, 0); OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); @@ -1578,8 +1579,8 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed // otherConsumers = 450 - 200 = 250 // Total = 100 (delta) + 200 (our messages) + 250 (other consumers) = 550 >= 500 threshold - com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes queueAttributes = - new com.uid2.optout.sqs.SqsMessageOperations.QueueAttributes(0, 450, 0); + SqsMessageOperations.QueueAttributes queueAttributes = + new SqsMessageOperations.QueueAttributes(0, 450, 0); OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); From 65d811b54cd0296471df228e2879138a753d3048 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 15:06:36 -0700 Subject: [PATCH 36/52] file rename/relocate --- .../OptOutSqsLogProducer.java | 12 ++++++------ .../OptOutTrafficCalculator.java | 1 + .../optout/delta/DeltaProductionOrchestrator.java | 12 ++++++------ ...errideService.java => ManualOverrideService.java} | 6 +++--- ...{DeltaUploadService.java => S3UploadService.java} | 7 +++---- 5 files changed, 19 insertions(+), 19 deletions(-) rename src/main/java/com/uid2/optout/{vertx => circuit-breaker}/OptOutSqsLogProducer.java (96%) rename src/main/java/com/uid2/optout/{vertx => circuit-breaker}/OptOutTrafficCalculator.java (99%) rename src/main/java/com/uid2/optout/delta/{DeltaManualOverrideService.java => ManualOverrideService.java} (94%) rename src/main/java/com/uid2/optout/delta/{DeltaUploadService.java => S3UploadService.java} (94%) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/circuit-breaker/OptOutSqsLogProducer.java similarity index 96% rename from src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java rename to src/main/java/com/uid2/optout/circuit-breaker/OptOutSqsLogProducer.java index cfc66af1..c43c0883 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/circuit-breaker/OptOutSqsLogProducer.java @@ -3,12 +3,12 @@ import com.uid2.optout.Const; import com.uid2.optout.auth.InternalAuthMiddleware; import com.uid2.optout.delta.DeltaFileWriter; -import com.uid2.optout.delta.DeltaManualOverrideService; +import com.uid2.optout.delta.ManualOverrideService; import com.uid2.optout.delta.DeltaProductionJobStatus; -import com.uid2.optout.delta.DeltaProductionResult; import com.uid2.optout.delta.DeltaProductionMetrics; import com.uid2.optout.delta.DeltaProductionOrchestrator; -import com.uid2.optout.delta.DeltaUploadService; +import com.uid2.optout.delta.DeltaProductionResult; +import com.uid2.optout.delta.S3UploadService; import com.uid2.optout.delta.StopReason; import com.uid2.optout.sqs.SqsWindowReader; import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; @@ -128,9 +128,9 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I // Orchestrator setup DeltaFileWriter deltaFileWriter = new DeltaFileWriter(bufferSize); - DeltaUploadService deltaUploadService = new DeltaUploadService(cloudStorage, this.sqsClient, queueUrl); - DeltaUploadService droppedRequestUploadService = new DeltaUploadService(cloudStorageDroppedRequests, this.sqsClient, queueUrl) ; - DeltaManualOverrideService manualOverrideService = new DeltaManualOverrideService(cloudStorage, jsonConfig.getString(Const.Config.ManualOverrideS3PathProp)); + S3UploadService deltaUploadService = new S3UploadService(cloudStorage, this.sqsClient, queueUrl); + S3UploadService droppedRequestUploadService = new S3UploadService(cloudStorageDroppedRequests, this.sqsClient, queueUrl) ; + ManualOverrideService manualOverrideService = new ManualOverrideService(cloudStorage, jsonConfig.getString(Const.Config.ManualOverrideS3PathProp)); SqsWindowReader windowReader = new SqsWindowReader( this.sqsClient, queueUrl, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds, maxMessagesPerFile diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java similarity index 99% rename from src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java rename to src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java index 2658f395..f6fd12af 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java @@ -6,6 +6,7 @@ import com.uid2.shared.optout.OptOutUtils; import com.uid2.optout.Const; import com.uid2.optout.sqs.SqsMessageOperations; + import io.vertx.core.json.JsonObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index b1990083..4c99cfd8 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -46,9 +46,9 @@ public class DeltaProductionOrchestrator { private final SqsWindowReader windowReader; private final DeltaFileWriter deltaFileWriter; - private final DeltaUploadService deltaUploadService; - private final DeltaUploadService droppedRequestUploadService; - private final DeltaManualOverrideService manualOverrideService; + private final S3UploadService deltaUploadService; + private final S3UploadService droppedRequestUploadService; + private final ManualOverrideService manualOverrideService; private final OptOutTrafficFilter trafficFilter; private final OptOutTrafficCalculator trafficCalculator; private final OptOutCloudSync cloudSync; @@ -62,9 +62,9 @@ public DeltaProductionOrchestrator( int jobTimeoutSeconds, SqsWindowReader windowReader, DeltaFileWriter deltaFileWriter, - DeltaUploadService deltaUploadService, - DeltaUploadService droppedRequestUploadService, - DeltaManualOverrideService manualOverrideService, + S3UploadService deltaUploadService, + S3UploadService droppedRequestUploadService, + ManualOverrideService manualOverrideService, OptOutTrafficFilter trafficFilter, OptOutTrafficCalculator trafficCalculator, OptOutCloudSync cloudSync, diff --git a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java b/src/main/java/com/uid2/optout/delta/ManualOverrideService.java similarity index 94% rename from src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java rename to src/main/java/com/uid2/optout/delta/ManualOverrideService.java index f6f038ea..1dd8ca78 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaManualOverrideService.java +++ b/src/main/java/com/uid2/optout/delta/ManualOverrideService.java @@ -20,8 +20,8 @@ * {"manual_override": "DELAYED_PROCESSING"} * */ -public class DeltaManualOverrideService { - private static final Logger LOGGER = LoggerFactory.getLogger(DeltaManualOverrideService.class); +public class ManualOverrideService { + private static final Logger LOGGER = LoggerFactory.getLogger(ManualOverrideService.class); private static final String OVERRIDE_KEY = "manual_override"; private static final String DELAYED_PROCESSING_VALUE = "DELAYED_PROCESSING"; @@ -35,7 +35,7 @@ public class DeltaManualOverrideService { * @param cloudStorage Cloud storage client for reading/writing override file * @param overrideS3Path S3 path where the override file is stored */ - public DeltaManualOverrideService(ICloudStorage cloudStorage, String overrideS3Path) { + public ManualOverrideService(ICloudStorage cloudStorage, String overrideS3Path) { this.cloudStorage = cloudStorage; this.overrideS3Path = overrideS3Path; } diff --git a/src/main/java/com/uid2/optout/delta/DeltaUploadService.java b/src/main/java/com/uid2/optout/delta/S3UploadService.java similarity index 94% rename from src/main/java/com/uid2/optout/delta/DeltaUploadService.java rename to src/main/java/com/uid2/optout/delta/S3UploadService.java index 6929e333..4062f488 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaUploadService.java +++ b/src/main/java/com/uid2/optout/delta/S3UploadService.java @@ -17,8 +17,8 @@ * This class encapsulates the critical "upload then delete" pattern that ensures * data is persisted to S3 before messages are removed from the queue. */ -public class DeltaUploadService { - private static final Logger LOGGER = LoggerFactory.getLogger(DeltaUploadService.class); +public class S3UploadService { + private static final Logger LOGGER = LoggerFactory.getLogger(S3UploadService.class); private final ICloudStorage cloudStorage; private final SqsClient sqsClient; @@ -44,7 +44,7 @@ public interface UploadSuccessCallback { * @param sqsClient SQS client for message deletion * @param queueUrl SQS queue URL */ - public DeltaUploadService(ICloudStorage cloudStorage, SqsClient sqsClient, String queueUrl) { + public S3UploadService(ICloudStorage cloudStorage, SqsClient sqsClient, String queueUrl) { this.cloudStorage = cloudStorage; this.sqsClient = sqsClient; this.queueUrl = queueUrl; @@ -81,4 +81,3 @@ public void uploadAndDeleteMessages(byte[] data, String s3Path, List me } } } - From 04654afdcc9adbd4a5cfef2d9129a58b0cf42b3f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 15:09:26 -0700 Subject: [PATCH 37/52] move files --- .../optout/{vertx => circuit-breaker}/OptOutTrafficFilter.java | 0 .../optout/{circuit-breaker => vertx}/OptOutSqsLogProducer.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/main/java/com/uid2/optout/{vertx => circuit-breaker}/OptOutTrafficFilter.java (100%) rename src/main/java/com/uid2/optout/{circuit-breaker => vertx}/OptOutSqsLogProducer.java (100%) diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficFilter.java similarity index 100% rename from src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java rename to src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficFilter.java diff --git a/src/main/java/com/uid2/optout/circuit-breaker/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java similarity index 100% rename from src/main/java/com/uid2/optout/circuit-breaker/OptOutSqsLogProducer.java rename to src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java From aadf9c6aeed39d97f5065df77756da7742dbd138 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 7 Dec 2025 22:22:03 +0000 Subject: [PATCH 38/52] [CI Pipeline] Released Snapshot version: 4.5.9-alpha-123-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 21b5420b..caf9a28a 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.8-alpha-122-SNAPSHOT + 4.5.9-alpha-123-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 8aeb191e3198329594f91670088bb930ddc9b3d6 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 15:54:50 -0700 Subject: [PATCH 39/52] improve calculator visibility --- .../OptOutTrafficCalculator.java | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java index f6fd12af..447e3850 100644 --- a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java @@ -247,15 +247,27 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // Process delta files and count records in [deltaWindowStart, newestDeltaTs] int sum = 0; + int deltaRecordsCount = 0; + int filesProcessed = 0; + int cacheHits = 0; + int cacheMisses = 0; for (String s3Path : deltaS3Paths) { + boolean wasCached = isCached(s3Path); + if (wasCached) { + cacheHits++; + } else { + cacheMisses++; + } + List timestamps = getTimestampsFromFile(s3Path); + filesProcessed++; boolean shouldStop = false; for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { - LOGGER.debug("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); + LOGGER.info("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); break; } @@ -266,6 +278,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // increment sum if record is in delta window if (ts >= deltaWindowStart) { + deltaRecordsCount++; sum++; } @@ -276,6 +289,9 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat } } + LOGGER.info("delta files: processed={}, deltaRecords={}, cache hits={}, misses={}, cacheSize={}", + filesProcessed, deltaRecordsCount, cacheHits, cacheMisses, deltaFileCache.size()); + // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering int sqsCount = 0; if (sqsMessages != null && !sqsMessages.isEmpty()) { @@ -328,7 +344,7 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept } long newestTs = Collections.max(timestamps); - LOGGER.debug("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); + LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath); return newestTs; } @@ -341,10 +357,13 @@ private List listDeltaFiles() { List allFiles = cloudStorage.list(s3DeltaPrefix); // Filter to only .dat delta files and sort newest to oldest - return allFiles.stream() + List deltaFiles = allFiles.stream() .filter(OptOutUtils::isDeltaFile) .sorted(OptOutUtils.DeltaFilenameComparatorDescending) .collect(Collectors.toList()); + + LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix); + return deltaFiles; } catch (Exception e) { LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); @@ -352,6 +371,14 @@ private List listDeltaFiles() { } } + /** + * Check if a delta file is already cached + */ + private boolean isCached(String s3Path) { + String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1); + return deltaFileCache.containsKey(filename); + } + /** * Get timestamps from a delta file (S3 path), using cache if available */ @@ -362,16 +389,17 @@ private List getTimestampsFromFile(String s3Path) throws IOException { // Check cache first FileRecordCache cached = deltaFileCache.get(filename); if (cached != null) { - LOGGER.debug("using cached timestamps for file: {}", filename); + LOGGER.info("using cached timestamps for file: {}", filename); return cached.timestamps; } // Cache miss - download from S3 - LOGGER.debug("downloading and reading timestamps from s3: {}", s3Path); + LOGGER.info("downloading and reading timestamps from s3: {}", s3Path); List timestamps = readTimestampsFromS3(s3Path); // Store in cache deltaFileCache.put(filename, new FileRecordCache(timestamps)); + LOGGER.info("cached delta file: {} ({} records)", filename, timestamps.size()); return timestamps; } @@ -481,7 +509,7 @@ private Long extractTimestampFromMessage(Message msg) { try { return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds } catch (NumberFormatException e) { - LOGGER.debug("invalid sentTimestamp: {}", sentTimestamp); + LOGGER.info("invalid sentTimestamp: {}", sentTimestamp); } } From f1f252f1fbacc9a04975323f90d69b9ac9d50acb Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 7 Dec 2025 22:57:15 +0000 Subject: [PATCH 40/52] [CI Pipeline] Released Snapshot version: 4.5.10-alpha-124-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index caf9a28a..f5cf5831 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.9-alpha-123-SNAPSHOT + 4.5.10-alpha-124-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From d3a121bfb89e23415ad394284844ca82d9c305ff Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 16:17:59 -0700 Subject: [PATCH 41/52] update logging --- src/main/java/com/uid2/optout/Main.java | 4 ++-- .../optout/delta/DeltaProductionOrchestrator.java | 4 ++-- .../OptOutTrafficCalculator.java | 14 +++++--------- .../OptOutTrafficFilter.java | 2 +- .../uid2/optout/vertx/OptOutSqsLogProducer.java | 6 ++++-- .../OptOutTrafficCalculatorTest.java | 5 +++-- .../OptOutTrafficFilterTest.java | 3 ++- 7 files changed, 19 insertions(+), 19 deletions(-) rename src/main/java/com/uid2/optout/{circuit-breaker => traffic}/OptOutTrafficCalculator.java (96%) rename src/main/java/com/uid2/optout/{circuit-breaker => traffic}/OptOutTrafficFilter.java (99%) rename src/test/java/com/uid2/optout/{vertx => traffic}/OptOutTrafficCalculatorTest.java (99%) rename src/test/java/com/uid2/optout/{vertx => traffic}/OptOutTrafficFilterTest.java (99%) diff --git a/src/main/java/com/uid2/optout/Main.java b/src/main/java/com/uid2/optout/Main.java index 216cb391..0af2dc21 100644 --- a/src/main/java/com/uid2/optout/Main.java +++ b/src/main/java/com/uid2/optout/Main.java @@ -1,8 +1,8 @@ package com.uid2.optout; import com.uid2.optout.vertx.*; -import com.uid2.optout.vertx.OptOutTrafficFilter.MalformedTrafficFilterConfigException; -import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import com.uid2.optout.traffic.OptOutTrafficFilter.MalformedTrafficFilterConfigException; +import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import com.uid2.shared.ApplicationVersion; import com.uid2.shared.Utils; import com.uid2.shared.attest.AttestationResponseHandler; diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index 4c99cfd8..a6232bfb 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -3,8 +3,8 @@ import com.uid2.optout.sqs.SqsMessageOperations; import com.uid2.optout.sqs.SqsParsedMessage; import com.uid2.optout.sqs.SqsWindowReader; -import com.uid2.optout.vertx.OptOutTrafficCalculator; -import com.uid2.optout.vertx.OptOutTrafficFilter; +import com.uid2.optout.traffic.OptOutTrafficCalculator; +import com.uid2.optout.traffic.OptOutTrafficFilter; import com.uid2.shared.optout.OptOutCloudSync; import com.uid2.shared.optout.OptOutUtils; import io.vertx.core.json.JsonArray; diff --git a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java similarity index 96% rename from src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java rename to src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java index 447e3850..b9b09b09 100644 --- a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.traffic; import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.optout.OptOutCollection; @@ -267,7 +267,6 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { - LOGGER.info("stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart); break; } @@ -314,8 +313,8 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // Determine status TrafficStatus status = determineStatus(sum, this.baselineTraffic); - LOGGER.info("traffic calculation complete: sum={} (deltaRecords + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", - sum, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); + LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}", + sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status); return status; @@ -389,17 +388,14 @@ private List getTimestampsFromFile(String s3Path) throws IOException { // Check cache first FileRecordCache cached = deltaFileCache.get(filename); if (cached != null) { - LOGGER.info("using cached timestamps for file: {}", filename); return cached.timestamps; } // Cache miss - download from S3 - LOGGER.info("downloading and reading timestamps from s3: {}", s3Path); List timestamps = readTimestampsFromS3(s3Path); // Store in cache deltaFileCache.put(filename, new FileRecordCache(timestamps)); - LOGGER.info("cached delta file: {} ({} records)", filename, timestamps.size()); return timestamps; } @@ -509,7 +505,7 @@ private Long extractTimestampFromMessage(Message msg) { try { return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds } catch (NumberFormatException e) { - LOGGER.info("invalid sentTimestamp: {}", sentTimestamp); + LOGGER.warn("invalid sentTimestamp: {}", sentTimestamp); } } @@ -595,7 +591,7 @@ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { } if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.warn("delayed_processing threshold breached: sumCurrent={} >= {}×baselineTraffic={}", + LOGGER.error("delayed_processing threshold breached: sumCurrent={} >= {}×baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } diff --git a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java similarity index 99% rename from src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficFilter.java rename to src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java index b566c09e..80c5923e 100644 --- a/src/main/java/com/uid2/optout/circuit-breaker/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.traffic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index c43c0883..c7b40af8 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -11,8 +11,10 @@ import com.uid2.optout.delta.S3UploadService; import com.uid2.optout.delta.StopReason; import com.uid2.optout.sqs.SqsWindowReader; -import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; -import com.uid2.optout.vertx.OptOutTrafficFilter.MalformedTrafficFilterConfigException; +import com.uid2.optout.traffic.OptOutTrafficCalculator; +import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import com.uid2.optout.traffic.OptOutTrafficFilter; +import com.uid2.optout.traffic.OptOutTrafficFilter.MalformedTrafficFilterConfigException; import com.uid2.shared.Utils; import com.uid2.shared.cloud.ICloudStorage; import com.uid2.shared.health.HealthComponent; diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java similarity index 99% rename from src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java rename to src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java index 611270ab..c10a8b08 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java @@ -1,4 +1,4 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.traffic; import com.uid2.shared.cloud.CloudStorageException; import com.uid2.shared.cloud.ICloudStorage; @@ -24,7 +24,8 @@ import software.amazon.awssdk.services.sqs.model.Message; import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; -import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; +import com.uid2.optout.traffic.OptOutTrafficCalculator; +import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException; import java.io.ByteArrayInputStream; import java.util.*; diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java similarity index 99% rename from src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java rename to src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java index ccf425b5..127cd041 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java @@ -1,10 +1,11 @@ -package com.uid2.optout.vertx; +package com.uid2.optout.traffic; import org.junit.After; import org.junit.Before; import org.junit.Test; import com.uid2.optout.sqs.SqsParsedMessage; +import com.uid2.optout.traffic.OptOutTrafficFilter; import software.amazon.awssdk.services.sqs.model.Message; From 01474e42c328e93b45d2f76e666361df35f92fa2 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 7 Dec 2025 23:20:56 +0000 Subject: [PATCH 42/52] [CI Pipeline] Released Snapshot version: 4.5.11-alpha-126-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f5cf5831..2db11220 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.10-alpha-124-SNAPSHOT + 4.5.11-alpha-126-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From e8a8f1a83db974e1ce186e69a9c322f171c4fda8 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 16:43:12 -0700 Subject: [PATCH 43/52] fix traffic calulator processing too many files --- src/main/java/com/uid2/optout/Const.java | 1 + .../traffic/OptOutTrafficCalculator.java | 18 ++++++++++-------- .../optout/vertx/OptOutSqsLogProducer.java | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/uid2/optout/Const.java b/src/main/java/com/uid2/optout/Const.java index a4182507..5b047018 100644 --- a/src/main/java/com/uid2/optout/Const.java +++ b/src/main/java/com/uid2/optout/Const.java @@ -32,6 +32,7 @@ public static class Config extends com.uid2.shared.Const.Config { public static final String OptOutTrafficCalcThresholdMultiplierProp = "traffic_calc_threshold_multiplier"; public static final String OptOutTrafficCalcEvaluationWindowSecondsProp = "traffic_calc_evaluation_window_seconds"; public static final String OptOutTrafficCalcAllowlistRangesProp = "traffic_calc_allowlist_ranges"; + public static final String OptOutSqsDeltaWindowSecondsProp = "optout_sqs_delta_window_seconds"; } public static class Event { diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java index b9b09b09..16a80b23 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -246,6 +246,8 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat evictOldCacheEntries(deltaWindowStart); // Process delta files and count records in [deltaWindowStart, newestDeltaTs] + // Files are sorted newest to oldest, records within files are sorted newest to oldest + // Stop when the newest record in a file is older than the window int sum = 0; int deltaRecordsCount = 0; int filesProcessed = 0; @@ -263,7 +265,12 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat List timestamps = getTimestampsFromFile(s3Path); filesProcessed++; - boolean shouldStop = false; + // Check newest record in file - if older than window, stop processing remaining files + long newestRecordTs = timestamps.get(0); + if (newestRecordTs < deltaWindowStart) { + break; + } + for (long ts : timestamps) { // Stop condition: record is older than our window if (ts < deltaWindowStart) { @@ -280,11 +287,6 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat deltaRecordsCount++; sum++; } - - } - - if (shouldStop) { - break; } } @@ -591,12 +593,12 @@ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { } if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.error("delayed_processing threshold breached: sumCurrent={} >= {}×baselineTraffic={}", + LOGGER.error("delayed_processing threshold breached: sumCurrent={}, thresholdMultiplier={}, baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } - LOGGER.info("traffic within normal range: sumCurrent={} < {}×baselineTraffic={}", + LOGGER.info("traffic within normal range: sumCurrent={}, thresholdMultiplier={}, baselineTraffic={}", sumCurrent, thresholdMultiplier, baselineTraffic); return TrafficStatus.DEFAULT; } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index c7b40af8..72776284 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -122,7 +122,7 @@ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, I // Configuration values for orchestrator setup int replicaId = OptOutUtils.getReplicaId(jsonConfig); int maxMessagesPerPoll = 10; // SQS max is 10 - int deltaWindowSeconds = 300; // Fixed 5 minutes for all deltas + int deltaWindowSeconds = jsonConfig.getInteger(Const.Config.OptOutSqsDeltaWindowSecondsProp, 300); // fixed 5 minutes, allow config for testing int visibilityTimeout = jsonConfig.getInteger(Const.Config.OptOutSqsVisibilityTimeoutProp, 240); int jobTimeoutSeconds = jsonConfig.getInteger(Const.Config.OptOutDeltaJobTimeoutSecondsProp, 10800); int maxMessagesPerFile = jsonConfig.getInteger(Const.Config.OptOutMaxMessagesPerFileProp, 10000); From e3a8085d3c8de5e39d37ed11ab2ad5cd4390056d Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Sun, 7 Dec 2025 23:45:27 +0000 Subject: [PATCH 44/52] [CI Pipeline] Released Snapshot version: 4.5.12-alpha-127-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2db11220..18134240 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.11-alpha-126-SNAPSHOT + 4.5.12-alpha-127-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From 337ca99b3687f8a324c7a1a1115b8b4906c9e51b Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 18:29:00 -0700 Subject: [PATCH 45/52] use sliding window, not fixed boundaries --- src/main/java/com/uid2/optout/sqs/SqsWindowReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index ff41ab27..746f2e34 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -117,7 +117,7 @@ public WindowReadResult readWindow() { // Add eligible messages to current window boolean newWindow = false; for (SqsParsedMessage msg : batchResult.getMessages()) { - long msgWindowStart = (msg.getTimestamp() / this.deltaWindowSeconds) * this.deltaWindowSeconds; + long msgWindowStart = msg.getTimestamp(); // Discover start of window if (currentWindowStart == 0) { From d0511dac99274127e929d87882d8e85cd85301a4 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Sun, 7 Dec 2025 18:35:33 -0700 Subject: [PATCH 46/52] denylisted deduplication --- .../delta/DeltaProductionOrchestrator.java | 4 +-- .../traffic/OptOutTrafficCalculator.java | 8 +++-- .../traffic/OptOutTrafficCalculatorTest.java | 34 +++++++++---------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index a6232bfb..671b067e 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -179,9 +179,9 @@ private boolean processWindow(SqsWindowReader.WindowReadResult windowResult, } } - // check traffic calculator + // check traffic calculator - pass denylisted count for accurate invisible message deduplication SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); - OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(deltaMessages, queueAttributes); + OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(deltaMessages, queueAttributes, droppedMessages.size()); if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { LOGGER.error("optout delta production has hit DELAYED_PROCESSING status, stopping production and setting manual override"); diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java index 16a80b23..91ed2f0e 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -214,11 +214,12 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic * - SQS messages passed in (with allowlist filtering) * - Invisible messages from other consumers (from queue attributes, avoiding double count) * - * @param sqsMessages List of SQS messages this consumer has read + * @param sqsMessages List of SQS messages this consumer has read (non-denylisted) * @param queueAttributes Queue attributes including invisible message count (can be null) + * @param denylistedCount Number of denylisted messages read by this consumer (for invisible deduplication) * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ - public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes) { + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount) { try { // Get list of delta files from S3 (sorted newest to oldest) @@ -302,10 +303,11 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // Add invisible messages being processed by OTHER consumers // (notVisible count includes our messages, so subtract what we've read to avoid double counting) + // ourMessages = delta messages + denylisted messages (all messages we've read from the queue) int otherConsumersMessages = 0; if (queueAttributes != null) { int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); - int ourMessages = sqsMessages != null ? sqsMessages.size() : 0; + int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount; otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); sum += otherConsumersMessages; LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java index c10a8b08..ed6b9e11 100644 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java @@ -1123,7 +1123,7 @@ void testCalculateStatus_noDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); // Assert - should return DEFAULT when no delta files assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1154,7 +1154,7 @@ void testCalculateStatus_normalTraffic() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1185,7 +1185,7 @@ void testCalculateStatus_delayedProcessing() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1208,7 +1208,7 @@ void testCalculateStatus_noSqsMessages() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - null SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1231,7 +1231,7 @@ void testCalculateStatus_emptySqsMessages() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1263,7 +1263,7 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { for (int i = 0; i < 30; i++) { sqsMessages.add(createSqsMessage(t - i * 10)); } - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1306,7 +1306,7 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - should filter out entries in traffic calc config ranges // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 @@ -1332,13 +1332,13 @@ void testCalculateStatus_cacheUtilization() throws Exception { // Act - first call should populate cache List sqsMessages = Arrays.asList(createSqsMessage(t)); - calculator.calculateStatus(sqsMessages, null); + calculator.calculateStatus(sqsMessages, null, 0); Map stats = calculator.getCacheStats(); int cachedFiles = (Integer) stats.get("cached_files"); // Second call should use cache (no additional S3 download) - calculator.calculateStatus(sqsMessages, null); + calculator.calculateStatus(sqsMessages, null, 0); Map stats2 = calculator.getCacheStats(); int cachedFiles2 = (Integer) stats2.get("cached_files"); @@ -1360,7 +1360,7 @@ void testCalculateStatus_s3Exception() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should not throw exception - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1377,7 +1377,7 @@ void testCalculateStatus_deltaFileReadException() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1401,7 +1401,7 @@ void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { // Act - SQS message without timestamp (should use current time) List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1441,7 +1441,7 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1475,7 +1475,7 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1499,7 +1499,7 @@ void testCalculateStatus_timestampsCached() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); // Assert assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1543,7 +1543,7 @@ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Excep SqsMessageOperations.QueueAttributes queueAttributes = new SqsMessageOperations.QueueAttributes(0, 600, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0); // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1583,7 +1583,7 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce SqsMessageOperations.QueueAttributes queueAttributes = new SqsMessageOperations.QueueAttributes(0, 450, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0); // Assert - DELAYED_PROCESSING due to combined count exceeding threshold assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); From 9e8e6aba2cbdb0303ec940941104e6999da98648 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 8 Dec 2025 11:40:20 -0700 Subject: [PATCH 47/52] increase calculator accuracy --- .../delta/DeltaProductionOrchestrator.java | 5 +-- .../com/uid2/optout/sqs/SqsWindowReader.java | 33 +++++++++++------- .../traffic/OptOutTrafficCalculator.java | 9 ++--- .../traffic/OptOutTrafficCalculatorTest.java | 34 +++++++++---------- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index 671b067e..3425e2a2 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -179,9 +179,10 @@ private boolean processWindow(SqsWindowReader.WindowReadResult windowResult, } } - // check traffic calculator - pass denylisted count for accurate invisible message deduplication + // check traffic calculator - pass counts for accurate invisible message deduplication + int filteredAsTooRecentCount = windowResult.getRawMessagesRead() - messages.size(); SqsMessageOperations.QueueAttributes queueAttributes = SqsMessageOperations.getQueueAttributes(this.sqsClient, this.queueUrl); - OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(deltaMessages, queueAttributes, droppedMessages.size()); + OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(deltaMessages, queueAttributes, droppedMessages.size(), filteredAsTooRecentCount); if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { LOGGER.error("optout delta production has hit DELAYED_PROCESSING status, stopping production and setting manual override"); diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index 746f2e34..61ee2ce7 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -45,33 +45,37 @@ public static class WindowReadResult { private final List messages; private final long windowStart; private final StopReason stopReason; + private final int rawMessagesRead; // total messages pulled from SQS - private WindowReadResult(List messages, long windowStart, StopReason stopReason) { + private WindowReadResult(List messages, long windowStart, StopReason stopReason, int rawMessagesRead) { this.messages = messages; this.windowStart = windowStart; this.stopReason = stopReason; + this.rawMessagesRead = rawMessagesRead; } - public static WindowReadResult withMessages(List messages, long windowStart) { - return new WindowReadResult(messages, windowStart, StopReason.NONE); + public static WindowReadResult withMessages(List messages, long windowStart, int rawMessagesRead) { + return new WindowReadResult(messages, windowStart, StopReason.NONE, rawMessagesRead); } - public static WindowReadResult queueEmpty(List messages, long windowStart) { - return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY); + public static WindowReadResult queueEmpty(List messages, long windowStart, int rawMessagesRead) { + return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY, rawMessagesRead); } - public static WindowReadResult messagesTooRecent(List messages, long windowStart) { - return new WindowReadResult(messages, windowStart, StopReason.MESSAGES_TOO_RECENT); + public static WindowReadResult messagesTooRecent(List messages, long windowStart, int rawMessagesRead) { + return new WindowReadResult(messages, windowStart, StopReason.MESSAGES_TOO_RECENT, rawMessagesRead); } - public static WindowReadResult messageLimitExceeded(List messages, long windowStart) { - return new WindowReadResult(messages, windowStart, StopReason.MESSAGE_LIMIT_EXCEEDED); + public static WindowReadResult messageLimitExceeded(List messages, long windowStart, int rawMessagesRead) { + return new WindowReadResult(messages, windowStart, StopReason.MESSAGE_LIMIT_EXCEEDED, rawMessagesRead); } public List getMessages() { return messages; } public long getWindowStart() { return windowStart; } public boolean isEmpty() { return messages.isEmpty(); } public StopReason getStopReason() { return stopReason; } + /** Total raw messages pulled from SQS */ + public int getRawMessagesRead() { return rawMessagesRead; } } /** @@ -88,11 +92,12 @@ public WindowReadResult readWindow() { List windowMessages = new ArrayList<>(); long currentWindowStart = 0; int batchNumber = 0; + int rawMessagesRead = 0; // track total messages pulled from SQS while (true) { if (windowMessages.size() >= maxMessagesPerWindow) { LOGGER.warn("message limit exceeded: {} messages >= limit {}", windowMessages.size(), maxMessagesPerWindow); - return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart); + return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart, rawMessagesRead); } // Read one batch from SQS (up to 10 messages) @@ -100,15 +105,17 @@ public WindowReadResult readWindow() { this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout); if (rawBatch.isEmpty()) { - return WindowReadResult.queueEmpty(windowMessages, currentWindowStart); + return WindowReadResult.queueEmpty(windowMessages, currentWindowStart, rawMessagesRead); } + rawMessagesRead += rawBatch.size(); + // parse, validate, filter SqsBatchProcessor.BatchProcessingResult batchResult = batchProcessor.processBatch(rawBatch, batchNumber++); if (!batchResult.hasMessages()) { if (batchResult.getStopReason() == StopReason.MESSAGES_TOO_RECENT) { - return WindowReadResult.messagesTooRecent(windowMessages, currentWindowStart); + return WindowReadResult.messagesTooRecent(windowMessages, currentWindowStart, rawMessagesRead); } // Corrupt messages were deleted, continue reading continue; @@ -133,7 +140,7 @@ public WindowReadResult readWindow() { } if (newWindow) { - return WindowReadResult.withMessages(windowMessages, currentWindowStart); + return WindowReadResult.withMessages(windowMessages, currentWindowStart, rawMessagesRead); } } } diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java index 91ed2f0e..769596ba 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -216,10 +216,11 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic * * @param sqsMessages List of SQS messages this consumer has read (non-denylisted) * @param queueAttributes Queue attributes including invisible message count (can be null) - * @param denylistedCount Number of denylisted messages read by this consumer (for invisible deduplication) + * @param denylistedCount Number of denylisted messages read by this consumer + * @param filteredAsTooRecentCount Number of messages filtered as "too recent" by window reader * @return TrafficStatus (DELAYED_PROCESSING or DEFAULT) */ - public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount) { + public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) { try { // Get list of delta files from S3 (sorted newest to oldest) @@ -303,11 +304,11 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // Add invisible messages being processed by OTHER consumers // (notVisible count includes our messages, so subtract what we've read to avoid double counting) - // ourMessages = delta messages + denylisted messages (all messages we've read from the queue) + // ourMessages = delta messages + denylisted messages + filtered "too recent" messages int otherConsumersMessages = 0; if (queueAttributes != null) { int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible(); - int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount; + int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount + filteredAsTooRecentCount; otherConsumersMessages = Math.max(0, totalInvisible - ourMessages); sum += otherConsumersMessages; LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})", diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java index ed6b9e11..b20f0879 100644 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java @@ -1123,7 +1123,7 @@ void testCalculateStatus_noDeltaFiles() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); // Assert - should return DEFAULT when no delta files assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1154,7 +1154,7 @@ void testCalculateStatus_normalTraffic() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1185,7 +1185,7 @@ void testCalculateStatus_delayedProcessing() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1208,7 +1208,7 @@ void testCalculateStatus_noSqsMessages() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - null SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1231,7 +1231,7 @@ void testCalculateStatus_emptySqsMessages() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); // Assert - should still calculate based on delta files, DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1263,7 +1263,7 @@ void testCalculateStatus_multipleSqsMessages() throws Exception { for (int i = 0; i < 30; i++) { sqsMessages.add(createSqsMessage(t - i * 10)); } - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DELAYED_PROCESSING assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1306,7 +1306,7 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - should filter out entries in traffic calc config ranges // Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301 @@ -1332,13 +1332,13 @@ void testCalculateStatus_cacheUtilization() throws Exception { // Act - first call should populate cache List sqsMessages = Arrays.asList(createSqsMessage(t)); - calculator.calculateStatus(sqsMessages, null, 0); + calculator.calculateStatus(sqsMessages, null, 0, 0); Map stats = calculator.getCacheStats(); int cachedFiles = (Integer) stats.get("cached_files"); // Second call should use cache (no additional S3 download) - calculator.calculateStatus(sqsMessages, null, 0); + calculator.calculateStatus(sqsMessages, null, 0, 0); Map stats2 = calculator.getCacheStats(); int cachedFiles2 = (Integer) stats2.get("cached_files"); @@ -1360,7 +1360,7 @@ void testCalculateStatus_s3Exception() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - should not throw exception - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1377,7 +1377,7 @@ void testCalculateStatus_deltaFileReadException() throws Exception { cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); // Assert - DEFAULT on error assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1401,7 +1401,7 @@ void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception { // Act - SQS message without timestamp (should use current time) List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp()); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1441,7 +1441,7 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1475,7 +1475,7 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert - DEFAULT assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1499,7 +1499,7 @@ void testCalculateStatus_timestampsCached() throws Exception { // Act List sqsMessages = Arrays.asList(createSqsMessage(t)); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0); // Assert assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); @@ -1543,7 +1543,7 @@ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Excep SqsMessageOperations.QueueAttributes queueAttributes = new SqsMessageOperations.QueueAttributes(0, 600, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); @@ -1583,7 +1583,7 @@ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exce SqsMessageOperations.QueueAttributes queueAttributes = new SqsMessageOperations.QueueAttributes(0, 450, 0); - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0); + OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0); // Assert - DELAYED_PROCESSING due to combined count exceeding threshold assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status); From 5b99a60dd69ca5e850520972ec69a605a88b1392 Mon Sep 17 00:00:00 2001 From: Release Workflow Date: Mon, 8 Dec 2025 18:42:20 +0000 Subject: [PATCH 48/52] [CI Pipeline] Released Snapshot version: 4.5.13-alpha-128-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 18134240..9af7dfbe 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.uid2 uid2-optout - 4.5.12-alpha-127-SNAPSHOT + 4.5.13-alpha-128-SNAPSHOT uid2-optout https://github.com/IABTechLab/uid2-optout From df98d9d0907b23eca2c50798d3b0b13a3a73b1a1 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 8 Dec 2025 12:20:52 -0700 Subject: [PATCH 49/52] improve calculator logs and visibility --- .../traffic/OptOutTrafficCalculator.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java index 769596ba..c8d1f441 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -252,6 +252,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // Stop when the newest record in a file is older than the window int sum = 0; int deltaRecordsCount = 0; + int deltaAllowlistedCount = 0; int filesProcessed = 0; int cacheHits = 0; int cacheMisses = 0; @@ -281,6 +282,7 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat // skip records in allowlisted ranges if (isInAllowlist(ts)) { + deltaAllowlistedCount++; continue; } @@ -292,8 +294,8 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat } } - LOGGER.info("delta files: processed={}, deltaRecords={}, cache hits={}, misses={}, cacheSize={}", - filesProcessed, deltaRecordsCount, cacheHits, cacheMisses, deltaFileCache.size()); + LOGGER.info("delta files: processed={}, deltaRecords={}, allowlisted={}, cache hits={}, misses={}, cacheSize={}", + filesProcessed, deltaRecordsCount, deltaAllowlistedCount, cacheHits, cacheMisses, deltaFileCache.size()); // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering int sqsCount = 0; @@ -522,25 +524,27 @@ private Long extractTimestampFromMessage(Message msg) { * Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes */ private int countSqsMessages(List sqsMessages, long oldestQueueTs) { - + int count = 0; + int allowlistedCount = 0; long windowEnd = oldestQueueTs + 5 * 60; - + for (Message msg : sqsMessages) { Long ts = extractTimestampFromMessage(msg); if (ts < oldestQueueTs || ts > windowEnd) { continue; } - + if (isInAllowlist(ts)) { + allowlistedCount++; continue; } count++; - + } - - LOGGER.info("sqs messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd); + + LOGGER.info("sqs messages: {} in window, {} allowlisted [oldestQueueTs={}, oldestQueueTs+5m={}]", count, allowlistedCount, oldestQueueTs, windowEnd); return count; } From 7282a03b132c1ae23ca2ca2bc399bf08356c064f Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 8 Dec 2025 14:44:30 -0700 Subject: [PATCH 50/52] standardizing logs --- src/main/java/com/uid2/optout/Main.java | 6 +++--- .../java/com/uid2/optout/vertx/OptOutSqsLogProducer.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/uid2/optout/Main.java b/src/main/java/com/uid2/optout/Main.java index 0af2dc21..4d52e879 100644 --- a/src/main/java/com/uid2/optout/Main.java +++ b/src/main/java/com/uid2/optout/Main.java @@ -308,11 +308,11 @@ public void run(String[] args) throws IOException { LOGGER.info("SQS log producer deployed - bucket: {}, folder: {}", this.config.getString(Const.Config.OptOutS3BucketProp), sqsFolder); } catch (IOException e) { - LOGGER.error("failed to initialize SQS log producer, delta production will be disabled: {}", e.getMessage(), e); + LOGGER.error("circuit_breaker_config_error: failed to initialize SQS log producer, delta production will be disabled: {}", e.getMessage(), e); } catch (MalformedTrafficFilterConfigException e) { - LOGGER.error("the traffic filter config is malformed, refusing to process messages, delta production will be disabled: {}", e.getMessage(), e); + LOGGER.error("circuit_breaker_config_error: traffic filter config is malformed, delta production will be disabled: {}", e.getMessage(), e); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.error("the traffic calc config is malformed, refusing to process messages, delta production will be disabled: {}", e.getMessage(), e); + LOGGER.error("circuit_breaker_config_error: traffic calc config is malformed, delta production will be disabled: {}", e.getMessage(), e); } } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 72776284..1bca5688 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -253,7 +253,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { try { this.trafficFilter.reloadTrafficFilterConfig(); } catch (MalformedTrafficFilterConfigException e) { - LOGGER.error("error reloading traffic filter config", e); + LOGGER.error("circuit_breaker_config_error: failed to reload traffic filter config: {}", e.getMessage(), e); sendError(resp, e); return; } @@ -261,7 +261,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { try { this.trafficCalculator.reloadTrafficCalcConfig(); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.error("error reloading traffic calculator config", e); + LOGGER.error("circuit_breaker_config_error: failed to reload traffic calc config: {}", e.getMessage(), e); sendError(resp, e); return; } From bf665a0f3a4ed2da33eb460a77f7fb13f25882b7 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Mon, 8 Dec 2025 16:44:43 -0700 Subject: [PATCH 51/52] standardize error logging for alerts --- .../delta/DeltaProductionOrchestrator.java | 6 +- .../optout/delta/ManualOverrideService.java | 4 +- .../uid2/optout/delta/S3UploadService.java | 2 +- .../uid2/optout/sqs/SqsBatchProcessor.java | 4 +- .../uid2/optout/sqs/SqsMessageOperations.java | 6 +- .../com/uid2/optout/sqs/SqsMessageParser.java | 8 +-- .../com/uid2/optout/sqs/SqsWindowReader.java | 2 +- .../traffic/OptOutTrafficCalculator.java | 61 ++++++++++++------- .../optout/traffic/OptOutTrafficFilter.java | 16 ++--- .../optout/vertx/OptOutSqsLogProducer.java | 4 +- .../traffic/OptOutTrafficCalculatorTest.java | 39 ++++-------- .../vertx/OptOutSqsLogProducerTest.java | 36 +++++++++-- 12 files changed, 106 insertions(+), 82 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java index 3425e2a2..6bec569c 100644 --- a/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java +++ b/src/main/java/com/uid2/optout/delta/DeltaProductionOrchestrator.java @@ -185,7 +185,7 @@ private boolean processWindow(SqsWindowReader.WindowReadResult windowResult, OptOutTrafficCalculator.TrafficStatus trafficStatus = this.trafficCalculator.calculateStatus(deltaMessages, queueAttributes, droppedMessages.size(), filteredAsTooRecentCount); if (trafficStatus == OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING) { - LOGGER.error("optout delta production has hit DELAYED_PROCESSING status, stopping production and setting manual override"); + LOGGER.error("circuit_breaker_triggered: traffic spike detected, stopping production and setting manual override"); manualOverrideService.setDelayedProcessing(); return true; } @@ -262,11 +262,11 @@ private boolean isJobTimedOut(long jobStartTime) { long elapsedTime = OptOutUtils.nowEpochSeconds() - jobStartTime; if (elapsedTime > 3600) { // 1 hour - log warning - LOGGER.error("delta production job has been running for {} seconds", elapsedTime); + LOGGER.error("delta_job_timeout: job has been running for {} seconds", elapsedTime); } if (elapsedTime > this.jobTimeoutSeconds) { - LOGGER.error("delta production job has been running for {} seconds (exceeds timeout of {}s)", + LOGGER.error("delta_job_timeout: job exceeded timeout, running for {} seconds (timeout: {}s)", elapsedTime, this.jobTimeoutSeconds); return true; } diff --git a/src/main/java/com/uid2/optout/delta/ManualOverrideService.java b/src/main/java/com/uid2/optout/delta/ManualOverrideService.java index 1dd8ca78..db39e5e3 100644 --- a/src/main/java/com/uid2/optout/delta/ManualOverrideService.java +++ b/src/main/java/com/uid2/optout/delta/ManualOverrideService.java @@ -62,7 +62,7 @@ public boolean setDelayedProcessing() { LOGGER.info("set manual override to DELAYED_PROCESSING: {}", overrideS3Path); return true; } catch (Exception e) { - LOGGER.error("error setting manual override: {}", overrideS3Path, e); + LOGGER.error("manual_override_error: failed to set override at {}", overrideS3Path, e); return false; } } @@ -76,7 +76,7 @@ private String getOverrideValue() { JsonObject configJson = Utils.toJsonObject(inputStream); return configJson.getString(OVERRIDE_KEY, ""); } catch (Exception e) { - LOGGER.error("no manual override found: {}", overrideS3Path); + LOGGER.error("manual_override_error: no manual override file found at {}", overrideS3Path); return ""; } } diff --git a/src/main/java/com/uid2/optout/delta/S3UploadService.java b/src/main/java/com/uid2/optout/delta/S3UploadService.java index 4062f488..9bdbcc74 100644 --- a/src/main/java/com/uid2/optout/delta/S3UploadService.java +++ b/src/main/java/com/uid2/optout/delta/S3UploadService.java @@ -72,7 +72,7 @@ public void uploadAndDeleteMessages(byte[] data, String s3Path, List me onSuccess.onSuccess(messages.size()); } } catch (Exception e) { - LOGGER.error("failed to upload to s3: path={}", s3Path, e); + LOGGER.error("s3_error: failed to upload delta or dropped requests to path={}", s3Path, e); throw new IOException("s3 upload failed: " + s3Path, e); } diff --git a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java index 5db15890..d77f561a 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java +++ b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java @@ -82,14 +82,14 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN if (parsedBatch.size() < messageBatch.size()) { List invalidMessages = identifyInvalidMessages(messageBatch, parsedBatch); if (!invalidMessages.isEmpty()) { - LOGGER.error("found {} invalid messages in batch {}, deleting", invalidMessages.size(), batchNumber); + LOGGER.error("sqs_error: found {} invalid messages in batch {}, deleting", invalidMessages.size(), batchNumber); SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages); } } // No valid messages after deleting corrupt ones, continue reading if (parsedBatch.isEmpty()) { - LOGGER.error("no valid messages in batch {}", batchNumber); + LOGGER.info("no valid messages in batch {} after removing invalid messages", batchNumber); return BatchProcessingResult.corruptMessagesDeleted(); } diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java index 5db890d4..a25806bd 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java @@ -136,7 +136,7 @@ public static List receiveMessagesFromSqs( return response.messages(); } catch (Exception e) { - LOGGER.error("error receiving messages", e); + LOGGER.error("sqs_error: failed to receive messages", e); return new ArrayList<>(); } } @@ -174,7 +174,7 @@ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, L DeleteMessageBatchResponse deleteResponse = sqsClient.deleteMessageBatch(deleteRequest); if (!deleteResponse.failed().isEmpty()) { - LOGGER.error("failed to delete {} messages", deleteResponse.failed().size()); + LOGGER.error("sqs_error: failed to delete {} messages", deleteResponse.failed().size()); } else { totalDeleted += entries.size(); } @@ -186,7 +186,7 @@ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, L LOGGER.info("deleted {} messages", totalDeleted); } catch (Exception e) { - LOGGER.error("error deleting messages", e); + LOGGER.error("sqs_error: exception during message deletion", e); } } } diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java index 1bcf6751..315ff611 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java +++ b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java @@ -40,7 +40,7 @@ public static List parseAndSortMessages(List messages String phone = body.getString("phone"); if (identityHash == null || advertisingId == null) { - LOGGER.error("invalid message format: {}", message.body()); + LOGGER.error("sqs_error: invalid message format: {}", message.body()); continue; } @@ -48,13 +48,13 @@ public static List parseAndSortMessages(List messages byte[] idBytes = OptOutUtils.base64StringTobyteArray(advertisingId); if (hashBytes == null || idBytes == null) { - LOGGER.error("invalid base64 encoding"); + LOGGER.error("sqs_error: invalid base64 encoding"); continue; } parsedMessages.add(new SqsParsedMessage(message, hashBytes, idBytes, timestampSeconds, email, phone, clientIp, traceId)); } catch (Exception e) { - LOGGER.error("error parsing message", e); + LOGGER.error("sqs_error: error parsing message", e); } } @@ -73,7 +73,7 @@ public static List parseAndSortMessages(List messages private static long extractTimestamp(Message message) { String sentTimestampStr = message.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP); if (sentTimestampStr == null) { - LOGGER.warn("message missing SentTimestamp, using current time"); + LOGGER.info("message missing SentTimestamp, using current time"); return OptOutUtils.nowEpochSeconds(); } return Long.parseLong(sentTimestampStr) / 1000; // ms to seconds diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index 61ee2ce7..f5f74d87 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -96,7 +96,7 @@ public WindowReadResult readWindow() { while (true) { if (windowMessages.size() >= maxMessagesPerWindow) { - LOGGER.warn("message limit exceeded: {} messages >= limit {}", windowMessages.size(), maxMessagesPerWindow); + LOGGER.warn("high_message_volume: message limit exceeded while reading window, {} messages >= limit {}", windowMessages.size(), maxMessagesPerWindow); return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart, rawMessagesRead); } diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java index c8d1f441..ab6a2c00 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java @@ -139,10 +139,10 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size()); } catch (MalformedTrafficCalcConfigException e) { - LOGGER.warn("failed to load traffic calc config, config is malformed: {}", trafficCalcConfigPath, e); + LOGGER.error("circuit_breaker_config_error: config is malformed, configPath={}", trafficCalcConfigPath, e); throw e; } catch (Exception e) { - LOGGER.warn("failed to load traffic calc config, config is malformed or missing: {}", trafficCalcConfigPath, e); + LOGGER.error("circuit_breaker_config_error: config is malformed or missing, configPath={}", trafficCalcConfigPath, e); throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage()); } } @@ -163,12 +163,12 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long end = rangeArray.getLong(1); if(start >= end) { - LOGGER.error("invalid allowlist range: start must be less than end: [{}, {}]", start, end); + LOGGER.error("circuit_breaker_config_error: allowlist range start must be less than end, range=[{}, {}]", start, end); throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end"); } if (end - start > 86400) { - LOGGER.error("invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end); + LOGGER.error("circuit_breaker_config_error: allowlist range must be less than 24 hours, range=[{}, {}]", start, end); throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours"); } @@ -186,7 +186,7 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic long currentEnd = ranges.get(i).get(1); long nextStart = ranges.get(i + 1).get(0); if (currentEnd >= nextStart) { - LOGGER.error("overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]", + LOGGER.error("circuit_breaker_config_error: overlapping allowlist ranges, range=[{}, {}] overlaps with range=[{}, {}]", ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1)); throw new MalformedTrafficCalcConfigException( "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1)); @@ -196,7 +196,7 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic } catch (MalformedTrafficCalcConfigException e) { throw e; } catch (Exception e) { - LOGGER.error("failed to parse allowlist ranges", e); + LOGGER.error("circuit_breaker_config_error: failed to parse allowlist ranges", e); throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage()); } @@ -227,8 +227,8 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat List deltaS3Paths = listDeltaFiles(); if (deltaS3Paths.isEmpty()) { - LOGGER.warn("no delta files found in s3 with prefix: {}", s3DeltaPrefix); - return TrafficStatus.DEFAULT; + LOGGER.error("s3_error: no delta files found in s3 at prefix={}", s3DeltaPrefix); + throw new RuntimeException("no delta files found in s3 at prefix=" + s3DeltaPrefix); } // Find newest delta file timestamp for delta traffic window @@ -326,8 +326,8 @@ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperat return status; } catch (Exception e) { - LOGGER.error("error calculating traffic status", e); - return TrafficStatus.DEFAULT; + LOGGER.error("delta_job_failed: error calculating traffic status", e); + throw new RuntimeException("error calculating traffic status", e); } } @@ -345,7 +345,7 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept List timestamps = getTimestampsFromFile(newestDeltaPath); if (timestamps.isEmpty()) { - LOGGER.warn("newest delta file has no timestamps: {}", newestDeltaPath); + LOGGER.error("s3_error: newest delta file has no timestamps, path={}", newestDeltaPath); return System.currentTimeMillis() / 1000; } @@ -372,7 +372,7 @@ private List listDeltaFiles() { return deltaFiles; } catch (Exception e) { - LOGGER.error("failed to list delta files from s3 with prefix: {}", s3DeltaPrefix, e); + LOGGER.error("s3_error: failed to list delta files at prefix={}", s3DeltaPrefix, e); return Collections.emptyList(); } } @@ -429,7 +429,7 @@ private List readTimestampsFromS3(String s3Path) throws IOException { return timestamps; } catch (Exception e) { - LOGGER.error("failed to read delta file from s3: {}", s3Path, e); + LOGGER.error("s3_error: failed to read delta file at path={}", s3Path, e); throw new IOException("failed to read delta file from s3: " + s3Path, e); } } @@ -512,7 +512,7 @@ private Long extractTimestampFromMessage(Message msg) { try { return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds } catch (NumberFormatException e) { - LOGGER.warn("invalid sentTimestamp: {}", sentTimestamp); + LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp); } } @@ -590,23 +590,38 @@ private void evictOldCacheEntries(long cutoffTimestamp) { } /** - * Determine traffic status based on current vs past counts + * Determine traffic status based on current vs baseline traffic. + * Logs warnings at 50%, 75%, and 90% of the circuit breaker threshold. */ TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) { if (baselineTraffic == 0 || thresholdMultiplier == 0) { - // Avoid division by zero - if no baseline traffic, return DEFAULT status - LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0, returning default status"); - return TrafficStatus.DEFAULT; + LOGGER.error("circuit_breaker_config_error: baselineTraffic is 0 or thresholdMultiplier is 0"); + throw new RuntimeException("invalid circuit breaker config: baselineTraffic=" + baselineTraffic + ", thresholdMultiplier=" + thresholdMultiplier); } - if (sumCurrent >= thresholdMultiplier * baselineTraffic) { - LOGGER.error("delayed_processing threshold breached: sumCurrent={}, thresholdMultiplier={}, baselineTraffic={}", - sumCurrent, thresholdMultiplier, baselineTraffic); + int threshold = thresholdMultiplier * baselineTraffic; + double thresholdPercent = (double) sumCurrent / threshold * 100; + + // Log warnings at increasing thresholds before circuit breaker triggers + if (thresholdPercent >= 90.0) { + LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } else if (thresholdPercent >= 75.0) { + LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } else if (thresholdPercent >= 50.0) { + LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); + } + + if (sumCurrent >= threshold) { + LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic); return TrafficStatus.DELAYED_PROCESSING; } - LOGGER.info("traffic within normal range: sumCurrent={}, thresholdMultiplier={}, baselineTraffic={}", - sumCurrent, thresholdMultiplier, baselineTraffic); + LOGGER.info("traffic within normal range: sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%", + sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent)); return TrafficStatus.DEFAULT; } diff --git a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java index 80c5923e..59a60a9f 100644 --- a/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java +++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java @@ -89,7 +89,7 @@ public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigExcep LOGGER.info("loaded traffic filter config: filterRules={}", filterRules.size()); } catch (Exception e) { - LOGGER.warn("no traffic filter config found at: {}", trafficFilterConfigPath, e); + LOGGER.error("circuit_breaker_config_error: no traffic filter config found at {}", trafficFilterConfigPath, e); throw new MalformedTrafficFilterConfigException(e.getMessage()); } } @@ -102,7 +102,7 @@ List parseFilterRules(JsonObject config) throws MalformedTraf try { JsonArray denylistRequests = config.getJsonArray("denylist_requests"); if (denylistRequests == null) { - LOGGER.error("invalid traffic filter config: denylist_requests is null"); + LOGGER.error("circuit_breaker_config_error: denylist_requests is null"); throw new MalformedTrafficFilterConfigException("invalid traffic filter config: denylist_requests is null"); } for (int i = 0; i < denylistRequests.size(); i++) { @@ -116,7 +116,7 @@ List parseFilterRules(JsonObject config) throws MalformedTraf long end = rangeJson.getLong(1); if (start >= end) { - LOGGER.error("invalid traffic filter rule, range start must be less than end: {}", ruleJson.encode()); + LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end"); } range.add(start); @@ -125,7 +125,7 @@ List parseFilterRules(JsonObject config) throws MalformedTraf // log error and throw exception if range is not 2 elements if (range.size() != 2) { - LOGGER.error("invalid traffic filter rule, range is not 2 elements: {}", ruleJson.encode()); + LOGGER.error("circuit_breaker_config_error: rule range is not 2 elements, rule={}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements"); } @@ -140,13 +140,13 @@ List parseFilterRules(JsonObject config) throws MalformedTraf // log error and throw exception if IPs is empty if (ipAddresses.size() == 0) { - LOGGER.error("invalid traffic filter rule, IPs is empty: {}", ruleJson.encode()); + LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty"); } // log error and throw exception if rule is invalid if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less - LOGGER.error("invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode()); + LOGGER.error("circuit_breaker_config_error: rule range must be 24 hours or less, rule={}", ruleJson.encode()); throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less"); } @@ -157,7 +157,7 @@ List parseFilterRules(JsonObject config) throws MalformedTraf } return rules; } catch (Exception e) { - LOGGER.error("failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage()); + LOGGER.error("circuit_breaker_config_error: failed to parse rules, config={}, error={}", config.encode(), e.getMessage()); throw new MalformedTrafficFilterConfigException(e.getMessage()); } } @@ -167,7 +167,7 @@ public boolean isDenylisted(SqsParsedMessage message) { String clientIp = message.getClientIp(); if (clientIp == null || clientIp.isEmpty()) { - LOGGER.error("request does not contain client ip, timestamp={}", timestamp); + LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.getOriginalMessage().messageId()); return false; } diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java index 1bca5688..7b8a7082 100644 --- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java +++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java @@ -271,7 +271,7 @@ private void handleDeltaProduceStart(RoutingContext routingContext) { // If there's an existing job, check if it's still running if (existingJob != null) { if (existingJob.getState() == DeltaProductionJobStatus.JobState.RUNNING) { - LOGGER.warn("job already running"); + LOGGER.info("job already running, returning conflict"); sendConflict(resp, "job already running on this pod"); return; } @@ -304,7 +304,7 @@ private void startDeltaProductionJob(DeltaProductionJobStatus job) { job.complete(ar.result()); } else { job.fail(ar.cause().getMessage()); - LOGGER.error("job failed", ar.cause()); + LOGGER.error("delta_job_failed: {}", ar.cause().getMessage(), ar.cause()); } }); } diff --git a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java index b20f0879..fd435a9d 100644 --- a/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java +++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java @@ -858,11 +858,8 @@ void testDetermineStatus_sumPastZero() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - should return DEFAULT to avoid crash - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(100, 0); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + // Act & Assert - should throw exception for invalid config + assertThrows(RuntimeException.class, () -> calculator.determineStatus(100, 0)); } @Test @@ -871,11 +868,8 @@ void testDetermineStatus_bothZero() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - should return DEFAULT to avoid crash - OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 0); - - // Assert - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + // Act & Assert - should throw exception for invalid config + assertThrows(RuntimeException.class, () -> calculator.determineStatus(0, 0)); } @Test @@ -1122,11 +1116,8 @@ void testCalculateStatus_noDeltaFiles() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); - - // Assert - should return DEFAULT when no delta files - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + // Act & Assert - should throw exception when no delta files + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); } @Test @@ -1297,8 +1288,8 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception { byte[] deltaFileBytes = createDeltaFileBytes(timestamps); createConfigFromPartialJson(trafficCalcConfigJson); - when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat")); - when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat")) + when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat")) .thenReturn(new ByteArrayInputStream(deltaFileBytes)); OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( @@ -1359,11 +1350,8 @@ void testCalculateStatus_s3Exception() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - should not throw exception - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); - - // Assert - DEFAULT on error - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + // Act & Assert - should throw exception on S3 error + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); } @Test @@ -1376,11 +1364,8 @@ void testCalculateStatus_deltaFileReadException() throws Exception { OptOutTrafficCalculator calculator = new OptOutTrafficCalculator( cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH); - // Act - empty SQS messages - OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0); - - // Assert - DEFAULT on error - assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status); + // Act & Assert - should throw exception on S3 download error + assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0)); } @Test diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java index 3df0a96b..b874db9d 100644 --- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java +++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java @@ -25,6 +25,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.*; import static org.mockito.Mockito.doAnswer; import org.mockito.ArgumentCaptor; @@ -85,6 +86,13 @@ public void setup(TestContext context) throws Exception { doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString()); + // Mock S3 list for traffic calculator (required for delta production) + when(cloudStorage.list(anyString())).thenReturn(Arrays.asList("sqs-delta/delta/optout-delta-001_2025-01-01T00.00.00Z_aaaaaaaa.dat")); + // Mock S3 download - manual override returns null (not found), delta files return valid bytes + when(cloudStorage.download(MANUAL_OVERRIDE_S3_PATH)).thenReturn(null); + when(cloudStorage.download(argThat(path -> path != null && path.contains("optout-delta")))) + .thenAnswer(inv -> new ByteArrayInputStream(createMinimalDeltaFileBytes())); + // Mock getQueueAttributes by default (returns zero messages) Map defaultQueueAttrs = new HashMap<>(); defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "0"); @@ -921,6 +929,12 @@ public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throw doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString()); doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString()); + // Re-mock S3 list and download for traffic calculator after reset + when(cloudStorage.list(anyString())).thenReturn(Arrays.asList("sqs-delta/delta/optout-delta-001_2025-01-01T00.00.00Z_aaaaaaaa.dat")); + when(cloudStorage.download(MANUAL_OVERRIDE_S3_PATH)).thenReturn(null); + when(cloudStorage.download(argThat(path -> path != null && path.contains("optout-delta")))) + .thenAnswer(inv -> new ByteArrayInputStream(createMinimalDeltaFileBytes())); + // Act & Assert - second request - should now be denylisted vertx.createHttpClient() .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1", @@ -966,9 +980,7 @@ public void testTrafficCalculator_defaultStatus(TestContext context) throws Exce """; createTrafficCalcConfigFile(trafficCalcConfig); - // Setup - no manual override - when(cloudStorage.download(anyString())) - .thenReturn(null); + // Note: manual override is already mocked to return null in setup // Setup - create messages that will result in DEFAULT status long oldTime = System.currentTimeMillis() - 400_000; @@ -1190,9 +1202,7 @@ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext contex public void testManualOverride_notSet(TestContext context) throws Exception { Async async = context.async(); - // Setup - mock no manual override file - when(cloudStorage.download(anyString())) - .thenReturn(null); + // Note: manual override is already mocked to return null in setup // Setup - create messages long oldTime = System.currentTimeMillis() - 400_000; @@ -1445,5 +1455,19 @@ private byte[] createDeltaFileBytes(List timestamps) throws Exception { OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0])); return collection.getStore(); } + + /** + * Create minimal delta file bytes with a single recent timestamp for traffic calculator + */ + private static byte[] createMinimalDeltaFileBytes() { + try { + long timestamp = System.currentTimeMillis() / 1000 - 3600; // 1 hour ago + OptOutEntry entry = OptOutEntry.newTestEntry(1, timestamp); + OptOutCollection collection = new OptOutCollection(new OptOutEntry[]{entry}); + return collection.getStore(); + } catch (Exception e) { + throw new RuntimeException("Failed to create minimal delta file bytes", e); + } + } } From e31e2b3677b76fc8fedbb2d7fb1bada36ed7f634 Mon Sep 17 00:00:00 2001 From: Ian-Nara Date: Tue, 9 Dec 2025 19:32:24 -0700 Subject: [PATCH 52/52] add todos --- src/main/java/com/uid2/optout/delta/S3UploadService.java | 2 +- src/main/java/com/uid2/optout/delta/StopReason.java | 2 +- src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java | 2 +- src/main/java/com/uid2/optout/sqs/SqsWindowReader.java | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/uid2/optout/delta/S3UploadService.java b/src/main/java/com/uid2/optout/delta/S3UploadService.java index 9bdbcc74..bae03520 100644 --- a/src/main/java/com/uid2/optout/delta/S3UploadService.java +++ b/src/main/java/com/uid2/optout/delta/S3UploadService.java @@ -71,7 +71,7 @@ public void uploadAndDeleteMessages(byte[] data, String s3Path, List me if (onSuccess != null) { onSuccess.onSuccess(messages.size()); } - } catch (Exception e) { + } catch (Exception e) { // TODO: catch specific exceptions LOGGER.error("s3_error: failed to upload delta or dropped requests to path={}", s3Path, e); throw new IOException("s3 upload failed: " + s3Path, e); } diff --git a/src/main/java/com/uid2/optout/delta/StopReason.java b/src/main/java/com/uid2/optout/delta/StopReason.java index 0f61d7d5..4473228e 100644 --- a/src/main/java/com/uid2/optout/delta/StopReason.java +++ b/src/main/java/com/uid2/optout/delta/StopReason.java @@ -27,7 +27,7 @@ public enum StopReason { MESSAGE_LIMIT_EXCEEDED, /** - * Pre-existing manual override was set (checked at job start). + * Pre-existing manual override was set to DELAYED_PROCESSING (checked at job start). */ MANUAL_OVERRIDE_ACTIVE, diff --git a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java index d77f561a..41373d0f 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java +++ b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java @@ -83,7 +83,7 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN List invalidMessages = identifyInvalidMessages(messageBatch, parsedBatch); if (!invalidMessages.isEmpty()) { LOGGER.error("sqs_error: found {} invalid messages in batch {}, deleting", invalidMessages.size(), batchNumber); - SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages); + SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages); // TODO: send to a folder in the dropped requests bucket before deleting. } } diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java index f5f74d87..eff5bf29 100644 --- a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java +++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java @@ -29,10 +29,10 @@ public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerP int visibilityTimeout, int deltaWindowSeconds, int maxMessagesPerWindow) { this.sqsClient = sqsClient; this.queueUrl = queueUrl; - this.maxMessagesPerPoll = maxMessagesPerPoll; - this.visibilityTimeout = visibilityTimeout; + this.maxMessagesPerPoll = maxMessagesPerPoll; // 10 max + this.visibilityTimeout = visibilityTimeout; // TODO: ensure we can process all messages before visibility timeout this.deltaWindowSeconds = deltaWindowSeconds; - this.maxMessagesPerWindow = maxMessagesPerWindow; + this.maxMessagesPerWindow = maxMessagesPerWindow; // TODO: ensure we can process all messages before visibility timeout this.batchProcessor = new SqsBatchProcessor(sqsClient, queueUrl, deltaWindowSeconds); LOGGER.info("initialized: maxMessagesPerWindow={}, maxMessagesPerPoll={}, visibilityTimeout={}, deltaWindowSeconds={}", maxMessagesPerWindow, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds); @@ -94,7 +94,7 @@ public WindowReadResult readWindow() { int batchNumber = 0; int rawMessagesRead = 0; // track total messages pulled from SQS - while (true) { + while (true) { // TODO: add a timeout to the loop if (windowMessages.size() >= maxMessagesPerWindow) { LOGGER.warn("high_message_volume: message limit exceeded while reading window, {} messages >= limit {}", windowMessages.size(), maxMessagesPerWindow); return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart, rawMessagesRead);