messages) throws IOException {
+ byte[] droppedRequestData = droppedRequestStream.encode().getBytes();
+
+ droppedRequestUploadService.uploadAndDeleteMessages(droppedRequestData, droppedRequestName, messages,
+ metrics::recordDroppedRequestsProduced);
+ }
+
+ /**
+ * Writes a dropped request entry to the JSON array.
+ */
+ private void writeDroppedRequestEntry(JsonArray droppedRequestArray, SqsParsedMessage parsed) {
+ String messageBody = parsed.getOriginalMessage().body();
+ JsonObject messageJson = new JsonObject(messageBody);
+ droppedRequestArray.add(messageJson);
+ }
+
+ /**
+ * Generates a unique filename for dropped requests.
+ */
+ private String generateDroppedRequestFileName() {
+ return String.format("%s%03d_%s_%08x.json",
+ "optout-dropped-",
+ replicaId,
+ Instant.now().truncatedTo(ChronoUnit.SECONDS).toString().replace(':', '.'),
+ OptOutUtils.rand.nextInt());
+ }
+
+ /**
+ * Checks if the job has exceeded its timeout.
+ */
+ private boolean isJobTimedOut(long jobStartTime) {
+ long elapsedTime = OptOutUtils.nowEpochSeconds() - jobStartTime;
+
+ if (elapsedTime > 3600) { // 1 hour - log warning
+ LOGGER.error("delta_job_timeout: job has been running for {} seconds", elapsedTime);
+ }
+
+ if (elapsedTime > this.jobTimeoutSeconds) {
+ LOGGER.error("delta_job_timeout: job exceeded timeout, running for {} seconds (timeout: {}s)",
+ elapsedTime, this.jobTimeoutSeconds);
+ return true;
+ }
+ return false;
+ }
+}
+
diff --git a/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java
new file mode 100644
index 00000000..d88ef128
--- /dev/null
+++ b/src/main/java/com/uid2/optout/delta/DeltaProductionResult.java
@@ -0,0 +1,139 @@
+package com.uid2.optout.delta;
+
+import io.vertx.core.json.JsonObject;
+
+/**
+ * Immutable result containing statistics from a delta production job.
+ *
+ * This class holds production counts and the stop reason, with methods for JSON serialization.
+ * Use {@link Builder} to accumulate statistics during production, then call {@link Builder#build()}
+ * to create the immutable result.
+ *
+ * Note: Job duration is tracked by {@link DeltaProductionJobStatus}, not this class.
+ */
+public class DeltaProductionResult {
+ private final int deltasProduced;
+ private final int entriesProcessed;
+ private final int droppedRequestFilesProduced;
+ private final int droppedRequestsProcessed;
+ private final StopReason stopReason;
+
+ /**
+ * Private constructor. Use {@link #builder()} to create instances.
+ */
+ private DeltaProductionResult(int deltasProduced, int entriesProcessed,
+ int droppedRequestFilesProduced, int droppedRequestsProcessed,
+ StopReason stopReason) {
+ this.deltasProduced = deltasProduced;
+ this.entriesProcessed = entriesProcessed;
+ this.droppedRequestFilesProduced = droppedRequestFilesProduced;
+ this.droppedRequestsProcessed = droppedRequestsProcessed;
+ this.stopReason = stopReason;
+ }
+
+ /**
+ * Creates a new Builder for accumulating production statistics.
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ // ==================== Getters ====================
+
+ public int getDeltasProduced() {
+ return deltasProduced;
+ }
+
+ public int getEntriesProcessed() {
+ return entriesProcessed;
+ }
+
+ public int getDroppedRequestFilesProduced() {
+ return droppedRequestFilesProduced;
+ }
+
+ public int getDroppedRequestsProcessed() {
+ return droppedRequestsProcessed;
+ }
+
+ public StopReason getStopReason() {
+ return stopReason;
+ }
+
+ // ==================== JSON Serialization ====================
+
+ public JsonObject toJson() {
+ return new JsonObject()
+ .put("deltas_produced", deltasProduced)
+ .put("entries_processed", entriesProcessed)
+ .put("dropped_request_files_produced", droppedRequestFilesProduced)
+ .put("dropped_requests_processed", droppedRequestsProcessed)
+ .put("stop_reason", stopReason.name());
+ }
+
+ public JsonObject toJsonWithStatus(String status) {
+ return toJson().put("status", status);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "DeltaProductionResult{deltasProduced=%d, entriesProcessed=%d, " +
+ "droppedRequestFilesProduced=%d, droppedRequestsProcessed=%d, stopReason=%s}",
+ deltasProduced, entriesProcessed, droppedRequestFilesProduced,
+ droppedRequestsProcessed, stopReason);
+ }
+
+ // ==================== Builder ====================
+
+ /**
+ * Mutable builder for accumulating production statistics.
+ *
+ * Use this builder to track stats during delta production jobs,
+ * then call {@link #build()} to create the immutable result.
+ */
+ public static class Builder {
+ private int deltasProduced;
+ private int entriesProcessed;
+ private int droppedRequestFilesProduced;
+ private int droppedRequestsProcessed;
+ private StopReason stopReason = StopReason.NONE;
+
+ public Builder incrementDeltasProduced() {
+ deltasProduced++;
+ return this;
+ }
+
+ public Builder incrementEntriesProcessed(int count) {
+ entriesProcessed += count;
+ return this;
+ }
+
+ public Builder incrementDroppedRequestFilesProduced() {
+ droppedRequestFilesProduced++;
+ return this;
+ }
+
+ public Builder incrementDroppedRequestsProcessed(int count) {
+ droppedRequestsProcessed += count;
+ return this;
+ }
+
+ public Builder stopReason(StopReason reason) {
+ this.stopReason = reason;
+ return this;
+ }
+
+ /**
+ * Builds the DeltaProductionResult with the accumulated statistics.
+ */
+ public DeltaProductionResult build() {
+ return new DeltaProductionResult(
+ deltasProduced,
+ entriesProcessed,
+ droppedRequestFilesProduced,
+ droppedRequestsProcessed,
+ stopReason);
+ }
+ }
+}
diff --git a/src/main/java/com/uid2/optout/delta/ManualOverrideService.java b/src/main/java/com/uid2/optout/delta/ManualOverrideService.java
new file mode 100644
index 00000000..db39e5e3
--- /dev/null
+++ b/src/main/java/com/uid2/optout/delta/ManualOverrideService.java
@@ -0,0 +1,84 @@
+package com.uid2.optout.delta;
+
+import com.uid2.shared.Utils;
+import com.uid2.shared.cloud.ICloudStorage;
+import io.vertx.core.json.JsonObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+
+/**
+ * Service for managing manual override status in S3.
+ *
+ * The manual override allows operators to force DELAYED_PROCESSING status,
+ * which stops delta production until manually cleared.
+ *
+ * Override file format:
+ *
+ * {"manual_override": "DELAYED_PROCESSING"}
+ *
+ */
+public class ManualOverrideService {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ManualOverrideService.class);
+
+ private static final String OVERRIDE_KEY = "manual_override";
+ private static final String DELAYED_PROCESSING_VALUE = "DELAYED_PROCESSING";
+
+ private final ICloudStorage cloudStorage;
+ private final String overrideS3Path;
+
+ /**
+ * Create a ManualOverrideService.
+ *
+ * @param cloudStorage Cloud storage client for reading/writing override file
+ * @param overrideS3Path S3 path where the override file is stored
+ */
+ public ManualOverrideService(ICloudStorage cloudStorage, String overrideS3Path) {
+ this.cloudStorage = cloudStorage;
+ this.overrideS3Path = overrideS3Path;
+ }
+
+ /**
+ * Check if DELAYED_PROCESSING override is currently set.
+ *
+ * @return true if manual override is set to DELAYED_PROCESSING
+ */
+ public boolean isDelayedProcessing() {
+ return DELAYED_PROCESSING_VALUE.equals(getOverrideValue());
+ }
+
+ /**
+ * Set the manual override to DELAYED_PROCESSING.
+ * This will stop delta production until manually cleared.
+ *
+ * @return true if override was set successfully
+ */
+ public boolean setDelayedProcessing() {
+ try {
+ JsonObject config = new JsonObject().put(OVERRIDE_KEY, DELAYED_PROCESSING_VALUE);
+ cloudStorage.upload(new ByteArrayInputStream(config.encode().getBytes()), overrideS3Path);
+ LOGGER.info("set manual override to DELAYED_PROCESSING: {}", overrideS3Path);
+ return true;
+ } catch (Exception e) {
+ LOGGER.error("manual_override_error: failed to set override at {}", overrideS3Path, e);
+ return false;
+ }
+ }
+
+ /**
+ * Get the current manual override value
+ */
+ private String getOverrideValue() {
+ try {
+ InputStream inputStream = cloudStorage.download(overrideS3Path);
+ JsonObject configJson = Utils.toJsonObject(inputStream);
+ return configJson.getString(OVERRIDE_KEY, "");
+ } catch (Exception e) {
+ LOGGER.error("manual_override_error: no manual override file found at {}", overrideS3Path);
+ return "";
+ }
+ }
+}
+
diff --git a/src/main/java/com/uid2/optout/delta/S3UploadService.java b/src/main/java/com/uid2/optout/delta/S3UploadService.java
new file mode 100644
index 00000000..bae03520
--- /dev/null
+++ b/src/main/java/com/uid2/optout/delta/S3UploadService.java
@@ -0,0 +1,83 @@
+package com.uid2.optout.delta;
+
+import com.uid2.optout.sqs.SqsMessageOperations;
+import com.uid2.shared.cloud.ICloudStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.sqs.SqsClient;
+import software.amazon.awssdk.services.sqs.model.Message;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Service for uploading data to S3 and deleting messages from SQS after successful upload.
+ *
+ * This class encapsulates the critical "upload then delete" pattern that ensures
+ * data is persisted to S3 before messages are removed from the queue.
+ */
+public class S3UploadService {
+ private static final Logger LOGGER = LoggerFactory.getLogger(S3UploadService.class);
+
+ private final ICloudStorage cloudStorage;
+ private final SqsClient sqsClient;
+ private final String queueUrl;
+
+ /**
+ * Callback interface for after successful upload.
+ */
+ @FunctionalInterface
+ public interface UploadSuccessCallback {
+ /**
+ * Called after successful S3 upload, before SQS message deletion.
+ *
+ * @param messageCount Number of messages in the uploaded batch
+ */
+ void onSuccess(int messageCount);
+ }
+
+ /**
+ * Create an S3UploadService.
+ *
+ * @param cloudStorage Cloud storage client for S3 operations
+ * @param sqsClient SQS client for message deletion
+ * @param queueUrl SQS queue URL
+ */
+ public S3UploadService(ICloudStorage cloudStorage, SqsClient sqsClient, String queueUrl) {
+ this.cloudStorage = cloudStorage;
+ this.sqsClient = sqsClient;
+ this.queueUrl = queueUrl;
+ }
+
+ /**
+ * Upload data to S3 and delete associated messages from SQS after successful upload.
+ *
+ * Critical behavior: Messages are ONLY deleted from SQS after
+ * the S3 upload succeeds. This ensures no data loss if upload fails.
+ *
+ * @param data Data to upload
+ * @param s3Path S3 path (key) for the upload
+ * @param messages SQS messages to delete after successful upload
+ * @param onSuccess Callback invoked after successful upload (before message deletion)
+ * @throws IOException if upload fails
+ */
+ public void uploadAndDeleteMessages(byte[] data, String s3Path, List messages, UploadSuccessCallback onSuccess) throws IOException {
+ LOGGER.info("uploading to s3: path={}, size={} bytes, messages={}", s3Path, data.length, messages.size());
+
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(data)) {
+ cloudStorage.upload(inputStream, s3Path);
+
+ if (onSuccess != null) {
+ onSuccess.onSuccess(messages.size());
+ }
+ } catch (Exception e) { // TODO: catch specific exceptions
+ LOGGER.error("s3_error: failed to upload delta or dropped requests to path={}", s3Path, e);
+ throw new IOException("s3 upload failed: " + s3Path, e);
+ }
+
+ if (!messages.isEmpty()) {
+ SqsMessageOperations.deleteMessagesFromSqs(sqsClient, queueUrl, messages);
+ }
+ }
+}
diff --git a/src/main/java/com/uid2/optout/delta/StopReason.java b/src/main/java/com/uid2/optout/delta/StopReason.java
new file mode 100644
index 00000000..4473228e
--- /dev/null
+++ b/src/main/java/com/uid2/optout/delta/StopReason.java
@@ -0,0 +1,39 @@
+package com.uid2.optout.delta;
+
+/**
+ * Represents why delta production stopped.
+ * Used across all layers (batch, window, orchestrator) for consistent stop reason tracking.
+ */
+public enum StopReason {
+
+ /**
+ * Processing completed normally with work done, or still in progress.
+ */
+ NONE,
+
+ /**
+ * No messages available in the SQS queue.
+ */
+ QUEUE_EMPTY,
+
+ /**
+ * Messages exist in the queue but are too recent (less than deltaWindowSeconds old).
+ */
+ MESSAGES_TOO_RECENT,
+
+ /**
+ * Hit the maximum messages per window limit.
+ */
+ MESSAGE_LIMIT_EXCEEDED,
+
+ /**
+ * Pre-existing manual override was set to DELAYED_PROCESSING (checked at job start).
+ */
+ MANUAL_OVERRIDE_ACTIVE,
+
+ /**
+ * Circuit breaker triggered during processing (traffic spike detected).
+ */
+ CIRCUIT_BREAKER_TRIGGERED
+}
+
diff --git a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java
similarity index 51%
rename from src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java
rename to src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java
index 0a656e98..41373d0f 100644
--- a/src/main/java/com/uid2/optout/vertx/SqsBatchProcessor.java
+++ b/src/main/java/com/uid2/optout/sqs/SqsBatchProcessor.java
@@ -1,15 +1,15 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.sqs;
+import com.uid2.optout.delta.StopReason;
import com.uid2.shared.optout.OptOutUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.sqs.SqsClient;
import software.amazon.awssdk.services.sqs.model.Message;
-import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Applies parsing, validation, filtering, and deletion of corrupted SQS messages.
@@ -29,46 +29,46 @@ public SqsBatchProcessor(SqsClient sqsClient, String queueUrl, int deltaWindowSe
}
/**
- * Result of processing a batch of messages from SQS.
- * Encapsulates eligible messages and metadata about the processing.
+ * Result of processing a batch of (10) messages from SQS.
+ * Encapsulates eligible messages and the reason for stopping (if any).
*/
public static class BatchProcessingResult {
private final List eligibleMessages;
- private final boolean shouldStopProcessing;
+ private final StopReason stopReason;
- private BatchProcessingResult(List eligibleMessages, boolean shouldStopProcessing) {
+ private BatchProcessingResult(List eligibleMessages, StopReason stopReason) {
this.eligibleMessages = eligibleMessages;
- this.shouldStopProcessing = shouldStopProcessing;
+ this.stopReason = stopReason;
}
- public static BatchProcessingResult withEligibleMessages(List messages) {
- return new BatchProcessingResult(messages, false);
+ public static BatchProcessingResult withMessages(List messages) {
+ return new BatchProcessingResult(messages, StopReason.NONE);
}
- public static BatchProcessingResult stopProcessing() {
- return new BatchProcessingResult(new ArrayList<>(), true);
+ public static BatchProcessingResult messagesTooRecent() {
+ return new BatchProcessingResult(List.of(), StopReason.MESSAGES_TOO_RECENT);
}
- public static BatchProcessingResult empty() {
- return new BatchProcessingResult(new ArrayList<>(), false);
+ public static BatchProcessingResult corruptMessagesDeleted() {
+ return new BatchProcessingResult(List.of(), StopReason.NONE);
}
- public boolean isEmpty() {
- return eligibleMessages.isEmpty();
+ public boolean hasMessages() {
+ return !eligibleMessages.isEmpty();
}
- public boolean shouldStopProcessing() {
- return shouldStopProcessing;
+ public StopReason getStopReason() {
+ return stopReason;
}
- public List getEligibleMessages() {
+ public List getMessages() {
return eligibleMessages;
}
}
/**
* Processes a batch of messages: parses, validates, cleans up invalid messages,
- * and filters for eligible messages based on age threshold (message is less than 5 minutes old)
+ * and filters for eligible messages based on age threshold (message is older than deltaWindowSeconds)
*
* @param messageBatch Raw messages from SQS
* @param batchNumber The batch number (for logging)
@@ -82,59 +82,53 @@ public BatchProcessingResult processBatch(List messageBatch, int batchN
if (parsedBatch.size() < messageBatch.size()) {
List invalidMessages = identifyInvalidMessages(messageBatch, parsedBatch);
if (!invalidMessages.isEmpty()) {
- LOGGER.error("Found {} invalid messages in batch {} (failed parsing). Deleting from queue.",
- invalidMessages.size(), batchNumber);
- SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages);
+ LOGGER.error("sqs_error: found {} invalid messages in batch {}, deleting", invalidMessages.size(), batchNumber);
+ SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, invalidMessages); // TODO: send to a folder in the dropped requests bucket before deleting.
}
}
- // If no valid messages, return empty result
+ // No valid messages after deleting corrupt ones, continue reading
if (parsedBatch.isEmpty()) {
- LOGGER.warn("No valid messages in batch {} (all failed parsing)", batchNumber);
- return BatchProcessingResult.empty();
+ LOGGER.info("no valid messages in batch {} after removing invalid messages", batchNumber);
+ return BatchProcessingResult.corruptMessagesDeleted();
}
// Check if the oldest message in this batch is too recent
long currentTime = OptOutUtils.nowEpochSeconds();
SqsParsedMessage oldestMessage = parsedBatch.get(0);
- long messageAge = currentTime - oldestMessage.getTimestamp();
- if (messageAge < this.deltaWindowSeconds) {
- // Signal to stop processing - messages are too recent
- return BatchProcessingResult.stopProcessing();
+ if (!isMessageEligible(oldestMessage, currentTime)) {
+ return BatchProcessingResult.messagesTooRecent();
}
- // Filter for eligible messages (>= 5 minutes old)
+ // Filter for eligible messages (>= deltaWindowSeconds old)
List eligibleMessages = filterEligibleMessages(parsedBatch, currentTime);
- if (eligibleMessages.isEmpty()) {
- LOGGER.debug("No eligible messages in batch {} (all too recent)", batchNumber);
- return BatchProcessingResult.empty();
- }
+ return BatchProcessingResult.withMessages(eligibleMessages);
+ }
- return BatchProcessingResult.withEligibleMessages(eligibleMessages);
+ /**
+ * Checks if a message is old enough to be processed.
+ *
+ * @param message The parsed message to check
+ * @param currentTime Current time in epoch seconds
+ * @return true if the message is at least deltaWindowSeconds old
+ */
+ private boolean isMessageEligible(SqsParsedMessage message, long currentTime) {
+ return currentTime - message.getTimestamp() >= this.deltaWindowSeconds;
}
/**
* Filters messages to only include those where sufficient time has elapsed.
-. *
+ *
* @param messages List of parsed messages
* @param currentTime Current time in seconds
* @return List of messages that meet the time threshold
*/
- public List filterEligibleMessages(
- List messages,
- long currentTime) {
-
- List eligibleMessages = new ArrayList<>();
-
- for (SqsParsedMessage pm : messages) {
- if (currentTime - pm.getTimestamp() >= this.deltaWindowSeconds) {
- eligibleMessages.add(pm);
- }
- }
-
- return eligibleMessages;
+ List filterEligibleMessages(List messages, long currentTime) {
+ return messages.stream()
+ .filter(msg -> isMessageEligible(msg, currentTime))
+ .collect(Collectors.toList());
}
/**
@@ -145,21 +139,12 @@ public List filterEligibleMessages(
* @return List of messages that failed to parse
*/
private List identifyInvalidMessages(List originalBatch, List parsedBatch) {
- // Create a set of message IDs from successfully parsed messages
- Set validMessageIds = new HashSet<>();
- for (SqsParsedMessage parsed : parsedBatch) {
- validMessageIds.add(parsed.getOriginalMessage().messageId());
- }
+ Set validIds = parsedBatch.stream()
+ .map(p -> p.getOriginalMessage().messageId())
+ .collect(Collectors.toSet());
- // Find messages that were not successfully parsed
- List invalidMessages = new ArrayList<>();
- for (Message msg : originalBatch) {
- if (!validMessageIds.contains(msg.messageId())) {
- invalidMessages.add(msg);
- }
- }
-
- return invalidMessages;
+ return originalBatch.stream()
+ .filter(msg -> !validIds.contains(msg.messageId()))
+ .collect(Collectors.toList());
}
}
-
diff --git a/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java
new file mode 100644
index 00000000..a25806bd
--- /dev/null
+++ b/src/main/java/com/uid2/optout/sqs/SqsMessageOperations.java
@@ -0,0 +1,193 @@
+package com.uid2.optout.sqs;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.sqs.SqsClient;
+import software.amazon.awssdk.services.sqs.model.*;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility class for SQS message operations.
+ */
+public class SqsMessageOperations {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SqsMessageOperations.class);
+ private static final int SQS_MAX_DELETE_BATCH_SIZE = 10;
+
+ /**
+ * Result of getting queue attributes from SQS.
+ */
+ public static class QueueAttributes {
+ private final int approximateNumberOfMessages;
+ private final int approximateNumberOfMessagesNotVisible;
+ private final int approximateNumberOfMessagesDelayed;
+
+ public QueueAttributes(int approximateNumberOfMessages,
+ int approximateNumberOfMessagesNotVisible,
+ int approximateNumberOfMessagesDelayed) {
+ this.approximateNumberOfMessages = approximateNumberOfMessages;
+ this.approximateNumberOfMessagesNotVisible = approximateNumberOfMessagesNotVisible;
+ this.approximateNumberOfMessagesDelayed = approximateNumberOfMessagesDelayed;
+ }
+
+ /** Number of messages available for retrieval from the queue (visible messages) */
+ public int getApproximateNumberOfMessages() {
+ return approximateNumberOfMessages;
+ }
+
+ /** Number of messages that are in flight (being processed by consumers, invisible) */
+ public int getApproximateNumberOfMessagesNotVisible() {
+ return approximateNumberOfMessagesNotVisible;
+ }
+
+ /** Number of messages in the queue that are delayed and not available yet */
+ public int getApproximateNumberOfMessagesDelayed() {
+ return approximateNumberOfMessagesDelayed;
+ }
+
+ /** Total messages in queue = visible + invisible + delayed */
+ public int getTotalMessages() {
+ return approximateNumberOfMessages + approximateNumberOfMessagesNotVisible + approximateNumberOfMessagesDelayed;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("QueueAttributes{visible=%d, invisible=%d, delayed=%d, total=%d}",
+ approximateNumberOfMessages, approximateNumberOfMessagesNotVisible,
+ approximateNumberOfMessagesDelayed, getTotalMessages());
+ }
+ }
+
+ /**
+ * Gets queue attributes from SQS including message counts.
+ *
+ * @param sqsClient The SQS client
+ * @param queueUrl The queue URL
+ * @return QueueAttributes with message counts, or null if failed
+ */
+ public static QueueAttributes getQueueAttributes(SqsClient sqsClient, String queueUrl) {
+ try {
+ GetQueueAttributesRequest request = GetQueueAttributesRequest.builder()
+ .queueUrl(queueUrl)
+ .attributeNames(
+ QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES,
+ QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE,
+ QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED
+ )
+ .build();
+
+ GetQueueAttributesResponse response = sqsClient.getQueueAttributes(request);
+ Map attrs = response.attributes();
+
+ int visible = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES), 0);
+ int invisible = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE), 0);
+ int delayed = parseIntOrDefault(attrs.get(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED), 0);
+
+ QueueAttributes queueAttributes = new QueueAttributes(visible, invisible, delayed);
+ LOGGER.info("queue attributes: {}", queueAttributes);
+ return queueAttributes;
+
+ } catch (Exception e) {
+ LOGGER.info("error getting queue attributes", e);
+ return null;
+ }
+ }
+
+ private static int parseIntOrDefault(String value, int defaultValue) {
+ if (value == null) {
+ return defaultValue;
+ }
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ return defaultValue;
+ }
+ }
+
+ /**
+ * Receives a batch of messages from SQS.
+ *
+ * @param sqsClient The SQS client
+ * @param queueUrl The queue URL
+ * @param maxMessages Maximum number of messages to receive (max 10)
+ * @param visibilityTimeout Visibility timeout in seconds
+ * @return List of received messages
+ */
+ public static List receiveMessagesFromSqs(
+ SqsClient sqsClient,
+ String queueUrl,
+ int maxMessages,
+ int visibilityTimeout) {
+
+ try {
+ ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder()
+ .queueUrl(queueUrl)
+ .maxNumberOfMessages(maxMessages)
+ .visibilityTimeout(visibilityTimeout)
+ .waitTimeSeconds(0) // Non-blocking poll
+ .messageSystemAttributeNames(MessageSystemAttributeName.SENT_TIMESTAMP) // Request SQS system timestamp
+ .build();
+
+ ReceiveMessageResponse response = sqsClient.receiveMessage(receiveRequest);
+
+ LOGGER.info("received {} messages", response.messages().size());
+ return response.messages();
+
+ } catch (Exception e) {
+ LOGGER.error("sqs_error: failed to receive messages", e);
+ return new ArrayList<>();
+ }
+ }
+
+ /**
+ * Deletes messages from SQS in batches (max 10 per batch).
+ *
+ * @param sqsClient The SQS client
+ * @param queueUrl The queue URL
+ * @param messages Messages to delete
+ */
+ public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, List messages) {
+ if (messages.isEmpty()) {
+ return;
+ }
+
+ try {
+ List entries = new ArrayList<>();
+ int batchId = 0;
+ int totalDeleted = 0;
+
+ for (Message msg : messages) {
+ entries.add(DeleteMessageBatchRequestEntry.builder()
+ .id(String.valueOf(batchId++))
+ .receiptHandle(msg.receiptHandle())
+ .build());
+
+ // Send batch when we reach 10 messages or at the end
+ if (entries.size() == SQS_MAX_DELETE_BATCH_SIZE || batchId == messages.size()) {
+ DeleteMessageBatchRequest deleteRequest = DeleteMessageBatchRequest.builder()
+ .queueUrl(queueUrl)
+ .entries(entries)
+ .build();
+
+ DeleteMessageBatchResponse deleteResponse = sqsClient.deleteMessageBatch(deleteRequest);
+
+ if (!deleteResponse.failed().isEmpty()) {
+ LOGGER.error("sqs_error: failed to delete {} messages", deleteResponse.failed().size());
+ } else {
+ totalDeleted += entries.size();
+ }
+
+ entries.clear();
+ }
+ }
+
+ LOGGER.info("deleted {} messages", totalDeleted);
+
+ } catch (Exception e) {
+ LOGGER.error("sqs_error: exception during message deletion", e);
+ }
+ }
+}
+
diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java
similarity index 89%
rename from src/main/java/com/uid2/optout/vertx/SqsMessageParser.java
rename to src/main/java/com/uid2/optout/sqs/SqsMessageParser.java
index 44a6c5e9..315ff611 100644
--- a/src/main/java/com/uid2/optout/vertx/SqsMessageParser.java
+++ b/src/main/java/com/uid2/optout/sqs/SqsMessageParser.java
@@ -1,4 +1,4 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.sqs;
import com.uid2.shared.optout.OptOutUtils;
import io.vertx.core.json.JsonObject;
@@ -40,7 +40,7 @@ public static List parseAndSortMessages(List messages
String phone = body.getString("phone");
if (identityHash == null || advertisingId == null) {
- LOGGER.error("Invalid message format, skipping: {}", message.body());
+ LOGGER.error("sqs_error: invalid message format: {}", message.body());
continue;
}
@@ -48,13 +48,13 @@ public static List parseAndSortMessages(List messages
byte[] idBytes = OptOutUtils.base64StringTobyteArray(advertisingId);
if (hashBytes == null || idBytes == null) {
- LOGGER.error("Invalid base64 encoding, skipping message");
+ LOGGER.error("sqs_error: invalid base64 encoding");
continue;
}
parsedMessages.add(new SqsParsedMessage(message, hashBytes, idBytes, timestampSeconds, email, phone, clientIp, traceId));
} catch (Exception e) {
- LOGGER.error("Error parsing SQS message", e);
+ LOGGER.error("sqs_error: error parsing message", e);
}
}
@@ -73,7 +73,7 @@ public static List parseAndSortMessages(List messages
private static long extractTimestamp(Message message) {
String sentTimestampStr = message.attributes().get(MessageSystemAttributeName.SENT_TIMESTAMP);
if (sentTimestampStr == null) {
- LOGGER.warn("Message missing SentTimestamp attribute, using current time");
+ LOGGER.info("message missing SentTimestamp, using current time");
return OptOutUtils.nowEpochSeconds();
}
return Long.parseLong(sentTimestampStr) / 1000; // ms to seconds
diff --git a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java b/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java
similarity index 97%
rename from src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java
rename to src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java
index 1ad8ba77..850cd7a1 100644
--- a/src/main/java/com/uid2/optout/vertx/SqsParsedMessage.java
+++ b/src/main/java/com/uid2/optout/sqs/SqsParsedMessage.java
@@ -1,4 +1,4 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.sqs;
import software.amazon.awssdk.services.sqs.model.Message;
diff --git a/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java
new file mode 100644
index 00000000..eff5bf29
--- /dev/null
+++ b/src/main/java/com/uid2/optout/sqs/SqsWindowReader.java
@@ -0,0 +1,147 @@
+package com.uid2.optout.sqs;
+
+import com.uid2.optout.delta.StopReason;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.services.sqs.SqsClient;
+import software.amazon.awssdk.services.sqs.model.Message;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Reads messages from SQS for complete 5-minute time windows.
+ * Handles accumulation of all messages for a window before returning.
+ * Limits messages per window to prevent memory issues.
+ */
+public class SqsWindowReader {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SqsWindowReader.class);
+
+ private final SqsClient sqsClient;
+ private final String queueUrl;
+ private final int maxMessagesPerPoll;
+ private final int visibilityTimeout;
+ private final int deltaWindowSeconds;
+ private final SqsBatchProcessor batchProcessor;
+ private int maxMessagesPerWindow;
+
+ public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerPoll,
+ int visibilityTimeout, int deltaWindowSeconds, int maxMessagesPerWindow) {
+ this.sqsClient = sqsClient;
+ this.queueUrl = queueUrl;
+ this.maxMessagesPerPoll = maxMessagesPerPoll; // 10 max
+ this.visibilityTimeout = visibilityTimeout; // TODO: ensure we can process all messages before visibility timeout
+ this.deltaWindowSeconds = deltaWindowSeconds;
+ this.maxMessagesPerWindow = maxMessagesPerWindow; // TODO: ensure we can process all messages before visibility timeout
+ this.batchProcessor = new SqsBatchProcessor(sqsClient, queueUrl, deltaWindowSeconds);
+ LOGGER.info("initialized: maxMessagesPerWindow={}, maxMessagesPerPoll={}, visibilityTimeout={}, deltaWindowSeconds={}",
+ maxMessagesPerWindow, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds);
+ }
+
+ /**
+ * Result of reading messages for a 5-minute window.
+ */
+ public static class WindowReadResult {
+ private final List messages;
+ private final long windowStart;
+ private final StopReason stopReason;
+ private final int rawMessagesRead; // total messages pulled from SQS
+
+ private WindowReadResult(List messages, long windowStart, StopReason stopReason, int rawMessagesRead) {
+ this.messages = messages;
+ this.windowStart = windowStart;
+ this.stopReason = stopReason;
+ this.rawMessagesRead = rawMessagesRead;
+ }
+
+ public static WindowReadResult withMessages(List messages, long windowStart, int rawMessagesRead) {
+ return new WindowReadResult(messages, windowStart, StopReason.NONE, rawMessagesRead);
+ }
+
+ public static WindowReadResult queueEmpty(List messages, long windowStart, int rawMessagesRead) {
+ return new WindowReadResult(messages, windowStart, StopReason.QUEUE_EMPTY, rawMessagesRead);
+ }
+
+ public static WindowReadResult messagesTooRecent(List messages, long windowStart, int rawMessagesRead) {
+ return new WindowReadResult(messages, windowStart, StopReason.MESSAGES_TOO_RECENT, rawMessagesRead);
+ }
+
+ public static WindowReadResult messageLimitExceeded(List messages, long windowStart, int rawMessagesRead) {
+ return new WindowReadResult(messages, windowStart, StopReason.MESSAGE_LIMIT_EXCEEDED, rawMessagesRead);
+ }
+
+ public List getMessages() { return messages; }
+ public long getWindowStart() { return windowStart; }
+ public boolean isEmpty() { return messages.isEmpty(); }
+ public StopReason getStopReason() { return stopReason; }
+ /** Total raw messages pulled from SQS */
+ public int getRawMessagesRead() { return rawMessagesRead; }
+ }
+
+ /**
+ * Reads messages from SQS for one complete 5-minute window.
+ * Keeps reading batches and accumulating messages until:
+ * - We discover the next window
+ * - Queue is empty (no more messages)
+ * - Messages are too recent (all messages younger than deltaWindowSeconds)
+ * - Message count exceeds maxMessagesPerWindow
+ *
+ * @return WindowReadResult with messages for the window, or empty if done
+ */
+ public WindowReadResult readWindow() {
+ List windowMessages = new ArrayList<>();
+ long currentWindowStart = 0;
+ int batchNumber = 0;
+ int rawMessagesRead = 0; // track total messages pulled from SQS
+
+ while (true) { // TODO: add a timeout to the loop
+ if (windowMessages.size() >= maxMessagesPerWindow) {
+ LOGGER.warn("high_message_volume: message limit exceeded while reading window, {} messages >= limit {}", windowMessages.size(), maxMessagesPerWindow);
+ return WindowReadResult.messageLimitExceeded(windowMessages, currentWindowStart, rawMessagesRead);
+ }
+
+ // Read one batch from SQS (up to 10 messages)
+ List rawBatch = SqsMessageOperations.receiveMessagesFromSqs(
+ this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout);
+
+ if (rawBatch.isEmpty()) {
+ return WindowReadResult.queueEmpty(windowMessages, currentWindowStart, rawMessagesRead);
+ }
+
+ rawMessagesRead += rawBatch.size();
+
+ // parse, validate, filter
+ SqsBatchProcessor.BatchProcessingResult batchResult = batchProcessor.processBatch(rawBatch, batchNumber++);
+
+ if (!batchResult.hasMessages()) {
+ if (batchResult.getStopReason() == StopReason.MESSAGES_TOO_RECENT) {
+ return WindowReadResult.messagesTooRecent(windowMessages, currentWindowStart, rawMessagesRead);
+ }
+ // Corrupt messages were deleted, continue reading
+ continue;
+ }
+
+ // Add eligible messages to current window
+ boolean newWindow = false;
+ for (SqsParsedMessage msg : batchResult.getMessages()) {
+ long msgWindowStart = msg.getTimestamp();
+
+ // Discover start of window
+ if (currentWindowStart == 0) {
+ currentWindowStart = msgWindowStart;
+ }
+
+ // Discover next window
+ if (msgWindowStart > currentWindowStart + this.deltaWindowSeconds) {
+ newWindow = true;
+ }
+
+ windowMessages.add(msg);
+ }
+
+ if (newWindow) {
+ return WindowReadResult.withMessages(windowMessages, currentWindowStart, rawMessagesRead);
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java
similarity index 68%
rename from src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java
rename to src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java
index f0c7a7c5..ab6a2c00 100644
--- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficCalculator.java
+++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficCalculator.java
@@ -1,10 +1,12 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.traffic;
import com.uid2.shared.cloud.ICloudStorage;
import com.uid2.shared.optout.OptOutCollection;
import com.uid2.shared.optout.OptOutEntry;
import com.uid2.shared.optout.OptOutUtils;
import com.uid2.optout.Const;
+import com.uid2.optout.sqs.SqsMessageOperations;
+
import io.vertx.core.json.JsonObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,7 +89,7 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix,
this.trafficCalcConfigPath = trafficCalcConfigPath;
reloadTrafficCalcConfig(); // Load ConfigMap
- LOGGER.info("OptOutTrafficCalculator initialized: s3DeltaPrefix={}, threshold={}x",
+ LOGGER.info("initialized: s3DeltaPrefix={}, threshold={}x",
s3DeltaPrefix, thresholdMultiplier);
}
@@ -107,23 +109,23 @@ public OptOutTrafficCalculator(ICloudStorage cloudStorage, String s3DeltaPrefix,
* Can be called periodically to pick up config changes without restarting.
*/
public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException {
- LOGGER.info("Loading traffic calc config from ConfigMap");
+ LOGGER.info("loading traffic calc config from configmap");
try (InputStream is = Files.newInputStream(Paths.get(trafficCalcConfigPath))) {
String content = new String(is.readAllBytes(), StandardCharsets.UTF_8);
JsonObject trafficCalcConfig = new JsonObject(content);
// Validate required fields exist
if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp)) {
- throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_evaluation_window_seconds");
+ throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_evaluation_window_seconds");
}
if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcBaselineTrafficProp)) {
- throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_baseline_traffic");
+ throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_baseline_traffic");
}
if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcThresholdMultiplierProp)) {
- throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_threshold_multiplier");
+ throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_threshold_multiplier");
}
if (!trafficCalcConfig.containsKey(Const.Config.OptOutTrafficCalcAllowlistRangesProp)) {
- throw new MalformedTrafficCalcConfigException("Missing required field: traffic_calc_allowlist_ranges");
+ throw new MalformedTrafficCalcConfigException("missing required field: traffic_calc_allowlist_ranges");
}
this.evaluationWindowSeconds = trafficCalcConfig.getInteger(Const.Config.OptOutTrafficCalcEvaluationWindowSecondsProp);
@@ -133,15 +135,15 @@ public void reloadTrafficCalcConfig() throws MalformedTrafficCalcConfigException
List> ranges = parseAllowlistRanges(trafficCalcConfig);
this.allowlistRanges = ranges;
- LOGGER.info("Successfully loaded traffic calc config from ConfigMap: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}",
+ LOGGER.info("loaded traffic calc config: evaluationWindowSeconds={}, baselineTraffic={}, thresholdMultiplier={}, allowlistRanges={}",
this.evaluationWindowSeconds, this.baselineTraffic, this.thresholdMultiplier, ranges.size());
} catch (MalformedTrafficCalcConfigException e) {
- LOGGER.warn("Failed to load traffic calc config. Config is malformed: {}", trafficCalcConfigPath, e);
+ LOGGER.error("circuit_breaker_config_error: config is malformed, configPath={}", trafficCalcConfigPath, e);
throw e;
} catch (Exception e) {
- LOGGER.warn("Failed to load traffic calc config. Config is malformed or missing: {}", trafficCalcConfigPath, e);
- throw new MalformedTrafficCalcConfigException("Failed to load traffic calc config: " + e.getMessage());
+ LOGGER.error("circuit_breaker_config_error: config is malformed or missing, configPath={}", trafficCalcConfigPath, e);
+ throw new MalformedTrafficCalcConfigException("failed to load traffic calc config: " + e.getMessage());
}
}
@@ -161,18 +163,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic
long end = rangeArray.getLong(1);
if(start >= end) {
- LOGGER.error("Invalid allowlist range: start must be less than end: [{}, {}]", start, end);
- throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": start must be less than end");
+ LOGGER.error("circuit_breaker_config_error: allowlist range start must be less than end, range=[{}, {}]", start, end);
+ throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": start must be less than end");
}
if (end - start > 86400) {
- LOGGER.error("Invalid allowlist range: range must be less than 24 hours: [{}, {}]", start, end);
- throw new MalformedTrafficCalcConfigException("Invalid allowlist range at index " + i + ": range must be less than 24 hours");
+ LOGGER.error("circuit_breaker_config_error: allowlist range must be less than 24 hours, range=[{}, {}]", start, end);
+ throw new MalformedTrafficCalcConfigException("invalid allowlist range at index " + i + ": range must be less than 24 hours");
}
List range = Arrays.asList(start, end);
ranges.add(range);
- LOGGER.info("Loaded allowlist range: [{}, {}]", start, end);
+ LOGGER.info("loaded allowlist range: [{}, {}]", start, end);
}
}
}
@@ -184,18 +186,18 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic
long currentEnd = ranges.get(i).get(1);
long nextStart = ranges.get(i + 1).get(0);
if (currentEnd >= nextStart) {
- LOGGER.error("Overlapping allowlist ranges detected: [{}, {}] overlaps with [{}, {}]",
+ LOGGER.error("circuit_breaker_config_error: overlapping allowlist ranges, range=[{}, {}] overlaps with range=[{}, {}]",
ranges.get(i).get(0), currentEnd, nextStart, ranges.get(i + 1).get(1));
throw new MalformedTrafficCalcConfigException(
- "Overlapping allowlist ranges detected at indices " + i + " and " + (i + 1));
+ "overlapping allowlist ranges detected at indices " + i + " and " + (i + 1));
}
}
} catch (MalformedTrafficCalcConfigException e) {
throw e;
} catch (Exception e) {
- LOGGER.error("Failed to parse allowlist ranges", e);
- throw new MalformedTrafficCalcConfigException("Failed to parse allowlist ranges: " + e.getMessage());
+ LOGGER.error("circuit_breaker_config_error: failed to parse allowlist ranges", e);
+ throw new MalformedTrafficCalcConfigException("failed to parse allowlist ranges: " + e.getMessage());
}
return ranges;
@@ -207,27 +209,35 @@ List> parseAllowlistRanges(JsonObject config) throws MalformedTraffic
* Uses the newest delta file timestamp to anchor the 24-hour delta traffic window,
* and the oldest queue timestamp to anchor the 5-minute queue window.
*
- * @param sqsMessages List of SQS messages
+ * Counts:
+ * - Delta file records (with allowlist filtering)
+ * - SQS messages passed in (with allowlist filtering)
+ * - Invisible messages from other consumers (from queue attributes, avoiding double count)
+ *
+ * @param sqsMessages List of SQS messages this consumer has read (non-denylisted)
+ * @param queueAttributes Queue attributes including invisible message count (can be null)
+ * @param denylistedCount Number of denylisted messages read by this consumer
+ * @param filteredAsTooRecentCount Number of messages filtered as "too recent" by window reader
* @return TrafficStatus (DELAYED_PROCESSING or DEFAULT)
*/
- public TrafficStatus calculateStatus(List sqsMessages) {
+ public TrafficStatus calculateStatus(List sqsMessages, SqsMessageOperations.QueueAttributes queueAttributes, int denylistedCount, int filteredAsTooRecentCount) {
try {
// Get list of delta files from S3 (sorted newest to oldest)
List deltaS3Paths = listDeltaFiles();
if (deltaS3Paths.isEmpty()) {
- LOGGER.warn("No delta files found in S3 with prefix: {}", s3DeltaPrefix);
- return TrafficStatus.DEFAULT;
+ LOGGER.error("s3_error: no delta files found in s3 at prefix={}", s3DeltaPrefix);
+ throw new RuntimeException("no delta files found in s3 at prefix=" + s3DeltaPrefix);
}
// Find newest delta file timestamp for delta traffic window
long newestDeltaTs = findNewestDeltaTimestamp(deltaS3Paths);
- LOGGER.info("Traffic calculation: newestDeltaTs={}", newestDeltaTs);
+ LOGGER.info("traffic calculation: newestDeltaTs={}", newestDeltaTs);
// Find oldest SQS queue message timestamp for queue window
long oldestQueueTs = findOldestQueueTimestamp(sqsMessages);
- LOGGER.info("Traffic calculation: oldestQueueTs={}", oldestQueueTs);
+ LOGGER.info("traffic calculation: oldestQueueTs={}", oldestQueueTs);
// Define start time of the delta evaluation window
// We need evaluationWindowSeconds of non-allowlisted time, so we iteratively extend
@@ -238,53 +248,86 @@ public TrafficStatus calculateStatus(List sqsMessages) {
evictOldCacheEntries(deltaWindowStart);
// Process delta files and count records in [deltaWindowStart, newestDeltaTs]
+ // Files are sorted newest to oldest, records within files are sorted newest to oldest
+ // Stop when the newest record in a file is older than the window
int sum = 0;
+ int deltaRecordsCount = 0;
+ int deltaAllowlistedCount = 0;
+ int filesProcessed = 0;
+ int cacheHits = 0;
+ int cacheMisses = 0;
for (String s3Path : deltaS3Paths) {
+ boolean wasCached = isCached(s3Path);
+ if (wasCached) {
+ cacheHits++;
+ } else {
+ cacheMisses++;
+ }
+
List timestamps = getTimestampsFromFile(s3Path);
+ filesProcessed++;
+
+ // Check newest record in file - if older than window, stop processing remaining files
+ long newestRecordTs = timestamps.get(0);
+ if (newestRecordTs < deltaWindowStart) {
+ break;
+ }
- boolean shouldStop = false;
for (long ts : timestamps) {
// Stop condition: record is older than our window
if (ts < deltaWindowStart) {
- LOGGER.debug("Stopping delta file processing at timestamp {} (older than window start {})", ts, deltaWindowStart);
break;
}
// skip records in allowlisted ranges
if (isInAllowlist(ts)) {
+ deltaAllowlistedCount++;
continue;
}
// increment sum if record is in delta window
if (ts >= deltaWindowStart) {
+ deltaRecordsCount++;
sum++;
}
-
- }
-
- if (shouldStop) {
- break;
}
}
- // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m]
+ LOGGER.info("delta files: processed={}, deltaRecords={}, allowlisted={}, cache hits={}, misses={}, cacheSize={}",
+ filesProcessed, deltaRecordsCount, deltaAllowlistedCount, cacheHits, cacheMisses, deltaFileCache.size());
+
+ // Count SQS messages in [oldestQueueTs, oldestQueueTs + 5m] with allowlist filtering
+ int sqsCount = 0;
if (sqsMessages != null && !sqsMessages.isEmpty()) {
- int sqsCount = countSqsMessages(sqsMessages, oldestQueueTs);
+ sqsCount = countSqsMessages(sqsMessages, oldestQueueTs);
sum += sqsCount;
}
+ // Add invisible messages being processed by OTHER consumers
+ // (notVisible count includes our messages, so subtract what we've read to avoid double counting)
+ // ourMessages = delta messages + denylisted messages + filtered "too recent" messages
+ int otherConsumersMessages = 0;
+ if (queueAttributes != null) {
+ int totalInvisible = queueAttributes.getApproximateNumberOfMessagesNotVisible();
+ int ourMessages = (sqsMessages != null ? sqsMessages.size() : 0) + denylistedCount + filteredAsTooRecentCount;
+ otherConsumersMessages = Math.max(0, totalInvisible - ourMessages);
+ sum += otherConsumersMessages;
+ LOGGER.info("traffic calculation: adding {} invisible messages from other consumers (totalInvisible={}, ourMessages={})",
+ otherConsumersMessages, totalInvisible, ourMessages);
+ }
+
// Determine status
TrafficStatus status = determineStatus(sum, this.baselineTraffic);
- LOGGER.info("Traffic calculation complete: sum={}, baselineTraffic={}, thresholdMultiplier={}, status={}",
- sum, this.baselineTraffic, this.thresholdMultiplier, status);
+ LOGGER.info("traffic calculation complete: sum={} (deltaRecords={} + sqsMessages={} + otherConsumers={}), baselineTraffic={}, thresholdMultiplier={}, status={}",
+ sum, deltaRecordsCount, sqsCount, otherConsumersMessages, this.baselineTraffic, this.thresholdMultiplier, status);
return status;
} catch (Exception e) {
- LOGGER.error("Error calculating traffic status", e);
- return TrafficStatus.DEFAULT;
+ LOGGER.error("delta_job_failed: error calculating traffic status", e);
+ throw new RuntimeException("error calculating traffic status", e);
}
}
@@ -302,12 +345,12 @@ private long findNewestDeltaTimestamp(List deltaS3Paths) throws IOExcept
List timestamps = getTimestampsFromFile(newestDeltaPath);
if (timestamps.isEmpty()) {
- LOGGER.warn("Newest delta file has no timestamps: {}", newestDeltaPath);
+ LOGGER.error("s3_error: newest delta file has no timestamps, path={}", newestDeltaPath);
return System.currentTimeMillis() / 1000;
}
long newestTs = Collections.max(timestamps);
- LOGGER.debug("Found newest delta timestamp {} from file {}", newestTs, newestDeltaPath);
+ LOGGER.info("found newest delta timestamp {} from file {}", newestTs, newestDeltaPath);
return newestTs;
}
@@ -320,17 +363,28 @@ private List listDeltaFiles() {
List allFiles = cloudStorage.list(s3DeltaPrefix);
// Filter to only .dat delta files and sort newest to oldest
- return allFiles.stream()
+ List deltaFiles = allFiles.stream()
.filter(OptOutUtils::isDeltaFile)
.sorted(OptOutUtils.DeltaFilenameComparatorDescending)
.collect(Collectors.toList());
+
+ LOGGER.info("listed {} delta files from s3 (prefix={})", deltaFiles.size(), s3DeltaPrefix);
+ return deltaFiles;
} catch (Exception e) {
- LOGGER.error("Failed to list delta files from S3 with prefix: {}", s3DeltaPrefix, e);
+ LOGGER.error("s3_error: failed to list delta files at prefix={}", s3DeltaPrefix, e);
return Collections.emptyList();
}
}
+ /**
+ * Check if a delta file is already cached
+ */
+ private boolean isCached(String s3Path) {
+ String filename = s3Path.substring(s3Path.lastIndexOf('/') + 1);
+ return deltaFileCache.containsKey(filename);
+ }
+
/**
* Get timestamps from a delta file (S3 path), using cache if available
*/
@@ -341,12 +395,10 @@ private List getTimestampsFromFile(String s3Path) throws IOException {
// Check cache first
FileRecordCache cached = deltaFileCache.get(filename);
if (cached != null) {
- LOGGER.debug("Using cached timestamps for file: {}", filename);
return cached.timestamps;
}
// Cache miss - download from S3
- LOGGER.debug("Downloading and reading timestamps from S3: {}", s3Path);
List timestamps = readTimestampsFromS3(s3Path);
// Store in cache
@@ -377,8 +429,8 @@ private List readTimestampsFromS3(String s3Path) throws IOException {
return timestamps;
} catch (Exception e) {
- LOGGER.error("Failed to read delta file from S3: {}", s3Path, e);
- throw new IOException("Failed to read delta file from S3: " + s3Path, e);
+ LOGGER.error("s3_error: failed to read delta file at path={}", s3Path, e);
+ throw new IOException("failed to read delta file from s3: " + s3Path, e);
}
}
@@ -460,7 +512,7 @@ private Long extractTimestampFromMessage(Message msg) {
try {
return Long.parseLong(sentTimestamp) / 1000; // Convert ms to seconds
} catch (NumberFormatException e) {
- LOGGER.debug("Invalid SentTimestamp: {}", sentTimestamp);
+ LOGGER.error("sqs_error: invalid sentTimestamp, messageId={}, sentTimestamp={}", msg.messageId(), sentTimestamp);
}
}
@@ -472,25 +524,27 @@ private Long extractTimestampFromMessage(Message msg) {
* Count SQS messages from oldestQueueTs to oldestQueueTs + 5 minutes
*/
private int countSqsMessages(List sqsMessages, long oldestQueueTs) {
-
+
int count = 0;
+ int allowlistedCount = 0;
long windowEnd = oldestQueueTs + 5 * 60;
-
+
for (Message msg : sqsMessages) {
Long ts = extractTimestampFromMessage(msg);
if (ts < oldestQueueTs || ts > windowEnd) {
continue;
}
-
+
if (isInAllowlist(ts)) {
+ allowlistedCount++;
continue;
}
count++;
-
+
}
-
- LOGGER.info("SQS messages: {} in window [oldestQueueTs={}, oldestQueueTs+5m={}]", count, oldestQueueTs, windowEnd);
+
+ LOGGER.info("sqs messages: {} in window, {} allowlisted [oldestQueueTs={}, oldestQueueTs+5m={}]", count, allowlistedCount, oldestQueueTs, windowEnd);
return count;
}
@@ -530,29 +584,44 @@ private void evictOldCacheEntries(long cutoffTimestamp) {
int afterSize = deltaFileCache.size();
if (beforeSize != afterSize) {
- LOGGER.info("Evicted {} old cache entries (before={}, after={})",
+ LOGGER.info("evicted {} old cache entries (before={}, after={})",
beforeSize - afterSize, beforeSize, afterSize);
}
}
/**
- * Determine traffic status based on current vs past counts
+ * Determine traffic status based on current vs baseline traffic.
+ * Logs warnings at 50%, 75%, and 90% of the circuit breaker threshold.
*/
TrafficStatus determineStatus(int sumCurrent, int baselineTraffic) {
if (baselineTraffic == 0 || thresholdMultiplier == 0) {
- // Avoid division by zero - if no baseline traffic, return DEFAULT status
- LOGGER.warn("baselineTraffic is 0 or thresholdMultiplier is 0 returning DEFAULT status.");
- return TrafficStatus.DEFAULT;
+ LOGGER.error("circuit_breaker_config_error: baselineTraffic is 0 or thresholdMultiplier is 0");
+ throw new RuntimeException("invalid circuit breaker config: baselineTraffic=" + baselineTraffic + ", thresholdMultiplier=" + thresholdMultiplier);
+ }
+
+ int threshold = thresholdMultiplier * baselineTraffic;
+ double thresholdPercent = (double) sumCurrent / threshold * 100;
+
+ // Log warnings at increasing thresholds before circuit breaker triggers
+ if (thresholdPercent >= 90.0) {
+ LOGGER.warn("high_message_volume: 90% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%",
+ sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent));
+ } else if (thresholdPercent >= 75.0) {
+ LOGGER.warn("high_message_volume: 75% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%",
+ sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent));
+ } else if (thresholdPercent >= 50.0) {
+ LOGGER.warn("high_message_volume: 50% of threshold reached, sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%",
+ sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent));
}
- if (sumCurrent >= thresholdMultiplier * baselineTraffic) {
- LOGGER.warn("DELAYED_PROCESSING threshold breached: sumCurrent={} >= {}×baselineTraffic={}",
- sumCurrent, thresholdMultiplier, baselineTraffic);
+ if (sumCurrent >= threshold) {
+ LOGGER.error("circuit_breaker_triggered: traffic threshold breached, sumCurrent={}, threshold={} ({}x{})",
+ sumCurrent, threshold, thresholdMultiplier, baselineTraffic);
return TrafficStatus.DELAYED_PROCESSING;
}
- LOGGER.info("Traffic within normal range: sumCurrent={} < {}×baselineTraffic={}",
- sumCurrent, thresholdMultiplier, baselineTraffic);
+ LOGGER.info("traffic within normal range: sumCurrent={}, threshold={} ({}x{}), thresholdPercent={}%",
+ sumCurrent, threshold, thresholdMultiplier, baselineTraffic, String.format("%.1f", thresholdPercent));
return TrafficStatus.DEFAULT;
}
@@ -571,4 +640,4 @@ public Map getCacheStats() {
return stats;
}
-}
+}
\ No newline at end of file
diff --git a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java
similarity index 72%
rename from src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java
rename to src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java
index e8bd04b8..59a60a9f 100644
--- a/src/main/java/com/uid2/optout/vertx/OptOutTrafficFilter.java
+++ b/src/main/java/com/uid2/optout/traffic/OptOutTrafficFilter.java
@@ -1,8 +1,10 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.traffic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.uid2.optout.sqs.SqsParsedMessage;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Collections;
@@ -60,8 +62,7 @@ public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTraff
this.filterRules = Collections.emptyList(); // start empty
reloadTrafficFilterConfig(); // load ConfigMap
- LOGGER.info("OptOutTrafficFilter initialized: filterRules={}",
- filterRules.size());
+ LOGGER.info("initialized: filterRules={}", filterRules.size());
}
/**
@@ -78,18 +79,17 @@ public OptOutTrafficFilter(String trafficFilterConfigPath) throws MalformedTraff
* Can be called periodically to pick up config changes without restarting.
*/
public void reloadTrafficFilterConfig() throws MalformedTrafficFilterConfigException {
- LOGGER.info("Loading traffic filter config from ConfigMap");
+ LOGGER.info("loading traffic filter config");
try (InputStream is = Files.newInputStream(Paths.get(trafficFilterConfigPath))) {
String content = new String(is.readAllBytes(), StandardCharsets.UTF_8);
JsonObject filterConfigJson = new JsonObject(content);
this.filterRules = parseFilterRules(filterConfigJson);
- LOGGER.info("Successfully loaded traffic filter config from ConfigMap: filterRules={}",
- filterRules.size());
+ LOGGER.info("loaded traffic filter config: filterRules={}", filterRules.size());
} catch (Exception e) {
- LOGGER.warn("No traffic filter config found at: {}", trafficFilterConfigPath, e);
+ LOGGER.error("circuit_breaker_config_error: no traffic filter config found at {}", trafficFilterConfigPath, e);
throw new MalformedTrafficFilterConfigException(e.getMessage());
}
}
@@ -102,8 +102,8 @@ List parseFilterRules(JsonObject config) throws MalformedTraf
try {
JsonArray denylistRequests = config.getJsonArray("denylist_requests");
if (denylistRequests == null) {
- LOGGER.error("Invalid traffic filter config: denylist_requests is null");
- throw new MalformedTrafficFilterConfigException("Invalid traffic filter config: denylist_requests is null");
+ LOGGER.error("circuit_breaker_config_error: denylist_requests is null");
+ throw new MalformedTrafficFilterConfigException("invalid traffic filter config: denylist_requests is null");
}
for (int i = 0; i < denylistRequests.size(); i++) {
JsonObject ruleJson = denylistRequests.getJsonObject(i);
@@ -116,13 +116,19 @@ List parseFilterRules(JsonObject config) throws MalformedTraf
long end = rangeJson.getLong(1);
if (start >= end) {
- LOGGER.error("Invalid traffic filter rule: range start must be less than end: {}", ruleJson.encode());
- throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule: range start must be less than end");
+ LOGGER.error("circuit_breaker_config_error: rule range start must be less than end, rule={}", ruleJson.encode());
+ throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range start must be less than end");
}
range.add(start);
range.add(end);
}
+ // log error and throw exception if range is not 2 elements
+ if (range.size() != 2) {
+ LOGGER.error("circuit_breaker_config_error: rule range is not 2 elements, rule={}", ruleJson.encode());
+ throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range is not 2 elements");
+ }
+
// parse IPs
var ipAddressesJson = ruleJson.getJsonArray("IPs");
List ipAddresses = new ArrayList<>();
@@ -132,20 +138,26 @@ List parseFilterRules(JsonObject config) throws MalformedTraf
}
}
+ // log error and throw exception if IPs is empty
+ if (ipAddresses.size() == 0) {
+ LOGGER.error("circuit_breaker_config_error: rule IPs is empty, rule={}", ruleJson.encode());
+ throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: IPs is empty");
+ }
+
// log error and throw exception if rule is invalid
- if (range.size() != 2 || ipAddresses.size() == 0 || range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less
- LOGGER.error("Invalid traffic filter rule, range must be 24 hours or less: {}", ruleJson.encode());
- throw new MalformedTrafficFilterConfigException("Invalid traffic filter rule, range must be 24 hours or less");
+ if (range.get(1) - range.get(0) > 86400) { // range must be 24 hours or less
+ LOGGER.error("circuit_breaker_config_error: rule range must be 24 hours or less, rule={}", ruleJson.encode());
+ throw new MalformedTrafficFilterConfigException("invalid traffic filter rule: range must be 24 hours or less");
}
TrafficFilterRule rule = new TrafficFilterRule(range, ipAddresses);
- LOGGER.info("Loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses());
+ LOGGER.info("loaded traffic filter rule: range=[{}, {}], IPs={}", rule.getRangeStart(), rule.getRangeEnd(), rule.getIpAddresses());
rules.add(rule);
}
return rules;
} catch (Exception e) {
- LOGGER.error("Failed to parse traffic filter rules: config={}, error={}", config.encode(), e.getMessage());
+ LOGGER.error("circuit_breaker_config_error: failed to parse rules, config={}, error={}", config.encode(), e.getMessage());
throw new MalformedTrafficFilterConfigException(e.getMessage());
}
}
@@ -155,7 +167,7 @@ public boolean isDenylisted(SqsParsedMessage message) {
String clientIp = message.getClientIp();
if (clientIp == null || clientIp.isEmpty()) {
- LOGGER.error("Request does not contain client IP, timestamp={}", timestamp);
+ LOGGER.error("sqs_error: request does not contain client ip, messageId={}", message.getOriginalMessage().messageId());
return false;
}
diff --git a/src/main/java/com/uid2/optout/util/HttpResponseHelper.java b/src/main/java/com/uid2/optout/util/HttpResponseHelper.java
new file mode 100644
index 00000000..6d82f568
--- /dev/null
+++ b/src/main/java/com/uid2/optout/util/HttpResponseHelper.java
@@ -0,0 +1,70 @@
+package com.uid2.optout.util;
+
+import io.vertx.core.http.HttpHeaders;
+import io.vertx.core.http.HttpServerResponse;
+import io.vertx.core.json.JsonObject;
+
+/**
+ * Utility class for HTTP JSON response handling.
+ * Ensures consistent response format across handlers.
+ */
+public class HttpResponseHelper {
+
+ /**
+ * Send a JSON response with the specified status code.
+ */
+ public static void sendJson(HttpServerResponse resp, int statusCode, JsonObject body) {
+ resp.setStatusCode(statusCode)
+ .putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
+ .end(body.encode());
+ }
+
+ /**
+ * Send a 200 OK response with JSON body.
+ */
+ public static void sendSuccess(HttpServerResponse resp, JsonObject body) {
+ sendJson(resp, 200, body);
+ }
+
+ /**
+ * Send a 200 OK response with status and message.
+ */
+ public static void sendSuccess(HttpServerResponse resp, String status, String message) {
+ sendJson(resp, 200, new JsonObject().put("status", status).put("message", message));
+ }
+
+ /**
+ * Send a 200 OK response with idle status and message.
+ */
+ public static void sendIdle(HttpServerResponse resp, String message) {
+ sendJson(resp, 200, new JsonObject().put("status", "idle").put("message", message));
+ }
+ /**
+ * Send a 202 Accepted response indicating async job started.
+ */
+ public static void sendAccepted(HttpServerResponse resp, String message) {
+ sendJson(resp, 202, new JsonObject().put("status", "accepted").put("message", message));
+ }
+
+ /**
+ * Send a 409 Conflict response.
+ */
+ public static void sendConflict(HttpServerResponse resp, String reason) {
+ sendJson(resp, 409, new JsonObject().put("status", "conflict").put("reason", reason));
+ }
+
+ /**
+ * Send a 500 Internal Server Error response.
+ */
+ public static void sendError(HttpServerResponse resp, String error) {
+ sendJson(resp, 500, new JsonObject().put("status", "failed").put("error", error));
+ }
+
+ /**
+ * Send a 500 Internal Server Error response from an exception.
+ */
+ public static void sendError(HttpServerResponse resp, Exception e) {
+ sendError(resp, e.getMessage());
+ }
+}
+
diff --git a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java b/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java
deleted file mode 100644
index 7f2eb8fb..00000000
--- a/src/main/java/com/uid2/optout/vertx/DeltaProductionResult.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package com.uid2.optout.vertx;
-
-/**
- * Result object containing statistics from delta production.
- */
-public class DeltaProductionResult {
- private final int deltasProduced;
- private final int entriesProcessed;
-
- /*
- * indicates that there are still messages in the queue, however,
- * not enough time has elapsed to produce a delta file.
- * We produce in batches of (5 minutes)
- */
- private final boolean stoppedDueToMessagesTooRecent;
-
- public DeltaProductionResult(int deltasProduced, int entriesProcessed, boolean stoppedDueToMessagesTooRecent) {
- this.deltasProduced = deltasProduced;
- this.entriesProcessed = entriesProcessed;
- this.stoppedDueToMessagesTooRecent = stoppedDueToMessagesTooRecent;
- }
-
- public int getDeltasProduced() {
- return deltasProduced;
- }
-
- public int getEntriesProcessed() {
- return entriesProcessed;
- }
-
- public boolean stoppedDueToMessagesTooRecent() {
- return stoppedDueToMessagesTooRecent;
- }
-}
-
diff --git a/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java b/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java
index 75073239..e01c49e1 100644
--- a/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java
+++ b/src/main/java/com/uid2/optout/vertx/OptOutServiceVerticle.java
@@ -365,21 +365,23 @@ private void handleQueue(RoutingContext routingContext) {
String email = body != null ? body.getString(EMAIL) : null;
String phone = body != null ? body.getString(PHONE) : null;
- HttpServerResponse resp = routingContext.response();
-
// while old delta production is enabled, response is handled by replicate logic
// Validate parameters - same as replicate
if (identityHash == null || params.getAll(IDENTITY_HASH).size() != 1) {
+ LOGGER.warn("handleQueue: Invalid identity_hash parameter");
// this.sendBadRequestError(resp);
return;
}
if (advertisingId == null || params.getAll(ADVERTISING_ID).size() != 1) {
+ LOGGER.warn("handleQueue: Invalid advertising_id parameter");
+
// this.sendBadRequestError(resp);
return;
}
if (!this.isGetOrPost(req)) {
+ LOGGER.warn("handleQueue: Invalid HTTP method: {}", req.method());
// this.sendBadRequestError(resp);
return;
}
@@ -408,8 +410,6 @@ private void handleQueue(RoutingContext routingContext) {
}
}, res -> {
if (res.failed()) {
- // this.sendInternalServerError(resp, "Failed to queue message: " + res.cause().getMessage());
- LOGGER.error("Failed to queue message: " + res.cause().getMessage());
} else {
String messageId = (String) res.result();
@@ -426,8 +426,7 @@ private void handleQueue(RoutingContext routingContext) {
}
});
} catch (Exception ex) {
- // this.sendInternalServerError(resp, ex.getMessage());
- LOGGER.error("Error processing queue request: " + ex.getMessage(), ex);
+ LOGGER.error("handleQueue: Error processing queue request");
}
}
diff --git a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java
index 910a2c61..7b8a7082 100644
--- a/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java
+++ b/src/main/java/com/uid2/optout/vertx/OptOutSqsLogProducer.java
@@ -2,30 +2,39 @@
import com.uid2.optout.Const;
import com.uid2.optout.auth.InternalAuthMiddleware;
+import com.uid2.optout.delta.DeltaFileWriter;
+import com.uid2.optout.delta.ManualOverrideService;
+import com.uid2.optout.delta.DeltaProductionJobStatus;
+import com.uid2.optout.delta.DeltaProductionMetrics;
+import com.uid2.optout.delta.DeltaProductionOrchestrator;
+import com.uid2.optout.delta.DeltaProductionResult;
+import com.uid2.optout.delta.S3UploadService;
+import com.uid2.optout.delta.StopReason;
+import com.uid2.optout.sqs.SqsWindowReader;
+import com.uid2.optout.traffic.OptOutTrafficCalculator;
+import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException;
+import com.uid2.optout.traffic.OptOutTrafficFilter;
+import com.uid2.optout.traffic.OptOutTrafficFilter.MalformedTrafficFilterConfigException;
import com.uid2.shared.Utils;
import com.uid2.shared.cloud.ICloudStorage;
import com.uid2.shared.health.HealthComponent;
import com.uid2.shared.health.HealthManager;
-import com.uid2.shared.optout.*;
-import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
+import com.uid2.shared.optout.OptOutCloudSync;
+import com.uid2.shared.optout.OptOutUtils;
+
import io.vertx.core.*;
import io.vertx.core.http.*;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.web.Router;
import io.vertx.ext.web.RoutingContext;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.sqs.SqsClient;
-import software.amazon.awssdk.services.sqs.model.*;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import static com.uid2.optout.util.HttpResponseHelper.*;
+
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -76,92 +85,83 @@ public class OptOutSqsLogProducer extends AbstractVerticle {
private final HealthComponent healthComponent = HealthManager.instance.registerComponent("sqs-log-producer");
private final SqsClient sqsClient;
- private final String queueUrl;
private final String eventDeltaProduced;
- private final int replicaId;
- private final ICloudStorage cloudStorage;
- private final OptOutCloudSync cloudSync;
- private final int maxMessagesPerPoll;
- private final int visibilityTimeout;
- private final int deltaWindowSeconds; // Time window for each delta file (5 minutes = 300 seconds)
- private final int jobTimeoutSeconds;
- private final int maxMessagesPerFile; // Memory protection: max messages per delta file
private final int listenPort;
- private final String internalApiKey;
private final InternalAuthMiddleware internalAuth;
-
- private Counter counterDeltaProduced = Counter
- .builder("uid2_optout_sqs_delta_produced_total")
- .description("counter for how many optout delta files are produced from SQS")
- .register(Metrics.globalRegistry);
-
- private Counter counterEntriesProcessed = Counter
- .builder("uid2_optout_sqs_entries_processed_total")
- .description("counter for how many optout entries are processed from SQS")
- .register(Metrics.globalRegistry);
-
- private ByteBuffer buffer;
- private boolean shutdownInProgress = false;
+ private final OptOutTrafficFilter trafficFilter;
+ private final OptOutTrafficCalculator trafficCalculator;
+ private final DeltaProductionOrchestrator orchestrator;
- //Tracks the current delta production job status for this pod.
- private final AtomicReference currentJob = new AtomicReference<>(null);
+ // Tracks the current delta production job status for this pod
+ private final AtomicReference currentJob = new AtomicReference<>(null);
- // Helper for reading complete 5-minute windows from SQS
- private final SqsWindowReader windowReader;
+ private volatile boolean shutdownInProgress = false;
- public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync) throws IOException {
- this(jsonConfig, cloudStorage, cloudSync, Const.Event.DeltaProduce);
- }
-
- public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced) throws IOException {
- this(jsonConfig, cloudStorage, cloudSync, eventDeltaProduced, null);
- }
-
- // Constructor for testing - allows injecting mock SqsClient
- public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, OptOutCloudSync cloudSync, String eventDeltaProduced, SqsClient sqsClient) throws IOException {
+ public OptOutSqsLogProducer(JsonObject jsonConfig, ICloudStorage cloudStorage, ICloudStorage cloudStorageDroppedRequests, OptOutCloudSync cloudSync, String eventDeltaProduced, SqsClient sqsClient) throws IOException, MalformedTrafficCalcConfigException, MalformedTrafficFilterConfigException {
this.eventDeltaProduced = eventDeltaProduced;
- this.replicaId = OptOutUtils.getReplicaId(jsonConfig);
- this.cloudStorage = cloudStorage;
- this.cloudSync = cloudSync;
-
+
// Initialize SQS client
- this.queueUrl = jsonConfig.getString(Const.Config.OptOutSqsQueueUrlProp);
- if (this.queueUrl == null || this.queueUrl.isEmpty()) {
- throw new IOException("SQS queue URL not configured");
+ String queueUrl = jsonConfig.getString(Const.Config.OptOutSqsQueueUrlProp);
+ if (queueUrl == null || queueUrl.isEmpty()) {
+ throw new IOException("sqs queue url not configured");
}
-
- // Use injected client for testing, or create new one
this.sqsClient = sqsClient != null ? sqsClient : SqsClient.builder().build();
- LOGGER.info("SQS client initialized for queue: " + this.queueUrl);
-
- // SQS Configuration
- this.maxMessagesPerPoll = 10; // SQS max is 10
- this.visibilityTimeout = jsonConfig.getInteger(Const.Config.OptOutSqsVisibilityTimeoutProp, 240); // 4 minutes default
- this.deltaWindowSeconds = 300; // Fixed 5 minutes for all deltas
- this.jobTimeoutSeconds = jsonConfig.getInteger(Const.Config.OptOutDeltaJobTimeoutSecondsProp, 10800); // 3 hours default
- this.maxMessagesPerFile = jsonConfig.getInteger(Const.Config.OptOutMaxMessagesPerFileProp, 10000); // Memory protection limit
+ LOGGER.info("sqs client initialized for queue: {}", queueUrl);
- // HTTP server configuration - use port offset + 1 to avoid conflicts
+ // HTTP server configuration
this.listenPort = Const.Port.ServicePortForOptOut + Utils.getPortOffset() + 1;
// Authentication
- this.internalApiKey = jsonConfig.getString(Const.Config.OptOutInternalApiTokenProp);
- this.internalAuth = new InternalAuthMiddleware(this.internalApiKey, "optout-sqs");
-
+ String internalApiKey = jsonConfig.getString(Const.Config.OptOutInternalApiTokenProp);
+ this.internalAuth = new InternalAuthMiddleware(internalApiKey, "optout-sqs");
+
+ // Circuit breaker tools
+ this.trafficFilter = new OptOutTrafficFilter(jsonConfig.getString(Const.Config.TrafficFilterConfigPathProp));
+ this.trafficCalculator = new OptOutTrafficCalculator(cloudStorage, jsonConfig.getString(Const.Config.OptOutSqsS3FolderProp), jsonConfig.getString(Const.Config.TrafficCalcConfigPathProp));
+
+ // Configuration values for orchestrator setup
+ int replicaId = OptOutUtils.getReplicaId(jsonConfig);
+ int maxMessagesPerPoll = 10; // SQS max is 10
+ int deltaWindowSeconds = jsonConfig.getInteger(Const.Config.OptOutSqsDeltaWindowSecondsProp, 300); // fixed 5 minutes, allow config for testing
+ int visibilityTimeout = jsonConfig.getInteger(Const.Config.OptOutSqsVisibilityTimeoutProp, 240);
+ int jobTimeoutSeconds = jsonConfig.getInteger(Const.Config.OptOutDeltaJobTimeoutSecondsProp, 10800);
+ int maxMessagesPerFile = jsonConfig.getInteger(Const.Config.OptOutMaxMessagesPerFileProp, 10000);
int bufferSize = jsonConfig.getInteger(Const.Config.OptOutProducerBufferSizeProp);
- this.buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN);
-
- // Initialize window reader with memory protection limit
- this.windowReader = new SqsWindowReader(
- this.sqsClient, this.queueUrl, this.maxMessagesPerPoll,
- this.visibilityTimeout, this.deltaWindowSeconds, this.maxMessagesPerFile
+
+ // Orchestrator setup
+ DeltaFileWriter deltaFileWriter = new DeltaFileWriter(bufferSize);
+ S3UploadService deltaUploadService = new S3UploadService(cloudStorage, this.sqsClient, queueUrl);
+ S3UploadService droppedRequestUploadService = new S3UploadService(cloudStorageDroppedRequests, this.sqsClient, queueUrl) ;
+ ManualOverrideService manualOverrideService = new ManualOverrideService(cloudStorage, jsonConfig.getString(Const.Config.ManualOverrideS3PathProp));
+ SqsWindowReader windowReader = new SqsWindowReader(
+ this.sqsClient, queueUrl, maxMessagesPerPoll,
+ visibilityTimeout, deltaWindowSeconds, maxMessagesPerFile
);
- LOGGER.info("OptOutSqsLogProducer initialized with maxMessagesPerFile: {}", this.maxMessagesPerFile);
+
+ this.orchestrator = new DeltaProductionOrchestrator(
+ this.sqsClient,
+ queueUrl,
+ replicaId,
+ deltaWindowSeconds,
+ jobTimeoutSeconds,
+ windowReader,
+ deltaFileWriter,
+ deltaUploadService,
+ droppedRequestUploadService,
+ manualOverrideService,
+ this.trafficFilter,
+ this.trafficCalculator,
+ cloudSync,
+ new DeltaProductionMetrics()
+ );
+
+ LOGGER.info("initialized with maxMessagesPerFile={}, maxMessagesPerPoll={}, visibilityTimeout={}, deltaWindowSeconds={}, jobTimeoutSeconds={}",
+ maxMessagesPerFile, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds, jobTimeoutSeconds);
}
@Override
public void start(Promise startPromise) {
- LOGGER.info("Starting SQS Log Producer with HTTP endpoint...");
+ LOGGER.info("starting http server on port {}", listenPort);
try {
vertx.createHttpServer()
@@ -169,18 +169,17 @@ public void start(Promise startPromise) {
.listen(listenPort, result -> {
if (result.succeeded()) {
this.healthComponent.setHealthStatus(true);
- LOGGER.info("SQS Log Producer HTTP server started on port: {} (delta window: {}s)",
- listenPort, this.deltaWindowSeconds);
+ LOGGER.info("http server started on port {}", listenPort);
startPromise.complete();
} else {
- LOGGER.error("Failed to start SQS Log Producer HTTP server", result.cause());
+ LOGGER.error("failed to start http server", result.cause());
this.healthComponent.setHealthStatus(false, result.cause().getMessage());
startPromise.fail(result.cause());
}
});
} catch (Exception e) {
- LOGGER.error("Failed to start SQS Log Producer", e);
+ LOGGER.error("failed to start http server", e);
this.healthComponent.setHealthStatus(false, e.getMessage());
startPromise.fail(e);
}
@@ -188,20 +187,19 @@ public void start(Promise startPromise) {
@Override
public void stop(Promise stopPromise) {
- LOGGER.info("Stopping SQS Log Producer...");
+ LOGGER.info("stopping");
this.shutdownInProgress = true;
if (this.sqsClient != null) {
try {
this.sqsClient.close();
- LOGGER.info("SQS client closed");
} catch (Exception e) {
- LOGGER.error("Error closing SQS client", e);
+ LOGGER.error("error closing sqs client", e);
}
}
stopPromise.complete();
- LOGGER.info("SQS Log Producer stopped");
+ LOGGER.info("stopped");
}
private Router createRouter() {
@@ -226,21 +224,14 @@ private Router createRouter() {
private void handleDeltaProduceStatus(RoutingContext routingContext) {
HttpServerResponse resp = routingContext.response();
- DeltaProduceJobStatus job = currentJob.get();
+ DeltaProductionJobStatus job = currentJob.get();
if (job == null) {
- resp.setStatusCode(200)
- .putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
- .end(new JsonObject()
- .put("state", "idle")
- .put("message", "No job running on this pod")
- .encode());
+ sendIdle(resp, "no job running on this pod");
return;
}
- resp.setStatusCode(200)
- .putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
- .end(job.toJson().encode());
+ sendSuccess(resp, job.toJson());
}
/**
@@ -257,72 +248,63 @@ private void handleDeltaProduceStatus(RoutingContext routingContext) {
private void handleDeltaProduceStart(RoutingContext routingContext) {
HttpServerResponse resp = routingContext.response();
- LOGGER.info("Delta production job requested via /deltaproduce endpoint");
+ LOGGER.info("delta production job requested");
-
- DeltaProduceJobStatus existingJob = currentJob.get();
+ try {
+ this.trafficFilter.reloadTrafficFilterConfig();
+ } catch (MalformedTrafficFilterConfigException e) {
+ LOGGER.error("circuit_breaker_config_error: failed to reload traffic filter config: {}", e.getMessage(), e);
+ sendError(resp, e);
+ return;
+ }
+
+ try {
+ this.trafficCalculator.reloadTrafficCalcConfig();
+ } catch (MalformedTrafficCalcConfigException e) {
+ LOGGER.error("circuit_breaker_config_error: failed to reload traffic calc config: {}", e.getMessage(), e);
+ sendError(resp, e);
+ return;
+ }
+
+ DeltaProductionJobStatus existingJob = currentJob.get();
// If there's an existing job, check if it's still running
if (existingJob != null) {
- if (existingJob.getState() == DeltaProduceJobStatus.JobState.RUNNING) {
- // Cannot replace a running job - 409 Conflict
- LOGGER.warn("Delta production job already running on this pod");
- resp.setStatusCode(409)
- .putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
- .end(new JsonObject()
- .put("status", "conflict")
- .put("message", "A delta production job is already running on this pod")
- .put("current_job", existingJob.toJson())
- .encode());
+ if (existingJob.getState() == DeltaProductionJobStatus.JobState.RUNNING) {
+ LOGGER.info("job already running, returning conflict");
+ sendConflict(resp, "job already running on this pod");
return;
}
- LOGGER.info("Auto-clearing previous {} job to start new one", existingJob.getState());
+ LOGGER.info("clearing previous {} job", existingJob.getState());
}
- DeltaProduceJobStatus newJob = new DeltaProduceJobStatus();
+ DeltaProductionJobStatus newJob = new DeltaProductionJobStatus();
// Try to set the new job
if (!currentJob.compareAndSet(existingJob, newJob)) {
- resp.setStatusCode(409)
- .putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
- .end(new JsonObject()
- .put("status", "conflict")
- .put("message", "Job state changed, please retry")
- .encode());
+ sendConflict(resp, "job state changed, please retry");
return;
}
- // Start the job asynchronously
- LOGGER.info("Starting delta production job");
+ LOGGER.info("starting new job");
this.startDeltaProductionJob(newJob);
// Return immediately with 202 Accepted
- resp.setStatusCode(202)
- .putHeader(HttpHeaders.CONTENT_TYPE, "application/json")
- .end(new JsonObject()
- .put("status", "accepted")
- .put("message", "Delta production job started on this pod")
- .encode());
+ sendAccepted(resp, "job started");
}
/**
* Starts the delta production job asynchronously
* The job runs on a worker thread and updates the DeltaProduceJobStatus when complete
*/
- private void startDeltaProductionJob(DeltaProduceJobStatus job) {
- vertx.executeBlocking(() -> {
- LOGGER.info("Executing delta production job");
- return produceDeltasBlocking();
- }).onComplete(ar -> {
+ private void startDeltaProductionJob(DeltaProductionJobStatus job) {
+ vertx.executeBlocking(() -> produceDeltasBlocking()).onComplete(ar -> {
if (ar.succeeded()) {
- JsonObject result = ar.result();
- job.complete(result);
- LOGGER.info("Delta production job succeeded: {}", result.encode());
+ job.complete(ar.result());
} else {
- String errorMsg = ar.cause().getMessage();
- job.fail(errorMsg);
- LOGGER.error("Delta production job failed: {}", errorMsg, ar.cause());
+ job.fail(ar.cause().getMessage());
+ LOGGER.error("delta_job_failed: {}", ar.cause().getMessage(), ar.cause());
}
});
}
@@ -337,221 +319,25 @@ private void startDeltaProductionJob(DeltaProduceJobStatus job) {
*/
private JsonObject produceDeltasBlocking() throws Exception {
if (this.shutdownInProgress) {
- throw new Exception("Producer is shutting down");
+ throw new Exception("producer is shutting down");
}
- JsonObject result = new JsonObject();
- LOGGER.info("Starting delta production from SQS queue");
-
- // Process messages until queue is empty or messages are too recent
- DeltaProductionResult deltaResult = this.produceBatchedDeltas();
-
- // Determine status based on results
- if (deltaResult.getDeltasProduced() == 0 && deltaResult.stoppedDueToMessagesTooRecent()) {
- // No deltas produced because all messages were too recent
- result.put("status", "skipped");
- result.put("reason", "All messages too recent");
- LOGGER.info("Delta production skipped: all messages too recent");
- } else {
- result.put("status", "success");
- LOGGER.info("Delta production complete: {} deltas, {} entries",
- deltaResult.getDeltasProduced(), deltaResult.getEntriesProcessed());
- }
-
- result.put("deltas_produced", deltaResult.getDeltasProduced());
- result.put("entries_processed", deltaResult.getEntriesProcessed());
-
- return result;
- }
+ DeltaProductionResult result = orchestrator.produceBatchedDeltas(this::publishDeltaProducedEvent);
+ StopReason stopReason = result.getStopReason();
+ boolean producedWork = result.getDeltasProduced() > 0 || result.getDroppedRequestFilesProduced() > 0;
+ boolean halted = stopReason == StopReason.CIRCUIT_BREAKER_TRIGGERED || stopReason == StopReason.MANUAL_OVERRIDE_ACTIVE;
- /**
- * Reads messages from SQS and produces delta files in 5 minute batches.
- * Continues until queue is empty or messages are too recent.
- * Windows are limited to maxMessagesPerFile for memory protection.
- *
- * @return DeltaProductionResult with counts and stop reason
- * @throws IOException if delta production fails
- */
- private DeltaProductionResult produceBatchedDeltas() throws IOException {
- int deltasProduced = 0;
- int totalEntriesProcessed = 0;
- boolean stoppedDueToMessagesTooRecent = false;
+ String status = halted ? "halted" : (producedWork ? "success" : "skipped");
- long jobStartTime = OptOutUtils.nowEpochSeconds();
- LOGGER.info("Starting delta production from SQS queue (maxMessagesPerFile: {})", this.maxMessagesPerFile);
+ LOGGER.info("delta production {}: {} deltas, {} entries, {} dropped files, {} dropped requests, reason={}",
+ status, result.getDeltasProduced(), result.getEntriesProcessed(),
+ result.getDroppedRequestFilesProduced(), result.getDroppedRequestsProcessed(), stopReason);
- // Read and process windows until done
- while (true) {
- if(checkJobTimeout(jobStartTime)){
- break;
- }
-
- // Read one complete 5-minute window (limited to maxMessagesPerFile)
- SqsWindowReader.WindowReadResult windowResult = windowReader.readWindow();
-
- // If no messages, we're done (queue empty or messages too recent)
- if (windowResult.isEmpty()) {
- stoppedDueToMessagesTooRecent = windowResult.stoppedDueToMessagesTooRecent();
- LOGGER.info("Delta production complete - no more eligible messages");
- break;
- }
-
- // Produce delta for this window
- long windowStart = windowResult.getWindowStart();
- List messages = windowResult.getMessages();
-
- // Create delta file
- String deltaName = OptOutUtils.newDeltaFileName(this.replicaId);
- ByteArrayOutputStream deltaStream = new ByteArrayOutputStream();
- writeStartOfDelta(deltaStream, windowStart);
-
- // Write all messages
- List sqsMessages = new ArrayList<>();
- for (SqsParsedMessage msg : messages) {
- writeOptOutEntry(deltaStream, msg.getHashBytes(), msg.getIdBytes(), msg.getTimestamp());
- sqsMessages.add(msg.getOriginalMessage());
- }
-
- // Upload and delete
- uploadDeltaAndDeleteMessages(deltaStream, deltaName, windowStart, sqsMessages);
- deltasProduced++;
- totalEntriesProcessed += messages.size();
-
- LOGGER.info("Produced delta for window [{}, {}] with {} messages",
- windowStart, windowStart + this.deltaWindowSeconds, messages.size());
- }
-
- long totalDuration = OptOutUtils.nowEpochSeconds() - jobStartTime;
- LOGGER.info("Delta production complete: took {}s, produced {} deltas, processed {} entries",
- totalDuration, deltasProduced, totalEntriesProcessed);
-
- return new DeltaProductionResult(deltasProduced, totalEntriesProcessed, stoppedDueToMessagesTooRecent);
- }
-
- /**
- * Checks if job has exceeded timeout
- */
- private boolean checkJobTimeout(long jobStartTime) {
- long elapsedTime = OptOutUtils.nowEpochSeconds() - jobStartTime;
- if (elapsedTime > 3600) { // 1 hour - log warning message
- LOGGER.error("Delta production job has been running for {} seconds",
- elapsedTime);
- return false;
- }
- if (elapsedTime > this.jobTimeoutSeconds) {
- LOGGER.error("Delta production job has been running for {} seconds (exceeds timeout of {}s)",
- elapsedTime, this.jobTimeoutSeconds);
- return true; // deadline exceeded
- }
- return false;
- }
-
- /**
- * Writes the start-of-delta entry with null hash and window start timestamp.
- */
- private void writeStartOfDelta(ByteArrayOutputStream stream, long windowStart) throws IOException {
-
- this.checkBufferSize(OptOutConst.EntrySize);
-
- buffer.put(OptOutUtils.nullHashBytes);
- buffer.put(OptOutUtils.nullHashBytes);
- buffer.putLong(windowStart);
-
- buffer.flip();
- byte[] entry = new byte[buffer.remaining()];
- buffer.get(entry);
-
- stream.write(entry);
- buffer.clear();
- }
-
- /**
- * Writes a single opt-out entry to the delta stream.
- */
- private void writeOptOutEntry(ByteArrayOutputStream stream, byte[] hashBytes, byte[] idBytes, long timestamp) throws IOException {
- this.checkBufferSize(OptOutConst.EntrySize);
- OptOutEntry.writeTo(buffer, hashBytes, idBytes, timestamp);
- buffer.flip();
- byte[] entry = new byte[buffer.remaining()];
- buffer.get(entry);
- stream.write(entry);
- buffer.clear();
+ return result.toJsonWithStatus(status);
}
- /**
- * Writes the end-of-delta sentinel entry with ones hash and window end timestamp.
- */
- private void writeEndOfDelta(ByteArrayOutputStream stream, long windowEnd) throws IOException {
- this.checkBufferSize(OptOutConst.EntrySize);
- buffer.put(OptOutUtils.onesHashBytes);
- buffer.put(OptOutUtils.onesHashBytes);
- buffer.putLong(windowEnd);
- buffer.flip();
- byte[] entry = new byte[buffer.remaining()];
- buffer.get(entry);
- stream.write(entry);
- buffer.clear();
- }
-
-
-
- // Upload a delta to S3 and delete messages from SQS after successful upload
- private void uploadDeltaAndDeleteMessages(ByteArrayOutputStream deltaStream, String deltaName, Long windowStart, List messages) throws IOException {
- try {
- // Add end-of-delta entry
- long endTimestamp = windowStart + this.deltaWindowSeconds;
- this.writeEndOfDelta(deltaStream, endTimestamp);
-
- // upload
- byte[] deltaData = deltaStream.toByteArray();
- String s3Path = this.cloudSync.toCloudPath(deltaName);
-
- LOGGER.info("SQS Delta Upload - fileName: {}, s3Path: {}, size: {} bytes, messages: {}, window: [{}, {})",
- deltaName, s3Path, deltaData.length, messages.size(), windowStart, endTimestamp);
-
- boolean uploadSucceeded = false;
- try (ByteArrayInputStream inputStream = new ByteArrayInputStream(deltaData)) {
- this.cloudStorage.upload(inputStream, s3Path);
- LOGGER.info("Successfully uploaded delta to S3: {}", s3Path);
- uploadSucceeded = true;
-
- // publish event
- this.publishDeltaProducedEvent(deltaName);
- this.counterDeltaProduced.increment();
- this.counterEntriesProcessed.increment(messages.size());
-
- } catch (Exception uploadEx) {
- LOGGER.error("Failed to upload delta to S3: " + uploadEx.getMessage(), uploadEx);
- throw new IOException("S3 upload failed", uploadEx);
- }
-
- // CRITICAL: Only delete messages from SQS after successful S3 upload
- if (uploadSucceeded && !messages.isEmpty()) {
- LOGGER.info("Deleting {} messages from SQS after successful S3 upload", messages.size());
- SqsMessageOperations.deleteMessagesFromSqs(this.sqsClient, this.queueUrl, messages);
- }
-
- // Close the stream
- deltaStream.close();
-
- } catch (Exception ex) {
- LOGGER.error("Error uploading delta: " + ex.getMessage(), ex);
- throw new IOException("Delta upload failed", ex);
- }
- }
-
- private void publishDeltaProducedEvent(String newDelta) {
- vertx.eventBus().publish(this.eventDeltaProduced, newDelta);
- LOGGER.info("Published delta.produced event for: {}", newDelta);
- }
-
- private void checkBufferSize(int dataSize) {
- ByteBuffer b = this.buffer;
- if (b.capacity() < dataSize) {
- int newCapacity = Integer.highestOneBit(dataSize) << 1;
- LOGGER.warn("Expanding buffer size: current {}, need {}, new {}", b.capacity(), dataSize, newCapacity);
- this.buffer = ByteBuffer.allocate(newCapacity).order(ByteOrder.LITTLE_ENDIAN);
- }
+ private void publishDeltaProducedEvent(String deltaName) {
+ vertx.eventBus().publish(this.eventDeltaProduced, deltaName);
}
}
diff --git a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java b/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java
deleted file mode 100644
index 6c2715b1..00000000
--- a/src/main/java/com/uid2/optout/vertx/SqsMessageOperations.java
+++ /dev/null
@@ -1,140 +0,0 @@
-package com.uid2.optout.vertx;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.services.sqs.SqsClient;
-import software.amazon.awssdk.services.sqs.model.*;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Utility class for SQS message operations.
- */
-public class SqsMessageOperations {
- private static final Logger LOGGER = LoggerFactory.getLogger(SqsMessageOperations.class);
- private static final int SQS_MAX_DELETE_BATCH_SIZE = 10;
-
- /**
- * Receives all available messages from an SQS queue up to a maximum number of batches.
- *
- * @param sqsClient The SQS client
- * @param queueUrl The queue URL
- * @param maxMessagesPerPoll Maximum messages to receive per poll (max 10)
- * @param visibilityTimeout Visibility timeout in seconds
- * @param maxBatches Maximum number of receive batches
- * @return List of all received messages
- */
- public static List receiveAllAvailableMessages(
- SqsClient sqsClient,
- String queueUrl,
- int maxMessagesPerPoll,
- int visibilityTimeout,
- int maxBatches) {
-
- List allMessages = new ArrayList<>();
- int batchCount = 0;
-
- // Keep receiving messages until we get an empty batch or hit the limit
- while (batchCount < maxBatches) {
- List batch = receiveMessagesFromSqs(sqsClient, queueUrl, maxMessagesPerPoll, visibilityTimeout);
- if (batch.isEmpty()) {
- break;
- }
- allMessages.addAll(batch);
- batchCount++;
-
- // If we got fewer messages than the max (of 10), the queue is likely empty
- if (batch.size() < maxMessagesPerPoll) {
- break;
- }
- }
-
- return allMessages;
- }
-
- /**
- * Receives a batch of messages from SQS.
- *
- * @param sqsClient The SQS client
- * @param queueUrl The queue URL
- * @param maxMessages Maximum number of messages to receive (max 10)
- * @param visibilityTimeout Visibility timeout in seconds
- * @return List of received messages
- */
- public static List receiveMessagesFromSqs(
- SqsClient sqsClient,
- String queueUrl,
- int maxMessages,
- int visibilityTimeout) {
-
- try {
- ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder()
- .queueUrl(queueUrl)
- .maxNumberOfMessages(maxMessages)
- .visibilityTimeout(visibilityTimeout)
- .waitTimeSeconds(0) // Non-blocking poll
- .messageSystemAttributeNames(MessageSystemAttributeName.SENT_TIMESTAMP) // Request SQS system timestamp
- .build();
-
- ReceiveMessageResponse response = sqsClient.receiveMessage(receiveRequest);
-
- LOGGER.debug("Received {} messages from SQS", response.messages().size());
- return response.messages();
-
- } catch (Exception e) {
- LOGGER.error("Error receiving messages from SQS", e);
- return new ArrayList<>();
- }
- }
-
- /**
- * Deletes messages from SQS in batches (max 10 per batch).
- *
- * @param sqsClient The SQS client
- * @param queueUrl The queue URL
- * @param messages Messages to delete
- */
- public static void deleteMessagesFromSqs(SqsClient sqsClient, String queueUrl, List messages) {
- if (messages.isEmpty()) {
- return;
- }
-
- try {
- List entries = new ArrayList<>();
- int batchId = 0;
- int totalDeleted = 0;
-
- for (Message msg : messages) {
- entries.add(DeleteMessageBatchRequestEntry.builder()
- .id(String.valueOf(batchId++))
- .receiptHandle(msg.receiptHandle())
- .build());
-
- // Send batch when we reach 10 messages or at the end
- if (entries.size() == SQS_MAX_DELETE_BATCH_SIZE || batchId == messages.size()) {
- DeleteMessageBatchRequest deleteRequest = DeleteMessageBatchRequest.builder()
- .queueUrl(queueUrl)
- .entries(entries)
- .build();
-
- DeleteMessageBatchResponse deleteResponse = sqsClient.deleteMessageBatch(deleteRequest);
-
- if (!deleteResponse.failed().isEmpty()) {
- LOGGER.error("Failed to delete {} messages from SQS", deleteResponse.failed().size());
- } else {
- totalDeleted += entries.size();
- }
-
- entries.clear();
- }
- }
-
- LOGGER.info("Deleted {} messages from SQS", totalDeleted);
-
- } catch (Exception e) {
- LOGGER.error("Error deleting messages from SQS", e);
- }
- }
-}
-
diff --git a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java b/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java
deleted file mode 100644
index 75368c62..00000000
--- a/src/main/java/com/uid2/optout/vertx/SqsWindowReader.java
+++ /dev/null
@@ -1,129 +0,0 @@
-package com.uid2.optout.vertx;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.services.sqs.SqsClient;
-import software.amazon.awssdk.services.sqs.model.Message;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Reads messages from SQS for complete 5-minute time windows.
- * Handles accumulation of all messages for a window before returning.
- * Limits messages per window to prevent memory issues.
- */
-public class SqsWindowReader {
- private static final Logger LOGGER = LoggerFactory.getLogger(SqsWindowReader.class);
-
- private final SqsClient sqsClient;
- private final String queueUrl;
- private final int maxMessagesPerPoll;
- private final int visibilityTimeout;
- private final int deltaWindowSeconds;
- private final int maxMessagesPerFile;
- private final SqsBatchProcessor batchProcessor;
-
- public SqsWindowReader(SqsClient sqsClient, String queueUrl, int maxMessagesPerPoll,
- int visibilityTimeout, int deltaWindowSeconds, int maxMessagesPerFile) {
- this.sqsClient = sqsClient;
- this.queueUrl = queueUrl;
- this.maxMessagesPerPoll = maxMessagesPerPoll;
- this.visibilityTimeout = visibilityTimeout;
- this.deltaWindowSeconds = deltaWindowSeconds;
- this.maxMessagesPerFile = maxMessagesPerFile;
- this.batchProcessor = new SqsBatchProcessor(sqsClient, queueUrl, deltaWindowSeconds);
- LOGGER.info("SqsWindowReader initialized with: maxMessagesPerFile: {}, maxMessagesPerPoll: {}, visibilityTimeout: {}, deltaWindowSeconds: {}",
- maxMessagesPerFile, maxMessagesPerPoll, visibilityTimeout, deltaWindowSeconds);
- }
-
- /**
- * Result of reading messages for a 5-minute window.
- */
- public static class WindowReadResult {
- private final List messages;
- private final long windowStart;
- private final boolean stoppedDueToMessagesTooRecent;
-
- public WindowReadResult(List messages, long windowStart,
- boolean stoppedDueToMessagesTooRecent) {
- this.messages = messages;
- this.windowStart = windowStart;
- this.stoppedDueToMessagesTooRecent = stoppedDueToMessagesTooRecent;
- }
-
- public List getMessages() { return messages; }
- public long getWindowStart() { return windowStart; }
- public boolean isEmpty() { return messages.isEmpty(); }
- public boolean stoppedDueToMessagesTooRecent() { return stoppedDueToMessagesTooRecent; }
- }
-
- /**
- * Reads messages from SQS for one complete 5-minute window.
- * Keeps reading batches and accumulating messages until:
- * - We discover the next window
- * - Queue is empty (no more messages)
- * - Messages are too recent (all messages younger than 5 minutes)
- * - Message limit is reached (memory protection)
- *
- * @return WindowReadResult with messages for the window, or empty if done
- */
- public WindowReadResult readWindow() {
- List windowMessages = new ArrayList<>();
- long currentWindowStart = 0;
-
- while (true) {
- // Check if we've hit the message limit
- if (windowMessages.size() >= this.maxMessagesPerFile) {
- LOGGER.warn("Window message limit reached ({} messages). Truncating window starting at {} for memory protection.",
- this.maxMessagesPerFile, currentWindowStart);
- return new WindowReadResult(windowMessages, currentWindowStart, false);
- }
-
- // Read one batch from SQS (up to 10 messages)
- List rawBatch = SqsMessageOperations.receiveMessagesFromSqs(
- this.sqsClient, this.queueUrl, this.maxMessagesPerPoll, this.visibilityTimeout);
-
- if (rawBatch.isEmpty()) {
- // Queue empty - return what we have
- return new WindowReadResult(windowMessages, currentWindowStart, false);
- }
-
- // Process batch: parse, validate, filter
- SqsBatchProcessor.BatchProcessingResult batchResult = batchProcessor.processBatch(rawBatch, 0);
-
- if (batchResult.isEmpty()) {
- if (batchResult.shouldStopProcessing()) {
- // Messages too recent - return what we have
- return new WindowReadResult(windowMessages, currentWindowStart, true);
- }
- // corrupt messages deleted, read next messages
- continue;
- }
-
- // Add eligible messages to current window
- boolean newWindow = false;
- for (SqsParsedMessage msg : batchResult.getEligibleMessages()) {
- long msgWindowStart = (msg.getTimestamp() / this.deltaWindowSeconds) * this.deltaWindowSeconds;
-
- // discover start of window
- if (currentWindowStart == 0) {
- currentWindowStart = msgWindowStart;
- }
-
- // discover new window
- if (msgWindowStart > currentWindowStart + this.deltaWindowSeconds) {
- newWindow = true;
- }
-
- windowMessages.add(msg);
- }
-
- if (newWindow) {
- // close current window and return
- return new WindowReadResult(windowMessages, currentWindowStart, false);
- }
- }
- }
-}
-
diff --git a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java b/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java
similarity index 99%
rename from src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java
rename to src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java
index 6fc1c865..778f858c 100644
--- a/src/test/java/com/uid2/optout/vertx/SqsBatchProcessorTest.java
+++ b/src/test/java/com/uid2/optout/sqs/SqsBatchProcessorTest.java
@@ -1,8 +1,9 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.sqs;
import io.vertx.core.json.JsonObject;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+
import software.amazon.awssdk.services.sqs.model.Message;
import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName;
diff --git a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java b/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java
similarity index 99%
rename from src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java
rename to src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java
index 810a7a41..5d6810db 100644
--- a/src/test/java/com/uid2/optout/vertx/SqsMessageParserTest.java
+++ b/src/test/java/com/uid2/optout/sqs/SqsMessageParserTest.java
@@ -1,7 +1,8 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.sqs;
import io.vertx.core.json.JsonObject;
import org.junit.jupiter.api.Test;
+
import software.amazon.awssdk.services.sqs.model.Message;
import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName;
import java.util.*;
@@ -269,3 +270,4 @@ public void testParseAndSortMessages_multipleValidMessages() {
}
}
+
diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java
similarity index 90%
rename from src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java
rename to src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java
index f977233d..fd435a9d 100644
--- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficCalculatorTest.java
+++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficCalculatorTest.java
@@ -1,9 +1,10 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.traffic;
import com.uid2.shared.cloud.CloudStorageException;
import com.uid2.shared.cloud.ICloudStorage;
import com.uid2.shared.optout.OptOutCollection;
import com.uid2.shared.optout.OptOutEntry;
+import com.uid2.optout.sqs.SqsMessageOperations;
import com.uid2.optout.Const;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
@@ -23,7 +24,8 @@
import software.amazon.awssdk.services.sqs.model.Message;
import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName;
-import com.uid2.optout.vertx.OptOutTrafficCalculator.MalformedTrafficCalcConfigException;
+import com.uid2.optout.traffic.OptOutTrafficCalculator;
+import com.uid2.optout.traffic.OptOutTrafficCalculator.MalformedTrafficCalcConfigException;
import java.io.ByteArrayInputStream;
import java.util.*;
@@ -856,11 +858,8 @@ void testDetermineStatus_sumPastZero() throws Exception {
OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
- // Act - should return DEFAULT to avoid crash
- OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(100, 0);
-
- // Assert
- assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
+ // Act & Assert - should throw exception for invalid config
+ assertThrows(RuntimeException.class, () -> calculator.determineStatus(100, 0));
}
@Test
@@ -869,11 +868,8 @@ void testDetermineStatus_bothZero() throws Exception {
OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
- // Act - should return DEFAULT to avoid crash
- OptOutTrafficCalculator.TrafficStatus status = calculator.determineStatus(0, 0);
-
- // Assert
- assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
+ // Act & Assert - should throw exception for invalid config
+ assertThrows(RuntimeException.class, () -> calculator.determineStatus(0, 0));
}
@Test
@@ -1120,11 +1116,8 @@ void testCalculateStatus_noDeltaFiles() throws Exception {
OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
- // Act
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList());
-
- // Assert - should return DEFAULT when no delta files
- assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
+ // Act & Assert - should throw exception when no delta files
+ assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0));
}
@Test
@@ -1152,7 +1145,7 @@ void testCalculateStatus_normalTraffic() throws Exception {
// Act
List sqsMessages = Arrays.asList(createSqsMessage(t));
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - 100+1 < 5 * 50 = 250, so should be DEFAULT
assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
@@ -1183,7 +1176,7 @@ void testCalculateStatus_delayedProcessing() throws Exception {
// Act
List sqsMessages = Arrays.asList(createSqsMessage(t));
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - 100+1 >= 5 * 10 = 50, DELAYED_PROCESSING
assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status);
@@ -1206,7 +1199,7 @@ void testCalculateStatus_noSqsMessages() throws Exception {
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
// Act - null SQS messages
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(null, null, 0, 0);
// Assert - should still calculate based on delta files, DEFAULT
assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
@@ -1229,7 +1222,7 @@ void testCalculateStatus_emptySqsMessages() throws Exception {
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
// Act - empty SQS messages
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList());
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList(), null, 0, 0);
// Assert - should still calculate based on delta files, DEFAULT
assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
@@ -1261,7 +1254,7 @@ void testCalculateStatus_multipleSqsMessages() throws Exception {
for (int i = 0; i < 30; i++) {
sqsMessages.add(createSqsMessage(t - i * 10));
}
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - DELAYED_PROCESSING
assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status);
@@ -1295,8 +1288,8 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception {
byte[] deltaFileBytes = createDeltaFileBytes(timestamps);
createConfigFromPartialJson(trafficCalcConfigJson);
- when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/delta-001.dat"));
- when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"))
+ when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat"));
+ when(cloudStorage.download("optout-v2/delta/optout-delta-001_2025-11-13T00.00.00Z_aaaaaaaa.dat"))
.thenReturn(new ByteArrayInputStream(deltaFileBytes));
OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
@@ -1304,7 +1297,7 @@ void testCalculateStatus_withTrafficCalcConfig() throws Exception {
// Act
List sqsMessages = Arrays.asList(createSqsMessage(t));
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - should filter out entries in traffic calc config ranges
// Only 300 from window count (not in traffic calc config ranges) + 1 SQS = 301
@@ -1330,13 +1323,13 @@ void testCalculateStatus_cacheUtilization() throws Exception {
// Act - first call should populate cache
List sqsMessages = Arrays.asList(createSqsMessage(t));
- calculator.calculateStatus(sqsMessages);
+ calculator.calculateStatus(sqsMessages, null, 0, 0);
Map stats = calculator.getCacheStats();
int cachedFiles = (Integer) stats.get("cached_files");
// Second call should use cache (no additional S3 download)
- calculator.calculateStatus(sqsMessages);
+ calculator.calculateStatus(sqsMessages, null, 0, 0);
Map stats2 = calculator.getCacheStats();
int cachedFiles2 = (Integer) stats2.get("cached_files");
@@ -1357,11 +1350,8 @@ void testCalculateStatus_s3Exception() throws Exception {
OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
- // Act - should not throw exception
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList());
-
- // Assert - DEFAULT on error
- assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
+ // Act & Assert - should throw exception on S3 error
+ assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0));
}
@Test
@@ -1374,11 +1364,8 @@ void testCalculateStatus_deltaFileReadException() throws Exception {
OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
- // Act - empty SQS messages
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(Collections.emptyList());
-
- // Assert - DEFAULT on error
- assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
+ // Act & Assert - should throw exception on S3 download error
+ assertThrows(RuntimeException.class, () -> calculator.calculateStatus(Collections.emptyList(), null, 0, 0));
}
@Test
@@ -1399,7 +1386,7 @@ void testCalculateStatus_invalidSqsMessageTimestamp() throws Exception {
// Act - SQS message without timestamp (should use current time)
List sqsMessages = Arrays.asList(createSqsMessageWithoutTimestamp());
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - DEFAULT
assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
@@ -1439,7 +1426,7 @@ void testCalculateStatus_multipleDeltaFiles() throws Exception {
// Act
List sqsMessages = Arrays.asList(createSqsMessage(t));
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - DEFAULT
assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
@@ -1473,7 +1460,7 @@ void testCalculateStatus_windowBoundaryTimestamp() throws Exception {
// Act
List sqsMessages = Arrays.asList(createSqsMessage(t));
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert - DEFAULT
assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status);
@@ -1497,7 +1484,7 @@ void testCalculateStatus_timestampsCached() throws Exception {
// Act
List sqsMessages = Arrays.asList(createSqsMessage(t));
- OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages);
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, null, 0, 0);
// Assert
assertEquals(OptOutTrafficCalculator.TrafficStatus.DEFAULT, status);
@@ -1507,4 +1494,84 @@ void testCalculateStatus_timestampsCached() throws Exception {
assertEquals(2, stats.get("total_cached_timestamps"));
}
-}
+ // ============================================================================
+ // SECTION 10: Tests for queue attributes (invisible messages from other consumers)
+ // ============================================================================
+
+ @Test
+ void testCalculateStatus_delayedProcessingFromQueueAttributesOnly() throws Exception {
+ // Setup - delta files with low traffic (10 records)
+ // Threshold = 100 * 5 = 500
+ // Queue attributes will have 600 invisible messages (other consumers processing)
+ long currentTime = System.currentTimeMillis() / 1000;
+ long t = currentTime;
+
+ List timestamps = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ timestamps.add(t - 3600 + i); // 10 entries from 1 hour ago
+ }
+ byte[] deltaFileBytes = createDeltaFileBytes(timestamps);
+
+ when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"));
+ when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"))
+ .thenReturn(new ByteArrayInputStream(deltaFileBytes));
+
+ OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
+ cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
+
+ // Act - 1 message read by us, but 600 invisible messages from other consumers
+ List sqsMessages = Arrays.asList(createSqsMessage(t));
+
+ // QueueAttributes: 0 visible, 600 invisible (other consumers), 0 delayed
+ // Since we read 1 message, otherConsumers = 600 - 1 = 599
+ // Total = 10 (delta) + 1 (our message) + 599 (other consumers) = 610 >= 500 threshold
+ SqsMessageOperations.QueueAttributes queueAttributes =
+ new SqsMessageOperations.QueueAttributes(0, 600, 0);
+
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0);
+
+ // Assert - DELAYED_PROCESSING due to high invisible message count from other consumers
+ assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status);
+ }
+
+ @Test
+ void testCalculateStatus_delayedProcessingFromBothQueueAndMessages() throws Exception {
+ // Setup - delta files with moderate traffic (100 records)
+ // Threshold = 100 * 5 = 500
+ // We'll have 200 messages + 250 invisible from other consumers = 550 > 500
+ long currentTime = System.currentTimeMillis() / 1000;
+ long t = currentTime;
+
+ List timestamps = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ timestamps.add(t - 3600 + i); // 100 entries from 1 hour ago
+ }
+ byte[] deltaFileBytes = createDeltaFileBytes(timestamps);
+
+ when(cloudStorage.list(S3_DELTA_PREFIX)).thenReturn(Arrays.asList("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"));
+ when(cloudStorage.download("optout-v2/delta/optout-delta--01_2025-11-13T00.00.00Z_aaaaaaaa.dat"))
+ .thenReturn(new ByteArrayInputStream(deltaFileBytes));
+
+ OptOutTrafficCalculator calculator = new OptOutTrafficCalculator(
+ cloudStorage, S3_DELTA_PREFIX, TRAFFIC_CONFIG_PATH);
+
+ // Act - 200 messages read by us + 450 invisible (200 are ours + 250 from others)
+ // Messages must be within 5-minute window to be counted, so use 1-second spacing
+ List sqsMessages = new ArrayList<>();
+ for (int i = 0; i < 200; i++) {
+ sqsMessages.add(createSqsMessage(t - i)); // 1 second apart, all within 5-minute window
+ }
+
+ // QueueAttributes: 0 visible, 450 invisible (200 ours + 250 others), 0 delayed
+ // otherConsumers = 450 - 200 = 250
+ // Total = 100 (delta) + 200 (our messages) + 250 (other consumers) = 550 >= 500 threshold
+ SqsMessageOperations.QueueAttributes queueAttributes =
+ new SqsMessageOperations.QueueAttributes(0, 450, 0);
+
+ OptOutTrafficCalculator.TrafficStatus status = calculator.calculateStatus(sqsMessages, queueAttributes, 0, 0);
+
+ // Assert - DELAYED_PROCESSING due to combined count exceeding threshold
+ assertEquals(OptOutTrafficCalculator.TrafficStatus.DELAYED_PROCESSING, status);
+ }
+
+}
\ No newline at end of file
diff --git a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java
similarity index 99%
rename from src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java
rename to src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java
index 63f6807c..127cd041 100644
--- a/src/test/java/com/uid2/optout/vertx/OptOutTrafficFilterTest.java
+++ b/src/test/java/com/uid2/optout/traffic/OptOutTrafficFilterTest.java
@@ -1,8 +1,12 @@
-package com.uid2.optout.vertx;
+package com.uid2.optout.traffic;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+
+import com.uid2.optout.sqs.SqsParsedMessage;
+import com.uid2.optout.traffic.OptOutTrafficFilter;
+
import software.amazon.awssdk.services.sqs.model.Message;
import java.nio.file.Files;
diff --git a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java
index a46023f6..b874db9d 100644
--- a/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java
+++ b/src/test/java/com/uid2/optout/vertx/OptOutSqsLogProducerTest.java
@@ -4,6 +4,8 @@
import com.uid2.shared.cloud.ICloudStorage;
import com.uid2.shared.optout.OptOutCloudSync;
import com.uid2.shared.vertx.VertxUtils;
+import com.uid2.shared.optout.OptOutEntry;
+import com.uid2.shared.optout.OptOutCollection;
import io.vertx.core.Vertx;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.unit.Async;
@@ -15,11 +17,18 @@
import software.amazon.awssdk.services.sqs.model.*;
import java.io.InputStream;
+import java.io.ByteArrayInputStream;
import java.util.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.CountDownLatch;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.doAnswer;
+import org.mockito.ArgumentCaptor;
/**
* Integration tests for OptOutSqsLogProducer deltaproduce endpoint.
@@ -33,12 +42,18 @@ public class OptOutSqsLogProducerTest {
private SqsClient sqsClient;
private ICloudStorage cloudStorage;
+ private ICloudStorage cloudStorageDroppedRequests;
private OptOutCloudSync cloudSync;
private static final String TEST_QUEUE_URL = "https://sqs.test.amazonaws.com/123456789/test";
private static final String TEST_API_KEY = "test-api-key";
private static final String VALID_HASH_BASE64 = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
private static final String VALID_ID_BASE64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=";
+ private static final String TRAFFIC_FILTER_CONFIG_PATH = "./traffic-filter.json";
+ private static final String TRAFFIC_CALC_CONFIG_PATH = "./traffic-calc.json";
+ private static final String MANUAL_OVERRIDE_S3_PATH = "manual-override.json";
+ private static final String S3_DELTA_PREFIX = "sqs-delta";
+ private static final String TEST_BUCKET_DROPPED_REQUESTS = "test-bucket-dropped-requests";
@Before
public void setup(TestContext context) throws Exception {
@@ -47,6 +62,7 @@ public void setup(TestContext context) throws Exception {
// Create mocks
sqsClient = mock(SqsClient.class);
cloudStorage = mock(ICloudStorage.class);
+ cloudStorageDroppedRequests = mock(ICloudStorage.class);
cloudSync = mock(OptOutCloudSync.class);
JsonObject config = VertxUtils.getJsonConfig(vertx);
@@ -54,7 +70,13 @@ public void setup(TestContext context) throws Exception {
.put(Const.Config.OptOutSqsVisibilityTimeoutProp, 240)
.put(Const.Config.OptOutProducerBufferSizeProp, 65536)
.put(Const.Config.OptOutProducerReplicaIdProp, 1)
- .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY);
+ .put(Const.Config.OptOutInternalApiTokenProp, TEST_API_KEY)
+ .put(Const.Config.TrafficFilterConfigPathProp, TRAFFIC_FILTER_CONFIG_PATH)
+ .put(Const.Config.TrafficCalcConfigPathProp, TRAFFIC_CALC_CONFIG_PATH)
+ .put(Const.Config.ManualOverrideS3PathProp, MANUAL_OVERRIDE_S3_PATH)
+ .put(Const.Config.OptOutS3BucketDroppedRequestsProp, TEST_BUCKET_DROPPED_REQUESTS)
+ .put(Const.Config.OptOutSqsS3FolderProp, S3_DELTA_PREFIX)
+ .put(Const.Config.OptOutMaxMessagesPerFileProp, 100); // Low limit for circuit breaker tests
// Mock cloud sync to return proper S3 paths
when(cloudSync.toCloudPath(anyString()))
@@ -62,9 +84,50 @@ public void setup(TestContext context) throws Exception {
// Mock S3 upload to succeed by default
doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString());
+ doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString());
+
+ // Mock S3 list for traffic calculator (required for delta production)
+ when(cloudStorage.list(anyString())).thenReturn(Arrays.asList("sqs-delta/delta/optout-delta-001_2025-01-01T00.00.00Z_aaaaaaaa.dat"));
+ // Mock S3 download - manual override returns null (not found), delta files return valid bytes
+ when(cloudStorage.download(MANUAL_OVERRIDE_S3_PATH)).thenReturn(null);
+ when(cloudStorage.download(argThat(path -> path != null && path.contains("optout-delta"))))
+ .thenAnswer(inv -> new ByteArrayInputStream(createMinimalDeltaFileBytes()));
+
+ // Mock getQueueAttributes by default (returns zero messages)
+ Map defaultQueueAttrs = new HashMap<>();
+ defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, "0");
+ defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE, "0");
+ defaultQueueAttrs.put(QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, "0");
+ doReturn(GetQueueAttributesResponse.builder()
+ .attributes(defaultQueueAttrs)
+ .build())
+ .when(sqsClient).getQueueAttributes(any(GetQueueAttributesRequest.class));
+
+
+ try {
+ String traficFilterConfig = """
+ {
+ "denylist_requests": [
+ ]
+ }
+ """;
+ createTrafficConfigFile(traficFilterConfig);
+
+ String trafficCalcConfig = """
+ {
+ "traffic_calc_evaluation_window_seconds": 86400,
+ "traffic_calc_baseline_traffic": 100,
+ "traffic_calc_threshold_multiplier": 5,
+ "traffic_calc_allowlist_ranges": []
+ }
+ """;
+ createTrafficCalcConfigFile(trafficCalcConfig);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
// Create producer with mock SqsClient
- producer = new OptOutSqsLogProducer(config, cloudStorage, cloudSync, Const.Event.DeltaProduce, sqsClient);
+ producer = new OptOutSqsLogProducer(config, cloudStorage, cloudStorageDroppedRequests, cloudSync, Const.Event.DeltaProduce, sqsClient);
// Deploy verticle
Async async = context.async();
@@ -76,12 +139,53 @@ public void tearDown(TestContext context) {
if (vertx != null) {
vertx.close(context.asyncAssertSuccess());
}
+ if (Files.exists(Path.of(TRAFFIC_FILTER_CONFIG_PATH))) {
+ try {
+ Files.delete(Path.of(TRAFFIC_FILTER_CONFIG_PATH));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ if (Files.exists(Path.of(TRAFFIC_CALC_CONFIG_PATH))) {
+ try {
+ Files.delete(Path.of(TRAFFIC_CALC_CONFIG_PATH));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
-
+
+ private void createTrafficConfigFile(String content) {
+ try {
+ Path configPath = Path.of(TRAFFIC_FILTER_CONFIG_PATH);
+ Files.writeString(configPath, content);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void createTrafficCalcConfigFile(String content) {
+ try {
+ Path configPath = Path.of(TRAFFIC_CALC_CONFIG_PATH);
+ Files.writeString(configPath, content);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private Message createMessage(String hash, String id, long timestampMs) {
+ return createMessage(hash, id, timestampMs, null, null, null, null);
+ }
+
+ private Message createMessage(String hash, String id, long timestampMs, String email, String phone, String clientIp, String traceId) {
JsonObject body = new JsonObject()
.put("identity_hash", hash)
.put("advertising_id", id);
+
+ if (email != null) body.put("email", email);
+ if (phone != null) body.put("phone", phone);
+ if (clientIp != null) body.put("client_ip", clientIp);
+ if (traceId != null) body.put("trace_id", traceId);
Map attrs = new HashMap<>();
attrs.put(MessageSystemAttributeName.SENT_TIMESTAMP, String.valueOf(timestampMs));
@@ -232,7 +336,7 @@ public void testDeltaProduceEndpoint_noMessages(TestContext context) {
context.assertEquals("completed", finalStatus.getString("state"));
JsonObject result = finalStatus.getJsonObject("result");
context.assertNotNull(result);
- context.assertEquals("success", result.getString("status"));
+ context.assertEquals("skipped", result.getString("status"));
context.assertEquals(0, result.getInteger("deltas_produced"));
context.assertEquals(0, result.getInteger("entries_processed"));
@@ -287,7 +391,7 @@ public void testDeltaProduceEndpoint_allMessagesTooRecent(TestContext context) {
JsonObject result = finalStatus.getJsonObject("result");
context.assertNotNull(result);
context.assertEquals("skipped", result.getString("status"));
- context.assertEquals("All messages too recent", result.getString("reason"));
+ context.assertEquals("MESSAGES_TOO_RECENT", result.getString("stop_reason"));
// No processing should occur
try {
@@ -331,9 +435,15 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context
createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime)
);
- // Mock SQS to return messages
+ // Use CountDownLatch to control when the mock returns - ensures job stays running
+ CountDownLatch processingLatch = new CountDownLatch(1);
+
+ // Mock SQS to wait on latch before returning - keeps job in RUNNING state
when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
- .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenAnswer(inv -> {
+ processingLatch.await(); // Wait until test releases
+ return ReceiveMessageResponse.builder().messages(messages).build();
+ })
.thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
@@ -373,8 +483,10 @@ public void testDeltaProduceEndpoint_concurrentJobPrevention(TestContext context
.onComplete(context.asyncAssertSuccess(body -> {
JsonObject response = new JsonObject(body.toString());
context.assertEquals("conflict", response.getString("status"));
- context.assertTrue(response.getString("message").contains("already running"));
+ context.assertTrue(response.getString("reason").contains("already running"));
+ // Release the latch so first job can complete
+ processingLatch.countDown();
async.complete();
}));
}
@@ -460,5 +572,902 @@ public void testDeltaProduceEndpoint_autoClearCompletedJob(TestContext context)
async.complete();
}));
}
+
+ @Test
+ public void testTrafficFilter_denylistedMessagesAreDropped(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - update traffic filter config to denylist specific IP and time range
+ long baseTime = System.currentTimeMillis() / 1000 - 400; // 400 seconds ago
+ String filterConfig = String.format("""
+ {
+ "denylist_requests": [
+ {
+ "range": [%d, %d],
+ "IPs": ["192.168.1.100"]
+ }
+ ]
+ }
+ """, baseTime - 100, baseTime + 100);
+ createTrafficConfigFile(filterConfig);
+
+ // Setup - create messages: some denylisted, some not
+ long denylistedTime = (baseTime) * 1000; // Within denylist range
+ long normalTime = (baseTime - 200) * 1000; // Outside denylist range
+ List messages = Arrays.asList(
+ // These should be dropped (denylisted)
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime, null, null, "192.168.1.100", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime + 1000, null, null, "192.168.1.100", null),
+ // These should be processed normally
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime + 1000, null, null, "10.0.0.2", null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("success", result.getString("status"));
+
+ // Should process 2 normal entries
+ context.assertEquals(2, result.getInteger("entries_processed"));
+
+ // Should have 2 dropped requests
+ context.assertEquals(2, result.getInteger("dropped_requests_processed"));
+
+ // Verify both delta and dropped request files were uploaded
+ try {
+ verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), anyString());
+ verify(cloudStorageDroppedRequests, atLeastOnce()).upload(any(InputStream.class), anyString());
+ } catch (Exception e) {
+ context.fail(e);
+ }
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testTrafficFilter_noBlacklistedMessages(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - traffic filter with a denylisted IP
+ long baseTime = System.currentTimeMillis() / 1000 - 400;
+ String filterConfig = String.format("""
+ {
+ "denylist_requests": [
+ {
+ "range": [%d, %d],
+ "IPs": ["192.168.1.100"]
+ }
+ ]
+ }
+ """, baseTime - 100, baseTime + 100);
+ createTrafficConfigFile(filterConfig);
+
+ // Setup - create messages that don't match denylist
+ long normalTime = (baseTime - 200) * 1000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime, null, null, "10.0.0.1", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, normalTime + 1000, null, null, "10.0.0.2", null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("success", result.getString("status"));
+ context.assertEquals(2, result.getInteger("entries_processed"));
+ context.assertEquals(0, result.getInteger("dropped_requests_processed"));
+
+ // Should not upload dropped request file
+ try {
+ verify(cloudStorageDroppedRequests, never()).upload(any(InputStream.class), anyString());
+ } catch (Exception e) {
+ context.fail(e);
+ }
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testTrafficFilter_allMessagesBlacklisted(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - traffic filter with a denylisted IP
+ long baseTime = System.currentTimeMillis() / 1000 - 400;
+ String filterConfig = String.format("""
+ {
+ "denylist_requests": [
+ {
+ "range": [%d, %d],
+ "IPs": ["192.168.1.100"]
+ }
+ ]
+ }
+ """, baseTime - 100, baseTime + 100);
+ createTrafficConfigFile(filterConfig);
+
+ // Setup - create messages that are denylisted
+ long denylistedTime = baseTime * 1000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime, null, null, "192.168.1.100", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, denylistedTime + 1000, null, null, "192.168.1.100", null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("success", result.getString("status"));
+
+ // No entries processed (all denylisted)
+ context.assertEquals(0, result.getInteger("entries_processed"));
+
+ // All messages dropped
+ context.assertEquals(2, result.getInteger("dropped_requests_processed"));
+
+ // Should upload dropped request file but not delta file
+ try {
+ verify(cloudStorageDroppedRequests, atLeastOnce()).upload(any(InputStream.class), anyString());
+ verify(cloudStorage, never()).upload(any(InputStream.class), anyString());
+ } catch (Exception e) {
+ context.fail(e);
+ }
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testTrafficFilter_messagesWithoutClientIp(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - traffic filter with a denylisted IP
+ long baseTime = System.currentTimeMillis() / 1000 - 400;
+ String filterConfig = String.format("""
+ {
+ "denylist_requests": [
+ {
+ "range": [%d, %d],
+ "IPs": ["192.168.1.100"]
+ }
+ ]
+ }
+ """, baseTime - 100, baseTime + 100);
+ createTrafficConfigFile(filterConfig);
+
+ // Create messages without client IP (should not be denylisted)
+ long time = baseTime * 1000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, time, null, null, null, null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("success", result.getString("status"));
+
+ // Message should be processed (not denylisted due to missing IP)
+ context.assertEquals(1, result.getInteger("entries_processed"));
+ context.assertEquals(0, result.getInteger("dropped_requests_processed"));
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testTrafficFilterConfig_reloadOnEachBatch(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - initial config with no denylist
+ String initialConfig = """
+ {
+ "denylist_requests": []
+ }
+ """;
+ createTrafficConfigFile(initialConfig);
+
+ // Setup - create messages
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "192.168.1.100", null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ // Act & Assert - first request - should process normally
+ int port = Const.Port.ServicePortForOptOut + 1;
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals(1, result.getInteger("entries_processed"));
+ context.assertEquals(0, result.getInteger("dropped_requests_processed"));
+
+ // Update config to denylist the IP
+ try {
+ long baseTime = System.currentTimeMillis() / 1000 - 400;
+ String updatedConfig = String.format("""
+ {
+ "denylist_requests": [
+ {
+ "range": [%d, %d],
+ "IPs": ["192.168.1.100"]
+ }
+ ]
+ }
+ """, baseTime - 100, baseTime + 100);
+ createTrafficConfigFile(updatedConfig);
+
+ // Reset mocks for second request
+ reset(sqsClient, cloudStorage, cloudStorageDroppedRequests);
+
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString());
+ doAnswer(inv -> null).when(cloudStorageDroppedRequests).upload(any(InputStream.class), anyString());
+
+ // Re-mock S3 list and download for traffic calculator after reset
+ when(cloudStorage.list(anyString())).thenReturn(Arrays.asList("sqs-delta/delta/optout-delta-001_2025-01-01T00.00.00Z_aaaaaaaa.dat"));
+ when(cloudStorage.download(MANUAL_OVERRIDE_S3_PATH)).thenReturn(null);
+ when(cloudStorage.download(argThat(path -> path != null && path.contains("optout-delta"))))
+ .thenAnswer(inv -> new ByteArrayInputStream(createMinimalDeltaFileBytes()));
+
+ // Act & Assert - second request - should now be denylisted
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body2 -> {
+ JsonObject response2 = new JsonObject(body2.toString());
+ context.assertEquals("accepted", response2.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus2 -> {
+ context.assertEquals("completed", finalStatus2.getString("state"));
+ JsonObject result2 = finalStatus2.getJsonObject("result");
+ // Now should be denylisted
+ context.assertEquals(0, result2.getInteger("entries_processed"));
+ context.assertEquals(1, result2.getInteger("dropped_requests_processed"));
+ async.complete();
+ }));
+ } catch (Exception e) {
+ context.fail(e);
+ }
+ }));
+ }
+
+ @Test
+ public void testTrafficCalculator_defaultStatus(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - traffic calc config with required fields
+ String trafficCalcConfig = """
+ {
+ "traffic_calc_evaluation_window_seconds": 86400,
+ "traffic_calc_baseline_traffic": 100,
+ "traffic_calc_threshold_multiplier": 5,
+ "traffic_calc_allowlist_ranges": []
+ }
+ """;
+ createTrafficCalcConfigFile(trafficCalcConfig);
+
+ // Note: manual override is already mocked to return null in setup
+
+ // Setup - create messages that will result in DEFAULT status
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime + 1000, null, null, "10.0.0.2", null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("success", result.getString("status"));
+
+ // Should process messages normally (DEFAULT status)
+ context.assertEquals(2, result.getInteger("entries_processed"));
+ context.assertTrue(result.getInteger("deltas_produced") >= 1);
+
+ // Verify upload happened
+ try {
+ verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), anyString());
+ } catch (Exception e) {
+ context.fail(e);
+ }
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testManualOverride_delayedProcessing(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Setup - mock manual override set to DELAYED_PROCESSING
+ JsonObject manualOverride = new JsonObject().put("manual_override", "DELAYED_PROCESSING");
+ doReturn(new java.io.ByteArrayInputStream(manualOverride.encode().getBytes()))
+ .when(cloudStorage).download(MANUAL_OVERRIDE_S3_PATH); // At root of bucket
+
+ // Setup - create messages (won't be processed due to override)
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime + 1000, null, null, "10.0.0.2", null)
+ );
+
+ List allMessages = new ArrayList<>(messages);
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(allMessages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("halted", result.getString("status"));
+ context.assertEquals("MANUAL_OVERRIDE_ACTIVE", result.getString("stop_reason"));
+
+ // Should not process anything - manual override checked at start
+ context.assertEquals(0, result.getInteger("entries_processed"));
+ context.assertEquals(0, result.getInteger("deltas_produced"));
+
+ // No SQS deletions should occur (messages not processed)
+ verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class));
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testTrafficCalculator_detectsSpikeInCurrentWindow(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Threshold = baseline * multiplier = 100 * 5 = 500
+ // Traffic calculator counts: delta file records + SQS messages (with allowlist filtering)
+ // We'll create 600 SQS messages to exceed threshold
+ String trafficCalcConfig = """
+ {
+ "traffic_calc_evaluation_window_seconds": 86400,
+ "traffic_calc_baseline_traffic": 100,
+ "traffic_calc_threshold_multiplier": 5,
+ "traffic_calc_allowlist_ranges": []
+ }
+ """;
+ createTrafficCalcConfigFile(trafficCalcConfig);
+
+ // Setup time
+ long currentTime = System.currentTimeMillis() / 1000;
+ long t = currentTime;
+
+ // Create historical delta file with recent timestamps (within 24h evaluation window)
+ List timestamps = new ArrayList<>();
+ timestamps.add(t - 3600); // 1 hour ago
+ timestamps.add(t - 3600 + 1000);
+ byte[] deltaFileBytes = createDeltaFileBytes(timestamps);
+
+ // Reset cloudStorage mock to ensure clean state
+ reset(cloudStorage);
+
+ // Re-mock S3 upload (needed after reset)
+ doAnswer(inv -> null).when(cloudStorage).upload(any(InputStream.class), anyString());
+
+ // Mock S3 operations for historical data
+ doReturn(Arrays.asList("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_baseline.dat"))
+ .when(cloudStorage).list("sqs-delta");
+ doAnswer(inv -> new ByteArrayInputStream(deltaFileBytes))
+ .when(cloudStorage).download("sqs-delta/delta/optout-delta--01_2025-11-13T00.00.00Z_baseline.dat");
+
+ // No manual override set (returns null)
+ doReturn(null).when(cloudStorage).download("manual-override.json");
+
+ // Setup SQS messages - 600 messages to exceed threshold (500)
+ // Traffic calculator counts SQS messages and can apply allowlist filtering
+ long baseTime = (t - 600) * 1000; // 10 minutes ago
+ List allMessages = new ArrayList<>();
+ for (int i = 0; i < 600; i++) {
+ long timestampMs = baseTime - (i * 100); // spread over ~60 seconds
+ allMessages.add(createMessage(VALID_HASH_BASE64, VALID_ID_BASE64,
+ timestampMs, null, null, "10.0.0." + (i % 256), null));
+ }
+
+ // Mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(allMessages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+
+ // Act & Assert
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("halted", result.getString("status"));
+ context.assertEquals("CIRCUIT_BREAKER_TRIGGERED", result.getString("stop_reason"));
+
+ // Expected behavior:
+ // Traffic calculator detects spike (delta records + SQS messages >= threshold 500)
+ // DELAYED_PROCESSING is triggered, no delta uploaded
+ context.assertEquals(0, result.getInteger("entries_processed"));
+ context.assertEquals(0, result.getInteger("deltas_produced"));
+
+ // Verify manual override was set to DELAYED_PROCESSING on S3
+ try {
+ ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class);
+ verify(cloudStorage, atLeastOnce()).upload(any(InputStream.class), pathCaptor.capture());
+
+ boolean overrideSet = pathCaptor.getAllValues().stream()
+ .anyMatch(path -> path.equals("manual-override.json"));
+ context.assertTrue(overrideSet, "Manual override should be set to DELAYED_PROCESSING after detecting spike");
+ } catch (Exception e) {
+ context.fail(e);
+ }
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testManualOverride_notSet(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Note: manual override is already mocked to return null in setup
+
+ // Setup - create messages
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null)
+ );
+
+ // Setup - mock SQS operations
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ // Act & Assert - call endpoint via HTTP
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ context.assertEquals("completed", finalStatus.getString("state"));
+ JsonObject result = finalStatus.getJsonObject("result");
+ context.assertEquals("success", result.getString("status"));
+
+ // Should process normally with traffic calc (no override)
+ context.assertEquals(1, result.getInteger("entries_processed"));
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testS3UploadFailure_messagesNotDeletedFromSqs(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Create messages to process
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime, null, null, "10.0.0.1", null),
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime + 1000, null, null, "10.0.0.2", null)
+ );
+
+ // Mock SQS to return messages
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ // Mock S3 upload to FAIL
+ doThrow(new RuntimeException("S3 upload failed - simulated error"))
+ .when(cloudStorage).upload(any(InputStream.class), anyString());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+
+ // Start job
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ JsonObject response = new JsonObject(body.toString());
+ context.assertEquals("accepted", response.getString("status"));
+ return pollForCompletion(context, port, 100, 50);
+ })
+ .onComplete(context.asyncAssertSuccess(finalStatus -> {
+ // Job should fail due to S3 error
+ context.assertEquals("failed", finalStatus.getString("state"));
+ context.assertTrue(finalStatus.getString("error").contains("S3") ||
+ finalStatus.getString("error").contains("upload"));
+
+ // CRITICAL: Messages should NOT be deleted from SQS when upload fails
+ verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class));
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testStatusEndpoint_showsRunningJob(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Create messages
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime)
+ );
+
+ // Use CountDownLatch to keep job running
+ CountDownLatch processingLatch = new CountDownLatch(1);
+
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenAnswer(inv -> {
+ processingLatch.await();
+ return ReceiveMessageResponse.builder().messages(messages).build();
+ })
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class)))
+ .thenReturn(DeleteMessageBatchResponse.builder().build());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+
+ // Start job
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> {
+ // Immediately check status - should show "running"
+ return vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.GET, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString() + "/status")
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send());
+ })
+ .compose(resp -> {
+ context.assertEquals(200, resp.statusCode());
+ return resp.body();
+ })
+ .onComplete(context.asyncAssertSuccess(body -> {
+ JsonObject status = new JsonObject(body.toString());
+ context.assertEquals("running", status.getString("state"));
+ context.assertNotNull(status.getString("start_time"));
+
+ // Release latch so job can complete
+ processingLatch.countDown();
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testStatusEndpoint_showsFailedJob(TestContext context) throws Exception {
+ Async async = context.async();
+
+ // Create messages
+ long oldTime = System.currentTimeMillis() - 400_000;
+ List messages = Arrays.asList(
+ createMessage(VALID_HASH_BASE64, VALID_ID_BASE64, oldTime)
+ );
+
+ when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class)))
+ .thenReturn(ReceiveMessageResponse.builder().messages(messages).build())
+ .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build());
+
+ // Make S3 upload fail
+ doThrow(new RuntimeException("Simulated S3 failure"))
+ .when(cloudStorage).upload(any(InputStream.class), anyString());
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+
+ // Start job and wait for it to fail
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send())
+ .compose(resp -> {
+ context.assertEquals(202, resp.statusCode());
+ return resp.body();
+ })
+ .compose(body -> pollForCompletion(context, port, 100, 50))
+ .compose(finalStatus -> {
+ context.assertEquals("failed", finalStatus.getString("state"));
+
+ // Now call status endpoint directly to verify failed state is persisted
+ return vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.GET, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString() + "/status")
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer " + TEST_API_KEY)
+ .send());
+ })
+ .compose(resp -> {
+ context.assertEquals(200, resp.statusCode());
+ return resp.body();
+ })
+ .onComplete(context.asyncAssertSuccess(body -> {
+ JsonObject status = new JsonObject(body.toString());
+ context.assertEquals("failed", status.getString("state"));
+ context.assertNotNull(status.getString("error"));
+ context.assertNotNull(status.getString("start_time"));
+ context.assertNotNull(status.getString("end_time"));
+ context.assertNotNull(status.getInteger("duration_seconds"));
+
+ async.complete();
+ }));
+ }
+
+ @Test
+ public void testDeltaProduceEndpoint_invalidApiKey(TestContext context) {
+ Async async = context.async();
+
+ int port = Const.Port.ServicePortForOptOut + 1;
+ vertx.createHttpClient()
+ .request(io.vertx.core.http.HttpMethod.POST, port, "127.0.0.1",
+ Endpoints.OPTOUT_DELTA_PRODUCE.toString())
+ .compose(req -> req
+ .putHeader("Authorization", "Bearer wrong-api-key")
+ .send())
+ .compose(resp -> {
+ context.assertEquals(401, resp.statusCode());
+ return resp.body();
+ })
+ .onComplete(context.asyncAssertSuccess(body -> {
+ // Should not call SQS when unauthorized
+ verify(sqsClient, never()).receiveMessage(any(ReceiveMessageRequest.class));
+ async.complete();
+ }));
+ }
+
+ /**
+ * Create delta file bytes with specified timestamps
+ */
+ private byte[] createDeltaFileBytes(List timestamps) throws Exception {
+ // Create OptOutEntry objects using newTestEntry
+ List entries = new ArrayList<>();
+
+ long idCounter = 1000; // Use incrementing IDs for test entries
+ for (long timestamp : timestamps) {
+ entries.add(OptOutEntry.newTestEntry(idCounter++, timestamp));
+ }
+
+ // Create OptOutCollection
+ OptOutCollection collection = new OptOutCollection(entries.toArray(new OptOutEntry[0]));
+ return collection.getStore();
+ }
+
+ /**
+ * Create minimal delta file bytes with a single recent timestamp for traffic calculator
+ */
+ private static byte[] createMinimalDeltaFileBytes() {
+ try {
+ long timestamp = System.currentTimeMillis() / 1000 - 3600; // 1 hour ago
+ OptOutEntry entry = OptOutEntry.newTestEntry(1, timestamp);
+ OptOutCollection collection = new OptOutCollection(new OptOutEntry[]{entry});
+ return collection.getStore();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create minimal delta file bytes", e);
+ }
+ }
}