Skip to content

Misc cleanups in S3BlobContainerRetriesTests #126101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 4, 2025
Merged

Misc cleanups in S3BlobContainerRetriesTests #126101

Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f48bff2
Misc cleanups in `S3BlobContainerRetriesTests`
DaveCTurner Apr 2, 2025
7b18bcf
Merge branch 'main' into 2025/04/02/S3BlobContainerRetriesTests-reque…
DaveCTurner Apr 2, 2025
3270fc6
[CI] Auto commit changes from spotless
Apr 2, 2025
c9198d5
Merge branch 'main' into 2025/04/02/S3BlobContainerRetriesTests-reque…
DaveCTurner Apr 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions ...itory-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,19 +114,19 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes

private static final int MAX_NUMBER_SNAPSHOT_DELETE_RETRIES = 10;
private S3Service service;
private AtomicBoolean shouldErrorOnDns;
private volatile boolean shouldErrorOnDns;
private RecordingMeterRegistry recordingMeterRegistry;

@Before
public void setUp() throws Exception {
shouldErrorOnDns = new AtomicBoolean(false);
shouldErrorOnDns = false;
service = new S3Service(Mockito.mock(Environment.class), Settings.EMPTY, Mockito.mock(ResourceWatcherService.class)) {
@Override
protected AmazonS3ClientBuilder buildClientBuilder(S3ClientSettings clientSettings) {
final AmazonS3ClientBuilder builder = super.buildClientBuilder(clientSettings);
final DnsResolver defaultDnsResolver = builder.getClientConfiguration().getDnsResolver();
builder.getClientConfiguration().setDnsResolver(host -> {
if (shouldErrorOnDns.get() && randomBoolean() && randomBoolean()) {
if (shouldErrorOnDns && randomBoolean() && randomBoolean()) {
throw new UnknownHostException(host);
}
return defaultDnsResolver.resolve(host);
Expand Down Expand Up @@ -667,7 +667,7 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {

final byte[] bytes = randomBlobContent(512);

shouldErrorOnDns.set(true);
shouldErrorOnDns = true;
final AtomicInteger failures = new AtomicInteger();
@SuppressForbidden(reason = "use a http server")
class FlakyReadHandler implements HttpHandler {
Expand Down Expand Up @@ -895,7 +895,7 @@ private int expectedNumberOfBatches(int blobsToDelete) {
}

@SuppressForbidden(reason = "use a http server")
private class ThrottlingDeleteHandler extends S3HttpHandler {
private static class ThrottlingDeleteHandler extends S3HttpHandler {

private static final String THROTTLING_ERROR_CODE = "SlowDown";

Expand All @@ -920,7 +920,7 @@ private class ThrottlingDeleteHandler extends S3HttpHandler {

@Override
public void handle(HttpExchange exchange) throws IOException {
if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
if (isMultiDeleteRequest(exchange)) {
onAttemptCallback.accept(numberOfDeleteAttempts.get());
numberOfDeleteAttempts.incrementAndGet();
if (throttleTimesBeforeSuccess.getAndDecrement() > 0) {
Expand Down Expand Up @@ -1024,7 +1024,7 @@ public void testSuppressedDeletionErrorsAreCapped() {
int maxBulkDeleteSize = randomIntBetween(1, 10);
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize);
httpServer.createContext("/", exchange -> {
if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
if (isMultiDeleteRequest(exchange)) {
exchange.sendResponseHeaders(
randomFrom(
HttpStatus.SC_INTERNAL_SERVER_ERROR,
Expand Down Expand Up @@ -1058,7 +1058,7 @@ public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException

final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
httpServer.createContext("/", exchange -> {
if (exchange.getRequestMethod().equals("POST") && exchange.getRequestURI().toString().startsWith("/bucket/?delete")) {
if (isMultiDeleteRequest(exchange)) {
final String requestBody = Streams.copyToString(new InputStreamReader(exchange.getRequestBody(), StandardCharsets.UTF_8));
final var matcher = pattern.matcher(requestBody);
final StringBuilder deletes = new StringBuilder();
Expand Down Expand Up @@ -1195,6 +1195,10 @@ private Map<String, Object> metricAttributes(String action) {
return Map.of("repo_type", "s3", "repo_name", "repository", "operation", "GetObject", "purpose", "Indices", "action", action);
}

private static boolean isMultiDeleteRequest(HttpExchange exchange) {
return new S3HttpHandler("bucket").parseRequest(exchange).isMultiObjectDeleteRequest();
}

/**
* Asserts that an InputStream is fully consumed, or aborted, when it is closed
*/
Expand Down