Limit parallel download requests

Author: Yeregorix

src/main/java/org/spongepowered/gradle/vanilla/resolver/jdk/JdkHttpClientDownloader.java

@@ -61,6 +61,14 @@ public class JdkHttpClientDownloader implements Downloader {
     private final ResolveMode resolveMode;
     private final boolean writeToDisk;
 
+    /**
+     * We limit the number of parallel requests otherwise:
+     * - The JDK throws an error when we reach HTTP/2 max_concurrent_streams (usually around 100).
+     * - Mojang servers randomly return error pages (200 HTTP code with HTML message "The request is blocked.").
+     * The maximum of 8 parallel requests has been determined purely empirically by trying several powers of two.
+     */
+    private final TaskQueue queue = new TaskQueue(8); // TODO configurable max, per host?
+
     /**
      * Create a downloader that does not cache.
      *
@@ -235,21 +243,23 @@ private <T> CompletableFuture<ResolutionResult<T>> sendRequest(final URI uri, fi
         if (etag != null) {
             requestBuilder.header(HttpConstants.HEADER_IF_NONE_MATCH, etag);
         }
-        return this.client.sendAsync(requestBuilder.build(), bodyHandler).thenApply(message -> {
-            switch (message.statusCode()) {
-                case HttpConstants.STATUS_NOT_FOUND:
-                    return ResolutionResult.notFound();
-                case HttpConstants.STATUS_OK:
-                    return ResolutionResult.result(message.body(), false); // Known invalid, hash does not match expected.
-                default:
-                    throw new CompletionException(new HttpErrorResponseException(uri, message.statusCode(), message.toString()));
-            }
-        });
+        final HttpRequest request = requestBuilder.build();
+        return this.queue.run(() -> this.client.sendAsync(request, bodyHandler))
+            .thenApply(message -> {
+                switch (message.statusCode()) {
+                    case HttpConstants.STATUS_NOT_FOUND:
+                        return ResolutionResult.notFound();
+                    case HttpConstants.STATUS_OK:
+                        return ResolutionResult.result(message.body(), false); // Known invalid, hash does not match expected.
+                    default:
+                        throw new CompletionException(new HttpErrorResponseException(uri, message.statusCode(), message.toString()));
+                }
+            });
     }
 
     @Override
     public void close() throws IOException {
-        // nothing needed, the client just relies on the executor
+        this.queue.close(); // abort pending requests
     }
 
     // body subscribers

src/main/java/org/spongepowered/gradle/vanilla/resolver/jdk/TaskQueue.java

@@ -0,0 +1,88 @@
+/*
+ * This file is part of VanillaGradle, licensed under the MIT License (MIT).
+ *
+ * Copyright (c) SpongePowered <https://www.spongepowered.org>
+ * Copyright (c) contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package org.spongepowered.gradle.vanilla.resolver.jdk;
+
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
+
+class TaskQueue implements AutoCloseable {
+    private final int max;
+    private boolean closed;
+    private int running;
+    private final Queue<CompletableFuture<Void>> pending = new LinkedList<>();
+
+    public TaskQueue(final int max) {
+        if (max < 1) {
+            throw new IllegalArgumentException("max must be >= 1");
+        }
+        this.max = max;
+    }
+
+    private synchronized CompletableFuture<Void> acquire() {
+        if (this.closed) {
+            return CompletableFuture.failedFuture(new IllegalStateException("queue closed"));
+        }
+        if (this.running < this.max) {
+            this.running++;
+            return CompletableFuture.completedFuture(null);
+        }
+        final CompletableFuture<Void> future = new CompletableFuture<>();
+        this.pending.add(future);
+        return future;
+    }
+
+    private synchronized void release() {
+        if (this.running > this.max) {
+            this.running--;
+            return;
+        }
+        final CompletableFuture<Void> next = this.pending.poll();
+        if (next == null) {
+            this.running--;
+        } else {
+            next.complete(null);
+        }
+    }
+
+    public <T> CompletableFuture<T> run(final Supplier<CompletableFuture<T>> task) {
+        return this.acquire().thenCompose(_ -> task.get()).whenComplete((_, _) -> this.release());
+    }
+
+    @Override
+    public synchronized void close() {
+        if (!this.closed) {
+            this.closed = true;
+            if (!this.pending.isEmpty()) {
+                final Exception ex = new IllegalStateException("queue closed");
+                for (final CompletableFuture<Void> future : this.pending) {
+                    future.completeExceptionally(ex);
+                }
+                this.pending.clear();
+            }
+        }
+    }
+}