Merge remote-tracking branch 'origin/master' into feat/use-hc5-file-download

# Conflicts:
#	src/main/java/me/itzg/helpers/http/FetchBuilderBase.java
#	src/main/java/me/itzg/helpers/http/SharedFetch.java

Author: itzg

.github/workflows/test.yml

@@ -18,3 +18,4 @@ jobs:
     with:
       arguments: test
       include-test-report: true
+      retest-showing-standard-streams: true

DEVELOPMENT.md

@@ -6,3 +6,23 @@ Beyond the unit tests, ad hoc "integration testing" can be done by running via G
 ./gradlew run --args="assert fileExists build.gradle"
 ```
 
+### Using IntelliJ
+
+Create an "Application" run configuration, such as shown here:
+
+![intellij-run-config](docs/intellij-run-config.png)
+
+### Build and use application script
+
+Build and install the distribution script
+
+```shell
+./gradlew installDist
+```
+
+Run the script using:
+
+```shell
+./build/install/mc-image-helper/bin/mc-image-helper ...args...
+```
+

README.md

@@ -761,15 +761,17 @@ Usage: mc-image-helper manage-users [-fh] [--existing=<existingFileBehavior>]
                                     >] -t=<type>
                                     [--user-api-provider=<userApiProvider>]
                                     [--version=<version>]
+                                    [--offline]
                                     [[--http-response-timeout=DURATION]
                                     [--tls-handshake-timeout=DURATION]
                                     [--connection-pool-pending-acquire-timeout=D
                                     URATION]
                                     [--connection-pool-max-idle-timeout=DURATION
                                     ]] [INPUT[,INPUT...]...]
       [INPUT[,INPUT...]...] One or more Mojang usernames, UUID, or ID (UUID
-                              without dashes); however, when offline, only
-                              UUID/IDs can be provided.
+                              without dashes); flags are listed after a colon
+                              separated by comma:
+                              <username/UUID/ID>:flag1,flag2
                             When input is a file, only one local file path or
                               URL can be provided
       --connection-pool-max-idle-timeout=DURATION
@@ -798,6 +800,8 @@ Usage: mc-image-helper manage-users [-fh] [--existing=<existingFileBehavior>]
                             Allowed: mojang, playerdb
       --version=<version>   Minecraft game version. If not provided, assumes
                               JSON format
+      --offline             Server is in offline mode, for users that have the
+                              offline flag the UUID is generated locally 
 ```
 
 ### maven-download
@@ -847,9 +851,11 @@ Downloads a maven artifact from a Maven repository
 ### mcopy
 
 ```
-Usage: mc-image-helper mcopy [-hz] [--file-is-listing] [--skip-existing]
-                             [--glob=GLOB] [--scope=<manifestId>] --to=<dest>
-                             SRC[,|<nl>SRC...]...
+Usage: mc-image-helper mcopy [-hz] [--file-is-listing]
+                             [--ignore-missing-sources] [--quiet-when-skipped]
+                             [--skip-existing] [--glob=GLOB]
+                             [--scope=<manifestId>] --to=<dest> SRC[,
+                             |<nl>SRC...]...
 Multi-source file copy operation with with managed cleanup. Supports
 auto-detected sourcing from file list, directories, and URLs
       SRC[,|<nl>SRC...]...   Any mix of source file, directory, or URLs.
@@ -861,6 +867,10 @@ auto-detected sourcing from file list, directories, and URLs
       --glob=GLOB            When a source is a directory, this filename glob
                                will be applied to select files.
   -h, --help
+      --ignore-missing-sources
+                             Don't log or fail exit code when any or all
+                               sources are missing
+      --quiet-when-skipped   Don't log when file exists or is up to date
       --scope, --manifest-id=<manifestId>
                              If managed cleanup is required, this is the
                                identifier used for qualifying manifest filename

build.gradle

@@ -2,7 +2,7 @@ plugins {
   id 'java'
   id 'application'
   id 'idea'
-  id "io.freefair.lombok" version '8.13.1'
+  id "io.freefair.lombok" version '8.14'
   // https://github.com/qoomon/gradle-git-versioning-plugin
   id 'me.qoomon.git-versioning' version '6.4.4'
   // https://github.com/itzg/github-releaser-gradle-plugin
@@ -80,7 +80,7 @@ repositories {
 dependencies {
   def picocliVersion = '4.7.7'
 
-  implementation(platform('com.fasterxml.jackson:jackson-bom:2.18.3'))
+  implementation(platform('com.fasterxml.jackson:jackson-bom:2.19.1'))
   // 1.4.+ requires Java 11 and we are stuck on 8
   implementation(platform('ch.qos.logback:logback-parent:[1.3.5,1.4)'))
 
@@ -97,8 +97,8 @@ dependencies {
   implementation 'com.kjetland:mbknor-jackson-jsonschema_2.13:1.0.39'
   implementation 'com.jayway.jsonpath:json-path:2.9.0'
   implementation 'org.apache.httpcomponents.client5:httpclient5:5.5'
-  implementation 'io.projectreactor.netty:reactor-netty-http:1.2.6'
-  implementation 'org.apache.maven:maven-artifact:3.9.9'
+  implementation 'io.projectreactor.netty:reactor-netty-http:1.2.7'
+  implementation 'org.apache.maven:maven-artifact:3.9.10'
   implementation 'commons-codec:commons-codec:1.18.0'
   // for RFC5987 parsing of content-disposition filename*
   // 3.5.x is last of Java8 compatible: https://cxf.apache.org/download.html

docs/intellij-run-config.png

@@ -8,6 +8,7 @@
 import static me.itzg.helpers.singles.MoreCollections.safeStreamFrom;
 
 import com.fasterxml.jackson.databind.JsonMappingException;
+import io.netty.channel.ChannelException;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -753,10 +754,12 @@ private  Mono<DownloadOrResolveResult> buildRetryableDownload(InstallContext con
             )
             // retry the deferred part above if one of the expected failure cases
             .retryWhen(
-                Retry.fixedDelay(BAD_FILE_ATTEMPTS, BAD_FILE_DELAY)
+                Retry.backoff(BAD_FILE_ATTEMPTS, BAD_FILE_DELAY)
                     .filter(throwable ->
                         throwable instanceof FileHashInvalidException ||
-                            throwable instanceof FailedRequestException
+                            throwable instanceof FailedRequestException ||
+                            throwable instanceof IOException ||
+                            throwable instanceof ChannelException
                     )
                     .doBeforeRetry(retrySignal ->
                         log.warn("Retrying to download {} @ {}:{}",

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -3,6 +3,7 @@
 import io.netty.handler.codec.http.HttpHeaders;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import java.net.URI;
+import java.net.URISyntaxException;
 import lombok.Getter;
 import lombok.ToString;
 
@@ -19,14 +20,23 @@ public class FailedRequestException extends RuntimeException {
      */
     public FailedRequestException(HttpResponseStatus status, URI uri, String body, String msg, HttpHeaders headers) {
         super(
-            String.format("HTTP request of %s failed with %s: %s", uri, status, msg)
+            String.format("HTTP request of %s failed with %s: %s", obfuscate(uri), status, msg)
         );
         this.uri = uri;
         this.statusCode = status.code();
         this.body = body;
         this.headers = headers;
     }
 
+    public static String obfuscate(URI uri) {
+        try {
+            return new URI(uri.getScheme(), uri.getUserInfo() != null ? "*:*" : null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())
+                .toString();
+        } catch (URISyntaxException e) {
+            return "???";
+        }
+    }
+
     @SuppressWarnings("unused")
     public static boolean isNotFound(Throwable throwable) {
         return isStatus(throwable, HttpResponseStatus.NOT_FOUND);

src/main/java/me/itzg/helpers/http/FailedRequestException.java

@@ -145,6 +145,16 @@ protected URI uri() {
         return state.uri;
     }
 
+    protected URI uriForFile() {
+        if (state.sharedFetch == null) {
+            return state.uri;
+        }
+
+        final URI filesViaUrl = state.sharedFetch.getFilesViaUrl();
+        return filesViaUrl != null ? filesViaUrl.resolve(state.uri.getPath())
+            : state.uri;
+    }
+
     public Set<String> getAcceptContentTypes() {
         return state.acceptContentTypes;
     }

src/main/java/me/itzg/helpers/http/FetchBuilderBase.java

@@ -38,6 +38,8 @@ public class SharedFetch implements AutoCloseable {
     private final CloseableHttpAsyncClient hcAsyncClient;
     private boolean hcAsyncClientStarted = false;
 
+    private final URI filesViaUrl;
+
     public SharedFetch(String forCommand, Options options) {
         final String userAgent = String.format("%s/%s/%s (cmd=%s)",
             "itzg",
@@ -80,6 +82,8 @@ public SharedFetch(String forCommand, Options options) {
 
         headers.put("x-fetch-session", fetchSessionId);
 
+        this.filesViaUrl = options.getFilesViaUrl();
+
         hcAsyncClient = HttpAsyncClients.createSystem();
     }
 
@@ -138,14 +142,20 @@ public static class Options {
 
         private final Map<String,String> extraHeaders;
 
+        /**
+         * Can be set for unit testing file downloads where the original URL's path is resolved
+         * against this given URL.
+         */
+        private final URI filesViaUrl;
+
         public Options withHeader(String key, String value) {
             final Map<String, String> newHeaders = extraHeaders != null ?
                 new HashMap<>(extraHeaders) : new HashMap<>();
             newHeaders.put(key, value);
 
             return new Options(
                 responseTimeout, tlsHandshakeTimeout, maxIdleTimeout, pendingAcquireTimeout,
-                newHeaders
+                newHeaders, filesViaUrl
             );
         }
     }

src/main/java/me/itzg/helpers/http/SharedFetch.java

@@ -54,7 +54,7 @@ public Path execute() throws IOException {
     }
 
     public Mono<Path> assemble() {
-        final URI uri = uri();
+        final URI uri = uriForFile();
 
         if (skipExisting && Files.exists(file)) {
             log.debug("Skipping file={} that already exists due to request", file);