Implementing multiple destinations for mcopy donwloads

.devcontainer/devcontainer.json

@@ -3,7 +3,7 @@
 {
 	"name": "Java",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/java:17",
+	"image": "mcr.microsoft.com/devcontainers/java:21",
 
 	"features": {
 		"ghcr.io/devcontainers/features/java:1": {

README.md

@@ -2,78 +2,92 @@
 [![test](https://github.com/itzg/mc-image-helper/actions/workflows/test.yml/badge.svg)](https://github.com/itzg/mc-image-helper/actions/workflows/test.yml)
 ![LOC](https://img.shields.io/endpoint?url=https%3A%2F%2Fshields-codetab-code-loc-bridge.vercel.app%2Fapi%2Fcodeloc%3Fgithub%3Ditzg%2Fmc-image-helper%26language%3Djava)
 
-This tool does the complicated bits for the [itzg/minecraft-server](https://github.com/itzg/docker-minecraft-server) and [itzg/bungeecord](https://github.com/itzg/docker-bungeecord/) Docker images.
+This tool does the complicated bits for the [itzg/minecraft-server](https://github.com/itzg/docker-minecraft-server) and [itzg/docker-mc-proxy](https://github.com/itzg/docker-mc-proxy) Docker images.
 
 ## Usage
 
 > **NOTE** The following documentation may not always be up-to-date. Please be sure to use `-h` or `--help` after any subcommand to view the current usage.
 
 ```
-Usage: mc-image-helper [-hs] [--debug] [COMMAND]
-      --debug    Enable debug output. Can also set environment variable
-                   DEBUG_HELPER
-  -h, --help     Show this usage and exit
-  -s, --silent   Don't output logs even if there's an error
+Usage: mc-image-helper [-hsV] [--debug | --logging=<loggingLevel>] [COMMAND]
+      --debug     Enable debug output. Can also set environment variables
+                    DEBUG_HELPER or DEBUG
+  -h, --help      Show this usage and exit
+      --logging=<loggingLevel>
+                  Set logging to specific level.
+                  Valid values:
+  -s, --silent    Don't output logs even if there's an error
+  -V, --version
 Commands:
-  asciify                    Converts UTF-8 on stdin to ASCII by escaping
-                               Unicode characters
-  assert                     Provides assertion operators for verifying
-                               container setup
-  compare-versions           Used for shell scripting, exits with success(0)
-                               when comparison is satisfied or 1 when not
-  curseforge-files           Download and manage individual mod/plugin files
-                               from CurseForge
-  find                       Specialized replacement for GNU's find
-  get                        Download a file
+  asciify                         Converts UTF-8 on stdin to ASCII by escaping
+                                    Unicode characters
+  assert                          Provides assertion operators for verifying
+                                    container setup
+  compare-versions                Used for shell scripting, exits with success
+                                    (0) when comparison is satisfied or 1 when
+                                    not
+  curseforge-files                Download and manage individual mod/plugin
+                                    files from CurseForge
+  find                            Specialized replacement for GNU's find
+  get                             Download a file
   github
-  hash                       Outputs an MD5 hash of the standard input
-  install-curseforge         Downloads, installs, and upgrades CurseForge
-                               modpacks
-  install-fabric-loader      Provides a few ways to obtain a Fabric loader with
-                               simple cleanup of previous loader instances
-  install-forge              Downloads and installs a requested version of Forge
-  install-modrinth-modpack   Supports installation of Modrinth modpacks along
-                               with the associated mod loader
-  install-neoforge           Downloads and installs a requested version of
-                               NeoForge
-  install-paper              Installs selected PaperMC
-  install-purpur             Downloads latest or selected version of Purpur
-  install-quilt              Installs Quilt mod loader
-  interpolate                Interpolates existing files in one or more
-                               directories
-  java-release               Outputs the Java release number, such as 8, 11, 17
+  hash                            Outputs an MD5 hash of the standard input
+  install-curseforge              Downloads, installs, and upgrades CurseForge
+                                    modpacks
+  install-fabric-loader           Provides a few ways to obtain a Fabric loader
+                                    with simple cleanup of previous loader
+                                    instances
+  install-forge                   Downloads and installs a requested version of
+                                    Forge
+  install-modrinth-modpack        Supports installation of Modrinth modpacks
+                                    along with the associated mod loader
+  install-neoforge                Downloads and installs a requested version of
+                                    NeoForge
+  install-paper                   Installs selected PaperMC
+  install-purpur                  Downloads latest or selected version of Purpur
+  install-quilt                   Installs Quilt mod loader
+  interpolate                     Interpolates existing files in one or more
+                                    directories
+  java-release                    Outputs the Java release number, such as 8,
+                                    11, 17
   manage-users
-  maven-download             Downloads a maven artifact from a Maven repository
-  modrinth                   Automates downloading of modrinth resources
-  mcopy                      Multi-source file copy operation with with managed
-                               cleanup. Supports auto-detected sourcing from
-                               file list, directories, and URLs
-  network-interfaces         Provides simple operations to list network
-                               interface names and check existence
-  patch                      Patches one or more existing files using JSON path
-                               based operations
-                             Supports the file formats:
-                             - JSON
-                             - JSON5
-                             - Yaml
-                             - TOML, but processed output is not pretty
-  resolve-minecraft-version  Resolves and validate latest, snapshot, and
-                               specific versions
-  set-properties             Maps environment variables to a properties file
-  show-all-subcommand-usage  Renders all of the subcommand usage as markdown
-                               sections for README
-  sync                       Synchronizes the contents of one directory to
-                               another.
-  sync-and-interpolate       Synchronizes the contents of one directory to
-                               another with conditional variable interpolation.
+  maven-download                  Downloads a maven artifact from a Maven
+                                    repository
+  modrinth                        Automates downloading of modrinth resources
+  mcopy                           Multi-source file copy operation with with
+                                    managed cleanup. Supports auto-detected
+                                    sourcing from file list, directories, and
+                                    URLs
+  network-interfaces              Provides simple operations to list network
+                                    interface names and check existence
+  patch                           Patches one or more existing files using JSON
+                                    path based operations
+                                  Supports the file formats:
+                                  - JSON
+                                  - JSON5
+                                  - Yaml
+                                  - TOML, but processed output is not pretty
+  resolve-minecraft-version       Resolves and validate latest, snapshot, and
+                                    specific versions
+  set-properties                  Maps environment variables to a properties
+                                    file
+  show-all-subcommand-usage       Renders all of the subcommand usage as
+                                    markdown sections for README
+  sync                            Synchronizes the contents of one directory to
+                                    another.
+  sync-and-interpolate            Synchronizes the contents of one directory to
+                                    another with conditional variable
+                                    interpolation.
   test-logging-levels
-  toml-path                  Extracts a path from a TOML file using json-path
-                               syntax
-  yaml-path                  Extracts a path from a YAML file using json-path
-                               syntax
-  vanillatweaks              Downloads Vanilla Tweaks resource packs, data
-                               packs, or crafting tweaks given a share code or
-                               pack file
+  toml-path                       Extracts a path from a TOML file using
+                                    json-path syntax
+  vanillatweaks                   Downloads Vanilla Tweaks resource packs, data
+                                    packs, or crafting tweaks given a share
+                                    code or pack file
+  version-from-modrinth-projects  Finds a compatible Minecraft version across
+                                    given Modrinth projects
+  yaml-path                       Extracts a path from a YAML file using
+                                    json-path syntax
 ```
 
 For [patch](#patch) command [see below](#patch-schemas) for a description of [PatchSet](#patchset) and [PatchDefinition](#patchdefinition) JSON schemas.
@@ -853,13 +867,19 @@ Downloads a maven artifact from a Maven repository
 ```
 Usage: mc-image-helper mcopy [-hz] [--file-is-listing]
                              [--ignore-missing-sources] [--quiet-when-skipped]
-                             [--skip-existing] [--glob=GLOB]
-                             [--scope=<manifestId>] --to=<dest> SRC[,
-                             |<nl>SRC...]...
+                             [--skip-existing] [--delimiter=<stringDelimiter>]
+                             [--glob=GLOB] [--scope=<manifestId>] --to=<dest>
+                             SRC[,|<nl>SRC...]...
 Multi-source file copy operation with with managed cleanup. Supports
 auto-detected sourcing from file list, directories, and URLs
       SRC[,|<nl>SRC...]...   Any mix of source file, directory, or URLs.
                              Can be optionally comma or newline separated.
+                             Per-file destinations can be assigned by
+                               destination@source
+      --delimiter=<stringDelimiter>
+                             When using per-file destinations, which symbol
+                               should be used to delimit
+                               destination<delimiter>source
       --file-is-listing      Source files or URLs are processed as a line
                                delimited list of sources.
                              For remote listing files, the contents must all be

src/main/java/me/itzg/helpers/sync/MulitCopyCommand.java

@@ -70,22 +70,25 @@ public class MulitCopyCommand implements Callable<Integer> {
     @Option(names = "--ignore-missing-sources", description = "Don't log or fail exit code when any or all sources are missing")
     boolean ignoreMissingSources;
 
+    @Option(names = "--delimiter", defaultValue = "@",
+        description = "When using per-file destinations, which symbol should be used to delimit destination<delimiter>source"
+    )
+    String stringDelimiter;
+
     @Parameters(split = SPLIT_COMMA_NL, splitSynopsisLabel = SPLIT_SYNOPSIS_COMMA_NL, arity = "1..*",
         paramLabel = "SRC",
         description = "Any mix of source file, directory, or URLs."
             + "%nCan be optionally comma or newline separated."
+            + "%nPer-file destinations can be assigned by destination@source"
     )
     List<String> sources;
 
     @Override
     public Integer call() throws Exception {
-
-        Files.createDirectories(dest);
-
         Flux.fromIterable(sources)
             .map(String::trim)
             .filter(s -> !s.isEmpty())
-            .flatMap(source -> processSource(source, fileIsListingOption))
+            .flatMap(source -> processSource(source, fileIsListingOption, null))
             .collectList()
             .flatMap(this::cleanupAndSaveManifest)
             .block();
@@ -113,24 +116,58 @@ private Mono<?> cleanupAndSaveManifest(List<Path> paths) {
             });
     }
 
-    private Publisher<Path> processSource(String source, boolean fileIsListing) {
+    @SuppressWarnings("BlockingMethodInNonBlockingContext") // idk if that is a good idea
+    private Publisher<Path> processSource(String source, boolean fileIsListing, Path parentDestination) {
+        Path destination = dest;
+
+        if (parentDestination != null) {
+            destination = parentDestination;
+        }
+
+        if (source.contains(stringDelimiter)) {
+            String[] split = source.split(stringDelimiter);
+            destination = destination.resolve(Paths.get(split[0]));
+            source = split[1];
+        }
+
+        if (fileIsListing) {
+            if (Uris.isUri(source)) {
+                return processRemoteListingFile(source, destination);
+            } else {
+                final Path path = Paths.get(source);
+                if (Files.isDirectory(path)) {
+                    throw new GenericException(String.format("Specified listing file '%s' is a directory", source));
+                }
+                if (!Files.exists(path)) {
+                    throw new GenericException(String.format("Source file '%s' does not exist", source));
+                }
+                return processListingFile(path, destination);
+            }
+        }
+
+        try {
+            Files.createDirectories(destination);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
         if (Uris.isUri(source)) {
-            return fileIsListing ? processRemoteListingFile(source) : processRemoteSource(source);
+            return processRemoteSource(source, destination);
         } else {
             final Path path = Paths.get(source);
             if (!Files.exists(path)) {
                 throw new GenericException(String.format("Source file '%s' does not exist", source));
             }
 
             if (Files.isDirectory(path)) {
-                return processDirectory(path);
+                return processDirectory(path, destination);
             } else {
-                return fileIsListing ? processListingFile(path) : processFile(path);
+                return processFile(path, destination);
             }
         }
     }
 
-    private Flux<Path> processListingFile(Path listingFile) {
+    private Flux<Path> processListingFile(Path listingFile, Path destination) {
         return Mono.just(listingFile)
             .publishOn(Schedulers.boundedElastic())
             .flatMapMany(path -> {
@@ -141,22 +178,23 @@ private Flux<Path> processListingFile(Path listingFile) {
                         .filter(this::isListingLine)
                         .flatMap(src -> processSource(src,
                             // avoid recursive file-listing processing
-                            false));
+                            false,
+                            destination));
                 } catch (IOException e) {
                     return Mono.error(new GenericException("Failed to read file listing from " + path));
                 }
             });
     }
 
-    private Mono<Path> processFile(Path source) {
+    private Mono<Path> processFile(Path source, Path destination) {
 
         return Mono.just(source)
             .publishOn(Schedulers.boundedElastic())
-            .map(path -> processFileImmediate(source, dest));
+            .map(path -> processFileImmediate(source, destination));
     }
 
     /**
-     * Non-mono version of {@link #processFile(Path)}
+     * Non-mono version of {@link #processFile(Path, Path)}
      *
      * @param scopedDest allows for sub-directory destinations
      */
@@ -203,7 +241,7 @@ private Path processFileImmediate(Path source, Path scopedDest) {
         return destFile;
     }
 
-    private Flux<Path> processDirectory(Path srcDir) {
+    private Flux<Path> processDirectory(Path srcDir, Path destination) {
         return Mono.just(srcDir)
             .publishOn(Schedulers.boundedElastic())
             .flatMapMany(path -> {
@@ -220,7 +258,7 @@ private Flux<Path> processDirectory(Path srcDir) {
                     try (DirectoryStream<Path> files = Files.newDirectoryStream(srcDir, fileGlob)) {
                         for (final Path file : files) {
                             //noinspection BlockingMethodInNonBlockingContext because IntelliJ is confused
-                            results.add(processFileImmediate(file, dest));
+                            results.add(processFileImmediate(file, destination));
                         }
                     }
                     return Flux.fromIterable(results);
@@ -230,10 +268,10 @@ private Flux<Path> processDirectory(Path srcDir) {
             });
     }
 
-    private Mono<Path> processRemoteSource(String source) {
+    private Mono<Path> processRemoteSource(String source, Path destination) {
         return Fetch.fetch(URI.create(source))
             .userAgentCommand("mcopy")
-            .toDirectory(dest)
+            .toDirectory(destination)
             .skipUpToDate(skipUpToDate)
             .skipExisting(skipExisting)
             .handleDownloaded((downloaded, uri, size) ->
@@ -263,7 +301,7 @@ private Mono<Path> processRemoteSource(String source) {
             .checkpoint("Retrieving " + source, true);
     }
 
-    private Flux<Path> processRemoteListingFile(String source) {
+    private Flux<Path> processRemoteListingFile(String source, Path destination) {
         @SuppressWarnings("resource") // closed on terminate
         SharedFetch sharedFetch = Fetch.sharedFetch("mcopy", SharedFetch.Options.builder().build());
         return Mono.just(source)
@@ -273,7 +311,7 @@ private Flux<Path> processRemoteListingFile(String source) {
                     .flatMapMany(content -> Flux.just(content.split("\\r?\\n")))
                     .filter(this::isListingLine)
             )
-            .flatMap(this::processRemoteSource)
+            .flatMap(url -> processSource(url, false, destination))
             .doOnTerminate(sharedFetch::close)
             .checkpoint("Processing remote listing at " + source, true);
     }