🐛 Better impl and tests for URIs; Resolves #421

ebullient · ebullient · commit 5ac05f32fdfb · 2024-04-21T14:44:42.000-04:00
diff --git a/src/main/java/dev/ebullient/convert/config/CompendiumConfig.java b/src/main/java/dev/ebullient/convert/config/CompendiumConfig.java
@@ -433,6 +433,7 @@ private enum ConfigKeys {
         useDiceRoller,
         exclude,
         excludePattern,
+        fallbackPaths(List.of("fallback-paths")),
         from,
         fullSource(List.of("convert", "full-source")),
         images,
@@ -502,6 +503,7 @@ static class ImageOptions {
         String internalRoot;
         Boolean copyInternal;
         Boolean copyExternal;
+        final Map<String, String> fallbackPaths = new HashMap<>();
 
         public ImageOptions() {
         }
@@ -511,6 +513,7 @@ public ImageOptions(ImageOptions images, ImageOptions images2) {
                 copyExternal = images.copyExternal;
                 copyInternal = images.copyInternal;
                 internalRoot = images.internalRoot;
+                fallbackPaths.putAll(images.fallbackPaths);
             }
             if (images2 != null) {
                 copyExternal = images2.copyExternal == null
@@ -522,6 +525,7 @@ public ImageOptions(ImageOptions images, ImageOptions images2) {
                 internalRoot = images2.internalRoot == null
                         ? internalRoot
                         : images2.internalRoot;
+                fallbackPaths.putAll(images2.fallbackPaths);
             }
         }
 
@@ -532,6 +536,10 @@ public boolean copyExternal() {
         public boolean copyInternal() {
             return copyInternal != null && copyInternal;
         }
+
+        public Map<String, String> fallbackPaths() {
+            return Collections.unmodifiableMap(fallbackPaths);
+        }
     }
 
     @RegisterForReflection
diff --git a/src/main/java/dev/ebullient/convert/config/TtrpgConfig.java b/src/main/java/dev/ebullient/convert/config/TtrpgConfig.java
@@ -95,9 +95,11 @@ public static class ImageRoot {
         final String internalImageRoot;
         final boolean copyInternal;
         final boolean copyExternal;
+        final Map<String, String> fallbackPaths;
 
         private ImageRoot(String cfgRoot, ImageOptions options) {
             this.copyExternal = options.copyExternal();
+            this.fallbackPaths = options.fallbackPaths();
 
             if (cfgRoot == null) {
                 this.internalImageRoot = "";
@@ -139,6 +141,10 @@ private String endWithSlash(String path) {
             }
             return path.endsWith("/") ? path : path + "/";
         }
+
+        public String getFallbackPath(String key) {
+            return fallbackPaths.getOrDefault(key, key);
+        }
     }
 
     public static ImageRoot internalImageRoot() {
@@ -154,10 +160,6 @@ public static ImageRoot internalImageRoot() {
         return root;
     }
 
-    public static Map<String, String> imageFallbackPaths() {
-        return activeDSConfig().fallbackImagePaths;
-    }
-
     public static JsonNode readIndex(String key) {
         String file = activeDSConfig().indexes.get(key);
         Optional<Path> root = file == null ? Optional.empty() : tui.resolvePath(Path.of(file));
diff --git a/src/main/java/dev/ebullient/convert/io/Tui.java b/src/main/java/dev/ebullient/convert/io/Tui.java
@@ -6,6 +6,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
 import java.nio.channels.Channels;
@@ -213,7 +215,7 @@ public void init(CommandSpec spec, boolean debug, boolean verbose, boolean log)
         this.debug = debug || log;
         this.verbose = verbose;
         if (log) {
-            Path p = Path.of("ttrpg-convert.out");
+            Path p = Path.of("ttrpg-convert.out.txt");
             try {
                 this.log = new PrintWriter(Files.newOutputStream(p));
                 VersionProvider vp = new VersionProvider();
@@ -436,6 +438,28 @@ private void copyImageResource(ImageRef image, Path targetPath) {
         }
     }
 
+    private final static String allowedCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~/";
+
+    String escapeUrlImagePath(String url) throws MalformedURLException, UnsupportedEncodingException {
+        URL urlObject = new URL(url);
+        String path = urlObject.getPath();
+
+        StringBuilder encodedPath = new StringBuilder();
+        for (char ch : path.toCharArray()) {
+            if (allowedCharacters.indexOf(ch) == -1) {
+                byte[] bytes = String.valueOf(ch).getBytes("UTF-8");
+                for (byte b : bytes) {
+                    encodedPath.append(String.format("%%%02X", b));
+                }
+            } else {
+                encodedPath.append(ch);
+            }
+        }
+
+        return url.replace(path, encodedPath.toString())
+                .replace("/imgur.com", "/i.imgur.com");
+    }
+
     private void copyRemoteImage(ImageRef image, Path targetPath) {
         targetPath.getParent().toFile().mkdirs();
 
@@ -445,12 +469,14 @@ private void copyRemoteImage(ImageRef image, Path targetPath) {
             return;
         }
         if (!url.startsWith("http") && !url.startsWith("file")) {
-            errorf("ImageRef %s has invalid URL %s", image.targetFilePath(), url);
+            errorf("Remote ImageRef %s has invalid URL %s", image.targetFilePath(), url);
             return;
         }
 
-        Tui.instance().debugf("copy image %s %n   to %s", url, targetPath);
         try {
+            url = escapeUrlImagePath(url);
+            Tui.instance().debugf("copy image %s", url);
+
             ReadableByteChannel readableByteChannel = Channels.newChannel(new URL(url).openStream());
             try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath.toFile())) {
                 fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
diff --git a/src/main/java/dev/ebullient/convert/qute/ImageRef.java b/src/main/java/dev/ebullient/convert/qute/ImageRef.java
@@ -1,5 +1,7 @@
 package dev.ebullient.convert.qute;
 
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 
 import dev.ebullient.convert.config.TtrpgConfig;
@@ -43,7 +45,7 @@ public class ImageRef {
     final String titleAttr;
 
     private ImageRef(String url, Path sourcePath, Path targetFilePath, String title, String vaultPath, Integer width) {
-        this.url = url == null ? null : url.replace(" ", "%20"); // catch any remaining spaces
+        this.url = url;
         this.sourcePath = sourcePath;
         this.targetFilePath = targetFilePath;
         title = title == null
@@ -59,10 +61,6 @@ private ImageRef(String url, Path sourcePath, Path targetFilePath, String title,
         }
         this.vaultPath = vaultPath;
         this.width = width;
-
-        if (url == null && vaultPath == null) {
-            Tui.instance().errorf("ImageRef (target=%s) has no url or vaultPath", targetFilePath);
-        }
     }
 
     String escape(String s) {
@@ -208,61 +206,52 @@ public Builder setUrl(String url) {
         }
 
         public ImageRef build() {
-            final ImageRoot imageRoot = TtrpgConfig.internalImageRoot();
-
-            if (url != null && !imageRoot.copyExternalToVault()) {
-                // leave external images alone (referenced as url)
-                return new ImageRef(url, null, null, title, null, width);
-            }
-
             if (url == null && sourcePath == null) {
                 Tui.instance().errorf("ImageRef build for internal image called without url or sourcePath set");
                 return null;
             }
-            if (relativeTarget == null || vaultRoot == null || rootFilePath == null) {
-                Tui.instance().errorf("ImageRef build called without target paths set");
-                return null;
+
+            final ImageRoot imageRoot = TtrpgConfig.internalImageRoot();
+            String sourceUrl = url == null ? sourcePath.toString() : url;
+
+            // Check for any URL replacements (to replace a not-found-image with a local one, e.g.)
+            // replace backslashes with forward slashes
+            sourceUrl = imageRoot.getFallbackPath(sourceUrl)
+                    .replace('\\', '/');
+
+            try {
+                // Remove escaped characters here (local file paths won't want it)
+                sourceUrl = java.net.URLDecoder.decode(sourceUrl, StandardCharsets.UTF_8.name());
+            } catch (UnsupportedEncodingException e) {
+                Tui.instance().errorf("Error decoding image URL: %s", e.getMessage());
             }
 
-            Path targetFilePath = rootFilePath.resolve(relativeTarget);
-            String vaultPath = String.format("%s%s", vaultRoot,
-                    relativeTarget.toString().replace('\\', '/'));
-
-            // Escaping spaces is a mess. Remove here (local file paths won't want it)
-            // It is changed back (from space to %20) if in URL form (file or http)
-            String remoteUrl = url == null
-                    ? sourcePath.toString().replace("%20", " ")
-                    : url;
-            if (remoteUrl.startsWith("http")) {
-                remoteUrl = remoteUrl
-                        .replaceAll("^(https?):/+", "$1://")
-                        .replace("/imgur.com", "/i.imgur.com");
-            } else if (!remoteUrl.startsWith("file:/")) {
-                remoteUrl = imageRoot.getRootPath() + remoteUrl;
+            boolean copyToVault = false;
+
+            if (sourceUrl.startsWith("http") || sourceUrl.startsWith("file")) {
+                sourceUrl = sourceUrl.replaceAll("^(https?):/+", "$1://");
+                copyToVault = imageRoot.copyExternalToVault();
+            } else if (!sourceUrl.startsWith("file:/")) {
+                sourceUrl = imageRoot.getRootPath() + sourceUrl;
+                copyToVault = imageRoot.copyInternalToVault();
             }
 
-            if (imageRoot.copyInternalToVault() || imageRoot.copyExternalToVault()) {
+            boolean localTargetSet = relativeTarget != null && vaultRoot != null && rootFilePath != null;
+            if (localTargetSet && copyToVault) {
+                Path targetFilePath = rootFilePath.resolve(relativeTarget);
+                String vaultPath = String.format("%s%s", vaultRoot,
+                        relativeTarget.toString().replace('\\', '/'));
+
                 // remote images to be copied into the vault
-                if (remoteUrl.startsWith("http") || remoteUrl.startsWith("file")) {
-                    String filename = remoteUrl.substring(remoteUrl.lastIndexOf('/') + 1);
-                    if (!filename.contains("%")) {
-                        try {
-                            String encoded = java.net.URLEncoder.encode(filename, "UTF-8")
-                                    .replace("+", "%20");
-                            remoteUrl = remoteUrl.replace(filename, encoded);
-                        } catch (java.io.UnsupportedEncodingException e) {
-                            Tui.instance().errorf("Failed to encode filename: %s", filename);
-                        }
-                    }
-                    // also replace any remaining spaces in the path
-                    return new ImageRef(remoteUrl,
-                            null, targetFilePath, title, vaultPath, width);
+                if (sourceUrl.startsWith("http") || sourceUrl.startsWith("file")) {
+                    return new ImageRef(sourceUrl, null, targetFilePath, title, vaultPath, width);
                 }
-                return new ImageRef(null, Path.of(remoteUrl), targetFilePath, title, vaultPath, width);
+                // local image to be copied into the vault
+                return new ImageRef(null, Path.of(sourceUrl), targetFilePath, title, vaultPath, width);
             }
 
-            // remote images are not copied to the vault --> url image ref
-            return new ImageRef(remoteUrl,
+            // remote images that are not copied to the vault --> url image ref, no target
+            return new ImageRef(sourceUrl,
                     null, null, title, null, width);
         }
 
diff --git a/src/main/resources/convertData.json b/src/main/resources/convertData.json
@@ -332,14 +332,6 @@
     "constants": {
       "internalImageRoot": "https://raw.githubusercontent.com/5etools-mirror-2/5etools-img/main/"
     },
-    "fallbackImage": {
-      "img/PSZ/Archon of Redemption.png": "img/PSZ/Archon Of Redemption.png",
-      "img/bestiary/ERLW/Inspired.png": "img/bestiary/ERLW/Inspired.webp",
-      "img/bestiary/MTF/Merrenoloth.jpg": "img/bestiary/MTF/Merrenoloth.webp",
-      "img/bestiary/SDW/Lhammaruntosz.jpg": "img/SDW/Lhammaruntosz.png",
-      "img/bestiary/VGM/Deep Scion.jpg": "img/bestiary/VGM/Deep Scion.webp",
-      "img/items/CRCotN/Medal of the Maze.jpg": "img/items/CRCotN/Medal of the Maze.webp"
-    },
     "markerFiles": [
       "bestiary/bestiary-mm.json",
       "cultsboons.json",
diff --git a/src/test/java/dev/ebullient/convert/io/ImgurUrlTest.java b/src/test/java/dev/ebullient/convert/io/ImgurUrlTest.java
@@ -0,0 +1,70 @@
+package dev.ebullient.convert.io;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.UnsupportedEncodingException;
+import java.net.MalformedURLException;
+import java.nio.file.Path;
+
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import dev.ebullient.convert.config.TtrpgConfig;
+import dev.ebullient.convert.qute.ImageRef;
+import dev.ebullient.convert.tools.dnd5e.Tools5eIndex;
+import dev.ebullient.convert.tools.dnd5e.Tools5eSources;
+import io.quarkus.arc.Arc;
+import io.quarkus.test.junit.QuarkusTest;
+
+@QuarkusTest
+public class ImgurUrlTest {
+    protected static Tui tui;
+    protected static Tools5eIndex index;
+
+    @BeforeAll
+    public static void prepare() {
+        tui = Arc.container().instance(Tui.class).get();
+        tui.init(null, true, false);
+        index = new Tools5eIndex(TtrpgConfig.getConfig());
+    }
+
+    @Test
+    public void testImgurUrl() throws MalformedURLException, UnsupportedEncodingException {
+        String input = "https://imgur.com/lQfZ1dF.png";
+
+        assertThat(tui.escapeUrlImagePath(input))
+                .isEqualTo("https://i.imgur.com/lQfZ1dF.png");
+    }
+
+    @Test
+    public void testAccentedCharacters() throws MalformedURLException, UnsupportedEncodingException {
+        String input = "https://whatever.com/áé.png?raw=true";
+
+        assertThat(tui.escapeUrlImagePath(input))
+                .isEqualTo("https://whatever.com/%C3%A1%C3%A9.png?raw=true");
+
+        Tools5eSources sources = Tools5eSources.findOrTemporary(
+                Tui.MAPPER.createObjectNode()
+                        .put("name", "Critter")
+                        .put("source", "DMG"));
+
+        ImageRef ref = sources.buildTokenImageRef(index,
+                "https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/Hill%20Giant%20Warlock%20Of%20Ogrémoch.jpg",
+                Path.of("something.png"),
+                false);
+
+        assertThat(tui.escapeUrlImagePath(ref.url()))
+                .isEqualTo(
+                        "https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/Hill%20Giant%20Warlock%20Of%20Ogr%C3%A9moch.jpg");
+
+        ref = sources.buildTokenImageRef(index,
+                "https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/token/Stone%20Giant%20Warlock%20Of%20Ogrémoch%20%28Token%29.png",
+                Path.of("something.png"),
+                false);
+
+        assertThat(tui.escapeUrlImagePath(ref.url()))
+                .isEqualTo(
+                        "https://raw.githubusercontent.com/TheGiddyLimit/homebrew/master/_img/MonsterManualExpanded3/creature/token/Stone%20Giant%20Warlock%20Of%20Ogr%C3%A9moch%20%28Token%29.png");
+    }
+
+}
diff --git a/src/test/java/dev/ebullient/convert/tools/ImgurUrlTest.java b/src/test/java/dev/ebullient/convert/tools/ImgurUrlTest.java
diff --git a/src/test/resources/sources-homebrew.json b/src/test/resources/sources-homebrew.json
