devonfw/ide-urls#17: fix UrlUpdater to auto-remove broken URLs and allow self-repair (#681)

Author: hohwille

cli/src/main/java/com/devonfw/tools/ide/io/HttpErrorResponse.java

@@ -0,0 +1,75 @@
+package com.devonfw.tools.ide.io;
+
+import java.net.URI;
+import java.net.http.HttpClient.Version;
+import java.net.http.HttpHeaders;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.Collections;
+import java.util.Optional;
+import javax.net.ssl.SSLSession;
+
+/**
+ * Implementation of {@link HttpResponse} in case of an {@link Throwable error} to prevent sub-sequent {@link NullPointerException}s.
+ */
+public class HttpErrorResponse implements HttpResponse<Throwable> {
+
+  private final Throwable error;
+
+  private final HttpRequest request;
+
+  private final URI uri;
+  private static final HttpHeaders NO_HEADERS = HttpHeaders.of(Collections.emptyMap(), (x, y) -> true);
+
+  /**
+   * @param error the {@link Throwable} that was preventing the HTTP request for {@link #body()}.
+   * @param request the {@link HttpRequest} for {@link #request()}.
+   * @param uri the {@link URI} for {@link #uri()}.
+   */
+  public HttpErrorResponse(Throwable error, HttpRequest request, URI uri) {
+    super();
+    this.error = error;
+    this.request = request;
+    this.uri = uri;
+  }
+
+  @Override
+  public int statusCode() {
+    return -1;
+  }
+
+  @Override
+  public HttpRequest request() {
+    return this.request;
+  }
+
+  @Override
+  public Optional<HttpResponse<Throwable>> previousResponse() {
+    return Optional.empty();
+  }
+
+  @Override
+  public HttpHeaders headers() {
+    return NO_HEADERS;
+  }
+
+  @Override
+  public Throwable body() {
+    return this.error;
+  }
+
+  @Override
+  public Optional<SSLSession> sslSession() {
+    return Optional.empty();
+  }
+
+  @Override
+  public URI uri() {
+    return this.uri;
+  }
+
+  @Override
+  public Version version() {
+    return Version.HTTP_2;
+  }
+}

cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseJavaUrlUpdater.java

@@ -6,12 +6,7 @@
 public class EclipseJavaUrlUpdater extends EclipseUrlUpdater {
 
   @Override
-  protected String getEdition() {
-
-    return "eclipse";
-  }
-
-  @Override
+  // getEdition() must still return "eclipse"
   protected String getEclipseEdition() {
 
     return "java";

cli/src/main/java/com/devonfw/tools/ide/tool/eclipse/EclipseUrlUpdater.java

@@ -13,9 +13,9 @@
  */
 public abstract class EclipseUrlUpdater extends WebsiteUrlUpdater {
 
-  private static final String[] MIRRORS = { "https://ftp.snt.utwente.nl/pub/software/eclipse/technology/epp/downloads",
-      "https://ftp.osuosl.org/pub/eclipse/technology/epp/downloads",
-      "https://archive.eclipse.org/technology/epp/downloads" };
+  private static final String[] MIRRORS = {
+      "https://archive.eclipse.org/technology/epp/downloads",
+      "https://ftp.osuosl.org/pub/eclipse/technology/epp/downloads" };
 
   @Override
   protected String getTool() {
@@ -24,7 +24,7 @@ protected String getTool() {
   }
 
   /**
-   * @return the eclipse edition name.
+   * @return the eclipse edition name. May be different from {@link #getEdition()} allowing a different edition name (e.g. eclipse) for IDEasy.
    */
   protected String getEclipseEdition() {
 
@@ -79,7 +79,6 @@ protected String getVersionUrl() {
   @Override
   protected Pattern getVersionPattern() {
 
-    // return Pattern.compile("\\d{4}-\\d{2}(\\s\\w{2})?");
     return Pattern.compile("\\d{4}-\\d{2}");
   }
 

cli/src/main/java/com/devonfw/tools/ide/url/UpdateInitiator.java

@@ -32,6 +32,7 @@ public static void main(String[] args) {
 
     String pathToRepo = args[0];
     Instant expirationTime = null;
+    String selectedTool = null;
 
     if (args.length < 2) {
       logger.warn("Timeout was not set, setting timeout to infinite instead.");
@@ -44,6 +45,9 @@ public static void main(String[] args) {
         logger.error("Error: Provided timeout format is not valid.", e);
         System.exit(1);
       }
+      if (args.length > 2) {
+        selectedTool = args[2];
+      }
     }
 
     Path repoPath = Path.of(pathToRepo);
@@ -56,7 +60,11 @@ public static void main(String[] args) {
     UrlFinalReport urlFinalReport = new UrlFinalReport();
 
     UpdateManager updateManager = new UpdateManager(repoPath, urlFinalReport, expirationTime);
-    updateManager.updateAll();
+    if (selectedTool == null) {
+      updateManager.updateAll();
+    } else {
+      updateManager.update(selectedTool);
+    }
 
     logger.info(urlFinalReport.toString());
   }

cli/src/main/java/com/devonfw/tools/ide/url/model/AbstractUrlArtifactWithParent.java

@@ -31,4 +31,8 @@ public P getParent() {
     return this.parent;
   }
 
+  @Override
+  public void delete() {
+    getParent().deleteChild(getName());
+  }
 }

cli/src/main/java/com/devonfw/tools/ide/url/model/UrlArtifactWithParent.java

@@ -13,4 +13,12 @@ public interface UrlArtifactWithParent<P extends UrlFolder<?>> extends UrlArtifa
    * @return the parent {@link UrlFolder} owning this artifact as child.
    */
   P getParent();
+
+
+  /**
+   * Physically deletes this artifact with all its potential children from the disc. Will also remove it from its {@link #getParent() parent}.
+   */
+  default void delete() {
+    getParent().deleteChild(getName());
+  }
 }

cli/src/main/java/com/devonfw/tools/ide/url/model/file/AbstractUrlFile.java

@@ -2,6 +2,8 @@
 
 import java.nio.file.Files;
 
+import org.jline.utils.Log;
+
 import com.devonfw.tools.ide.url.model.AbstractUrlArtifactWithParent;
 import com.devonfw.tools.ide.url.model.folder.AbstractUrlFolder;
 import com.devonfw.tools.ide.url.model.folder.UrlFolder;
@@ -54,6 +56,7 @@ public void load(boolean recursive) {
   public void save() {
 
     if (this.modified) {
+      Log.debug("Saving {}", getPath());
       doSave();
       this.modified = false;
     }

cli/src/main/java/com/devonfw/tools/ide/url/model/file/UrlStatusFile.java

@@ -4,7 +4,6 @@
 import java.io.BufferedWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
 
 import com.devonfw.tools.ide.json.JsonMapping;
 import com.devonfw.tools.ide.url.model.file.json.StatusJson;
@@ -23,7 +22,7 @@ public class UrlStatusFile extends AbstractUrlFile<UrlVersion> {
 
   private static final ObjectMapper MAPPER = JsonMapping.create();
 
-  private StatusJson statusJson = new StatusJson();
+  private StatusJson statusJson;
 
   /**
    * The constructor.
@@ -33,6 +32,7 @@ public class UrlStatusFile extends AbstractUrlFile<UrlVersion> {
   public UrlStatusFile(UrlVersion parent) {
 
     super(parent, STATUS_JSON);
+    this.statusJson = new StatusJson();
   }
 
   /**
@@ -70,11 +70,23 @@ protected void doLoad() {
   @Override
   protected void doSave() {
 
-    try (BufferedWriter writer = Files.newBufferedWriter(getPath(), StandardOpenOption.CREATE)) {
+    try (BufferedWriter writer = Files.newBufferedWriter(getPath())) {
       MAPPER.writeValue(writer, this.statusJson);
     } catch (Exception e) {
       throw new IllegalStateException("Failed to save file " + getPath(), e);
     }
   }
 
+  /**
+   * Performs a cleanup and removes all unused entries.
+   *
+   * @see StatusJson#cleanup()
+   */
+  public void cleanup() {
+
+    boolean changed = this.statusJson.cleanup();
+    if (changed) {
+      this.modified = true;
+    }
+  }
 }

cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/StatusJson.java

@@ -1,7 +1,9 @@
 package com.devonfw.tools.ide.url.model.file.json;
 
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import com.devonfw.tools.ide.url.model.file.UrlStatusFile;
 
@@ -27,7 +29,7 @@ public StatusJson() {
 
   /**
    * @return {@code true} if this file has been created manually and the containing version folder shall be ignored by the automatic update process,
-   * {@code false} otherwise.
+   *     {@code false} otherwise.
    */
   public boolean isManual() {
 
@@ -58,13 +60,70 @@ public void setUrls(Map<Integer, UrlStatus> urlStatuses) {
     this.urls = urlStatuses;
   }
 
+  /**
+   * @param url the URL to get the {@link UrlStatus} for.
+   * @return the existing {@link UrlStatus} for the given URL or a {@code null} if not found.
+   */
+  public UrlStatus getStatus(String url) {
+
+    return getStatus(url, false);
+  }
+
   /**
    * @param url the URL to get or create the {@link UrlStatus} for.
    * @return the existing {@link UrlStatus} for the given URL or a new {@link UrlStatus} associated with the given URL.
    */
   public UrlStatus getOrCreateUrlStatus(String url) {
 
+    return getStatus(url, true);
+  }
+
+  /**
+   * @param url the URL to get or create the {@link UrlStatus} for.
+   * @param create {@code true} for {@link #getOrCreateUrlStatus(String)} and {@code false} for {@link #getStatus(String)}.
+   * @return the existing {@link UrlStatus} for the given URL or {@code null} or created status according to {@code create} flag.
+   */
+  public UrlStatus getStatus(String url, boolean create) {
+
+    UrlStatus urlStatus;
+    Integer key = computeKey(url);
+    if (create) {
+      urlStatus = this.urls.computeIfAbsent(key, hash -> new UrlStatus());
+    } else {
+      urlStatus = this.urls.get(key);
+    }
+    if (urlStatus != null) {
+      urlStatus.markStillUsed();
+    }
+    return urlStatus;
+  }
+
+  static Integer computeKey(String url) {
     Integer key = Integer.valueOf(url.hashCode());
-    return this.urls.computeIfAbsent(key, hash -> new UrlStatus());
+    return key;
+  }
+
+  public void remove(String url) {
+
+    this.urls.remove(computeKey(url));
+  }
+
+  /**
+   * Performs a cleanup and removes all unused entries.
+   *
+   * @return {@code true} if something changed during cleanup, {@code false} otherwise.
+   */
+  public boolean cleanup() {
+
+    boolean changed = false;
+    Iterator<Entry<Integer, UrlStatus>> entryIterator = this.urls.entrySet().iterator();
+    while (entryIterator.hasNext()) {
+      Entry<Integer, UrlStatus> entry = entryIterator.next();
+      if (!entry.getValue().checkStillUsed()) {
+        entryIterator.remove();
+        changed = true;
+      }
+    }
+    return changed;
   }
 }

cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatus.java

@@ -9,6 +9,8 @@ public class UrlStatus {
 
   private UrlStatusState error;
 
+  private transient boolean stillUsed;
+
   /**
    * The constructor.
    */
@@ -48,4 +50,32 @@ public void setError(UrlStatusState error) {
 
     this.error = error;
   }
+
+  /**
+   * @return {@code true} if entirely empty, {@code false} otherwise.
+   */
+  public boolean checkEmpty() {
+
+    return (this.error == null) && (this.success == null);
+  }
+
+  /**
+   * ATTENTION: This is not a standard getter (isStillUsed()) since otherwise Jackson will write it to JSON.
+   *
+   * @return {@code true} if still {@link StatusJson#getOrCreateUrlStatus(String) used}, {@code false} otherwise (if the entire version has been processed, it
+   *     will be removed via {@link StatusJson#cleanup()}.
+   */
+  public boolean checkStillUsed() {
+
+    return this.stillUsed;
+  }
+
+  /**
+   * Sets {@link #checkStillUsed()} to {@code true}.
+   */
+  void markStillUsed() {
+
+    this.stillUsed = true;
+  }
+
 }