#687: prevent checksum recheck (#688)

Author: hohwille

cli/src/main/java/com/devonfw/tools/ide/url/model/file/json/UrlStatusState.java

@@ -21,7 +21,17 @@ public final class UrlStatusState {
    */
   public UrlStatusState() {
 
-    this.timestamp = Instant.now();
+    this(Instant.now());
+  }
+
+  /**
+   * The constructor.
+   *
+   * @param timestamp the {@link #getTimestamp() timestamp}.
+   */
+  public UrlStatusState(Instant timestamp) {
+
+    this.timestamp = timestamp;
   }
 
   /**
@@ -78,4 +88,4 @@ public String toString() {
     return getClass().getSimpleName() + "@" + this.timestamp
         + ((this.message == null || this.message.isEmpty()) ? "" : ":" + this.message);
   }
-}
+}

cli/src/main/java/com/devonfw/tools/ide/url/updater/AbstractUrlUpdater.java

@@ -173,7 +173,7 @@ protected HttpResponse<InputStream> doGetResponseAsStream(String url) {
    *
    * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
    * @param downloadUrl the URL of the download for the tool.
-   * @return true if the version was successfully updated, false otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) {
 
@@ -186,7 +186,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl) {
    * @param edition the edition of the tool.
    * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
    * @param downloadUrl the URL of the download for the tool.
-   * @return true if the version was successfully updated, false otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl) {
 
@@ -200,7 +200,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow
    * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
    * @param downloadUrl the URL of the download for the tool.
    * @param os the {@link OperatingSystem} for the tool (can be null).
-   * @return true if the version was successfully updated, false otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os) {
 
@@ -214,7 +214,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, Operat
    * @param urlVersion the {@link UrlVersion} with the {@link UrlVersion#getName() version-number} to process.
    * @param downloadUrl the URL of the download for the tool.
    * @param os the {@link OperatingSystem} for the tool (can be null).
-   * @return true if the version was successfully updated, false otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl, OperatingSystem os) {
 
@@ -228,7 +228,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow
    * @param downloadUrl the URL of the download for the tool.
    * @param os the {@link OperatingSystem} for the tool (can be null).
    * @param architecture the optional {@link SystemArchitecture}.
-   * @return true if the version was successfully updated, false otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, OperatingSystem os,
       SystemArchitecture architecture) {
@@ -244,7 +244,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String downloadUrl, Operat
    * @param downloadUrl the URL of the download for the tool.
    * @param os the {@link OperatingSystem} for the tool (can be null).
    * @param architecture the optional {@link SystemArchitecture}.
-   * @return true if the version was successfully updated, false otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(String edition, UrlVersion urlVersion, String downloadUrl, OperatingSystem os,
       SystemArchitecture architecture) {
@@ -261,7 +261,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String dow
    * @param os the optional {@link OperatingSystem}.
    * @param architecture the optional {@link SystemArchitecture}.
    * @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed.
-   * @return {@code true} if the version was successfully updated, {@code false} otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSystem os, SystemArchitecture architecture, String checksum) {
     return doAddVersion(getEdition(), urlVersion, url, os, architecture, checksum);
@@ -276,7 +276,7 @@ protected boolean doAddVersion(UrlVersion urlVersion, String url, OperatingSyste
    * @param os the optional {@link OperatingSystem}.
    * @param architecture the optional {@link SystemArchitecture}.
    * @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed.
-   * @return {@code true} if the version was successfully updated, {@code false} otherwise.
+   * @return {@code true} if the version was successfully added, {@code false} otherwise.
    */
   protected boolean doAddVersion(String edition, UrlVersion urlVersion, String url, OperatingSystem os, SystemArchitecture architecture,
       String checksum) {
@@ -297,8 +297,7 @@ protected boolean doAddVersion(String edition, UrlVersion urlVersion, String url
     }
     url = url.replace("${edition}", edition);
 
-    return checkDownloadUrl(edition, url, urlVersion, os, architecture, checksum);
-
+    return doAddVersionUrlIfNewAndValid(edition, url, urlVersion, os, architecture, checksum);
   }
 
   /**
@@ -388,9 +387,16 @@ protected boolean isValidContentType(String contentType) {
    * @param checksum the existing checksum (e.g. from JSON metadata) or the empty {@link String} if not available and computation needed.
    * @return {@code true} if the download was checked successfully, {@code false} otherwise.
    */
-  private boolean checkDownloadUrl(String edition, String url, UrlVersion urlVersion, OperatingSystem os,
+  private boolean doAddVersionUrlIfNewAndValid(String edition, String url, UrlVersion urlVersion, OperatingSystem os,
       SystemArchitecture architecture, String checksum) {
 
+    UrlDownloadFile urlDownloadFile = urlVersion.getUrls(os, architecture);
+    if (urlDownloadFile != null) {
+      if (urlDownloadFile.getUrls().contains(url)) {
+        logger.debug("Skipping add of already existing URL {}", url);
+        return false;
+      }
+    }
     HttpResponse<?> response = doCheckDownloadViaHeadRequest(url);
     int statusCode = response.statusCode();
     String toolWithEdition = getToolWithEdition(edition);
@@ -400,7 +406,6 @@ private boolean checkDownloadUrl(String edition, String url, UrlVersion urlVersi
 
     boolean update = false;
 
-    UrlDownloadFile urlDownloadFile = urlVersion.getUrls(os, architecture);
     if (success) {
       UrlChecksum urlChecksum = null;
       if (urlDownloadFile != null) {
@@ -661,7 +666,7 @@ protected Set<String> getUrlFilenames() {
   /**
    * Checks if we are dependent on OS URL file names, can be overridden to disable OS dependency
    *
-   * @return true if we want to check for missing OS URL file names, false if not
+   * @return {@code true} if we want to check for missing OS URL file names, {@code false} if not.
    */
   protected boolean isOsDependent() {
 
@@ -672,7 +677,7 @@ protected boolean isOsDependent() {
    * Checks if an OS URL file name was missing in {@link UrlVersion}
    *
    * @param urlVersion the {@link UrlVersion} to check
-   * @return true if an OS type was missing, false if not
+   * @return {@code true} if an OS type was missing, {@code false} if not.
    */
   public boolean isMissingOs(UrlVersion urlVersion) {
 

cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterMock.java

@@ -4,7 +4,6 @@
 import java.util.HashSet;
 import java.util.Set;
 
-import com.devonfw.tools.ide.url.model.folder.UrlRepository;
 import com.devonfw.tools.ide.url.model.folder.UrlVersion;
 import com.devonfw.tools.ide.url.updater.AbstractUrlUpdater;
 import com.devonfw.tools.ide.url.updater.UrlUpdater;
@@ -35,11 +34,6 @@ protected String getTool() {
     return "mocked";
   }
 
-  @Override
-  public void update(UrlRepository urlRepository) {
-    super.update(urlRepository);
-  }
-
   @Override
   protected Set<String> getVersions() {
     return versions;

cli/src/test/java/com/devonfw/tools/ide/tool/UrlUpdaterTest.java

@@ -8,14 +8,25 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
+import com.devonfw.tools.ide.os.OperatingSystem;
+import com.devonfw.tools.ide.os.SystemArchitecture;
+import com.devonfw.tools.ide.url.model.file.UrlChecksum;
+import com.devonfw.tools.ide.url.model.file.UrlDownloadFile;
+import com.devonfw.tools.ide.url.model.file.UrlStatusFile;
 import com.devonfw.tools.ide.url.model.file.json.StatusJson;
 import com.devonfw.tools.ide.url.model.file.json.UrlStatus;
+import com.devonfw.tools.ide.url.model.file.json.UrlStatusState;
+import com.devonfw.tools.ide.url.model.folder.UrlEdition;
 import com.devonfw.tools.ide.url.model.folder.UrlRepository;
+import com.devonfw.tools.ide.url.model.folder.UrlTool;
+import com.devonfw.tools.ide.url.model.folder.UrlVersion;
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
 import com.github.tomakehurst.wiremock.junit5.WireMockTest;
 
@@ -93,80 +104,153 @@ public void testUrlUpdaterIsNotUpdatingWhenStatusManualIsTrue(@TempDir Path temp
   }
 
   /**
-   * Tests if the timestamps of the status.json get updated properly. Creates an initial status.json with a success timestamp. Updates the status.json with an
-   * error timestamp and compares it with the success timestamp. Updates the status.json with a final success timestamp and compares it with the error
-   * timestamp.
+   * Tests if the timestamps of the status.json get updated properly if a first error is detected.
    * <p>
    * See: <a href="https://github.com/devonfw/ide/issues/1343">#1343</a> for reference.
    *
    * @param tempDir Temporary directory
    * @param wmRuntimeInfo wireMock server on a random port
    */
   @Test
-  public void testUrlUpdaterStatusJsonRefreshBugStillExisting(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {
+  public void testStatusJsonUpdateOnFirstError(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {
 
     // arrange
-    stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(200).withBody("aBody")));
-
-    UrlRepository urlRepository = UrlRepository.load(tempDir);
-    UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
-
-    String statusUrl = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
     String toolName = "mocked";
     String editionName = "mocked";
     String versionName = "1.0";
+    String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
+    Instant now = Instant.now();
+    Instant lastMonth = now.minus(31, ChronoUnit.DAYS);
+    UrlRepository urlRepository = UrlRepository.load(tempDir);
+    UrlTool urlTool = urlRepository.getOrCreateChild(toolName);
+    UrlEdition urlEdition = urlTool.getOrCreateChild(editionName);
+    UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName);
+    // we create the structure of our tool version and URL to simulate it was valid last moth
+    UrlStatusFile statusFile = urlVersion.getOrCreateStatus();
+    UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url);
+    UrlStatusState successState = new UrlStatusState(lastMonth); // ensure that we trigger a recheck of the URL
+    status.setSuccess(successState);
+    UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64);
+    urlDownloadFile.addUrl(url);
+    UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
+    urlChecksum.setChecksum("1234567890");
+    urlVersion.save();
+    UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
+    // now we want to simulate that the url got broken (404) and the updater is properly handling this
+    stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404)));
 
     // act
     updater.update(urlRepository);
 
-    Path versionsPath = tempDir.resolve(toolName).resolve(editionName).resolve(versionName);
-
     // assert
-    assertThat(versionsPath.resolve("status.json")).exists();
-
     StatusJson statusJson = retrieveStatusJson(urlRepository, toolName, editionName, versionName);
+    status = statusJson.getStatus(url);
+    successState = status.getSuccess();
+    assertThat(successState).isNotNull();
+    assertThat(successState.getTimestamp()).isEqualTo(lastMonth);
+    UrlStatusState errorState = status.getError();
+    assertThat(errorState).isNotNull();
+    assertThat(errorState.getCode()).isEqualTo(404);
+    assertThat(Duration.between(errorState.getTimestamp(), now)).isLessThan(Duration.ofSeconds(5));
+  }
 
-    UrlStatus urlStatus = statusJson.getOrCreateUrlStatus(statusUrl);
-
-    Instant successTimestamp = urlStatus.getSuccess().getTimestamp();
-
-    assertThat(successTimestamp).isNotNull();
-
-    stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404)));
-
-    // re-initialize UrlRepository for error timestamp
-    UrlRepository urlRepositoryWithError = UrlRepository.load(tempDir);
-    updater.update(urlRepositoryWithError);
-
-    statusJson = retrieveStatusJson(urlRepositoryWithError, toolName, editionName, versionName);
-
-    urlStatus = statusJson.getOrCreateUrlStatus(statusUrl);
-    successTimestamp = urlStatus.getSuccess().getTimestamp();
-    Instant errorTimestamp = urlStatus.getError().getTimestamp();
-    Integer errorCode = urlStatus.getError().getCode();
-
-    assertThat(errorCode).isEqualTo(404);
-    assertThat(errorTimestamp).isAfter(successTimestamp);
+  /**
+   * Tests if the timestamps of the status.json get updated properly on success after an error.
+   * <p>
+   * See: <a href="https://github.com/devonfw/ide/issues/1343">#1343</a> for reference.
+   *
+   * @param tempDir Temporary directory
+   * @param wmRuntimeInfo wireMock server on a random port
+   */
+  @Test
+  public void testSuccessStateUpdatedAfterError(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {
 
+    // arrange
+    String toolName = "mocked";
+    String editionName = "mocked";
+    String versionName = "1.0";
+    String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
+    Instant now = Instant.now();
+    Instant lastMonth = now.minus(31, ChronoUnit.DAYS);
+    Instant lastSuccess = lastMonth.minus(1, ChronoUnit.DAYS);
+    UrlRepository urlRepository = UrlRepository.load(tempDir);
+    UrlTool urlTool = urlRepository.getOrCreateChild(toolName);
+    UrlEdition urlEdition = urlTool.getOrCreateChild(editionName);
+    UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName);
+    // we create the structure of our tool version and URL to simulate it was valid last moth
+    UrlStatusFile statusFile = urlVersion.getOrCreateStatus();
+    UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url);
+    UrlStatusState successState = new UrlStatusState(lastSuccess);
+    status.setSuccess(successState);
+    UrlStatusState errorState = new UrlStatusState(lastMonth);
+    errorState.setCode(404);
+    status.setError(errorState);
+    UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64);
+    urlDownloadFile.addUrl(url);
+    UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
+    urlChecksum.setChecksum("1234567890");
+    urlVersion.save();
+    UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
+    // now we want to simulate that the broken url is working again
     stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(200).withHeader("Content-Type", "text/plain")));
 
-    // re-initialize UrlRepository for error timestamp
-    UrlRepository urlRepositoryWithSuccess = UrlRepository.load(tempDir);
-    updater.update(urlRepositoryWithSuccess);
-
-    assertThat(versionsPath.resolve("status.json")).exists();
+    // act
+    updater.update(urlRepository);
 
-    statusJson = retrieveStatusJson(urlRepositoryWithSuccess, toolName, editionName, versionName);
+    // assert
+    status = retrieveStatusJson(urlRepository, toolName, editionName, versionName).getStatus(url);
+    successState = status.getSuccess();
+    assertThat(successState).isNotNull();
+    assertThat(Duration.between(successState.getTimestamp(), now)).isLessThan(Duration.ofSeconds(5));
+    errorState = status.getError();
+    assertThat(errorState).isNotNull();
+    assertThat(errorState.getCode()).isEqualTo(404);
+    assertThat(errorState.getTimestamp()).isEqualTo(lastMonth);
+  }
 
-    urlStatus = statusJson.getOrCreateUrlStatus(statusUrl);
+  /**
+   * Tests if the the tool version gets entirely removed if all versions are broken for a long time.
+   *
+   * @param tempDir Temporary directory
+   * @param wmRuntimeInfo wireMock server on a random port
+   */
+  @Test
+  public void testVersionRemovedIfErrorPersists(@TempDir Path tempDir, WireMockRuntimeInfo wmRuntimeInfo) {
 
-    successTimestamp = urlStatus.getSuccess().getTimestamp();
-    errorTimestamp = urlStatus.getError().getTimestamp();
-    errorCode = urlStatus.getError().getCode();
+    // arrange
+    String toolName = "mocked";
+    String editionName = "mocked";
+    String versionName = "1.0";
+    String url = wmRuntimeInfo.getHttpBaseUrl() + "/os/windows_x64_url.tgz";
+    Instant now = Instant.now();
+    Instant lastMonth = now.minus(31, ChronoUnit.DAYS);
+    Instant lastSuccess = lastMonth.minus(1, ChronoUnit.DAYS);
+    UrlRepository urlRepository = UrlRepository.load(tempDir);
+    UrlTool urlTool = urlRepository.getOrCreateChild(toolName);
+    UrlEdition urlEdition = urlTool.getOrCreateChild(editionName);
+    UrlVersion urlVersion = urlEdition.getOrCreateChild(versionName);
+    // we create the structure of our tool version and URL to simulate it was valid last moth
+    UrlStatusFile statusFile = urlVersion.getOrCreateStatus();
+    UrlStatus status = statusFile.getStatusJson().getOrCreateUrlStatus(url);
+    UrlStatusState successState = new UrlStatusState(lastSuccess);
+    status.setSuccess(successState);
+    UrlStatusState errorState = new UrlStatusState(lastMonth);
+    errorState.setCode(404);
+    status.setError(errorState);
+    UrlDownloadFile urlDownloadFile = urlVersion.getOrCreateUrls(OperatingSystem.WINDOWS, SystemArchitecture.X64);
+    urlDownloadFile.addUrl(url);
+    UrlChecksum urlChecksum = urlVersion.getOrCreateChecksum(urlDownloadFile.getName());
+    urlChecksum.setChecksum("1234567890");
+    urlVersion.save();
+    UrlUpdaterMockSingle updater = new UrlUpdaterMockSingle(wmRuntimeInfo);
+    // now we want to simulate that the url got broken (404) and the updater is properly handling this
+    stubFor(any(urlMatching("/os/.*")).willReturn(aResponse().withStatus(404)));
 
-    assertThat(errorCode).isEqualTo(200);
-    assertThat(errorTimestamp).isAfter(successTimestamp);
+    // act
+    updater.update(urlRepository);
 
+    // assert
+    assertThat(urlVersion.getPath()).doesNotExist();
   }
 
   /**