feat(kube-api-test): use binaries from GitHub releases (6281)

improve: use new GitHub based binary repo for KubeAPITest

Signed-off-by: Attila Mészáros <[email protected]>
---
wip

Signed-off-by: Attila Mészáros <[email protected]>
---
cleanup

Signed-off-by: Attila Mészáros <[email protected]>
---
add missing license

Signed-off-by: Attila Mészáros <[email protected]>
---
naming

Signed-off-by: Attila Mészáros <[email protected]>
---
changelog update

Signed-off-by: Attila Mészáros <[email protected]>
---
test update

Signed-off-by: Attila Mészáros <[email protected]>
---
sonar fixes

Signed-off-by: Attila Mészáros <[email protected]>

Author: csviri

CHANGELOG.md

@@ -8,6 +8,7 @@
 * Fix #5264: Remove deprecated `Config.errorMessages` field
 * Fix #6008: removing the optional dependency on bouncy castle
 * Fix #6230: introduced Quantity.multiply(int) to allow for Quantity multiplication by an integer
+* Fix #6281: use GitHub binary repo for Kube API Tests
 
 #### Dependency Upgrade
 * Fix #6052: Removed dependency on no longer maintained com.github.mifmif:generex

junit/kube-api-test/core/pom.xml

@@ -106,5 +106,9 @@
             <groupId>io.javaoperatorsdk</groupId>
             <artifactId>kubernetes-webhooks-framework-core</artifactId>
         </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.dataformat</groupId>
+        <artifactId>jackson-dataformat-yaml</artifactId>
+      </dependency>
     </dependencies>
 </project>

junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/BinaryDownloader.java

@@ -39,8 +39,6 @@ public class BinaryDownloader {
 
   private static final Logger log = LoggerFactory.getLogger(BinaryDownloader.class);
 
-  private static final String OBJECT_TAR_PREFIX = "kubebuilder-tools-";
-
   private final String kubeAPITestDir;
   private final BinaryRepo binaryRepo;
   private final OSInfo osInfoProvider;
@@ -161,15 +159,8 @@ public String findLatestOfWildcard(String wildcardVersion) {
 
   private Stream<String> listAllRelevantVersions() {
     var objects = binaryRepo.listObjectNames();
-    return objects.filter(o -> o.contains(osInfoProvider.getOSName())
-        && o.contains(osInfoProvider.getOSArch()))
-        .map(o -> {
-          String stripped = o.replace(OBJECT_TAR_PREFIX, "");
-          String version = stripped.substring(0, stripped.indexOf("-"));
-          if (version.startsWith("v")) {
-            version = version.substring(1);
-          }
-          return version;
-        });
+    return objects.filter(o -> o.getOs().equals(osInfoProvider.getOSName())
+        && o.getArch().equals(osInfoProvider.getOSArch()))
+        .map(o -> o.getVersion());
   }
 }

junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/repo/ArchiveDescriptor.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.fabric8.kubeapitest.binary.repo;
+
+public class ArchiveDescriptor {
+
+  private final String version;
+  private final String arch;
+  private final String os;
+
+  public ArchiveDescriptor(String version, String arch, String os) {
+    this.version = version;
+    this.arch = arch;
+    this.os = os;
+  }
+
+  public String getVersion() {
+    return version;
+  }
+
+  public String getArch() {
+    return arch;
+  }
+
+  public String getOs() {
+    return os;
+  }
+}

junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/repo/ObjectListItem.java renamed to junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/repo/ArchiveReference.java

@@ -15,16 +15,24 @@
  */
 package io.fabric8.kubeapitest.binary.repo;
 
-public class ObjectListItem {
+public class ArchiveReference {
 
-  private String name;
+  private String selfLink;
+  private String hash;
 
-  public String getName() {
-    return name;
+  public String getSelfLink() {
+    return selfLink;
   }
 
-  public ObjectListItem setName(String name) {
-    this.name = name;
-    return this;
+  public void setSelfLink(String selfLink) {
+    this.selfLink = selfLink;
+  }
+
+  public String getHash() {
+    return hash;
+  }
+
+  public void setHash(String hash) {
+    this.hash = hash;
   }
 }

junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/repo/ObjectList.java renamed to junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/repo/BinaryIndex.java

@@ -15,18 +15,17 @@
  */
 package io.fabric8.kubeapitest.binary.repo;
 
-import java.util.List;
+import java.util.Map;
 
-public class ObjectList {
+public class BinaryIndex {
 
-  private List<ObjectListItem> items;
+  private Map<String, Map<String, ArchiveReference>> releases;
 
-  public List<ObjectListItem> getItems() {
-    return items;
+  public Map<String, Map<String, ArchiveReference>> getReleases() {
+    return releases;
   }
 
-  public ObjectList setItems(List<ObjectListItem> items) {
-    this.items = items;
-    return this;
+  public void setReleases(Map<String, Map<String, ArchiveReference>> releases) {
+    this.releases = releases;
   }
 }

junit/kube-api-test/core/src/main/java/io/fabric8/kubeapitest/binary/repo/BinaryRepo.java

@@ -15,8 +15,8 @@
  */
 package io.fabric8.kubeapitest.binary.repo;
 
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import io.fabric8.kubeapitest.KubeAPITestException;
 import io.fabric8.kubeapitest.binary.OSInfo;
 import org.slf4j.Logger;
@@ -25,36 +25,58 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URI;
 import java.net.URL;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.util.List;
-import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public class BinaryRepo {
 
   private static final Logger log = LoggerFactory.getLogger(BinaryRepo.class);
 
+  private static final String BINARY_INDEX_URL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml";
+  private static final ObjectMapper MAPPER = new ObjectMapper(new YAMLFactory());
+  public static final String TAR_GZ_SUFFIX = ".tar.gz";
+
+  private static List<ArchiveDescriptor> objectNames;
+
   private final OSInfo osInfo;
-  private static List<String> objectNames;
-  private static final ReentrantLock downloadLock = new ReentrantLock();
 
   public BinaryRepo(OSInfo osInfo) {
     this.osInfo = osInfo;
   }
 
+  public synchronized Stream<ArchiveDescriptor> listObjectNames() {
+    try {
+      if (objectNames == null) {
+        var index = MAPPER.readValue(new URL(BINARY_INDEX_URL), BinaryIndex.class);
+        objectNames = index.getReleases().values().stream().flatMap(v -> v.values().stream()).map(
+            a -> mapSelfLinkToArchiveDescriptor(a.getSelfLink()))
+            .collect(Collectors.toList());
+      }
+      return objectNames.stream();
+    } catch (IOException e) {
+      throw new KubeAPITestException(e);
+    }
+  }
+
+  static ArchiveDescriptor mapSelfLinkToArchiveDescriptor(String selfLink) {
+    var versionOsArch = selfLink.split("/")[8]
+        .replace("envtest-v", "")
+        .replace(TAR_GZ_SUFFIX, "")
+        .split("-");
+
+    return new ArchiveDescriptor(versionOsArch[0], versionOsArch[2], versionOsArch[1]);
+  }
+
   public File downloadVersionToTempFile(String version) {
     try {
-      String url = "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-" + version +
-          "-" + osInfo.getOSName() + "-" + osInfo.getOSArch() + ".tar.gz";
+      String url = "https://github.com/kubernetes-sigs/controller-tools/releases/download/envtest-v" + version +
+          "/envtest-v" + version + "-" + osInfo.getOSName() + "-" + osInfo.getOSArch() + TAR_GZ_SUFFIX;
 
-      File tempFile = File.createTempFile("kubebuilder-tools-" + version, ".tar.gz");
+      File tempFile = File.createTempFile("kubebuilder-tools-" + version, TAR_GZ_SUFFIX);
       log.debug("Downloading binary from url: {} to Temp file: {}", url, tempFile.getPath());
       copyURLToFile(url, tempFile);
       return tempFile;
@@ -68,35 +90,4 @@ private void copyURLToFile(String url, File tempFile) throws IOException {
       Files.copy(in, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
     }
   }
-
-  public Stream<String> listObjectNames() {
-    downloadLock.lock();
-    try {
-      if (objectNames == null) {
-        log.debug("Listing objects from storage");
-        var httpClient = HttpClient.newBuilder()
-            .build();
-
-        HttpRequest request = HttpRequest.newBuilder()
-            .GET()
-            .uri(URI.create("https://storage.googleapis.com/storage/v1/b/kubebuilder-tools/o"))
-            .build();
-
-        var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()).body();
-        ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        ObjectList objectList = mapper.readValue(response, ObjectList.class);
-        objectNames = objectList.getItems().stream().map(ObjectListItem::getName)
-            .collect(Collectors.toList());
-      }
-      return objectNames.stream();
-    } catch (IOException e) {
-      throw new KubeAPITestException(e);
-    } catch (InterruptedException e) {
-      Thread.currentThread().interrupt();
-      throw new KubeAPITestException(e);
-    } finally {
-      downloadLock.unlock();
-    }
-  }
-
 }

junit/kube-api-test/core/src/test/java/io/fabric8/kubeapitest/binary/BinaryDownloaderTest.java

@@ -15,6 +15,7 @@
  */
 package io.fabric8.kubeapitest.binary;
 
+import io.fabric8.kubeapitest.binary.repo.ArchiveDescriptor;
 import io.fabric8.kubeapitest.binary.repo.BinaryRepo;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
@@ -38,9 +39,9 @@ void findsLatestBinary() {
         when(mockOsInfo.getOSArch()).thenReturn("amd64");
 
         when(mockBinaryRepo.listObjectNames()).thenReturn(List.of(
-                        "kubebuilder-tools-1.17.9-linux-amd64.tar.gz"
-                        ,"kubebuilder-tools-1.26.1-darwin-amd64.tar.gz",
-                        "kubebuilder-tools-"+VERSION+"-linux-amd64.tar.gz")
+                        new ArchiveDescriptor("1.17.9","amd64","linux"),
+            new ArchiveDescriptor("1.26.1","amd64","darwin"),
+            new ArchiveDescriptor(VERSION,"amd64","linux"))
                 .stream());
 
         var latest = binaryDownloader.findLatestVersion();

junit/kube-api-test/core/src/test/java/io/fabric8/kubeapitest/binary/repo/BinaryRepoTest.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.fabric8.kubeapitest.binary.repo;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class BinaryRepoTest {
+
+  @Test
+  void parsesSelfLinkToArchiveDescriptor() {
+    var archiveDescriptor = BinaryRepo
+        .mapSelfLinkToArchiveDescriptor(
+            "https://github.com/kubernetes-sigs/controller-tools/releases/download/envtest-v1.28.0/envtest-v1.28.0-linux-arm64.tar.gz");
+
+    assertThat(archiveDescriptor.getOs()).isEqualTo("linux");
+    assertThat(archiveDescriptor.getArch()).isEqualTo("arm64");
+    assertThat(archiveDescriptor.getVersion()).isEqualTo("1.28.0");
+  }
+
+}