improve: parallel builds jvm locks and object list improvements (#84)

Author: csviri

core/src/main/java/io/javaoperatorsdk/jenvtest/binary/BinaryDownloader.java

@@ -5,6 +5,9 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -17,7 +20,6 @@
 import io.javaoperatorsdk.jenvtest.JenvtestException;
 import io.javaoperatorsdk.jenvtest.Utils;
 import io.javaoperatorsdk.jenvtest.binary.repo.BinaryRepo;
-import io.javaoperatorsdk.jenvtest.lock.LockFile;
 
 public class BinaryDownloader {
 
@@ -28,6 +30,7 @@ public class BinaryDownloader {
   private final String jenvtestDir;
   private final BinaryRepo binaryRepo;
   private final OSInfo osInfoProvider;
+  private static final Map<String, ReentrantLock> versionLocks = new ConcurrentHashMap<>();
 
   public BinaryDownloader(String jenvtestDir, OSInfo osInfoProvider) {
     this.jenvtestDir = jenvtestDir;
@@ -42,31 +45,27 @@ public BinaryDownloader(String jenvtestDir, OSInfo osInfoProvider) {
   }
 
   public File download(String version) {
-    log.info("Downloading binaries with version: {}", version);
-    var downloadDir = new File(jenvtestDir, BinaryManager.BINARY_LIST_DIR);
-    downloadDir.mkdirs();
-    LockFile lock =
-        new LockFile(version + ".lock", downloadDir.getPath());
+    var lock = versionLocks.computeIfAbsent(version, v -> new ReentrantLock());
     var dirForVersion = dirForVersion(version);
-    if (lock.tryLock()) {
+    lock.lock();
+    try {
       if (dirForVersion.exists()) {
         return dirForVersion;
       }
+      new File(jenvtestDir, BinaryManager.BINARY_LIST_DIR).mkdirs();
+      log.info("Downloading binaries with version: {}", version);
       var tempFile = binaryRepo.downloadVersionToTempFile(version);
       File dir = createDirForBinaries(version);
       extractFiles(tempFile, dir);
+      log.debug("Binary downloaded and extracted");
       var deleted = tempFile.delete();
       if (!deleted) {
         log.warn("Unable to delete temp file: {}", tempFile.getPath());
       }
-      lock.releaseLock();
-      return dir;
-    } else {
-      log.debug("Waiting for lock to be deleted for version: {}", version);
-      lock.waitUntilLockDeleted();
-      log.debug("Lock deleted for version: {}", version);
-      return dirForVersion;
+    } finally {
+      lock.unlock();
     }
+    return dirForVersion;
   }
 
   public File downloadLatest() {

core/src/main/java/io/javaoperatorsdk/jenvtest/binary/repo/BinaryRepo.java

@@ -7,6 +7,9 @@
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.commons.io.FileUtils;
@@ -23,15 +26,14 @@ public class BinaryRepo {
 
   private static final Logger log = LoggerFactory.getLogger(BinaryRepo.class);
 
-  private static final String BUCKET_NAME = "kubebuilder-tools";
-
   private final OSInfo osInfo;
+  private static List<String> objectNames;
+  private static final ReentrantLock downloadLock = new ReentrantLock();
 
   public BinaryRepo(OSInfo osInfo) {
     this.osInfo = osInfo;
   }
 
-
   public File downloadVersionToTempFile(String version) {
     try {
       String url = "https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-" + version +
@@ -47,25 +49,33 @@ public File downloadVersionToTempFile(String version) {
   }
 
   public Stream<String> listObjectNames() {
+    downloadLock.lock();
     try {
-      var httpClient = HttpClient.newBuilder()
-          .build();
-
-      HttpRequest request = HttpRequest.newBuilder()
-          .GET()
-          .uri(URI.create("https://storage.googleapis.com/storage/v1/b/kubebuilder-tools/o"))
-          .build();
-
-      var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()).body();
-      ObjectMapper mapper =
-          new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-      ObjectList objectList = mapper.readValue(response, ObjectList.class);
-      return objectList.getItems().stream().map(ObjectListItem::getName);
+      if (objectNames == null) {
+        log.debug("Listing objects from storage");
+        var httpClient = HttpClient.newBuilder()
+            .build();
+
+        HttpRequest request = HttpRequest.newBuilder()
+            .GET()
+            .uri(URI.create("https://storage.googleapis.com/storage/v1/b/kubebuilder-tools/o"))
+            .build();
+
+        var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()).body();
+        ObjectMapper mapper =
+            new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        ObjectList objectList = mapper.readValue(response, ObjectList.class);
+        objectNames = objectList.getItems().stream().map(ObjectListItem::getName)
+            .collect(Collectors.toList());
+      }
+      return objectNames.stream();
     } catch (IOException e) {
       throw new JenvtestException(e);
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
       throw new JenvtestException(e);
+    } finally {
+      downloadLock.unlock();
     }
   }
 

core/src/main/java/io/javaoperatorsdk/jenvtest/cert/CertManager.java

@@ -12,6 +12,7 @@
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.Date;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.bouncycastle.asn1.x500.X500Name;
 import org.bouncycastle.asn1.x509.Extension;
@@ -28,7 +29,6 @@
 import org.slf4j.LoggerFactory;
 
 import io.javaoperatorsdk.jenvtest.JenvtestException;
-import io.javaoperatorsdk.jenvtest.lock.LockFile;
 
 public class CertManager {
 
@@ -40,6 +40,8 @@ public class CertManager {
   public static final String CLIENT_KEY_NAME = "client.key";
   public static final String CLIENT_CERT_NAME = "client.crt";
 
+  private static final ReentrantLock generatorLock = new ReentrantLock();
+
   private String jenvtestDir;
 
   public CertManager(String jenvtestDir) {
@@ -50,20 +52,15 @@ public void createCertificatesIfNeeded() {
     if (certFilesPresent()) {
       return;
     }
-    // locking is for parallel execution
-    LockFile lockFile = new LockFile("cert.lock", jenvtestDir);
-    if (lockFile.tryLock()) {
+    generatorLock.lock();
+    try {
       if (certFilesPresent()) {
         return;
       }
-      try {
-        generateAPIServerCertificates();
-        generateUserCertificates();
-      } finally {
-        lockFile.releaseLock();
-      }
-    } else {
-      lockFile.waitUntilLockDeleted();
+      generateAPIServerCertificates();
+      generateUserCertificates();
+    } finally {
+      generatorLock.unlock();
     }
   }