startup wait

Author: csviri

devnotes.txt

@@ -1,4 +1,8 @@
 
+bins:
+https://github.com/etcd-io/etcd/releases/download/v3.4.24/etcd-v3.4.24-linux-amd64.tar.gz
+https://dl.k8s.io/v1.26.1/bin/linux/amd64/kube-apiserver
+
 Gen self signed for api server:
 https://raymii.org/s/tutorials/OpenSSL_generate_self_signed_cert_with_Subject_Alternative_name_oneliner.html
 
@@ -15,7 +19,7 @@ run etcd:
 
 run self signed:
 
- ./kube-apiserver --cert-dir /home/csviri/.jenvtest/  --etcd-servers http://0.0.0.0:2379 --authorization-mode RBAC --service-account-issuer https://localhost --service-account-signing-key-file /home/csviri/.jenvtest/apiserver.key --service-account-signing-key-file /home/csviri/.jenvtest/apiserver.key --service-account-key-file /home/csviri/Downloads/kubeapi/tempcerts/apiserver.key --service-account-issuer /home/csviri/.jenvtest/apiserver.cert --disable-admission-plugins ServiceAccount --client-ca-file /home/csviri/.jenvtest/client.crt
+ ./kube-apiserver --cert-dir /home/csviri/.jenvtest/  --etcd-servers http://0.0.0.0:2379 --authorization-mode RBAC --service-account-issuer https://localhost --service-account-signing-key-file /home/csviri/.jenvtest/apiserver.key --service-account-signing-key-file /home/csviri/.jenvtest/apiserver.key --service-account-key-file /home/csviri/.jenvtest/apiserver.key --service-account-issuer /home/csviri/.jenvtest/apiserver.cert --disable-admission-plugins ServiceAccount --client-ca-file /home/csviri/.jenvtest/client.crt
 
 // ./kube-apiserver --cert-dir .
 // --etcd-servers http://0.0.0.0:2379

src/main/java/com/csviri/kubeapi/APIServer.java

@@ -4,19 +4,22 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
+import java.io.*;
+import java.util.Scanner;
 
 public class APIServer {
 
     private static final Logger log = LoggerFactory.getLogger(APIServer.class);
+    public static final int STARTUP_TIMEOUT = 10_000;
 
     private final APIServerConfig config;
     private BinaryManager binaryManager;
     private CertManager certManager;
     private KubeConfigManager kubeConfigManager;
     private Process etcdProcess;
-    private Process apiServerProcess;
+    private volatile Process apiServerProcess;
+    private Thread startupWaiter;
+    private volatile boolean startedUpProperly;
 
     public APIServer() {
         this(new APIServerConfig());
@@ -30,13 +33,14 @@ public APIServer(APIServerConfig config) {
     }
 
     public void start() {
-        log.debug("Stating. Using jenvtest dir: {}", config.getJenvtestDirectory());
+        log.debug("Stating API Server. Using jenvtest dir: {}", config.getJenvtestDirectory());
         prepareLogDirectory();
         cleanEtcdData();
         startEtcd();
         startApiServer();
         kubeConfigManager.updateKubeConfig();
         waitUntilDefaultNamespaceCreated();
+        log.info("API Server ready to use");
     }
 
     private void prepareLogDirectory() {
@@ -58,7 +62,7 @@ public void stop() {
 
     private void stopApiServer() {
         if (apiServerProcess != null) {
-            apiServerProcess.destroy();
+            apiServerProcess.destroyForcibly();
         }
         log.debug("API Server stopped");
     }
@@ -71,7 +75,6 @@ private void cleanEtcdData() {
         }
     }
 
-
     private void stopEtcd() {
         if (etcdProcess != null) {
             etcdProcess.destroy();
@@ -80,7 +83,14 @@ private void stopEtcd() {
     }
 
     private void waitUntilDefaultNamespaceCreated() {
-       // todo
+        try {
+            startupWaiter.join(STARTUP_TIMEOUT);
+            if (!startedUpProperly) {
+                throw new KubeApiException("Something went wrong starting up KubeApi server. Check the logs");
+            }
+        } catch (InterruptedException e) {
+            throw new KubeApiException(e);
+        }
     }
 
     // todo detect if process not started up correctly
@@ -96,6 +106,7 @@ private void startEtcd() {
             etcdProcess = new ProcessBuilder(etcdBinary.getAbsolutePath(),
                     "--listen-client-urls=http://0.0.0.0:2379",
                     "--advertise-client-urls=http://0.0.0.0:2379")
+                    // todo log to a different logger on debug level
                     .redirectOutput(logsFile)
                     .redirectError(logsFile)
                     .start();
@@ -126,15 +137,34 @@ private void startApiServer() {
                     "--client-ca-file", certManager.getClientCertPath(),
                     "--service-cluster-ip-range", "10.0.0.0/24",
                     "--allow-privileged"
-                    )
-                    .redirectOutput(logsFile)
-                    .redirectError(logsFile)
+            )
                     .start();
-              // todo detect premature termination
+
+            addStartupReadyHandler();
+            // todo detect premature termination
 //            apiServerProcess.onExit()
             log.debug("API Server started");
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
     }
+
+    private void addStartupReadyHandler() {
+        // alternative would be to use health checks? https://kubernetes.io/docs/reference/using-api/health-checks/
+        // waits until fully started, otherwise default namespace might be missing
+        this.startupWaiter = new Thread(() -> {
+            // todo the scanner is not closed
+            Scanner sc = new Scanner(apiServerProcess.getErrorStream());
+                while (sc.hasNextLine()) {
+                    String line = sc.nextLine();
+//                    if (line.contains("Caches are synced")) {
+                    if (line.contains("all system priority classes are created successfully or already exist")) {
+                        startedUpProperly = true;
+                        return;
+                    }
+                }
+            });
+        this.startupWaiter.start();
+    }
+
 }

src/main/java/com/csviri/kubeapi/junit/APIServerExtension.java

@@ -1,4 +1,22 @@
 package com.csviri.kubeapi.junit;
 
-public class APIServerExtension {
+import com.csviri.kubeapi.APIServer;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+public class APIServerExtension implements BeforeAllCallback, AfterAllCallback {
+
+    private APIServer apiServer;
+
+    @Override
+    public void beforeAll(ExtensionContext extensionContext) throws Exception {
+        apiServer = new APIServer();
+        apiServer.start();
+    }
+
+    @Override
+    public void afterAll(ExtensionContext extensionContext) throws Exception {
+        apiServer.stop();
+    }
 }

src/main/java/com/csviri/kubeapi/junit/EnableAPIServer.java

@@ -1,4 +1,15 @@
 package com.csviri.kubeapi.junit;
 
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.*;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+@ExtendWith(APIServerExtension.class)
+@Target({ TYPE, ANNOTATION_TYPE })
+@Retention(RUNTIME)
 public @interface EnableAPIServer {
 }

src/test/java/com/csviri/kubeapi/ApiServerTest.java

@@ -17,7 +17,7 @@ void sanityTest() throws InterruptedException {
         var kubeApi = new APIServer();
         try {
             kubeApi.start();
-            Thread.sleep(3000);
+
             // todo cleanup before start
             var client = new KubernetesClientBuilder().build();
             client.resource(configMap()).createOrReplace();

src/test/java/com/csviri/kubeapi/JUnitExtensionTest.java

@@ -0,0 +1,36 @@
+package com.csviri.kubeapi;
+
+import com.csviri.kubeapi.junit.EnableAPIServer;
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.fabric8.kubernetes.client.KubernetesClientBuilder;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+@EnableAPIServer
+public class JUnitExtensionTest {
+
+    @Test
+    public void testCommunication() {
+        var client = new KubernetesClientBuilder().build();
+        client.resource(configMap()).createOrReplace();
+        var cm = client.resource(configMap()).get();
+
+        assertThat(cm).isNotNull();
+    }
+
+    private ConfigMap configMap() {
+        return new ConfigMapBuilder()
+                .withMetadata(new ObjectMetaBuilder()
+                        .withName("test1")
+                        .withNamespace("default")
+                        .build())
+                .withData(Map.of("key","data"))
+                .build();
+    }
+
+}