Convert pipeline to static and fill in values in java code

rnc · rnc · commit 19c96485d741 · 2024-11-27T15:00:55.000Z
diff --git a/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java b/java-components/build-request-processor/src/main/java/com/redhat/hacbs/container/build/preprocessor/AbstractPreprocessor.java
@@ -145,7 +145,20 @@ public void run() {
             fi
             echo "PATH:$PATH"
 
+            update-ca-trust
+
+            # Go through certificates and insert them into the cacerts
+            for cert in $(find /etc/pki/ca-trust/source/anchors -type f); do
+              echo "Inserting $cert into java cacerts"
+              keytool -import -alias $(basename $cert)-ca \\
+                -file $cert \\
+                -keystore /etc/pki/java/cacerts \\
+                -storepass changeit --noprompt
+            done
+
             # End of generic build script
+
+            echo "Building the project ..."
             """;
 
         if (isNotEmpty(buildScript)) {
diff --git a/java-components/cli/src/main/java/com/redhat/hacbs/cli/driver/Pipeline.java b/java-components/cli/src/main/java/com/redhat/hacbs/cli/driver/Pipeline.java
@@ -44,6 +44,8 @@ public void run() {
                 .repositoryDependencyUrl(dependencies)
                 .repositoryBuildContentId("test-maven-konflux-int-0001")
                 .recipeImage(recipeImage)
+                // Just use default from buildah-oci-ta for now.
+                .podMemoryOverride("4Gi")
                 .build();
         driver.create(request);
     }
diff --git a/java-components/driver/pom.xml b/java-components/driver/pom.xml
@@ -28,10 +28,10 @@
             <groupId>io.quarkiverse.tektonclient</groupId>
             <artifactId>quarkus-tekton-client</artifactId>
         </dependency>
-<!--        <dependency>-->
-<!--            <groupId>io.quarkus</groupId>-->
-<!--            <artifactId>quarkus-openshift-client</artifactId>-->
-<!--        </dependency>-->
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-openshift-client</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-kubernetes-client</artifactId>
diff --git a/java-components/driver/src/main/java/com/redhat/hacbs/driver/Driver.java b/java-components/driver/src/main/java/com/redhat/hacbs/driver/Driver.java
@@ -3,7 +3,6 @@
 import static org.apache.commons.lang3.StringUtils.isEmpty;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.Map;
@@ -12,24 +11,22 @@
 import jakarta.enterprise.context.RequestScoped;
 import jakarta.inject.Inject;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.text.StringSubstitutor;
 import org.eclipse.microprofile.config.ConfigProvider;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.eclipse.microprofile.rest.client.inject.RestClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import com.redhat.hacbs.driver.clients.IndyService;
 import com.redhat.hacbs.driver.clients.IndyTokenRequestDTO;
 import com.redhat.hacbs.driver.clients.IndyTokenResponseDTO;
 import com.redhat.hacbs.driver.dto.BuildRequest;
 
+import io.fabric8.kubernetes.api.model.Quantity;
 import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.tekton.client.TektonClient;
+import io.fabric8.tekton.pipeline.v1.ParamBuilder;
 import io.fabric8.tekton.pipeline.v1.PipelineRun;
 import io.quarkus.oidc.client.OidcClient;
 import lombok.Setter;
@@ -39,15 +36,12 @@ public class Driver {
 
     private static final Logger logger = LoggerFactory.getLogger(Driver.class);
 
-    private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
-
     @Inject
     OidcClient oidcClient;
 
     @RestClient
     IndyService indyService;
 
-    // TODO: Could use KubernetesClient or OpenShiftClient
     @Inject
     KubernetesClient client;
 
@@ -72,40 +66,55 @@ public void create(BuildRequest buildRequest) {
             tokenResponseDTO = indyService.getAuthToken(
                     new IndyTokenRequestDTO(buildRequest.getRepositoryBuildContentId()),
                     "Bearer " + getFreshAccessToken());
-            logger.debug("### new access token: {}", tokenResponseDTO.getToken()); // TODO: REMOVE
-
         }
 
         Map<String, String> templateProperties = new HashMap<>();
-        templateProperties.put("REQUEST_PROCESSOR", processor);
-        templateProperties.put("QUAY_REPO", quayRepo);
-        templateProperties.put("URL", buildRequest.getScmUrl());
-        templateProperties.put("REVISION", buildRequest.getScmRevision());
-        templateProperties.put("RECIPE_IMAGE", buildRequest.getRecipeImage());
+        templateProperties.put("ACCESS_TOKEN", tokenResponseDTO.getToken());
+        templateProperties.put("BUILD_ID", buildRequest.getRepositoryBuildContentId());
+        templateProperties.put("BUILD_SCRIPT", buildRequest.getBuildScript());
         templateProperties.put("BUILD_TOOL", buildRequest.getBuildTool());
         templateProperties.put("BUILD_TOOL_VERSION", buildRequest.getBuildToolVersion());
         templateProperties.put("JAVA_VERSION", buildRequest.getJavaVersion());
-        templateProperties.put("BUILD_SCRIPT", buildRequest.getBuildScript());
-        templateProperties.put("MVN_REPO_DEPLOY_URL", buildRequest.getRepositoryDeployUrl());
         templateProperties.put("MVN_REPO_DEPENDENCIES_URL", buildRequest.getRepositoryDependencyUrl());
-        templateProperties.put("ACCESS_TOKEN", tokenResponseDTO.getToken());
-        templateProperties.put("BUILD_ID", buildRequest.getRepositoryBuildContentId());
+        templateProperties.put("MVN_REPO_DEPLOY_URL", buildRequest.getRepositoryDeployUrl());
+        templateProperties.put("QUAY_REPO", quayRepo);
+        templateProperties.put("RECIPE_IMAGE", buildRequest.getRecipeImage());
+        templateProperties.put("JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE", processor);
+        templateProperties.put("REVISION", buildRequest.getScmRevision());
+        templateProperties.put("URL", buildRequest.getScmUrl());
 
-        String pipeline = "";
+        PipelineRun pipelineRun = null;
         try {
+            var tc = client.adapt(TektonClient.class);
+            // Various ways to create the initial PipelineRun object. We can use an objectmapper,
+            // client.getKubernetesSerialization() or the load calls on the Fabric8 objects.
             if (customPipeline.isEmpty()) {
-                pipeline = IOUtils.resourceToString("pipeline.yaml", StandardCharsets.UTF_8,
-                        Thread.currentThread().getContextClassLoader());
+                pipelineRun = tc.v1().pipelineRuns()
+                        .load(IOUtils.resourceToURL("pipeline.yaml", Thread.currentThread().getContextClassLoader())).item();
             } else {
-                pipeline = FileUtils.readFileToString(Path.of(customPipeline.get()).toFile(), StandardCharsets.UTF_8);
+                pipelineRun = tc.v1().pipelineRuns().load(Path.of(customPipeline.get()).toFile()).item();
             }
         } catch (IOException e) {
+            e.printStackTrace();
             // TODO: process
         }
-
-        PipelineRun run = createModelNode(pipeline, templateProperties, PipelineRun.class);
-
-        var created = client.resource(run).inNamespace(buildRequest.getNamespace()).create();
+        pipelineRun = pipelineRun.edit().editOrNewSpec()
+                .addAllToParams(templateProperties.entrySet().stream()
+                        .map(t -> new ParamBuilder().withName(t.getKey()).withNewValue(t.getValue()).build()).toList())
+                .editFirstTaskRunSpec()
+                .editFirstStepSpec()
+                .editComputeResources()
+                .addToLimits("memory", new Quantity(buildRequest.getPodMemoryOverride()))
+                .addToRequests("memory", new Quantity(buildRequest.getPodMemoryOverride()))
+                .endComputeResources()
+                .endStepSpec()
+                .endTaskRunSpec()
+                .endSpec().build();
+
+        System.err.println("### Got p " + pipelineRun);
+        // PipelineRun run = createModelNode(pipeline, templateProperties, PipelineRun.class);
+        //run.getSpec().setParams();
+        var created = client.resource(pipelineRun).inNamespace(buildRequest.getNamespace()).create();
     }
 
     /**
@@ -115,22 +124,6 @@ public void create(BuildRequest buildRequest) {
      * @return fresh access token
      */
     public String getFreshAccessToken() {
-        var result = oidcClient.getTokens().await().indefinitely().getAccessToken();
-        logger.debug("### access token: {}", result); // TODO: REMOVE
-        return result;
-    }
-
-    private <T> T createModelNode(String resourceDefinition, Map<String, String> properties, Class<T> clazz) {
-        String definition = StringSubstitutor.replace(resourceDefinition, properties, "%{", "}");
-
-        if (logger.isTraceEnabled()) {
-            logger.trace("Node definition: {}", definition);
-        }
-
-        try {
-            return yamlMapper.readValue(definition, clazz);
-        } catch (JsonProcessingException e) {
-            throw new RuntimeException(e);
-        }
+        return oidcClient.getTokens().await().indefinitely().getAccessToken();
     }
 }
diff --git a/java-components/driver/src/main/java/com/redhat/hacbs/driver/dto/BuildRequest.java b/java-components/driver/src/main/java/com/redhat/hacbs/driver/dto/BuildRequest.java
@@ -39,6 +39,5 @@ public class BuildRequest {
 
     private final String namespace;
 
-    // TODO: NYI
-    // private final String podMemoryOverride;
+    private final String podMemoryOverride;
 }
diff --git a/java-components/driver/src/main/resources/pipeline.yaml b/java-components/driver/src/main/resources/pipeline.yaml
@@ -9,8 +9,6 @@ spec:
       # TODO: This will eventually be build-definitions repository
       - name: url
         value: https://raw.githubusercontent.com/rnc/jvm-build-service/refs/heads/NCL8774/deploy/pipeline/mw-pipeline-v0.1.yaml
-        #value: https://raw.githubusercontent.com/rnc/jvm-build-service/refs/heads/NCL8774/deploy/pipeline/mw-pipeline-v0.1.yaml
-
   workspaces:
     - name: source
       # TODO: If we have a custom git step we can share this with prebuild thereby eliminating the need for a volumeClaimTemplate
@@ -27,50 +25,10 @@ spec:
               storage: 1Gi
         status: {}
   params:
-    - name: QUAY_REPO
-      value: "%{QUAY_REPO}"
-    - name: URL
-      value: "%{URL}"
-    - name: REVISION
-      value: "%{REVISION}"
-    - name: RECIPE_IMAGE
-      value: "%{RECIPE_IMAGE}"
-    - name: BUILD_TOOL
-      value: "%{BUILD_TOOL}"
-    - name: BUILD_TOOL_VERSION
-      value: "%{BUILD_TOOL_VERSION}"
-    - name: JAVA_VERSION
-      value: "%{JAVA_VERSION}"
-    - name: BUILD_SCRIPT
-      value: |
-        update-ca-trust
-
-        # Go through certificates and insert them into the cacerts
-        for cert in /etc/pki/ca-trust/source/anchors/*; do
-          echo "inserting $cert into java cacerts"
-          keytool -import -alias $(basename $cert)-ca \
-            -file $cert \
-            -keystore /etc/pki/java/cacerts \
-            -storepass changeit --noprompt
-        done;
-
-        echo "Building the project ..."
-
-        %{BUILD_SCRIPT}
-    - name: MVN_REPO_DEPLOY_URL
-      value: "%{MVN_REPO_DEPLOY_URL}"
-    - name: MVN_REPO_DEPENDENCIES_URL
-      value: "%{MVN_REPO_DEPENDENCIES_URL}"
-    - name: ACCESS_TOKEN
-      value: "%{ACCESS_TOKEN}"
-    - name: BUILD_ID
-      value: "%{BUILD_ID}"
     - name: caTrustConfigMapName
       value: custom-ca
     - name: ENABLE_INDY_PROXY
       value: "false"
-    - name: JVM_BUILD_SERVICE_REQPROCESSOR_IMAGE
-      value: "%{REQUEST_PROCESSOR}"
   # TODO: Should PNC set both limits and requests? See
   #   https://home.robusta.dev/blog/kubernetes-memory-limit
   #   https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
@@ -79,13 +37,7 @@ spec:
       stepSpecs:
         - name: build
           computeResources:
-            limits:
-              # Default in buildah-oci-ta is 4Gi
-              # TODO: Configure as %{MEMORY}
-              memory: 5Gi
-            requests:
-              # Default in buildah-oci-ta is 1Gi
-              memory: 5Gi
+            # limits/requests memory are configured dynamically in Driver.
   taskRunTemplate:
     podTemplate:
       env:

Original file line number	Diff line number	Diff line change
`@@ -44,6 +44,8 @@ public void run() {`
`44`	`44`	`.repositoryDependencyUrl(dependencies)`
`45`	`45`	`.repositoryBuildContentId("test-maven-konflux-int-0001")`
`46`	`46`	`.recipeImage(recipeImage)`
	`47`	`+ // Just use default from buildah-oci-ta for now.`
	`48`	`+ .podMemoryOverride("4Gi")`
`47`	`49`	`.build();`
`48`	`50`	`driver.create(request);`
`49`	`51`	`}`
Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,5 @@ public class BuildRequest {`
`39`	`39`
`40`	`40`	`private final String namespace;`
`41`	`41`
`42`		`- // TODO: NYI`
`43`		`- // private final String podMemoryOverride;`
	`42`	`+ private final String podMemoryOverride;`
`44`	`43`	`}`