fix pipeline bug and workflow

Signed-off-by: Sunyanan Choochotkaew <[email protected]>

Author: sunya-ch

.github/workflows/integration-test.yml

@@ -1,18 +1,15 @@
 name: Integration Test
 on:
   workflow_call:
-    secrets:
-      docker_username:
-        description: 'Docker username'
-        required: false
-      docker_password:
-        description: 'Docker password'
-        required: false
     inputs:
       base_change:
         description: 'Change flag on base image'
         required: true
         type: string
+      docker_secret:
+        description: 'Secret check'
+        required: true
+        type: string
       image_repo:
         description: 'The image repo to use'
         required: true
@@ -28,6 +25,7 @@ env:
 
 jobs:
   run-integration:
+    if: ${{ (inputs.base_change != 'true') || ( (inputs.base_change == 'true') && (inputs.docker_secret == 'true') ) }}
     runs-on: ubuntu-20.04
     steps:
       - name: use Kepler action to deploy cluster
@@ -42,25 +40,18 @@ jobs:
           kind load docker-image quay.io/sustainable_computing_io/kepler:latest
       - name: checkout
         uses: actions/checkout@v4
-      - name: Login to Docker
-        if: ${{ inputs.base_change == 'true' }} 
-        uses: docker/login-action@v3
-        with:
-            registry: ${{ inputs.image_repo }}
-            username: ${{ secrets.docker_username }}
-            password: ${{ secrets.docker_password }} 
-      - name: Replace value in Dockerfile
-        if: ${{ inputs.base_change == 'true' }} 
-        run: |
-          sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile
-      - name: Replace value in Dockerfile.test
-        if: ${{ inputs.base_change == 'true' }} 
-        run: |
-          sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile.test
       - name: set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+      - name: Replace value in Dockerfile if base changes
+        if: ${{ (inputs.base_change == 'true') && (inputs.docker_secret == 'true') }}
+        run: |
+          sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile
+      - name: Replace value in Dockerfile.test if base changes
+        if: ${{ (inputs.base_change == 'true') && (inputs.docker_secret == 'true') }}
+        run: |
+          sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile.test
       - name: build Kepler model server and test image and push to local registry
         run: make build build-test push push-test
       - name: set up Kustomize

.github/workflows/push-pr.yml

@@ -153,28 +153,22 @@ jobs:
           echo "change=true" >> "$GITHUB_OUTPUT"
 
   tekton-test:
-    needs: [check-branch, base-image]
-    # the changed base-image has been pushed or no change
-    if: ${{ needs.base-image.outputs.change == 'true' || needs.check-change.outputs.base != 'true' }}
+    needs: [check-secret, check-branch, base-image]
+    if: always()
     uses: ./.github/workflows/tekton-test.yml
     with:
-      base_change: ${{ needs.base-image.outputs.change }}
+      base_change: ${{ needs.check-change.outputs.base }}
+      docker_secret: ${{ needs.check-secret.outputs.docker-secret }}
       image_repo: ${{ vars.IMAGE_REPO || 'docker.io/library' }}
       image_tag: ${{ needs.check-branch.outputs.tag }}
       pipeline_name: std_v0.7
-    secrets:
-      docker_username: ${{ secrets.BOT_NAME }}
-      docker_password: ${{ secrets.BOT_TOKEN }}
 
   integration-test:
-    needs: [check-branch, base-image]
-    # the changed base-image has been pushed or no change
-    if: ${{ needs.base-image.outputs.change == 'true' || needs.check-change.outputs.base != 'true' }}
+    needs: [check-secret, check-branch, base-image]
+    if: always()
     uses: ./.github/workflows/integration-test.yml
     with:
-      base_change: ${{ needs.base-image.outputs.change }}
+      base_change: ${{ needs.check-change.outputs.base }}
+      docker_secret: ${{ needs.check-secret.outputs.docker-secret }}
       image_repo: ${{ vars.IMAGE_REPO || 'docker.io/library' }}
-      image_tag: ${{ needs.check-branch.outputs.tag }}
-    secrets:
-      docker_username: ${{ secrets.BOT_NAME }}
-      docker_password: ${{ secrets.BOT_TOKEN }}
+      image_tag: ${{ needs.check-branch.outputs.tag }}

.github/workflows/tekton-test.yml

@@ -2,18 +2,15 @@ name: Tekton Test
 
 on:
   workflow_call:
-    secrets:
-      docker_username:
-        description: 'Docker username'
-        required: false
-      docker_password:
-        description: 'Docker password'
-        required: false
     inputs:
       base_change:
         description: 'Change flag on base image'
         required: true
         type: string
+      docker_secret:
+        description: 'Secret check'
+        required: true
+        type: string
       image_repo:
         description: 'The image repo to use'
         required: true
@@ -43,6 +40,7 @@ env:
 
 jobs:
   tekton-test:
+    if: ${{ (inputs.base_change != 'true') || ( (inputs.base_change == 'true') && (inputs.docker_secret == 'true') ) }}
     runs-on: ubuntu-latest
     steps:
       - name: checkout
@@ -51,22 +49,13 @@ jobs:
         uses: docker/setup-qemu-action@v3
       - name: set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      - name: Login to Docker
-        if: ${{ inputs.base_change == 'true' }} 
-        uses: docker/login-action@v3
-        with:
-            registry: ${{ inputs.image_repo }}
-            username: ${{ secrets.docker_username }}
-            password: ${{ secrets.docker_password }} 
-      - name: Replace value in file
-        if: ${{ inputs.base_change == 'true' }} 
+      - name: Replace value in file if base changes
+        if: ${{ (inputs.base_change == 'true') && (inputs.docker_secret == 'true') }} 
         run: |
           sed -i "s|quay.io/sustainable_computing_io/kepler_model_server_base:v0.7|${{ env.BASE_IMAGE }}|" dockerfiles/Dockerfile
-
       - name: Build image
         run: |
           docker build -t $IMAGE -f dockerfiles/Dockerfile .
-
       - name: Prepare Cluster
         working-directory: model_training
         run: |
@@ -135,7 +124,7 @@ jobs:
             - name: STRESS_BREAK_INTERVAL
               value: 1
             - name: IDLE_COLLECT_INTERVAL
-              value: 20
+              value: 100
             - name: CPU_FREQUENCY_ENABLED
               value: false
             pipelineRef:

model_training/tekton/examples/complete-pipelinerun.yaml

@@ -24,7 +24,7 @@ spec:
   - name: STRESS_BREAK_INTERVAL
     value: 1
   - name: IDLE_COLLECT_INTERVAL
-    value: 20
+    value: 100
   - name: CPU_FREQUENCY_ENABLED
     value: false
   pipelineRef:

model_training/tekton/examples/single-train/abs-power.yaml

@@ -26,7 +26,7 @@ spec:
   - name: STRESS_BREAK_INTERVAL
     value: 1
   - name: IDLE_COLLECT_INTERVAL
-    value: 20
+    value: 100
   - name: CPU_FREQUENCY_ENABLED
     value: false
   pipelineRef:

model_training/tekton/examples/single-train/aws-push.yaml

@@ -32,7 +32,7 @@ spec:
   - name: STRESS_BREAK_INTERVAL
     value: 1
   - name: IDLE_COLLECT_INTERVAL
-    value: 20
+    value: 100
   - name: CPU_FREQUENCY_ENABLED
     value: false
   pipelineRef:

model_training/tekton/examples/single-train/dyn-power.yaml

@@ -26,7 +26,7 @@ spec:
   - name: STRESS_BREAK_INTERVAL
     value: 1
   - name: IDLE_COLLECT_INTERVAL
-    value: 20
+    value: 100
   - name: CPU_FREQUENCY_ENABLED
     value: false
   pipelineRef:

model_training/tekton/examples/single-train/ibmcloud-push.yaml

@@ -32,7 +32,7 @@ spec:
   - name: STRESS_BREAK_INTERVAL
     value: 1
   - name: IDLE_COLLECT_INTERVAL
-    value: 20
+    value: 100
   - name: CPU_FREQUENCY_ENABLED
     value: false
   pipelineRef:

model_training/tekton/examples/test-collect.yaml

@@ -23,7 +23,7 @@ spec:
   - name: STRESS_BREAK_INTERVAL
     value: 1
   - name: IDLE_COLLECT_INTERVAL
-    value: 20
+    value: 100
   - name: CPU_FREQUENCY_ENABLED
     value: false
   pipelineRef:

src/train/pipeline.py

@@ -143,7 +143,7 @@ def process(self, input_query_results, energy_components, energy_source, feature
         if abs_data is None and dyn_data is None:
             return False, None, None
         if replace_node_type is not None:
-            self.print_log("Replace Node Type: ", replace_node_type)
+            self.print_log("Replace Node Type:  {}".format(replace_node_type))
             abs_data[node_info_column] = replace_node_type
             dyn_data[node_info_column] = replace_node_type
         self._train(abs_data, dyn_data, power_labels, energy_source, feature_group)
@@ -156,7 +156,7 @@ def process_multiple_query(self, input_query_results_list, energy_components, en
         if (abs_data is None or len(abs_data) == 0) and (dyn_data is None or len(dyn_data) == 0):
             return False, None, None
         if replace_node_type is not None:
-            self.print_log("Replace Node Type: ", replace_node_type)
+            self.print_log("Replace Node Type: {}".format(replace_node_type))
             abs_data[node_info_column] = replace_node_type
             dyn_data[node_info_column] = replace_node_type
         self._train(abs_data, dyn_data, power_labels, energy_source, feature_group)