Merge pull request Jacoo-Zhao#12 from Jacoo-Zhao/feature-cicd

Feature: CICD in task level

Author: Jacoo-Zhao

.github/workflows/pipeline.yaml

@@ -1,13 +1,18 @@
-name: Run ClearML Pipeline
+name: Run ClearML Task
 
 on:
-  push:
-    branches: [ main ]
+  pull_request:
+    branches: [main]
 
 jobs:
   run_pipeline:
     runs-on: ubuntu-latest
 
+    env:  # ✅ env:  # Global environment variables for all steps
+      CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
+      CLEARML_API_SECRET_KEY: ${{ secrets.CLEARML_API_SECRET_KEY }}
+      CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
+
     steps:
       - uses: actions/checkout@v3
 
@@ -21,10 +26,12 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
+      - name: Debug ENV
+        run: |
+          echo "CLEARML_API_HOST=$CLEARML_API_HOST"
+          if [ -z "$CLEARML_API_HOST" ]; then echo "❌ HOST is empty!"; exit 1; fi
+          curl -I $CLEARML_API_HOST
+
       - name: Run pipeline
-        env:
-          CLEARML_API_ACCESS_KEY: ${{ secrets.CLEARML_API_ACCESS_KEY }}
-          CLEARML_API_SECRET_KEY: ${{ secrets.CLEARML_API_SECRET_KEY }}
-          CLEARML_API_HOST: ${{ secrets.CLEARML_API_HOST }}
         run: |
-          python main.py
+          python s1_dataset_artifact.py

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
-.idea
+.idea
+figs/

s1_dataset_artifact.py

@@ -0,0 +1,27 @@
+"s1_dataset_artifact.py"
+
+from clearml import Task, StorageManager
+import os
+
+# create an dataset experiment
+task = Task.init(project_name="AI_Studio_Demo", task_name="Pipeline step 1 dataset artifact")
+
+# only create the task, we will actually execute it later
+# task.execute_remotely()
+
+# Check if the local dataset file exists
+local_iris_csv_path = 'work_dataset/Iris.csv'
+if not os.path.exists(local_iris_csv_path):
+    print(f"Local file '{local_iris_csv_path}' not found. Downloading...")
+    local_iris_pkl = StorageManager.get_local_copy(
+        remote_url='https://github.com/allegroai/events/raw/master/odsc20-east/generic/iris_dataset.pkl'
+    )
+else:
+    print(f"Using existing local file: '{local_iris_csv_path}'")
+
+# Add and upload the dataset file
+task.upload_artifact('dataset', artifact_object=local_iris_csv_path)
+print('uploading artifacts in the background')
+
+# we are done
+print('Done')

step2_data_preprocessing.py s2_data_preprocessing.pystep2_data_preprocessing.py renamed to s2_data_preprocessing.py

@@ -112,6 +112,6 @@ def forward(self, x):
 disp.plot(cmap=plt.cm.Blues)
 
 plt.title('Confusion Matrix')
-plt.savefig('assets/confusion_matrix.png')
+plt.savefig('figs/confusion_matrix.png')
 
 print('Confusion matrix plotted and saved as confusion_matrix.png')