workflow file

Author: a5dur

.github/workflows/update-ckan-metadata.yml

@@ -1,149 +1,79 @@
-name: Update CKAN Extensions Metadata
+name: Update CKAN Metadata
+
 on:
   schedule:
-    # Runs every Sunday at 2:00 AM UTC
-    - cron: '0 2 * * 0'
-  workflow_dispatch:
-    # Allows manual triggering from GitHub UI
+    # Run every Sunday at 4 AM UTC
+    - cron: '0 4 * * 0'
+  workflow_dispatch:  # Allow manual triggering
+  push:
+    branches:
+      - main
+    paths:
+      - 'sites-workflow/**'
+      - 'Dockerfile'
+      - 'docker-entrypoint.sh'
+      - 'requirements.txt'
 
 env:
-  GITHUB_TOKEN: ${{ secrets.GH_METADATA_TOKEN }}
-  CKAN_API_KEY: ${{ secrets.CKAN_API_KEY }}
+  DOCKER_IMAGE: dathere/ckan-metadata-workflow
+  DOCKER_TAG: latest
 
 jobs:
-  update-metadata:
+  build-and-push:
     runs-on: ubuntu-latest
+    outputs:
+      image-digest: ${{ steps.build.outputs.digest }}
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
-    
-    - name: Set up Python
-      uses: actions/setup-python@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
       with:
-        python-version: '3.9'
-        cache: 'pip'
-    
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    
-    - name: Step 1 - Extract GitHub URLs from CKAN
-      run: |
-        echo "=== STEP 1: Extracting GitHub URLs from CKAN ==="
-        python 1getURL.py
-        echo "✓ URL extraction completed"
-        
-        # Check if output file was created
-        if [ -f "url_list.csv" ]; then
-          echo "✓ url_list.csv created successfully"
-          wc -l url_list.csv
-        else
-          echo "✗ url_list.csv not found"
-          exit 1
-        fi
-    
-    - name: Step 2 - Fetch Dynamic Metadata from GitHub
-      run: |
-        echo "=== STEP 2: Fetching dynamic metadata from GitHub ==="
-        python 2refresh.py
-        echo "✓ Metadata extraction completed"
-        
-        # Check if output file was created
-        if [ -f "dynamic_metadata_update.csv" ]; then
-          echo "✓ dynamic_metadata_update.csv created successfully"
-          wc -l dynamic_metadata_update.csv
-        else
-          echo "✗ dynamic_metadata_update.csv not found"
-          exit 1
-        fi
-    
-    - name: Step 3 - Update CKAN Catalog with Metadata
-      run: |
-        echo "=== STEP 3: Updating CKAN catalog with metadata ==="
-        python 3updateCatalog.py
-        echo "✓ CKAN catalog update completed"
-    
-    - name: Step 3.1 - Download Existing CSV from CKAN
-      run: |
-        echo "=== STEP 3.1: Downloading existing CSV from CKAN ==="
-        python 31download.py existing_metadata.csv
-        echo "✓ CSV download completed"
-        
-        # Check if download was successful
-        if [ -f "existing_metadata.csv" ]; then
-          echo "✓ existing_metadata.csv downloaded successfully"
-          wc -l existing_metadata.csv
-        else
-          echo "✗ existing_metadata.csv not found"
-          exit 1
-        fi
-    
-    - name: Step 3.2 - Merge CSVs
-      run: |
-        echo "=== STEP 3.2: Merging existing and new CSV data ==="
-        python 32merger.py existing_metadata.csv dynamic_metadata_update.csv dynamic_metadata_update.csv
-        echo "✓ CSV merge completed"
-        
-        # Check if merge was successful (32merger.py overwrites dynamic_metadata_update.csv)
-        if [ -f "dynamic_metadata_update.csv" ]; then
-          echo "✓ dynamic_metadata_update.csv merged successfully"
-          wc -l dynamic_metadata_update.csv
-        else
-          echo "✗ dynamic_metadata_update.csv not found after merge"
-          exit 1
-        fi
-    
-    - name: Step 3.3 - Delete Old Resource from CKAN
-      run: |
-        echo "=== STEP 3.3: Deleting old resource from CKAN ==="
-        python 33delete.py
-        echo "✓ Old resource deletion completed"
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Build and push Docker image
+      id: build
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./Dockerfile
+        push: true
+        tags: ${{ env.DOCKER_IMAGE }}:${{ env.DOCKER_TAG }}
+        platforms: linux/amd64,linux/arm64
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+
+  run-workflow:
+    needs: build-and-push
+    runs-on: ubuntu-latest
 
-    - name: Step 4 - Upload New CSV to CKAN Dataset
+    steps:
+    - name: Run CKAN metadata workflow
       run: |
-        echo "=== STEP 4: Uploading merged CSV to CKAN dataset ==="
-        # The upload script will use dynamic_metadata_update.csv (which is now merged)
-        python 4uploadDataset.py
-        echo "✓ CSV upload completed"
-    
-    - name: Upload artifacts on failure
-      if: failure()
+        docker run --rm \
+          -e CKAN_API_KEY="${{ secrets.CKAN_API_KEY }}" \
+          -v ${{ github.workspace }}/data:/app/data \
+          -v ${{ github.workspace }}/logs:/app/logs \
+          ${{ env.DOCKER_IMAGE }}:${{ env.DOCKER_TAG }}
+
+    - name: Upload workflow logs
       uses: actions/upload-artifact@v4
+      if: always()
       with:
-        name: debug-files
-        path: |
-          *.csv
-          *.log
-        retention-days: 7
-    
-    - name: Upload generated CSVs as artifacts
-      if: success()
+        name: workflow-logs-${{ github.run_number }}
+        path: logs/
+        retention-days: 30
+
+    - name: Upload generated data
       uses: actions/upload-artifact@v4
+      if: success()
       with:
-        name: metadata-csv-files
-        path: |
-          dynamic_metadata_update.csv
-          existing_metadata.csv
-        retention-days: 30
-    
-    - name: Workflow Summary
-      if: always()
-      run: |
-        echo "=== WORKFLOW SUMMARY ==="
-        echo "Status: ${{ job.status }}"
-        echo "Timestamp: $(date -u)"
-        
-        # Show file sizes if they exist
-        for file in url_list.csv dynamic_metadata_update.csv existing_metadata.csv; do
-          if [ -f "$file" ]; then
-            echo "$file: $(wc -l < "$file") lines, $(du -h "$file" | cut -f1)"
-          fi
-        done
-        
-        # Show any log files
-        if ls *.log >/dev/null 2>&1; then
-          echo "Log files created:"
-          ls -la *.log
-        fi
+        name: generated-data-${{ github.run_number }}
+        path: data/
+        retention-days: 7