changed workflow

Author: a5dur

docker-entrypoint.sh

@@ -15,30 +15,63 @@ log() {
 run_script() {
     local script_name=$1
     local description=$2
+    local criticality=$3  # "critical" or "optional"
 
     log "Starting: $description"
 
     if python "$script_name"; then
         log "Completed: $description"
     else
         log "Error: $description failed"
-        exit 1
+        if [ "$criticality" = "critical" ]; then
+            log "Critical error - stopping workflow"
+            exit 1
+        else
+            log "Non-critical error - continuing workflow"
+        fi
     fi
 }
 
 # Create logs directory if it doesn't exist
 mkdir -p /app/logs
 
+# Debug environment
+log "Environment check:"
+log "CKAN_API_KEY: $(if [ -n "$CKAN_API_KEY" ]; then echo "Set (${#CKAN_API_KEY} chars)"; else echo "NOT SET"; fi)"
+log "PYTHONPATH: $PYTHONPATH"
+log "Working directory: $(pwd)"
+log "Files in directory: $(ls -la)"
+
 log "Starting CKAN metadata workflow..."
 
-# Run scripts in sequential order
-run_script "1getSitesURL.py" "Fetching CKAN portal URLs"
-run_script "2CKANActionAPI.py" "Fetching metadata using CKAN API"
-run_script "31downloadDataset.py" "Downloading existing dataset"
-run_script "32merger.py" "Merging datasets"
-run_script "33delete.py" "Deleting old dataset"
-run_script "3updateSitesCatalog.py" "Updating sites catalog"
-run_script "4uploadDataset.py" "Uploading final dataset"
+# Run diagnostic script first
+log "Running diagnostics..."
+if python debug_api_access.py; then
+    log "Diagnostics completed"
+else
+    log "Diagnostics had issues - proceeding anyway"
+fi
+
+# Run scripts in sequential order with better error handling
+# Skip 1getSitesURL.py since we now use a static sites_urls.csv file
+log "Using pre-existing sites_urls.csv file"
+
+# Check if the static CSV file exists
+if [ ! -f "sites_urls.csv" ]; then
+    log "ERROR: sites_urls.csv file not found!"
+    exit 1
+fi
+
+# Show info about the CSV file
+csv_lines=$(wc -l < sites_urls.csv)
+log "Found sites_urls.csv with $csv_lines lines"
+
+run_script "2CKANActionAPI.py" "Fetching metadata using CKAN API" "critical"
+run_script "31downloadDataset.py" "Downloading existing dataset" "optional"
+run_script "32merger.py" "Merging datasets" "optional"
+run_script "33delete.py" "Deleting old dataset" "optional"
+run_script "3updateSitesCatalog.py" "Updating sites catalog" "optional"
+run_script "4uploadDataset.py" "Uploading final dataset" "optional"
 
 log "CKAN metadata workflow completed successfully!"