Merge pull request #163 from Qubinode/copilot/replace-manual-ssh-commands

Replace manual SSH commands with SSHOperator in Airflow DAGs

Author: tosin2013

.github/workflows/airflow-dag-ci.yml

@@ -37,27 +37,44 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
 
-      - name: Install Airflow and dependencies
+      - name: Install Airflow and initialize database
         run: |
+          # Use the CI setup script for consistent environment
+          chmod +x ./airflow/scripts/ci-setup.sh
+
+          # Install dependencies
           python -m pip install --upgrade pip
-          pip install "apache-airflow[postgres,celery]==${{ env.AIRFLOW_VERSION }}"
-          pip install pytest pytest-asyncio
+          echo "Installing Apache Airflow ${{ env.AIRFLOW_VERSION }}..."
+          # Use constraints to ensure compatible package versions
+          PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+          CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${{ env.AIRFLOW_VERSION }}/constraints-${PYTHON_VERSION}.txt"
+          echo "Using constraints: $CONSTRAINT_URL"
+
+          pip install "apache-airflow[postgres,celery]==${{ env.AIRFLOW_VERSION }}" --constraint "$CONSTRAINT_URL"
+          echo "Installing SSH provider..."
+          pip install "apache-airflow-providers-ssh" --constraint "$CONSTRAINT_URL"
+          pip install pytest pytest-asyncio pyyaml
+          echo "Airflow packages installed successfully"
 
           # Install project dependencies
           if [ -f requirements.txt ]; then
             pip install -r requirements.txt
           fi
 
-      - name: Initialize Airflow Database
-        run: |
+          # Setup PATH and initialize DB
+          SCRIPTS_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_path('scripts'))")
+          export PATH="$SCRIPTS_DIR:$PATH"
+          echo "$SCRIPTS_DIR" >> $GITHUB_PATH
+
+          # Verify airflow is available
+          which airflow
+          airflow version
+
           export AIRFLOW_HOME=${{ env.AIRFLOW_HOME }}
           airflow db init
 
-      - name: Install PyYAML for Registry Validation
-        run: |
-          pip install pyyaml
+          echo "[OK] Airflow installed and database initialized"
 
       - name: Validate DAG Registry (ADR-0046)
         run: |
@@ -192,6 +209,10 @@ jobs:
           mkdir -p ${{ env.AIRFLOW_HOME }}/dags
           cp -r airflow/dags/* ${{ env.AIRFLOW_HOME }}/dags/ || true
 
+          # Add scripts directory to PATH
+          SCRIPTS_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_path('scripts'))")
+          export PATH="$SCRIPTS_DIR:$PATH"
+
           # List and validate DAGs
           echo "Listing DAGs..."
           airflow dags list || echo "⚠️  No DAGs found or error listing"
@@ -244,6 +265,9 @@ jobs:
       - name: Initialize Airflow
         run: |
           export AIRFLOW_HOME=${{ env.AIRFLOW_HOME }}
+          # Add scripts directory to PATH for this step
+          SCRIPTS_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_path('scripts'))")
+          export PATH="$SCRIPTS_DIR:$PATH"
           airflow db init
 
       - name: Validate RAG Ingestion DAG

.github/workflows/airflow-validate.yml

@@ -102,6 +102,7 @@ jobs:
 
           pip install "apache-airflow==2.10.4" --constraint "$CONSTRAINT_URL"
           pip install "apache-airflow-providers-postgres" --constraint "$CONSTRAINT_URL"
+          pip install "apache-airflow-providers-ssh" --constraint "$CONSTRAINT_URL"
           pip install pyyaml
 
           # Find scripts directory
@@ -367,6 +368,7 @@ jobs:
 
           pip install "apache-airflow==2.10.4" --constraint "$CONSTRAINT_URL"
           pip install "apache-airflow-providers-postgres" --constraint "$CONSTRAINT_URL"
+          pip install "apache-airflow-providers-ssh" --constraint "$CONSTRAINT_URL"
           pip install openlineage-airflow
           pip install pyyaml
           pip install httpx  # For MCP server testing

airflow/Dockerfile

@@ -31,6 +31,11 @@ RUN pip install --no-cache-dir \
 RUN pip install --no-cache-dir \
     apache-airflow-providers-hashicorp>=3.0.0
 
+# SSH provider for remote host execution (replaces manual SSH in BashOperator)
+# Used for executing commands on host from container (kcli, virsh, ansible)
+RUN pip install --no-cache-dir \
+    apache-airflow-providers-ssh>=3.0.0
+
 # ADR-0038: FastMCP framework for MCP server
 # Required for MCP server integration with Claude/AI assistants
 RUN pip install --no-cache-dir \