Added changes to fix pod crash after deployment

nullchimp · nullchimp · commit 9530323cece2 · 2025-05-18T23:37:25.000+02:00
diff --git a/.github/workflows/deploy-azure.yml b/.github/workflows/deploy-azure.yml
@@ -107,17 +107,78 @@ jobs:
             done
           fi
           
+          # Create a credentials hash to force pod restart when credentials change using maximum security approach
+          # Create a random file name to prevent predictable access
+          TEMP_FILE=$(mktemp)
+          
+          # Write credentials directly to file descriptor to prevent command line visibility
+          {
+            printf "%s" "${{ secrets.MEMGRAPH_USERNAME }}"
+            printf "%s" "${{ secrets.MEMGRAPH_PASSWORD }}"
+          } > "$TEMP_FILE"
+          
+          # Generate hash and immediately remove the file
+          CREDENTIALS_HASH=$(sha256sum "$TEMP_FILE" | awk '{print $1}')
+          rm -f "$TEMP_FILE"
+          
+          # Store hash in environment variable for later use
+          echo "CREDENTIALS_HASH=$CREDENTIALS_HASH" >> $GITHUB_ENV
+          
+          # Apply the kubernetes secret with the new credentials
           kubectl create secret generic memgraph-credentials \
             --from-literal=username=${{ secrets.MEMGRAPH_USERNAME }} \
             --from-literal=password=${{ secrets.MEMGRAPH_PASSWORD }} \
             --dry-run=client -o yaml | kubectl apply -f -
       
       - name: Deploy to AKS
         run: |
-          kubectl apply -f infra/k8s/memgraph.yaml
+          # Replace the placeholder with the actual credentials hash
+          cat infra/k8s/memgraph.yaml | CREDENTIALS_HASH=${CREDENTIALS_HASH} envsubst > memgraph_deploy.yaml
+          
+          # Apply the updated deployment manifest
+          kubectl apply -f memgraph_deploy.yaml
+          
+          # Force restart if the deployment already exists
+          POD_NAME=$(kubectl get pods -l app=memgraph -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "")
+          if [[ -n "$POD_NAME" ]]; then
+            echo "Forcing restart of existing Memgraph pod..."
+            kubectl delete pod $POD_NAME
+          fi
+          
+          # Remove the temporary file
+          rm memgraph_deploy.yaml
       
       - name: Verify Deployment
         run: |
+          echo "Checking deployment status..."
           kubectl get pods
           kubectl get services
-          kubectl wait --for=condition=ready pod -l app=memgraph --timeout=5m
+          
+          echo "Waiting for Memgraph pod to be ready (may take up to 5 minutes)..."
+          if ! kubectl wait --for=condition=ready pod -l app=memgraph --timeout=5m; then
+            echo "Error: Memgraph pod did not become ready within the timeout period."
+            echo "Checking Memgraph pod logs:"
+            POD_NAME=$(kubectl get pods -l app=memgraph -o jsonpath="{.items[0].metadata.name}")
+            kubectl logs $POD_NAME
+            kubectl describe pod $POD_NAME
+            echo "Deployment verification failed!"
+            exit 1
+          fi
+          
+          echo "Memgraph deployment successful!"
+          
+          # Wait for the LoadBalancer service to get an external IP
+          echo "Waiting for LoadBalancer to get external IP..."
+          for i in {1..30}; do
+            EXTERNAL_IP=$(kubectl get service memgraph -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+            if [[ -n "$EXTERNAL_IP" ]]; then
+              echo "Memgraph is accessible at: ${EXTERNAL_IP}:7687"
+              break
+            fi
+            echo "Waiting for external IP (attempt $i)..."
+            sleep 10
+          done
+          
+          if [[ -z "$EXTERNAL_IP" ]]; then
+            echo "Warning: Could not obtain external IP for Memgraph service within timeout."
+          fi
diff --git a/README.md b/README.md
@@ -34,6 +34,8 @@ To deploy to Azure:
 3. Add `MEMGRAPH_USERNAME` and `MEMGRAPH_PASSWORD` to GitHub secrets
 4. Push to main branch to trigger deployment or manually trigger the workflow
 
+If you encounter issues connecting to Memgraph after deployment, see [Memgraph Troubleshooting Guide](docs/memgraph-troubleshooting.md).
+
 ## Architecture
 The project follows a component-based architecture where the AI Agent orchestrates interactions between users, language models, local tools, and MCP servers.
 
diff --git a/docs/memgraph-troubleshooting.md b/docs/memgraph-troubleshooting.md
@@ -0,0 +1,111 @@
+# Troubleshooting Memgraph Connections
+
+This guide provides troubleshooting steps for common issues when connecting to Memgraph.
+
+## Connection Issues
+
+If you're experiencing issues connecting to the Memgraph database deployed on Azure, follow these steps:
+
+### 1. Check if the Memgraph service is running
+
+```bash
+kubectl get pods -l app=memgraph
+```
+
+The output should show a pod in the `Running` state. If not, check the pod status:
+
+```bash
+kubectl describe pod -l app=memgraph
+```
+
+### 2. Check the Memgraph logs for errors
+
+```bash
+kubectl logs -l app=memgraph
+```
+
+Common errors include:
+- VM memory map count issues
+- Authentication failures
+- Networking problems
+
+### 3. Verify credential secret exists
+
+```bash
+kubectl get secret memgraph-credentials
+```
+
+### 4. Run the diagnostic script
+
+We provide a comprehensive diagnostic script that checks for common issues and attempts to fix them:
+
+```bash
+./scripts/memgraph_diagnostics.sh
+```
+
+This script will:
+- Check pod status
+- Verify service configuration
+- Validate credentials
+- Look for common errors in logs
+- Apply fixes when possible
+- Test connectivity
+
+### 5. Manual fix for credential issues
+
+If you've changed the Memgraph credentials in GitHub secrets but the pod doesn't reflect the changes:
+
+1. Update the Kubernetes secret:
+   ```bash
+   kubectl create secret generic memgraph-credentials \
+     --from-literal=username=YOUR_USERNAME \
+     --from-literal=password=YOUR_PASSWORD \
+     --dry-run=client -o yaml | kubectl apply -f -
+   ```
+
+2. Force the pod to restart with the new credentials:
+   ```bash
+   kubectl patch deployment memgraph -p \
+     "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"restart-at\":\"$(date +%s)\"}}}}}"
+   ```
+
+3. Wait for the new pod to be ready:
+   ```bash
+   kubectl wait --for=condition=ready pod -l app=memgraph --timeout=5m
+   ```
+
+### 6. Testing connectivity
+
+You can test connectivity using the provided test script:
+
+```bash
+# Make sure your .env file contains the correct credentials
+python examples/10-azure-memgraph-test.py
+```
+
+## Common Errors and Solutions
+
+### VM Max Map Count Error
+
+If you see an error like "Max virtual memory areas vm.max_map_count 65530 is too low", this is fixed in the latest deployment configuration with an init container.
+
+### Authentication Errors
+
+If you're getting authentication errors:
+1. Verify the credentials in your `.env` file match those in the Kubernetes secret
+2. Ensure the GitHub secrets have been properly updated
+3. Check if the deployment was updated after changing the credentials
+
+### Connection Timeouts
+
+If connections are timing out:
+1. Verify the Azure Network Security Group allows traffic on port 7687
+2. Check if the service has a valid external IP address
+3. Ensure your client can reach the Azure VM (no firewall blocking access)
+
+## Need Further Help?
+
+If the above steps don't resolve your issue, please:
+1. Gather the output from the diagnostic script
+2. Collect all relevant error messages
+3. Open an issue providing these details
diff --git a/examples/10-azure-memgraph-test.py b/examples/10-azure-memgraph-test.py
@@ -0,0 +1,64 @@
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+import os
+import mgclient
+import sys
+import time
+
+def test_memgraph_connection(max_retries=3, retry_delay=5) -> bool:
+    host = os.environ.get("MEMGRAPH_URI", "127.0.0.1")
+    port = int(os.environ.get("MEMGRAPH_PORT", "7687"))
+    username = os.environ.get("MEMGRAPH_USERNAME", "memgraph")
+    password = os.environ.get("MEMGRAPH_PASSWORD", "memgraph")
+    
+    print(f"Testing connection to Memgraph at {host}:{port} with username: {username}")
+
+    # Convert localhost to IP if needed
+    if host == "localhost":
+        host = "127.0.0.1"
+    
+    for attempt in range(1, max_retries + 1):
+        try:
+            print(f"Attempt {attempt}/{max_retries}: Connecting to Memgraph at {host}:{port}")
+            conn = mgclient.connect(
+                host=host, 
+                port=port,
+                username=username,
+                password=password,
+                connect_timeout_ms=10000  # 10 second timeout
+            )
+            
+            conn.autocommit = True
+            cursor = conn.cursor()
+            
+            # Test a basic query
+            cursor.execute("RETURN 'Connection successful!' AS message")
+            result = cursor.fetchone()[0]
+            print(f"Success: {result}")
+            
+            # Additional test query to verify data manipulation works
+            print("Testing data manipulation...")
+            cursor.execute("CREATE (n:TestNode {name: 'connection_test'}) RETURN n")
+            cursor.execute("MATCH (n:TestNode {name: 'connection_test'}) DELETE n")
+            print("Data manipulation successful")
+            
+            cursor.close()
+            conn.close()
+            return True
+        except Exception as e:
+            print(f"Connection attempt {attempt} failed: {str(e)}")
+            if attempt < max_retries:
+                print(f"Retrying in {retry_delay} seconds...")
+                time.sleep(retry_delay)
+            else:
+                print("All connection attempts failed")
+                return False
+
+if __name__ == "__main__":
+    success = test_memgraph_connection()
+    if not success:
+        print("Failed to connect to Memgraph")
+        sys.exit(1)
+    print("Successfully connected to Memgraph")
+    sys.exit(0)
diff --git a/infra/k8s/memgraph.yaml b/infra/k8s/memgraph.yaml
@@ -18,7 +18,16 @@ spec:
     metadata:
       labels:
         app: memgraph
+      annotations:
+        # This annotation will be updated whenever we want to force a pod restart
+        credentials-hash: "${CREDENTIALS_HASH}"
     spec:
+      initContainers:
+      - name: init-sysctl
+        image: busybox:1.28
+        command: ["sysctl", "-w", "vm.max_map_count=262144"]
+        securityContext:
+          privileged: true
       containers:
       - name: memgraph
         image: memgraph/memgraph-mage:latest
