Use insights ros ingress #20

Workflow file for this run

.github/workflows/docker-compose-test.yml at 84323d6

	name: Docker Compose Deployment Test

	on:
	pull_request:
	paths:
	- 'deployment/docker-compose/**'
	- '.github/workflows/docker-compose-test.yml'
	- 'internal/**' # Include internal code changes that might affect data processing
	workflow_dispatch:

	jobs:
	docker-compose-test:
	runs-on: ubuntu-latest
	timeout-minutes: 30

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Podman
	run: \|
	sudo apt-get update
	sudo apt-get install -y podman podman-compose
	# Configure podman for rootless operation
	sudo usermod -aG wheel $USER \|\| true
	# Enable lingering for systemd user services
	sudo loginctl enable-linger $USER \|\| true

	# Configure systemd for rootless KIND
	sudo mkdir -p /etc/systemd/system/user@$(id -u).service.d/
	echo -e "[Service]\nDelegate=yes" \| sudo tee /etc/systemd/system/user@$(id -u).service.d/delegate.conf
	sudo systemctl daemon-reload

	- name: Install additional dependencies
	run: \|
	sudo apt-get install -y uuid-runtime curl

	# Install KIND and kubectl for authentication setup
	curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
	chmod +x ./kind
	sudo mv ./kind /usr/local/bin/kind

	curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
	chmod +x kubectl
	sudo mv kubectl /usr/local/bin/kubectl

	- name: Configure environment variables
	run: \|
	echo "INGRESS_PORT=3000" >> $GITHUB_ENV
	echo "MINIO_ACCESS_KEY=minioaccesskey" >> $GITHUB_ENV
	echo "MINIO_SECRET_KEY=miniosecretkey" >> $GITHUB_ENV

	- name: Start Podman socket service
	run: \|
	# Start podman system service for docker-compose compatibility
	systemctl --user start podman.socket \|\| true
	# Set DOCKER_HOST for podman-compose compatibility
	echo "DOCKER_HOST=unix:///run/user/$UID/podman/podman.sock" >> $GITHUB_ENV

	# Verify podman socket is running
	systemctl --user status podman.socket \|\| true

	# Also set KIND to use podman provider globally
	echo "KIND_EXPERIMENTAL_PROVIDER=podman" >> $GITHUB_ENV

	- name: Verify podman installation
	run: \|
	podman --version
	podman-compose --version
	echo "Podman info:"
	podman info

	- name: Setup authentication for insights-ros-ingress
	working-directory: deployment/docker-compose
	run: \|
	# Make authentication setup script executable
	chmod +x ../../scripts/setup-ingress-auth.sh

	# Set up KIND cluster and authentication for insights-ros-ingress
	echo "Setting up Kubernetes authentication for insights-ros-ingress..."
	echo "Using KIND with podman rootless provider (KIND_EXPERIMENTAL_PROVIDER=$KIND_EXPERIMENTAL_PROVIDER)"
	(cd ../../scripts && ./setup-ingress-auth.sh)

	# Verify authentication setup
	if [ -f "../../scripts/.ingress-auth.env" ]; then
	echo "✅ Authentication environment created successfully"
	echo "Auth file contents (without sensitive data):"
	grep -v "DEV_SERVICE_ACCOUNT_TOKEN" ../../scripts/.ingress-auth.env \|\| true

	# Source the auth environment and export KUBECONFIG
	source ../../scripts/.ingress-auth.env
	echo "KUBECONFIG=$KUBECONFIG" >> $GITHUB_ENV
	echo "✅ KUBECONFIG exported for subsequent steps"
	else
	echo "❌ Authentication setup failed - environment file not found"
	exit 1
	fi

	- name: Pull required container images
	working-directory: deployment/docker-compose
	run: \|
	# Pre-pull images to avoid timeout issues during compose up
	podman-compose pull \|\| true

	- name: Start services with podman-compose
	working-directory: deployment/docker-compose
	run: \|
	echo "Starting services with podman-compose..."
	echo "KUBECONFIG is set to: $KUBECONFIG"
	echo "Environment variables: INGRESS_PORT=$INGRESS_PORT, MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY"
	podman-compose up -d

	echo "Waiting for services to initialize..."
	sleep 30

	echo "Service status:"
	podman-compose ps

	- name: Wait for core services
	working-directory: deployment/docker-compose
	run: \|
	# Wait for database services
	echo "Waiting for PostgreSQL services..."
	timeout 300 bash -c 'until podman exec db-ros_1 pg_isready -U postgres; do sleep 5; done'
	timeout 300 bash -c 'until podman exec db-kruize_1 pg_isready -U postgres; do sleep 5; done'
	timeout 300 bash -c 'until podman exec db-sources_1 pg_isready -U postgres; do sleep 5; done'

	# Wait for Kafka
	echo "Waiting for Kafka..."
	timeout 300 bash -c 'until podman exec kafka_1 kafka-broker-api-versions --bootstrap-server localhost:29092 &>/dev/null; do sleep 5; done'

	# Wait for MinIO
	echo "Waiting for MinIO..."
	timeout 180 bash -c 'until curl -f http://localhost:9000/minio/health/live &>/dev/null; do sleep 5; done'

	# Wait for Redis
	echo "Waiting for Redis..."
	timeout 180 bash -c 'until podman exec redis_1 redis-cli ping &>/dev/null; do sleep 5; done'

	- name: Wait for application services
	working-directory: deployment/docker-compose
	run: \|
	# Get actual ingress port
	ACTUAL_INGRESS_PORT=$(podman port ingress_1 2>/dev/null \| cut -d: -f2 \|\| echo "$INGRESS_PORT")
	echo "Using ingress port: $ACTUAL_INGRESS_PORT"

	# Wait for application services
	echo "Waiting for Ingress service..."
	timeout 300 bash -c "until curl -f http://localhost:${ACTUAL_INGRESS_PORT}/health &>/dev/null; do sleep 5; done"

	echo "Waiting for Kruize service..."
	timeout 300 bash -c 'until curl -f http://localhost:8080/listPerformanceProfiles &>/dev/null; do sleep 5; done'

	echo "Waiting for Sources API..."
	timeout 300 bash -c 'until curl -f http://localhost:8002/api/sources/v1.0/source_types &>/dev/null; do sleep 5; done'

	echo "Waiting for ROS-OCP API..."
	timeout 300 bash -c 'until curl -f http://localhost:8001/status &>/dev/null; do sleep 5; done'

	echo "Waiting for processor to start..."
	timeout 300 bash -c 'until podman logs rosocp-processor_1 2>/dev/null \| grep -q "Starting processor"; do sleep 5; done'

	echo "Waiting for recommendation poller to start..."
	timeout 300 bash -c 'until podman logs rosocp-recommendation-poller_1 2>/dev/null \| grep -q "Starting recommendation-poller"; do sleep 5; done'

	- name: Run data flow test
	working-directory: deployment/docker-compose
	run: \|
	# Make test script executable
	chmod +x test-ros-ocp-dataflow.sh

	# Verify new test data is available
	echo "Checking for ROS-OCP test data..."
	if [ -f "samples/ros-ocp-test-data.tar.gz" ]; then
	echo "✅ Found ros-ocp-test-data.tar.gz (proper 37-column CSV format)"
	else
	echo "❌ Missing ros-ocp-test-data.tar.gz - test may fail with CSV validation errors"
	fi

	# Fix kubeconfig IP address for GitHub Actions environment
	echo "Fixing kubeconfig for container network access..."
	if [ -f "/tmp/ros-ingress-kubeconfig" ]; then
	echo "Current kubeconfig server:"
	grep "server:" /tmp/ros-ingress-kubeconfig \|\| true

	# Get KIND container IP
	KIND_CONTAINER_IP=$(podman inspect ros-ingress-dev-control-plane 2>/dev/null \| grep -o '"IPAddress": "[^"]*"' \| grep -v '""' \| head -1 \| cut -d'"' -f4 \|\| echo "")

	if [ -n "$KIND_CONTAINER_IP" ]; then
	echo "Updating kubeconfig to use KIND container IP: $KIND_CONTAINER_IP"
	sed -i.bak "s\|server: https://.*:6443\|server: https://${KIND_CONTAINER_IP}:6443\|" /tmp/ros-ingress-kubeconfig
	echo "Updated kubeconfig server:"
	grep "server:" /tmp/ros-ingress-kubeconfig \|\| true

	# Restart ingress service to pick up the corrected kubeconfig
	echo "Restarting ingress service..."
	podman-compose restart ingress
	sleep 10
	else
	echo "❌ Could not determine KIND container IP"
	exit 1
	fi
	else
	echo "❌ Kubeconfig file not found at /tmp/ros-ingress-kubeconfig"
	exit 1
	fi

	# Environment variables are automatically available from GITHUB_ENV
	echo "Environment: INGRESS_PORT=$INGRESS_PORT, MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY"

	# Run the test script (it will use ros-ocp-test-data.tar.gz for proper validation)
	./test-ros-ocp-dataflow.sh

	- name: Check service health after test
	working-directory: deployment/docker-compose
	if: always()
	run: \|
	echo "=== Final service status ==="
	podman-compose ps

	echo "=== Service logs (last 20 lines each) ==="
	echo "--- Ingress logs ---"
	podman-compose logs --tail=20 ingress \|\| true

	echo "--- ROS-OCP API logs ---"
	podman-compose logs --tail=20 rosocp-api \|\| true

	echo "--- ROS-OCP Processor logs ---"
	podman-compose logs --tail=20 rosocp-processor \|\| true

	echo "--- Kruize logs ---"
	podman-compose logs --tail=20 kruize-autotune \|\| true

	- name: Cleanup services
	working-directory: deployment/docker-compose
	if: always()
	run: \|
	echo "Cleaning up services..."
	podman-compose down -v \|\| true

	# Clean up authentication resources
	echo "Cleaning up authentication resources..."
	kind delete cluster --name ros-ingress-dev \|\| true
	rm -f /tmp/ros-ingress-kubeconfig \|\| true
	rm -f ../../scripts/.ingress-auth.env \|\| true

	# Clean up any remaining containers
	podman container prune -f \|\| true

	# Clean up any remaining volumes
	podman volume prune -f \|\| true

	- name: Generate test report
	if: always()
	run: \|
	echo "## Docker Compose Test Results" >> $GITHUB_STEP_SUMMARY
	echo "" >> $GITHUB_STEP_SUMMARY
	if [ "${{ job.status }}" == "success" ]; then
	echo "✅ Test Status: PASSED" >> $GITHUB_STEP_SUMMARY
	echo "All services started successfully and data flow test completed." >> $GITHUB_STEP_SUMMARY
	else
	echo "❌ Test Status: FAILED" >> $GITHUB_STEP_SUMMARY
	echo "Check the logs above for details on what failed." >> $GITHUB_STEP_SUMMARY
	fi
	echo "" >> $GITHUB_STEP_SUMMARY
	echo "### Tested Components" >> $GITHUB_STEP_SUMMARY
	echo "- PostgreSQL databases (ROS, Kruize, Sources)" >> $GITHUB_STEP_SUMMARY
	echo "- Kafka message broker" >> $GITHUB_STEP_SUMMARY
	echo "- MinIO object storage (ros-data bucket)" >> $GITHUB_STEP_SUMMARY
	echo "- Redis cache" >> $GITHUB_STEP_SUMMARY
	echo "- insights-ros-ingress service (with Kubernetes authentication)" >> $GITHUB_STEP_SUMMARY
	echo "- ROS-OCP API service" >> $GITHUB_STEP_SUMMARY
	echo "- ROS-OCP Processor service" >> $GITHUB_STEP_SUMMARY
	echo "- ROS-OCP Recommendation Poller" >> $GITHUB_STEP_SUMMARY
	echo "- Kruize Autotune service" >> $GITHUB_STEP_SUMMARY
	echo "- Sources API service" >> $GITHUB_STEP_SUMMARY
	echo "- Authentication flow (KIND cluster + service account tokens)" >> $GITHUB_STEP_SUMMARY
	echo "- Complete ROS-OCP data flow (authenticated upload → CSV extraction → MinIO ros-data bucket → Kafka → processing → database)" >> $GITHUB_STEP_SUMMARY
	echo "- CSV validation and processing (37-column ROS-OCP format)" >> $GITHUB_STEP_SUMMARY

Navigation Menu

Use insights ros ingress #20

Workflow file

Use insights ros ingress #20

Uh oh!

Workflow file for this run