Simplify PgSTAC Bootstrap Process #825
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| branches: [ "main" ] | |
| types: [ opened, reopened, synchronize, labeled ] | |
| env: | |
| HELM_VERSION: v3.15.2 | |
| PGO_VERSION: 5.7.4 | |
| jobs: | |
| helm-tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: d3adb5/helm-unittest-action@v2 | |
| with: | |
| helm-version: ${{ env.HELM_VERSION }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: | | |
| cd helm-chart | |
| helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml | |
| k3s-integration-tests: | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| needs: helm-tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Start a local k3s cluster | |
| uses: jupyterhub/action-k3s-helm@v4 | |
| with: | |
| # See available: | |
| # - k3s release channels at https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml | |
| # - k3s versions at https://github.com/k3s-io/k3s/tags | |
| # - helm versions at https://github.com/helm/helm/tags | |
| k3s-channel: latest | |
| helm-version: ${{ env.HELM_VERSION }} | |
| metrics-enabled: false | |
| docker-enabled: true | |
| - name: last commit sha if PR | |
| if: ${{ github.event_name == 'pull_request' }} | |
| shell: bash | |
| run: | | |
| echo "LAST_COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> ${GITHUB_ENV} | |
| - name: last commit sha if push | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "LAST_COMMIT_SHA=${GITHUB_SHA}" >> ${GITHUB_ENV} | |
| - name: set k8s .release.name suffix | |
| run: | | |
| # salt for randomness per test run | |
| COMMITSHA=$(echo $LAST_COMMIT_SHA | cut -c 1-6) | |
| SALT=$(echo "${RANDOM}${RANDOM}${RANDOM}" | cut -c1-3) | |
| echo "RELEASE_NAME=eoapi$COMMITSHA$SALT" >> $GITHUB_ENV | |
| - name: helm install crunchydata postgres operator | |
| run: | | |
| helm upgrade --install \ | |
| --set disable_check_for_upgrades=true \ | |
| pgo \ | |
| oci://registry.developers.crunchydata.com/crunchydata/pgo \ | |
| --version ${{ env.PGO_VERSION }} | |
| - name: helm render/install eoapi templates | |
| run: | | |
| export GITSHA='${{github.sha}}' | |
| cd helm-chart | |
| helm dependency build eoapi | |
| helm install $RELEASE_NAME \ | |
| -f ./eoapi/values.yaml \ | |
| -f ./eoapi/test-k3s-unittest-values.yaml \ | |
| ./eoapi | |
| - name: sleep for 10s seconds while services boot | |
| shell: bash | |
| run: sleep 10s | |
| # - name: Setup upterm session | |
| # uses: lhotari/action-upterm@v1 | |
| - id: watchservices | |
| name: watch services boot | |
| timeout-minutes: 10 | |
| continue-on-error: true | |
| run: | | |
| while [[ -z "$(kubectl get pod | grep "^raster-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do | |
| echo "still waiting for raster service to start..." | |
| sleep 1 | |
| done | |
| echo "raster service has started, moving on..." | |
| while [[ -z "$(kubectl get pod | grep "^vector-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do | |
| echo "still waiting for vector service to start..." | |
| sleep 1 | |
| done | |
| echo "vector service has started, moving on..." | |
| while [[ -z "$(kubectl get pod | grep "^stac-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/_mgmt/ping" | head -n 1)" ]]; do | |
| echo "still waiting for stac service to start..." | |
| sleep 1 | |
| done | |
| echo "all services have started, moving on..." | |
| - name: cleanup if services fail to boot | |
| if: steps.watchservices.outcome == 'failure' | |
| run: | | |
| echo "The watchservices step failed or timed out. Extracting pod logs for debugging..." | |
| # Get and display all pods status | |
| echo "===== Pod Status =====" | |
| kubectl get pods | |
| # Extract logs from database pod | |
| echo "===== Database Pod Logs =====" | |
| kubectl get pod | grep "^db-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get database logs" | |
| # Extract logs from pgstacbootstrap pod | |
| echo "===== PGSTACBootstrap Pod Logs =====" | |
| kubectl get pod | grep "^pgstacbootstrap-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get pgstacbootstrap logs" | |
| # Extract logs from raster pod init container (wait-for-pgstacbootstrap) | |
| echo "===== Raster Pod Init Container Logs (wait-for-pgstacbootstrap) =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstacbootstrap --tail=100 || echo "Could not get raster init container logs" | |
| # Extract logs from raster pod main container | |
| echo "===== Raster Pod Main Container Logs =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get raster main container logs" | |
| # Extract logs from vector pod | |
| echo "===== Vector Pod Logs =====" | |
| kubectl get pod | grep "^vector-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get vector logs" | |
| # Extract logs from stac pod | |
| echo "===== STAC Pod Logs =====" | |
| kubectl get pod | grep "^stac-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get STAC logs" | |
| # Check if pods are in pending state or have issues | |
| echo "===== Pod Descriptions for Troubleshooting =====" | |
| kubectl get pod | grep "$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl describe pod/{} || echo "Could not describe pods" | |
| # force GH action to show failed result | |
| exit 128 | |
| - name: install python unit-test dependencies | |
| run: | | |
| python -m pip install pytest httpx | |
| - name: run the tests | |
| id: testrunner | |
| continue-on-error: true | |
| run: | | |
| kubectl get svc --all-namespaces | |
| kubectl get ingress --all-namespaces -o jsonpath='{range .items[0]}kubectl describe ingress {.metadata.name} -n {.metadata.namespace}{end}' | sh | |
| kubectl get middleware.traefik.io --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers | while read -r namespace name; do kubectl describe middleware.traefik.io "$name" -n "$namespace"; done | |
| PUBLICIP='http://'$(kubectl -n kube-system get svc traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}') | |
| export VECTOR_ENDPOINT=$PUBLICIP/vector$RELEASE_NAME | |
| export STAC_ENDPOINT=$PUBLICIP/stac$RELEASE_NAME | |
| export RASTER_ENDPOINT=$PUBLICIP/raster$RELEASE_NAME | |
| echo '#################################' | |
| echo $VECTOR_ENDPOINT | |
| echo $STAC_ENDPOINT | |
| echo $RASTER_ENDPOINT | |
| echo '#################################' | |
| pytest .github/workflows/tests/test_vector.py || kubectl logs svc/vector | |
| pytest .github/workflows/tests/test_stac.py || kubectl logs svc/stac | |
| # TODO: fix raster tests | |
| #pytest .github/workflows/tests/test_raster.py || kubectl logs svc/raster | |
| - name: error if tests failed | |
| if: steps.testrunner.outcome == 'failure' | |
| run: | | |
| echo "The tests failed. Extracting pod logs for debugging..." | |
| # Get and display all pods status | |
| echo "===== Pod Status =====" | |
| kubectl get pods | |
| # Extract logs from database pod | |
| echo "===== Database Pod Logs =====" | |
| kubectl get pod | grep "^db-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get database logs" | |
| # Extract logs from pgstacbootstrap pod | |
| echo "===== PGSTACBootstrap Pod Logs =====" | |
| kubectl get pod | grep "^pgstacbootstrap-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get pgstacbootstrap logs" | |
| # Extract logs from raster pod init container (wait-for-pgstacbootstrap) | |
| echo "===== Raster Pod Init Container Logs (wait-for-pgstacbootstrap) =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstacbootstrap --tail=100 || echo "Could not get raster init container logs" | |
| # Extract logs from raster pod main container | |
| echo "===== Raster Pod Main Container Logs =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get raster main container logs" | |
| # Extract logs from vector pod | |
| echo "===== Vector Pod Logs =====" | |
| kubectl get pod | grep "^vector-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get vector logs" | |
| # Extract logs from stac pod | |
| echo "===== STAC Pod Logs =====" | |
| kubectl get pod | grep "^stac-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get STAC logs" | |
| # Check if pods are in pending state or have issues | |
| echo "===== Pod Descriptions for Troubleshooting =====" | |
| kubectl get pod | grep "$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl describe pod/{} || echo "Could not describe pods" | |
| # force GH action to show failed result | |
| exit 128 | |
| - name: helm uninstall eoapi templates | |
| run: | | |
| helm uninstall $RELEASE_NAME |