build.yml

name: Build and test DataSQRL Examples

on:
  push:
    branches: [ "main" ]
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  TZ: 'America/Los_Angeles'
  SQRL_VERSION: '0.9.0-alpha2'

jobs:
  build:
    runs-on: ubuntu-latest-4-cores
    timeout-minutes: 30

    strategy:
      fail-fast: false
      matrix:
        include:
          - example: recommendation
            path: clickstream-ai-recommendation
            test_commands: |
              compile recommendation_package_api.json
              compile recommendation_package_kafka.json

          - example: credit-card-analytics
            path: finance-credit-card-chatbot/credit-card-analytics
            tag: credit-card-analytics
            test_commands: |
              compile creditcard_analytics_package_test.json
              compile creditcard_analytics_package_kafka.json
              test creditcard_analytics_package_test.json creditcard_analytics_package_jwt.json
          - example: credit-card-rewards
            path: finance-credit-card-chatbot/credit-card-rewards
            tag: credit-card-rewards
            test_commands: |
              compile creditcard_rewards_package_test.json
              compile creditcard_rewards_package_kafka.json
              test creditcard_rewards_package_test.json creditcard_rewards_package_jwt.json
          - example: credit-card-views
            path: finance-credit-card-chatbot/credit-card-views
            tag: credit-card-views
            test_commands: |
              compile creditcard_views_package_test.json
              test creditcard_views_package_test.json

          - example: healthcare-study-analytics
            path: healthcare-study/healthcare-study-analytics
            tag: healthcare-study-analytics
            test_commands: |
              compile study_analytics_package_test.json
              test study_analytics_package_test.json
          - example: healthcare-study-api
            path: healthcare-study/healthcare-study-api
            tag: healthcare-study-api
            test_commands: |
              compile study_api_package_api.json
              compile study_api_package_test.json
              test study_api_package_test.json
          - example: healthcare-study-stream
            path: healthcare-study/healthcare-study-stream
            tag: healthcare-study-stream
            test_commands: |
              compile study_stream_package_kafka.json
              compile study_stream_package_test.json
              test study_stream_package_test.json

          - example: iceberg-dedup
            path: iceberg-data-deduplication
            test_commands: |
              compile dedup_package.json

          - example: iot-sensor
            path: iot-sensor-metrics
            test_commands: |
              compile sensors_package_api.json
              compile sensors_package_test.json
              test sensors_package_test.json
              test sensors_package_api.json

          - example: law-enforcement
            path: law-enforcement
            test_commands: |
              compile baseball_card_package_test.json
              test baseball_card_package_test.json

          - example: logistics
            path: logistics-shipping-geodata
            test_commands: |
              compile logistics_package_test.json
              test logistics_package_test.json

          - example: oil-gas-monitoring
            path: oil-gas-agent-automation/oil-gas-agent-monitoring
            tag: oil-gas-agent-monitoring
            test_commands: |
              compile monitoring_agent_package_test.json
          - example: oil-gas-operations
            path: oil-gas-agent-automation/oil-gas-agent-operations
            tag: oil-gas-agent-operations
            test_commands: |
              compile operations_agent_package_test.json
              test operations_agent_package_test.json

          - example: usage-analytics
            path: usage-analytics
            test_commands: |
              compile usertokens_package_test.json
              test usertokens_package_test.json

          - example: canary-aggregation
            path: test-jobs/aggregation-tests
            tag: canary
            test_commands: |
              compile aggregation-test-package.json
              compile aggregation-query-test-package.json
              compile aggregation-query-test-iceberg-package.json
              compile aggregation-query-subscription-test-package.json

          - example: canary-failures
            path: test-jobs/failure-tests
            tag: canary
            test_commands: |
              compile runtime-failing-connector-package.json
              compile runtime-failing-udf-package.json

          - example: canary-batch
            path: test-jobs/batch-test
            tag: canary
            test_commands: |
              compile simple-batch-test-package.json

          - example: udf-jbang
            path: user-defined-function/jbang
            tag: udf-jbang
            test_commands: |
              compile udf_jbang_package.json

          - example: udf-maven
            path: user-defined-function/maven-project
            tag: udf-maven
            test_commands: |
              compile udf_maven_package.json

    steps:
    - uses: actions/checkout@v4

    - name: Setup Timezone
      uses: szenius/set-timezone@v1.1
      with:
        timezoneLinux: "America/Los_Angeles"

    - name: Cache Docker images
      uses: ScribeMD/docker-cache@0.5.0
      with:
        # one cache entry per SQRL version & OS
        key: sqrl-${{ runner.os }}-${{ env.SQRL_VERSION }}

    - name: Pull SQRL Docker image
      run: docker pull datasqrl/cmd:${{ env.SQRL_VERSION }}

    - name: Run ${{ matrix.example }} Tests
      working-directory: ${{ matrix.path }}
      run: |
        # Read raw lines from the matrix value
        mapfile -t raw_cmds <<< "${{ matrix.test_commands }}"
       
        # Keep only non‑empty lines
        cmds=()
        for c in "${raw_cmds[@]}"; do
          [[ -z "${c//[[:space:]]/}" ]] && continue   # skip blanks
          cmds+=("$c")
        done

        if (( ${#cmds[@]} == 0 )); then
          echo "ERROR: No test commands specified for ${{ matrix.example }}"
          exit 1
        fi

        for cmd in "${cmds[@]}"; do
          echo "::group::Running: $cmd"
          docker run -i -p 8888:8888 -p 8081:8081 -p 9092:9092 \
            -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \
            -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \
            -e AWS_REGION="${{ secrets.AWS_REGION }}" \
            --rm -v "$PWD":/build "datasqrl/cmd:${SQRL_VERSION}" $cmd
          echo "::endgroup::"
        done

    - name: Write Dockerfile
      run: |
        cat <<EOF > ${{ matrix.path }}/Dockerfile
        FROM datasqrl/cmd:${SQRL_VERSION}

        COPY . /build
        WORKDIR /build
        EOF

    - name: Set up QEMU
      uses: docker/setup-qemu-action@v2

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Login to Docker Hub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKER_USERNAME }}
        password: ${{ secrets.DOCKER_PASSWORD }}

    - name: Publish ${{ matrix.example }}
      uses: docker/build-push-action@v3
      with:
        context: ./${{ matrix.path }}
        file: ./${{ matrix.path }}/Dockerfile
        push: true
        tags: datasqrl/examples:${{ matrix.tag && matrix.tag != '' && matrix.tag || matrix.path }}
        platforms: linux/amd64,linux/arm64

  lint-graphql:
    name: Lint GraphQL Schemas
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Lint GraphQL schemas
        run: |
          SCHEMAS=$(find . -name "*.graphqls" -type f)
          if [ -z "$SCHEMAS" ]; then
            echo "No GraphQL schemas found"
            exit 0
          fi

          LINT_OPTS="--rules fields-are-camel-cased,types-are-capitalized,enum-values-all-caps,input-object-values-are-camel-cased"
          IGNORE='{"fields-are-camel-cased":["Query.Shipment","Query.Vehicle","Query.Customer"]}'
          FAILED=0

          for schema in $SCHEMAS; do
            echo "::group::Checking $schema"
            output=$(docker run --rm -v "$PWD":/app -w /app node:18-slim npx graphql-schema-linter $LINT_OPTS --ignore "$IGNORE" "$schema" 2>&1)
            result=$?
            echo "$output" | grep -v "^npm"
            echo "::endgroup::"
            if [ $result -ne 0 ]; then
              FAILED=1
            fi
          done

          exit $FAILED

  ci-summary:
    name: build ✅ summary
    needs: [build, lint-graphql]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Fail if any build job failed
        run: |
          failed=$(jq -r '.[] | select(.result=="failure")' <<< '${{ toJson(needs) }}' | wc -l)
          if [[ "$failed" -gt 0 ]]; then
            echo "::error ::${failed} matrix job(s) failed"
            exit 1
          fi