add duckdb performance test #181
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test DataSQRL Examples | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| TZ: 'America/Los_Angeles' | |
| SQRL_VERSION: '0.9.0-alpha2' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest-4-cores | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - example: recommendation | |
| path: clickstream-ai-recommendation | |
| test_commands: | | |
| compile recommendation_package_api.json | |
| compile recommendation_package_kafka.json | |
| - example: credit-card-analytics | |
| path: finance-credit-card-chatbot/credit-card-analytics | |
| tag: credit-card-analytics | |
| test_commands: | | |
| compile creditcard_analytics_package_test.json | |
| compile creditcard_analytics_package_kafka.json | |
| test creditcard_analytics_package_test.json creditcard_analytics_package_jwt.json | |
| - example: credit-card-rewards | |
| path: finance-credit-card-chatbot/credit-card-rewards | |
| tag: credit-card-rewards | |
| test_commands: | | |
| compile creditcard_rewards_package_test.json | |
| compile creditcard_rewards_package_kafka.json | |
| test creditcard_rewards_package_test.json creditcard_rewards_package_jwt.json | |
| - example: credit-card-views | |
| path: finance-credit-card-chatbot/credit-card-views | |
| tag: credit-card-views | |
| test_commands: | | |
| compile creditcard_views_package_test.json | |
| test creditcard_views_package_test.json | |
| - example: healthcare-study-analytics | |
| path: healthcare-study/healthcare-study-analytics | |
| tag: healthcare-study-analytics | |
| test_commands: | | |
| compile study_analytics_package_test.json | |
| test study_analytics_package_test.json | |
| - example: healthcare-study-api | |
| path: healthcare-study/healthcare-study-api | |
| tag: healthcare-study-api | |
| test_commands: | | |
| compile study_api_package_api.json | |
| compile study_api_package_test.json | |
| test study_api_package_test.json | |
| - example: healthcare-study-stream | |
| path: healthcare-study/healthcare-study-stream | |
| tag: healthcare-study-stream | |
| test_commands: | | |
| compile study_stream_package_kafka.json | |
| compile study_stream_package_test.json | |
| test study_stream_package_test.json | |
| - example: iceberg-dedup | |
| path: iceberg-data-deduplication | |
| test_commands: | | |
| compile dedup_package.json | |
| - example: iot-sensor | |
| path: iot-sensor-metrics | |
| test_commands: | | |
| compile sensors_package_api.json | |
| compile sensors_package_test.json | |
| test sensors_package_test.json | |
| test sensors_package_api.json | |
| - example: law-enforcement | |
| path: law-enforcement | |
| test_commands: | | |
| compile baseball_card_package_test.json | |
| test baseball_card_package_test.json | |
| - example: logistics | |
| path: logistics-shipping-geodata | |
| test_commands: | | |
| compile logistics_package_test.json | |
| test logistics_package_test.json | |
| - example: oil-gas-monitoring | |
| path: oil-gas-agent-automation/oil-gas-agent-monitoring | |
| tag: oil-gas-agent-monitoring | |
| test_commands: | | |
| compile monitoring_agent_package_test.json | |
| - example: oil-gas-operations | |
| path: oil-gas-agent-automation/oil-gas-agent-operations | |
| tag: oil-gas-agent-operations | |
| test_commands: | | |
| compile operations_agent_package_test.json | |
| test operations_agent_package_test.json | |
| - example: usage-analytics | |
| path: usage-analytics | |
| test_commands: | | |
| compile usertokens_package_test.json | |
| test usertokens_package_test.json | |
| - example: canary | |
| path: test-jobs | |
| tag: canary | |
| test_commands: | | |
| compile aggregation-test-package.json | |
| compile aggregation-query-test-package.json | |
| compile aggregation-query-test-iceberg-package.json | |
| compile aggregation-query-subscription-test-package.json | |
| compile runtime-failing-connector-package.json | |
| compile runtime-failing-udf-package.json | |
| - example: udf-jbang | |
| path: user-defined-function/jbang | |
| tag: udf-jbang | |
| test_commands: | | |
| compile udf_jbang_package.json | |
| - example: udf-maven | |
| path: user-defined-function/maven-project | |
| tag: udf-maven | |
| test_commands: | | |
| compile udf_maven_package.json | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Timezone | |
| uses: szenius/set-timezone@v1.1 | |
| with: | |
| timezoneLinux: "America/Los_Angeles" | |
| - name: Cache Docker images | |
| uses: ScribeMD/docker-cache@0.5.0 | |
| with: | |
| # one cache entry per SQRL version & OS | |
| key: sqrl-${{ runner.os }}-${{ env.SQRL_VERSION }} | |
| - name: Pull SQRL Docker image | |
| run: docker pull datasqrl/cmd:${{ env.SQRL_VERSION }} | |
| - name: Run ${{ matrix.example }} Tests | |
| working-directory: ${{ matrix.path }} | |
| run: | | |
| # Read raw lines from the matrix value | |
| mapfile -t raw_cmds <<< "${{ matrix.test_commands }}" | |
| # Keep only non‑empty lines | |
| cmds=() | |
| for c in "${raw_cmds[@]}"; do | |
| [[ -z "${c//[[:space:]]/}" ]] && continue # skip blanks | |
| cmds+=("$c") | |
| done | |
| if (( ${#cmds[@]} == 0 )); then | |
| echo "ERROR: No test commands specified for ${{ matrix.example }}" | |
| exit 1 | |
| fi | |
| for cmd in "${cmds[@]}"; do | |
| echo "::group::Running: $cmd" | |
| docker run -i -p 8888:8888 -p 8081:8081 -p 9092:9092 \ | |
| -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" \ | |
| -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" \ | |
| -e AWS_REGION="${{ secrets.AWS_REGION }}" \ | |
| --rm -v "$PWD":/build "datasqrl/cmd:${SQRL_VERSION}" $cmd | |
| echo "::endgroup::" | |
| done | |
| - name: Write Dockerfile | |
| run: | | |
| cat <<EOF > ${{ matrix.path }}/Dockerfile | |
| FROM datasqrl/cmd:${SQRL_VERSION} | |
| COPY . /build | |
| WORKDIR /build | |
| EOF | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Publish ${{ matrix.example }} | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: ./${{ matrix.path }} | |
| file: ./${{ matrix.path }}/Dockerfile | |
| push: true | |
| tags: datasqrl/examples:${{ matrix.tag && matrix.tag != '' && matrix.tag || matrix.path }} | |
| platforms: linux/amd64,linux/arm64 | |
| lint-graphql: | |
| name: Lint GraphQL Schemas | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Lint GraphQL schemas | |
| run: | | |
| SCHEMAS=$(find . -name "*.graphqls" -type f) | |
| if [ -z "$SCHEMAS" ]; then | |
| echo "No GraphQL schemas found" | |
| exit 0 | |
| fi | |
| LINT_OPTS="--rules fields-are-camel-cased,types-are-capitalized,enum-values-all-caps,input-object-values-are-camel-cased" | |
| IGNORE='{"fields-are-camel-cased":["Query.Shipment","Query.Vehicle","Query.Customer"]}' | |
| FAILED=0 | |
| for schema in $SCHEMAS; do | |
| echo "::group::Checking $schema" | |
| output=$(docker run --rm -v "$PWD":/app -w /app node:18-slim npx graphql-schema-linter $LINT_OPTS --ignore "$IGNORE" "$schema" 2>&1) | |
| result=$? | |
| echo "$output" | grep -v "^npm" | |
| echo "::endgroup::" | |
| if [ $result -ne 0 ]; then | |
| FAILED=1 | |
| fi | |
| done | |
| exit $FAILED | |
| ci-summary: | |
| name: build ✅ summary | |
| needs: [build, lint-graphql] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Fail if any build job failed | |
| run: | | |
| failed=$(jq -r '.[] | select(.result=="failure")' <<< '${{ toJson(needs) }}' | wc -l) | |
| if [[ "$failed" -gt 0 ]]; then | |
| echo "::error ::${failed} matrix job(s) failed" | |
| exit 1 | |
| fi |