integration.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: integration

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

# trigger for all PRs that touch certain files and changes to main
on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - .github/**
      - arrow-array/**
      - arrow-buffer/**
      - arrow-cast/**
      - arrow-csv/**
      - arrow-data/**
      - arrow-integration-test/**
      - arrow-integration-testing/**
      - arrow-ipc/**
      - arrow-json/**
      - arrow-avro/**
      - arrow-ord/**
      - arrow-pyarrow-integration-testing/**
      - arrow-pyarrow/**
      - arrow-schema/**
      - arrow-select/**
      - arrow-sort/**
      - arrow-string/**
      - arrow/**

jobs:
  integration:
    name: Archery test With other arrows
    runs-on: ubuntu-latest
    container:
      image: apache/arrow-dev:amd64-conda-integration
    env:
      ARROW_USE_CCACHE: OFF
      ARROW_CPP_EXE_PATH: /build/cpp/debug
      ARROW_NANOARROW_PATH: /build/nanoarrow
      ARROW_RUST_EXE_PATH: /build/rust/debug
      BUILD_DOCS_CPP: OFF
      ARROW_INTEGRATION_CPP: ON
      ARROW_INTEGRATION_CSHARP: ON
      ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS: "rust"
      ARCHERY_INTEGRATION_WITH_DOTNET: "1"
      ARCHERY_INTEGRATION_WITH_GO: "1"
      ARCHERY_INTEGRATION_WITH_JAVA: "1"
      ARCHERY_INTEGRATION_WITH_JS: "1"
      ARCHERY_INTEGRATION_WITH_NANOARROW: "1"
      # https://github.com/apache/arrow/pull/38403/files#r1371281630
      ARCHERY_INTEGRATION_WITH_RUST: "1"
      # These are necessary because the github runner overrides $HOME
      # https://github.com/actions/runner/issues/863
      RUSTUP_HOME: /root/.rustup
      CARGO_HOME: /root/.cargo
    defaults:
      run:
        shell: bash
    steps:
      - name: Monitor disk usage - Initial
        run: |
          echo "=== Initial Disk Usage ==="
          df -h /
          echo ""

      - name: Remove unnecessary preinstalled software
        run: |
          echo "=== Cleaning up host disk space ==="
          echo "Disk space before cleanup:"
          df -h /

          # Clean apt cache
          apt-get clean || true

          # Remove GitHub Actions tool cache
          rm -rf /__t/* || true

          # Remove large packages from host filesystem (mounted at /host/)
          rm -rf /host/usr/share/dotnet || true
          rm -rf /host/usr/local/lib/android || true
          rm -rf /host/usr/local/.ghcup || true
          rm -rf /host/opt/hostedtoolcache/CodeQL || true

          echo ""
          echo "Disk space after cleanup:"
          df -h /
          echo ""

      # This is necessary so that actions/checkout can find git
      - name: Export conda path
        run: echo "/opt/conda/envs/arrow/bin" >> $GITHUB_PATH
      # This is necessary so that Rust can find cargo
      - name: Export cargo path
        run: echo "/root/.cargo/bin" >> $GITHUB_PATH

      # Checkout repos (using shallow clones with fetch-depth: 1)
      - name: Checkout Arrow
        uses: actions/checkout@v6
        with:
          repository: apache/arrow
          submodules: true
          fetch-depth: 1
      - name: Checkout Arrow Rust
        uses: actions/checkout@v6
        with:
          path: rust
          submodules: true
          fetch-depth: 1
      - name: Checkout Arrow .NET
        uses: actions/checkout@v6
        with:
          repository: apache/arrow-dotnet
          path: dotnet
          fetch-depth: 1
      - name: Checkout Arrow Go
        uses: actions/checkout@v6
        with:
          repository: apache/arrow-go
          path: go
          fetch-depth: 1
      - name: Checkout Arrow Java
        uses: actions/checkout@v6
        with:
          repository: apache/arrow-java
          path: java
          fetch-depth: 1
      - name: Checkout Arrow JavaScript
        uses: actions/checkout@v6
        with:
          repository: apache/arrow-js
          path: js
          fetch-depth: 1
      - name: Checkout Arrow nanoarrow
        uses: actions/checkout@v6
        with:
          repository: apache/arrow-nanoarrow
          path: nanoarrow
          fetch-depth: 1

      - name: Monitor disk usage - After checkouts
        run: |
          echo "=== After Checkouts ==="
          df -h /
          echo ""

      - name: Build
        run: conda run --no-capture-output ci/scripts/integration_arrow_build.sh $PWD /build

      - name: Monitor disk usage - After build
        if: always()
        run: |
          echo "=== After Build ==="
          df -h /
          echo ""

      - name: Run
        run: conda run --no-capture-output ci/scripts/integration_arrow.sh $PWD /build

      - name: Monitor disk usage - After tests
        if: always()
        run: |
          echo "=== After Tests ==="
          df -h /
          echo ""

  # test FFI against the C-Data interface exposed by pyarrow
  pyarrow-integration-test:
    name: Pyarrow C Data Interface
    runs-on: ubuntu-latest
    strategy:
      matrix:
        rust: [stable]
        # PyArrow 15 was the first version to introduce StringView/BinaryView support
        pyarrow: ["15", "16", "17"]
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: true
      - name: Setup Rust toolchain
        run: |
          rustup toolchain install ${{ matrix.rust }}
          rustup default ${{ matrix.rust }}
          rustup component add rustfmt clippy
      - name: Cache Cargo
        uses: actions/cache@v5
        with:
          path: /home/runner/.cargo
          key: cargo-maturin-cache-
      - name: Cache Rust dependencies
        uses: actions/cache@v5
        with:
          path: /home/runner/target
          # this key is not equal because maturin uses different compilation flags.
          key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}-
      - uses: actions/setup-python@v6
        with:
          python-version: '3.8'
      - name: Upgrade pip and setuptools
        run: pip install --upgrade pip setuptools wheel virtualenv
      - name: Create virtualenv and install dependencies
        run: |
          virtualenv venv
          source venv/bin/activate
          pip install maturin toml pytest pytz pyarrow==${{ matrix.pyarrow }}
      - name: Run Rust tests
        run: |
          source venv/bin/activate
          cd arrow-pyarrow-testing
          cargo test
      - name: Run Python tests
        run: |
          source venv/bin/activate
          cd arrow-pyarrow-integration-testing
          maturin develop
          pytest -v .