Add support for MySQL databases & dialect in PyDough (#385)

Co-authored-by: knassre-bodo <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: knassre-bodo <[email protected]>
Co-authored-by: Hadia Ahmed <[email protected]>

Author: 4 people

.github/workflows/mysql_testing.yml

@@ -0,0 +1,74 @@
+name: Run MySQL Tests
+
+on:
+  workflow_call:
+    inputs:
+      python-versions:
+        description: "JSON string of Python versions"
+        type: string
+        required: true
+    secrets:
+      MYSQL_USERNAME:
+        required: true
+      MYSQL_PASSWORD:
+        required: true
+
+jobs:
+  mysql-tests:
+    name: MySQL Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ${{ fromJSON(inputs.python-versions) }}
+
+    # Define services here to run Docker containers alongside your job
+    services:
+      mysql:
+        image: bodoai1/pydough-mysql-tpch:latest
+        env:
+          # Set environment variables for MySQL container
+          MYSQL_ROOT_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
+          MYSQL_DATABASE: tpch
+        ports:
+          - 3306:3306
+    env:
+      MYSQL_USERNAME: ${{ secrets.MYSQL_USERNAME }}
+      MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
+      MYSQL_DATABASE: tpch
+      MYSQL_HOST: 127.0.0.1
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.23"
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install dependencies
+        run: uv pip install -e ".[mysql]" 
+
+      - name: Wait for MySQL to be ready
+        run: |
+          for i in $(seq 1 10); do
+            mysqladmin ping -h 127.0.0.1 -P 3306 -u root -p${{ env.MYSQL_PASSWORD }} && break
+            echo "Waiting for MySQL... ($i/10)"
+            sleep 5
+          done
+          mysqladmin ping -h 127.0.0.1 -P 3306 -u root -p${{ env.MYSQL_PASSWORD }} || exit 1
+
+      - name: Confirm MySQL connector is installed
+        run: uv run python -c "import mysql.connector; print('MySQL connector installed')"
+
+      - name: Run MySQL Tests
+
+        run: uv run pytest -m mysql tests/ -rs

.github/workflows/pr_testing.yml

@@ -26,6 +26,11 @@ on:
         type: boolean
         required: false
         default: false
+      run-mysql:
+        description: "Run MySQL Tests"
+        type: boolean
+        required: false
+        default: false
 
 # Limit CI to cancel previous runs in the same PR
 concurrency:
@@ -127,7 +132,7 @@ jobs:
         run: uv run ruff check .
 
       - name: Run Tests
-        run: uv run pytest tests/ -m "not snowflake" -rs
+        run: uv run pytest tests/ -m "not (snowflake or mysql)" -rs
 
   run-sf-tests:
     name: Snowflake Tests
@@ -144,3 +149,18 @@ jobs:
       python-versions: ${{ github.event_name == 'workflow_dispatch'
                       && needs.get-py-ver-matrix.outputs.matrix
                       || '["3.10", "3.11", "3.12"]' }}
+
+  run-mysql-tests:
+    name: MySQL Tests
+    needs: [get-msg, get-py-ver-matrix]
+    if: |
+      (github.event_name == 'pull_request' && contains(needs.get-msg.outputs.commitMsg, '[run mysql]')) ||
+      (github.event_name == 'workflow_dispatch' && inputs.run-mysql)
+    uses: ./.github/workflows/mysql_testing.yml # Path to MySQL workflow file
+    secrets:
+      MYSQL_USERNAME: ${{ secrets.MYSQL_USERNAME }}
+      MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }}
+    with:
+      python-versions: ${{ github.event_name == 'workflow_dispatch'
+                      && needs.get-py-ver-matrix.outputs.matrix
+                      || '["3.10", "3.11", "3.12"]' }}

demos/README.md

@@ -23,4 +23,5 @@ Once the introduction notebook is complete, you can explore the other notebooks:
 - [3_exploration.ipynb](notebooks/3_exploration.ipynb) shows how to use several user APIs for exploring PyDough.
 - [4_tpch.ipynb](notebooks/4_tpch.ipynb) provides PyDough translations for most of the TPC-H benchmark queries.
 - [5_what_if.ipynb](notebooks/5_what_if.ipynb) demonstrates how to do WHAT-IF analysis with PyDough.
+- [MySQL_TPCH.ipynb](notebooks/MySQL_TPCH.ipynb) demonstrates how to connect a MySQL database with PyDough.
 

demos/notebooks/1_introduction.ipynb

@@ -294,7 +294,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.6"
+   "version": "3.13.5"
   }
  },
  "nbformat": 4,

demos/notebooks/2_pydough_operations.ipynb

@@ -610,7 +610,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.6"
+   "version": "3.13.5"
   }
  },
  "nbformat": 4,