test: add integration tests (#11)

Author: timonviola

.containerignore

@@ -0,0 +1,2 @@
+!share
+*

.github/workflows/ci.yaml

@@ -12,12 +12,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
     - uses: actions/setup-python@v3
+    - name: Set up pip cache
+      if: runner.os == 'Linux'
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+        restore-keys: ${{ runner.os }}-pip-
+    - name: Install Hatch
+      run: pipx install hatch
     - uses: pre-commit/[email protected]
+      with:
+        extra_args: --color=always
 
   test:
     name: "tests & coverage 💊"
-    needs: precommit
+    # needs: precommit
     runs-on: ubuntu-latest
     strategy:
       matrix:

.gitignore

@@ -44,7 +44,6 @@ venv.bak/
 ######################
 *.log
 *.log.*
-*.sql
 *.sqlite
 
 # OS generated files #

.pre-commit-config.yaml

@@ -15,13 +15,15 @@ repos:
     - id: typos
       stages: [commit]
       exclude: "^typings/.*"
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.1  # Use the sha / tag you want to point at
+  - repo: local
     hooks:
-    -   id: mypy
-        language: system
-        pass_filenames: false
-        args: ['.']
+    - id: mypy-hatch
+      name: mypy-hatch
+      language: system
+      pass_filenames: false
+      args: ['.']
+      entry: hatch
+      args: ["run", "types:check"]
   - repo: https://github.com/compilerla/conventional-pre-commit
     rev: v3.3.0
     hooks:

CONTRIBUTING.md

@@ -10,21 +10,22 @@ Install hatch with `pipx` and configure it. As a minimum, configure it to create
 [dirs.env]
 virtual = ".venv"
 ```
+# Cheat sheet
+- tests: `hatch run test:test`
+- docs: `hatch run dev:docs`
 
-# VSCode
+# Dev tooling
+## VSCode
 1. Clone the repo
 1. Run `hatch env create && hatch env create dev && hatch env create test`
 1. Open VSCode `code .`
 1. Select the `dev` environment for development [See the documentation.](https://hatch.pypa.io/latest/how-to/integrate/vscode/)
 
 To enable test discovery and test debugging, change the *python interpreter path* to a test environments path e.g. `test.py3.11`.
 
-# No VSCode
+## No VSCode
 I suggest NeoVim or Zed. See the cheat-sheet below.
 
-# Cheat sheet
-- tests: `hatch test`
-- docs: `hatch run dev:docs`
 
 # Documentation
 The docs is built into "sites" folder. This is gitignored and the docs is built in CI.
@@ -54,19 +55,28 @@ The package follows semantic versioning. Breaking changes will occur unannounced
 
 
 # Docker
-The base Dockerfile can be used to run Airflow and install dagcellent in _editable_ mode, so it gives you a short feedback loop.
+It is recommended to use Podman with the container files. The base Dockerfile can be used to run Airflow and install dagcellent in _editable_ mode, so it gives you a short feedback loop.
+
+# Tests
+The testing suite uses Pytest.
 
-# tests
 ## Unit tests
 ### Fuzzing/hypothesis
+*coming soon* 👀
 
 ## Integration tests
-The CI will run integration tests, where external components are not mocked, but real containerized entities are used.
+The CI will run integration tests, where external components are not mocked, but real containerized entities are used. All integration tests are marked with `integration`.
+
+In general, prefer integration testing/system tests over mocking. E.g.: to guarantee that our tools work on various SQL engines, implement integration tests against those engines.
 
+The following structure illustrates where to find the various integration test.
+```
+tests
+├── dags
+└── integration
+    ├── mlflow
+    ├── mssql
+    └── psql
+```
 
-The following integrations are available (docker commands should be executed from the project root folder):
-- mssql: `docker compose -f docker-compose.yaml -f ./tests/integration/docker-compose.override.mssql.yaml up --detach`
-- psql: `docker compose -f docker-compose.yaml -f ./tests/integration/docker-compose.override.psql.yaml up --detach`
 
-To stop the running instances, it is a good idea to use the `volumes` flag to remove persistent data:
-`docker compose -f docker-compose.yaml -f ./tests/integration/docker-compose.override.mssql.yaml down`

docker-compose.yaml

@@ -9,12 +9,11 @@ version: '3.8'
 services:
   airflow:
     build: .
+    #image: pytest74406-airflow:latest
     ports:
       - "8080:8080"
     container_name: airflow
     restart: always
-    depends_on:
-      - db
     volumes:
       - ./tests/dags:/opt/airflow/dags
       - ./:/opt/dagcellent/

pyproject.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = ["hatchling", "versioningit"]
+requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [project]
 name = "dagcellent"
-dynamic = ["version"]
+version = "0.2.0"
 description = ''
 readme = "README.md"
 requires-python = ">=3.11"
@@ -28,7 +28,7 @@ classifiers = [
 dependencies = [
     "apache-airflow>=2.9.1",
     "apache-airflow-providers-amazon>=7.2.0",
-    "apache-airflow-providers-microsoft-mssql >=3.2.0",
+    "apache-airflow-providers-microsoft-mssql >=4.0.0",
     "apache-airflow-providers-snowflake >= 5.1",
     "apache-airflow-providers-common-sql[pandas,openlineage]",
     "tomli >= 2.0.1",
@@ -37,7 +37,9 @@ dependencies = [
     "pandas > 2.0.0",
     "awswrangler[redshift, postgres, sqlserver, deltalake] > 3.0.0",
     "pyarrow >= 17.0.0",
-    "mlflow-skinny >= 2.17.2"
+    "mlflow-skinny >= 2.17.2",
+    # TODO: https://github.com/xmlsec/python-xmlsec/issues/345
+    "xmlsec != 1.3.15"
 ]
 
 [project.urls]
@@ -64,18 +66,17 @@ extra-dependencies = [
     "pre-commit == 3.7.*",
     "ruff == 0.4.4",
     "mypy == 1.10.*",
-    "versioningit == 3.1.*",
     "towncrier == 23.11.*",
 ]
 
 [tool.hatch.envs.dev.scripts]
 install = "pre-commit install --hook-type commit-msg"
-version = "versioningit"
 changelog = "git-cliff -o CHANGELOG.md"
 
 [tool.hatch.envs.types]
 extra-dependencies = [
   "mypy>=1.0.0",
+  "types-requests",
 ]
 [tool.hatch.envs.types.scripts]
 check = "mypy --install-types --non-interactive {args:src/dagcellent tests}"
@@ -96,16 +97,21 @@ extra-dependencies = [
     "pytest >= 8.0.0",
     "pytest-cov",
     "pytest-mock >= 3.14.0",
+    "pytest-docker",
 ]
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12"]
 
 [tool.hatch.envs.test.scripts]
-test = "pytest --cov=dagcellent --cov-report=term-missing --cov-report=xml --cov-report=html tests"
+test = "pytest --cov=dagcellent --cov-report=term-missing --cov-report=xml --cov-report=html --container-scope module tests"
+
+[tool.pytest.ini_options]
+markers = [
+  "integration: Run tests against containers.",
+  "mlflow: Run MLFlow tests."
+]
 
-[tool.hatch.version]
-source = "versioningit"
 
 [tool.coverage.run]
 source_pkgs = ["dagcellent", "tests"]
@@ -311,24 +317,4 @@ exclude = [
     "^typings/*"
 ]
 
-[tool.versioningit]
-default-version = "0.0.0-unknown"
-
-[tool.versioningit.vcs]
-# The method key:
-method = "git"  # <- The method name
-
-# Parameters to pass to the method:
-match = ["v*"]
-default-tag = "0.0.1"
-
-[tool.hatch.build.hooks.versioningit-onbuild]
-source-file = "src/dagcellent/__init__.py"
-build-file = "dagcellent/__init__.py"
-require-match = true
-
-[tool.versioningit.format]
-distance = "{next_version}.dev{distance}+{vcs}{rev}"
-dirty = "{version}+dirty"
-distance-dirty = "{next_version}.dev{distance}+{vcs}{rev}.dirty"
 

src/dagcellent/dag/_dynamic.py

@@ -3,13 +3,15 @@
 from __future__ import annotations
 
 import json
-from collections.abc import Callable, Iterable
-from pathlib import Path
-from typing import TypeVar
+from typing import TYPE_CHECKING, TypeVar
 
 import tomli
 from pydantic import BaseModel, ConfigDict
 
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+    from pathlib import Path
+
 T = TypeVar("T", bound="Config")
 
 

src/dagcellent/data_utils/sql_reflection.py

@@ -151,7 +151,7 @@ def safe_add_database_to_connection(engine: Engine, database: str) -> Engine:
 def reflect_meta_data(engine: Engine, schema: str | None, table: str) -> Table | None:
     """Reflects the metadata from the engine."""
     meta_data = MetaData(schema=schema)
-    return Table(table, meta_data, autoload_with=engine)
+    return Table(table, meta_data, autoload_with=engine, schema=schema)
 
 
 def _log_reflected_table(meta_data: MetaData, table_name: str) -> None:

src/dagcellent/operators/external_table_arrow.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import Sequence
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from airflow.models.baseoperator import BaseOperator
 
@@ -10,6 +9,9 @@
     create_external_table_redshift_arrow,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
 
 class CreateExternalTableArrow(BaseOperator):
     """Generate SQL DDL to create external table in target database."""