Poetry --> UV (+ Makefile Added) (#22)

* chore: update lint-staged configuration and pyproject.toml

- Change command in .lintstagedrc from 'poetry run ruff' to 'uv run ruff'
- Migrate from [tool.poetry] to [project] section in pyproject.toml
- Update authors and license format in pyproject.toml
- Add dependencies under [project] and [project.optional-dependencies] sections
- Switch build system from poetry to hatchling

* chore: remove poetry.lock file

- Deleted the poetry.lock file as part of the transition to a new dependency management approach.

* chore: simplify dagster-dev command in Makefile

- Removed the clean-dagster target to streamline the dagster-dev command.
- Updated the dagster-dev command to remove the explicit workspace.yaml path, allowing for default behavior.

* chore: update CI workflow to use uv for dependency management

- Replaced Poetry setup with actions/setup-python for Python installation.
- Added installation step for uv and updated commands to use uv for running checks and tests.
- Streamlined the CI process by integrating uv for dependency synchronization and execution of tasks.

* chore: enhance development setup with Makefile commands

- Added `pyright` command to Makefile for type checking.
- Updated `pyright` script in package.json to use the correct Python path.
- Revised README to reflect the new Make commands for initializing and upgrading dependencies.
- Simplified instructions for running tests and the sample dagster project using Make commands.

* Update pyproject.toml

Co-authored-by: Reuven Gonzales <[email protected]>

* Update package.json

Co-authored-by: Reuven Gonzales <[email protected]>

* chore: streamline Makefile for consistent Python command usage

- Removed OS-specific checks for Windows in the Makefile.
- Unified Python, SQLMESH, and UV command definitions for a cleaner setup.
- Simplified activation and deactivation commands for the virtual environment.

* chore: update pyproject.toml and console.py for linter integration

- Added Pyright exclusion patterns in pyproject.toml to ignore specific directories.
- Introduced a new method `show_linter_violations` in EventConsole to handle and publish linting violations.
- Reorganized imports in console.py for better clarity and consistency.

---------

Co-authored-by: Kevin Longe <[email protected]>
Co-authored-by: Reuven Gonzales <[email protected]>

Author: 3 people

.github/workflows/ci-default.yml

@@ -52,25 +52,31 @@ jobs:
             - recursive: true
               args: [--frozen-lockfile, --strict-peer-dependencies]
 
-      - name: "Setup Python, Poetry and Dependencies"
-        uses: packetcoders/action-setup-cache-python-poetry@main
+      - name: Setup Python
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python_version }}
-          poetry-version: 1.8.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras
 
       - name: Type Check (Python)
         run: |
           pnpm pyright
 
       - name: Lint (Python)
         run: |
-          poetry run ruff check
+          uv run ruff check
 
       - name: Test (Python)
         run: |
-          poetry run pytest
+          uv run pytest
 
       - name: Run the sample dagster project
         run: |
-          poetry run dagster job execute -f sample/dagster_project/definitions.py -j all_assets_job
+          uv run dagster job execute -f sample/dagster_project/definitions.py -j all_assets_job
       

.lintstagedrc

@@ -1,6 +1,6 @@
 {
   "**/*.py": [
-    "poetry run ruff check --fix --force-exclude",
+    "uv run ruff check --fix --force-exclude",
     "pnpm pyright"
   ]
 }

Makefile

@@ -0,0 +1,57 @@
+# System detection
+UNAME := "$(shell uname)"
+PYTHON_VENV_NAME := ".venv"
+
+# Python virtual environment settings
+VENV_NAME := .venv
+PYTHON := python
+
+PYTHON_CMD := $(CURDIR)/$(VENV_NAME)/bin/python
+SQLMESH_CMD := $(CURDIR)/$(VENV_NAME)/bin/sqlmesh
+UV_CMD := uv
+ACTIVATE := source $(CURDIR)/$(VENV_NAME)/bin/activate
+DEACTIVATE := deactivate
+
+init-python:
+	@if [ ! -d "$(PYTHON_VENV_NAME)" ]; then \
+		echo "Creating virtual environment with Python 3.12..."; \
+		uv venv --python 3.12 $(PYTHON_VENV_NAME); \
+	fi
+
+install-python-deps:
+	uv sync --all-extras
+
+upgrade-python-deps:
+	uv lock --upgrade
+	make install-python-deps
+
+install-node-deps:
+	pnpm install
+
+upgrade-node-deps:
+	pnpm update
+
+# Base commands
+init: init-python install-python-deps check-pnpm install-node-deps
+
+clean:
+	find . \( -type d -name "__pycache__" -o -type f -name "*.pyc" -o -type d -name ".pytest_cache" -o -type d -name "*.egg-info" \) -print0 | xargs -0 rm -rf
+
+# Testing commands
+test:
+	$(PYTHON_CMD) -m pytest -vv --log-cli-level=INFO $(filter-out $@,$(MAKECMDGOALS))
+
+pyright:
+	pnpm pyright
+
+# Sample project commands
+
+dagster-dev: clean-dagster
+	DAGSTER_HOME=$(CURDIR)/sample/dagster_project $(PYTHON_CMD) -m dagster dev -h 0.0.0.0 -w
+
+dev: dagster-dev  # Alias for dagster-dev
+
+dagster-materialize:
+	$(PYTHON_CMD) -m dagster asset materialize -f sample/dagster_project/definitions.py --select '*'
+
+.PHONY: init init-python install-python-deps upgrade-python-deps clean test mypy check-pnpm install-node-deps upgrade-node-deps sample-dev dagster-dev dagster-materialize clean-dagster 

README.md

@@ -55,21 +55,33 @@ _Note: this is a python project but some of our dependent tools are in typescrip
 
 ### Installing
 
-To install everything you need for development just do the following:
+The project uses Make commands to simplify the development setup process. To get started:
 
 ```bash
-poetry install
-pnpm install
+make init
+```
+
+This will:
+- Set up a Python virtual environment with Python 3.12
+- Install all Python dependencies
+- Install Node.js dependencies via pnpm
+
+_Note: All Make commands automatically use the correct virtual environment - you don't need to activate it manually._
+
+To upgrade dependencies:
+```bash
+make upgrade-python-deps  # Upgrade Python dependencies
+make upgrade-node-deps   # Upgrade Node.js dependencies
 ```
 
 ### Running tests
 
 We have tests that should work entirely locally. You may see a `db.db` file appear in the root of the repository when these tests are run. It can be safely ignored or deleted.
 
-We run tests with `pytest` like so:
+To run tests:
 
 ```bash
-poetry run pytest
+make test
 ```
 
 ### Running the "sample" dagster project
@@ -79,13 +91,17 @@ accompanying sqlmesh project from `sample/sqlmesh_project` configured as an
 asset. To run the sample dagster project deployment with a UI:
 
 ```bash
-poetry run dagster dev -h 0.0.0.0 -f sample/dagster_project/definitions.py
+make dagster-dev 
+```
+or 
+```bash
+make dev
 ```
 
 If you'd like to materialize the dagster assets quickly on the CLI:
 
 ```bash
-dagster asset materialize -f sample/dagster_project/definitions.py --select '*'
+make dagster-materialize
 ```
 
 _Note: The sqlmesh project that is in the sample folder has a dependency on a

dagster_sqlmesh/console.py

@@ -1,22 +1,22 @@
+import logging
 import typing as t
-from typing import Dict, Union, Callable
-from dataclasses import dataclass
-import uuid
 import unittest
-import logging
+import uuid
+from dataclasses import dataclass
+from typing import Callable, Dict, Union
 
 from sqlglot.expressions import Alter
 from sqlmesh.core.console import Console
-from sqlmesh.core.plan import EvaluatablePlan
 from sqlmesh.core.context_diff import ContextDiff
-from sqlmesh.core.plan import PlanBuilder
-from sqlmesh.core.table_diff import RowDiff, SchemaDiff, TableDiff
 from sqlmesh.core.environment import EnvironmentNamingInfo
+from sqlmesh.core.linter.rule import RuleViolation
+from sqlmesh.core.plan import EvaluatablePlan, PlanBuilder
 from sqlmesh.core.snapshot import (
     Snapshot,
     SnapshotChangeCategory,
     SnapshotInfoLike,
 )
+from sqlmesh.core.table_diff import RowDiff, SchemaDiff, TableDiff
 from sqlmesh.utils.concurrency import NodeExecutionFailedError
 
 logger = logging.getLogger(__name__)
@@ -562,6 +562,12 @@ def print_environments(self, environments_summary: t.Dict[str, int]) -> None:
     def show_table_diff_summary(self, table_diff: TableDiff) -> None:
         self.publish(ShowTableDiffSummary(table_diff))
 
+    def show_linter_violations(
+        self,
+        violations: list[RuleViolation],
+    ) -> None:
+        self.publish(LogWarning("Linting violations found", str(violations)))
+
 
 class DebugEventConsole(EventConsole):
     """A console that wraps an existing console and logs all events to a logger"""

package.json

@@ -12,7 +12,7 @@
   "scripts": {
     "build": "turbo run build --concurrency=100%",
     "format:staged": "lint-staged",
-    "pyright": "pyright --pythonpath $(poetry env info --path)/bin/python",
+    "pyright": "uv run -q -c \"import sys; print(sys.executable)\" | xargs -r pyright --pythonpath",
     "prepare": "husky install"
   },
   "devDependencies": {