Make the entire project UV first

Author: kazmer97

.gitlab-ci.yml

@@ -31,12 +31,12 @@ developer_tests:
   before_script:
     - python --version
     - apt-get update -y
-    - apt-get install make -y
-    - pip install ruff
-    # Install dependencies needed by publish.py for test imports
-    - pip install typer rich boto3
-    # Install test dependencies
-    - cd lib/idp_common_pkg && pip install -e ".[test]" && cd ../..
+    - apt-get install make curl -y
+    # Install UV
+    - curl -LsSf https://astral.sh/uv/install.sh | sh
+    - export PATH="$HOME/.local/bin:$PATH"
+    # Sync all dependencies (includes ruff, pytest, publish deps, etc.)
+    - uv sync --all-extras --group dev
 
   script:
     - make lint-cicd
@@ -65,8 +65,10 @@ deployment_validation:
     - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
     - unzip awscliv2.zip
     - ./aws/install
-    # Install PyYAML for template analysis
-    - pip install PyYAML
+    # Install UV and PyYAML for template analysis
+    - curl -LsSf https://astral.sh/uv/install.sh | sh
+    - export PATH="$HOME/.local/bin:$PATH"
+    - uv pip install PyYAML
 
   script:
     # Check if service role has sufficient permissions for main stack deployment
@@ -97,13 +99,15 @@ integration_tests:
 
   before_script:
     - apt-get update -y
-    - apt-get install zip unzip curl python3-pip -y
+    - apt-get install zip unzip curl -y
     # Install AWS CLI
     - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
     - unzip awscliv2.zip
     - ./aws/install
-    # Install boto3 for Python script
-    - pip install boto3
+    # Install UV and boto3 for Python script
+    - curl -LsSf https://astral.sh/uv/install.sh | sh
+    - export PATH="$HOME/.local/bin:$PATH"
+    - uv pip install boto3
 
   script:
     - aws --version

Makefile

@@ -7,114 +7,128 @@ YELLOW := \033[1;33m
 BLUE := \033[0;34m
 NC := \033[0m  # No Color
 
-# Check for UV
+# Virtual environment and UV paths
+VENV := .venv
 UV := $(shell command -v uv 2> /dev/null)
 
-# Default target - run both lint and test
-all: check-uv lint test
+# Default target - ensure UV and venv, then run lint and test
+all: setup lint test
 
-# Ensure UV is installed
-check-uv:
+# Install UV if not present
+install-uv:
 ifndef UV
-	@echo -e "$(RED)❌ ERROR: uv is not installed$(NC)"
-	@echo -e "$(YELLOW)Install from: https://astral.sh/uv$(NC)"
-	@echo "  curl -LsSf https://astral.sh/uv/install.sh | sh"
+	@printf "$(YELLOW)📦 UV not found. Installing UV...$(NC)\n"
+	@curl -LsSf https://astral.sh/uv/install.sh | sh
+	@printf "$(GREEN)✅ UV installed!$(NC)\n"
+	@printf "$(YELLOW)⚠️  Please restart your shell or run: source ~/.bashrc (or ~/.zshrc)$(NC)\n"
+	@printf "$(YELLOW)⚠️  Then re-run make$(NC)\n"
 	@exit 1
+else
+	@printf "$(GREEN)✅ UV is already installed: $(UV)$(NC)\n"
 endif
-	@echo -e "$(GREEN)✅ UV is installed: $(UV)$(NC)"
 
-# Initialize UV environment
-init: check-uv
-	@echo -e "$(BLUE)🚀 Initializing UV workspace...$(NC)"
-	$(UV) sync --all-extras
-	@echo -e "$(GREEN)✅ Workspace initialized$(NC)"
+# Create .venv using UV if it doesn't exist
+$(VENV):
+	@printf "$(BLUE)🏗️  Creating virtual environment with UV...$(NC)\n"
+	@$(UV) venv
+	@printf "$(GREEN)✅ Virtual environment created at $(VENV)$(NC)\n"
+
+# Setup: ensure UV is installed and create venv
+setup: install-uv $(VENV)
+	@printf "$(GREEN)✅ Setup complete!$(NC)\n"
+
+# Initialize UV environment (creates .venv and syncs all dependencies)
+init: setup
+	@printf "$(BLUE)🚀 Initializing UV workspace...$(NC)\n"
+	@$(UV) sync --all-extras --group dev
+	@printf "$(GREEN)✅ Workspace initialized with all features + dev dependencies$(NC)\n"
 
 # Lock dependencies
-lock: check-uv
-	@echo -e "$(BLUE)🔒 Locking dependencies...$(NC)"
-	$(UV) lock
-	@echo -e "$(GREEN)✅ Dependencies locked (uv.lock updated)$(NC)"
+lock: install-uv
+	@printf "$(BLUE)🔒 Locking dependencies...$(NC)\n"
+	@$(UV) lock
+	@printf "$(GREEN)✅ Dependencies locked (uv.lock updated)$(NC)\n"
 
 # Sync local development environment
-sync: check-uv
-	@echo -e "$(BLUE)📦 Syncing development environment...$(NC)"
-	$(UV) sync --all-extras
-	@echo -e "$(GREEN)✅ Environment synced$(NC)"
+sync: setup
+	@printf "$(BLUE)📦 Syncing development environment...$(NC)\n"
+	@$(UV) sync --all-extras --group dev
+	@printf "$(GREEN)✅ Environment synced with all features$(NC)\n"
 
 # Update dependencies
-update: check-uv
-	@echo -e "$(BLUE)⬆️  Updating dependencies...$(NC)"
-	$(UV) lock --upgrade
-	@echo -e "$(GREEN)✅ Dependencies updated$(NC)"
+update: setup
+	@printf "$(BLUE)⬆️  Updating dependencies...$(NC)\n"
+	@$(UV) lock --upgrade
+	@printf "$(GREEN)✅ Dependencies updated$(NC)\n"
 
 # Update specific package
-update-package: check-uv
+update-package: setup
 ifndef PKG
-	@echo -e "$(RED)❌ ERROR: PKG variable not set$(NC)"
+	@printf "$(RED)❌ ERROR: PKG variable not set$(NC)\n"
 	@echo "Usage: make update-package PKG=boto3"
 	@exit 1
 endif
-	@echo -e "$(BLUE)⬆️  Updating $(PKG)...$(NC)"
-	$(UV) lock --upgrade-package $(PKG)
-	@echo -e "$(GREEN)✅ $(PKG) updated$(NC)"
+	@printf "$(BLUE)⬆️  Updating $(PKG)...$(NC)\n"
+	@$(UV) lock --upgrade-package $(PKG)
+	@printf "$(GREEN)✅ $(PKG) updated$(NC)\n"
 
 # Build idp_common package
-build-idp-common: check-uv
-	@echo -e "$(BLUE)🔨 Building idp_common package...$(NC)"
-	cd lib/idp_common_pkg && $(UV) build
-	@echo -e "$(GREEN)✅ idp_common built$(NC)"
+build-idp-common: setup
+	@printf "$(BLUE)🔨 Building idp_common package...$(NC)\n"
+	@cd lib/idp_common_pkg && $(UV) build
+	@printf "$(GREEN)✅ idp_common built$(NC)\n"
 
 # Build idp_cli package
-build-idp-cli: check-uv
-	@echo -e "$(BLUE)🔨 Building idp_cli package...$(NC)"
-	cd idp_cli && $(UV) build
-	@echo -e "$(GREEN)✅ idp_cli built$(NC)"
+build-idp-cli: setup
+	@printf "$(BLUE)🔨 Building idp_cli package...$(NC)\n"
+	@cd idp_cli && $(UV) build
+	@printf "$(GREEN)✅ idp_cli built$(NC)\n"
 
 # Build all Python packages
 build-packages: build-idp-common build-idp-cli
-	@echo -e "$(GREEN)✅ All packages built$(NC)"
+	@printf "$(GREEN)✅ All packages built$(NC)\n"
 
 # Run tests in idp_common_pkg and idp_cli directories
-test: check-uv
-	@echo -e "$(BLUE)🧪 Running tests...$(NC)"
-	$(MAKE) -C lib/idp_common_pkg test
-	cd idp_cli && $(UV) run pytest -v
-	@echo -e "$(GREEN)✅ All tests passed$(NC)"
+test: setup
+	@printf "$(BLUE)🧪 Running tests...$(NC)\n"
+	@cd lib/idp_common_pkg && $(UV) run --all-extras --group dev pytest -m "unit"
+	@cd idp_cli && $(UV) run --group dev pytest -v
+	@printf "$(GREEN)✅ All tests passed$(NC)\n"
 
 # Run linting checks and fix issues automatically
-ruff-lint: check-uv
-	@echo -e "$(BLUE)🔍 Running ruff linting...$(NC)"
-	$(UV) run ruff check --fix
-	@echo -e "$(GREEN)✅ Linting complete$(NC)"
+ruff-lint: setup
+	@printf "$(BLUE)🔍 Running ruff linting...$(NC)\n"
+	@$(UV) run --group dev ruff check --fix
+	@printf "$(GREEN)✅ Linting complete$(NC)\n"
 
 # Format code according to project standards
-format: check-uv
-	@echo -e "$(BLUE)✨ Formatting code...$(NC)"
-	$(UV) run ruff format
-	@echo -e "$(GREEN)✅ Formatting complete$(NC)"
+format: setup
+	@printf "$(BLUE)✨ Formatting code...$(NC)\n"
+	@$(UV) run --group dev ruff format
+	@printf "$(GREEN)✅ Formatting complete$(NC)\n"
 
 # Run both linting and formatting in one command
 lint: ruff-lint format check-arn-partitions
 
 # CI/CD version of lint that only checks but doesn't modify files
 # Used in CI pipelines to verify code quality without making changes
-lint-cicd:
-	@echo "Running code quality checks..."
-	@if ! ruff check; then \
-		echo -e "$(RED)ERROR: Ruff linting failed!$(NC)"; \
-		echo -e "$(YELLOW)Please run 'make ruff-lint' locally to fix these issues.$(NC)"; \
+lint-cicd: setup
+	@printf "$(BLUE)Running code quality checks...$(NC)\n"
+	@if ! $(UV) run --group dev ruff check; then \
+		printf "$(RED)ERROR: Ruff linting failed!$(NC)\n"; \
+		printf "$(YELLOW)Please run 'make ruff-lint' locally to fix these issues.$(NC)\n"; \
 		exit 1; \
 	fi
-	@if ! ruff format --check; then \
-		echo -e "$(RED)ERROR: Code formatting check failed!$(NC)"; \
-		echo -e "$(YELLOW)Please run 'make format' locally to fix these issues.$(NC)"; \
+	@if ! $(UV) run --group dev ruff format --check; then \
+		printf "$(RED)ERROR: Code formatting check failed!$(NC)\n"; \
+		printf "$(YELLOW)Please run 'make format' locally to fix these issues.$(NC)\n"; \
 		exit 1; \
 	fi
-	@echo -e "$(GREEN)All code quality checks passed!$(NC)"
+	@printf "$(GREEN)All code quality checks passed!$(NC)\n"
 
 # Check CloudFormation templates for hardcoded AWS partition ARNs and service principals
 check-arn-partitions:
-	@echo "Checking CloudFormation templates for hardcoded ARN partitions and service principals..."
+	@printf "$(BLUE)Checking CloudFormation templates for hardcoded ARN partitions and service principals...$(NC)\n"
 	@FOUND_ISSUES=0; \
 	for template in template.yaml patterns/*/template.yaml patterns/*/sagemaker_classifier_endpoint.yaml options/*/template.yaml; do \
 		if [ -f "$$template" ]; then \
@@ -143,10 +157,66 @@ check-arn-partitions:
 		exit 1; \
 	fi
 
+# Clean up build artifacts and caches
+clean:
+	@printf "$(BLUE)🧹 Cleaning build artifacts...$(NC)\n"
+	@rm -rf .venv
+	@rm -rf lib/idp_common_pkg/dist lib/idp_common_pkg/build lib/idp_common_pkg/*.egg-info
+	@rm -rf idp_cli/dist idp_cli/build idp_cli/*.egg-info
+	@find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+	@find . -type d -name ".pytest_cache" -exec rm -rf {} + 2>/dev/null || true
+	@find . -type d -name ".ruff_cache" -exec rm -rf {} + 2>/dev/null || true
+	@find . -type f -name "*.pyc" -delete 2>/dev/null || true
+	@printf "$(GREEN)✅ Cleanup complete$(NC)\n"
+
+# Clean everything including uv.lock (use with caution)
+clean-all: clean
+	@printf "$(YELLOW)⚠️  Removing uv.lock...$(NC)\n"
+	@rm -f uv.lock
+	@printf "$(GREEN)✅ Deep cleanup complete$(NC)\n"
+
+# Show help
+help:
+	@printf "$(BLUE)IDP Accelerator Makefile - UV-based Python Development$(NC)\n"
+	@echo ""
+	@printf "$(YELLOW)Setup Targets:$(NC)"
+	@echo "  make setup          - Install UV and create .venv (automatically done by other targets)"
+	@echo "  make init           - Initialize workspace with dev dependencies"
+	@echo "  make sync           - Sync development environment"
+	@echo ""
+	@printf "$(YELLOW)Development Targets:$(NC)"
+	@echo "  make lint           - Run linting and formatting"
+	@echo "  make ruff-lint      - Run ruff linting with auto-fix"
+	@echo "  make format         - Format code with ruff"
+	@echo "  make test           - Run all tests"
+	@echo ""
+	@printf "$(YELLOW)Dependency Management:$(NC)"
+	@echo "  make lock           - Lock dependencies (update uv.lock)"
+	@echo "  make update         - Update all dependencies"
+	@echo "  make update-package PKG=<name>  - Update specific package"
+	@echo ""
+	@printf "$(YELLOW)Build Targets:$(NC)"
+	@echo "  make build-idp-common  - Build idp_common package"
+	@echo "  make build-idp-cli     - Build idp_cli package"
+	@echo "  make build-packages    - Build all packages"
+	@echo ""
+	@printf "$(YELLOW)Cleanup Targets:$(NC)"
+	@echo "  make clean          - Remove .venv and build artifacts"
+	@echo "  make clean-all      - Remove .venv, build artifacts, and uv.lock"
+	@echo ""
+	@printf "$(YELLOW)Other Targets:$(NC)"
+	@echo "  make all            - Run setup, lint, and test"
+	@echo "  make check-arn-partitions  - Check CFN templates for GovCloud compatibility"
+	@echo "  make help           - Show this help message"
+
 # A convenience Makefile target that runs 
 commit: lint test
 	$(info Generating commit message...)
 	export COMMIT_MESSAGE="$(shell q chat --no-interactive --trust-all-tools "Understand pending local git change and changes to be committed, then infer a commit message. Return this commit message only" | tail -n 1 | sed 's/\x1b\[[0-9;]*m//g')" && \
 	git add . && \
 	git commit -am "$${COMMIT_MESSAGE}" && \
 	git push
+
+.PHONY: all setup install-uv init lock sync update update-package \
+        build-idp-common build-idp-cli build-packages test ruff-lint format \
+        lint lint-cicd check-arn-partitions clean clean-all help commit

README.md

@@ -233,16 +233,25 @@ pytest
 
 ### Publishing and Deployment
 
-The publish script automatically validates UV is installed:
+The publish script automatically syncs all required dependencies using UV:
 
 ```bash
+# One-time setup: sync development dependencies
+uv sync --group dev
+
 # Publish with parallel builds (faster with UV!)
 ./publish.sh my-bucket prefix us-east-1 --max-workers 4
 
 # UV enables safe parallel builds with built-in locking
 # Typical speedup: 4x faster than sequential builds
 ```
 
+The `publish.sh` script automatically:
+- Validates Python 3.12+, UV, and Node.js are installed
+- Syncs all required Python dependencies (boto3, PyYAML, rich, ruff, etc.)
+- Builds and packages Lambda functions
+- Uploads artifacts to S3
+
 For more details on the UV migration and dependency management, see:
 - [Setup Development Environment (macOS)](./docs/setup-development-env-macos.md)
 - [Setup Development Environment (Linux)](./docs/setup-development-env-linux.md)