Consolidate Dependencies and make UV default

.dockerignore

@@ -96,4 +96,6 @@ memory-bank/
 # Keep only source code and requirements
 !patterns/*/src/**
 !lib/idp_common_pkg/**
-!src/lambda/**
+!src/lambda/**
+!pyproject.toml
+!uv.lock

.gitignore

@@ -1,26 +1,50 @@
+# Environment
 .env
+.venv/
+venv/
+
+# AWS SAM
 .aws-sam
 build.toml
+
+# Build artifacts
 model.tar.gz
 .checksum
 .checksums/
 .build_checksum
 .lib_checksum
+dist/
+build/
+*.egg-info/
+
+# Python
+__pycache__
+*.py[cod]
+*$py.class
+*.so
+.Python
+
+# UV specific
+.python-version
+
+# Ruff
+.ruff_cache
+
+# IDEs
 .vscode/
+.idea/
+*.code-workspace
+
+# OS
 .DS_Store
-dist/
+
+# Project specific
 .sav/
 .delete/
 .data/
-*.egg-info/
-build/
-__pycache__
-*.code-workspace
-.ruff_cache
 .kiro
 rvl_cdip_*
 notebooks/examples/data
-.idea/
 .dsr/
 *tmp-dev-assets*
 scratch/

.gitlab-ci.yml

@@ -31,12 +31,12 @@ developer_tests:
   before_script:
     - python --version
     - apt-get update -y
-    - apt-get install make -y
-    - pip install ruff
-    # Install dependencies needed by publish.py for test imports
-    - pip install typer rich boto3
-    # Install test dependencies
-    - cd lib/idp_common_pkg && pip install -e ".[test]" && cd ../..
+    - apt-get install make curl -y
+    # Install UV
+    - curl -LsSf https://astral.sh/uv/install.sh | sh
+    - export PATH="$HOME/.local/bin:$PATH"
+    # Sync all dependencies (includes ruff, pytest, publish deps, etc.)
+    - uv sync --all-extras --group dev
 
   script:
     - make lint-cicd
@@ -65,8 +65,10 @@ deployment_validation:
     - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
     - unzip awscliv2.zip
     - ./aws/install
-    # Install PyYAML for template analysis
-    - pip install PyYAML
+    # Install UV and PyYAML for template analysis
+    - curl -LsSf https://astral.sh/uv/install.sh | sh
+    - export PATH="$HOME/.local/bin:$PATH"
+    - uv pip install PyYAML
 
   script:
     # Check if service role has sufficient permissions for main stack deployment
@@ -97,13 +99,15 @@ integration_tests:
 
   before_script:
     - apt-get update -y
-    - apt-get install zip unzip curl python3-pip -y
+    - apt-get install zip unzip curl -y
     # Install AWS CLI
     - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
     - unzip awscliv2.zip
     - ./aws/install
-    # Install boto3 for Python script
-    - pip install boto3
+    # Install UV and boto3 for Python script
+    - curl -LsSf https://astral.sh/uv/install.sh | sh
+    - export PATH="$HOME/.local/bin:$PATH"
+    - uv pip install boto3
 
   script:
     - aws --version

Dockerfile.optimized

@@ -1,5 +1,5 @@
-# Optimized Dockerfile for Lambda functions with minimal dependencies
-# This builds each function with ONLY the dependencies it needs
+# Optimized Dockerfile for Lambda functions with UV and pyproject.toml dependency groups
+# This builds each function with ONLY the dependencies it needs from the root pyproject.toml
 
 # checkov:skip=CKV_DOCKER_3: "The Dockerfile uses the official AWS Lambda Python base image (public.ecr.aws/lambda/python:3.12-arm64), which already configures the appropriate non-root user for Lambda execution"
 # checkov:skip=CKV_DOCKER_2: "The Dockerfile.optimized is specifically designed for AWS Lambda container images, which don't use Docker HEALTHCHECK instructions."
@@ -10,25 +10,25 @@ FROM public.ecr.aws/lambda/python:3.12-arm64 AS builder
 # Copy uv from official distroless image
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-# Build argument for function path
+# Build argument for dependency group from root pyproject.toml
+ARG DEPENDENCY_GROUP
 ARG FUNCTION_PATH
-ARG INSTALL_IDP_COMMON=true
 
 # Create working directory
 WORKDIR /build
 
-# Copy idp_common_pkg and requirements for installation
-COPY lib/idp_common_pkg /tmp/idp_common_pkg
-COPY ${FUNCTION_PATH}/requirements.txt* /build/
+# Copy root pyproject.toml, uv.lock, and idp_common_pkg
+COPY pyproject.toml uv.lock /build/
+COPY lib/idp_common_pkg /build/lib/idp_common_pkg
 
-# Install all dependencies including idp_common_pkg in one step
+# Install dependencies from the specified dependency group
 RUN --mount=type=cache,target=/root/.cache/uv \
-    if [ -f /build/requirements.txt ]; then \
-        sed 's|^\.\./\.\.\(/\.\.\)\?/lib/idp_common_pkg|/tmp/idp_common_pkg|' /build/requirements.txt > /tmp/requirements.txt && \
-        uv pip install --python python3.12 --target /opt/python -r /tmp/requirements.txt && \
-        rm /tmp/requirements.txt; \
-    fi && \
-    rm -rf /tmp/idp_common_pkg
+  if [ -n "$DEPENDENCY_GROUP" ]; then \
+  uv export --frozen --no-dev --no-editable --group "${DEPENDENCY_GROUP}" -o "requirements.txt"; \
+  uv pip install --target /opt/python --python 3.12 -r requirements.txt; \
+  else \
+  echo "ERROR: DEPENDENCY_GROUP not specified" && exit 1; \
+  fi
 
 # Final stage - minimal runtime
 FROM public.ecr.aws/lambda/python:3.12-arm64
@@ -44,4 +44,4 @@ COPY ${FUNCTION_PATH}/*.py ${LAMBDA_TASK_ROOT}/
 ENV PYTHONPATH=/opt/python:${LAMBDA_TASK_ROOT}
 
 # Set handler
-CMD ["index.handler"]
+CMD ["index.handler"]

Makefile

@@ -1,49 +1,134 @@
-# Makefile for code quality and formatting
+# Makefile for IDP Accelerator - UV + Hatchling build system
 
 # Define color codes
 RED := \033[0;31m
 GREEN := \033[0;32m
 YELLOW := \033[1;33m
+BLUE := \033[0;34m
 NC := \033[0m  # No Color
 
-# Default target - run both lint and test
-all: lint test
+# Virtual environment and UV paths
+VENV := .venv
+UV := $(shell command -v uv 2> /dev/null)
 
-# Run tests in idp_common_pkg and idp_cli directories
-test:
-	$(MAKE) -C lib/idp_common_pkg test
-	cd idp_cli && python -m pytest -v
+# Default target - ensure UV and venv, then run lint and test
+all: setup lint test
 
-# Run both linting and formatting in one command
-lint: ruff-lint format check-arn-partitions
+# Install UV if not present
+install-uv:
+ifndef UV
+	@printf "$(YELLOW)📦 UV not found. Installing UV...$(NC)\n"
+	@curl -LsSf https://astral.sh/uv/install.sh | sh
+	@printf "$(GREEN)✅ UV installed!$(NC)\n"
+	@printf "$(YELLOW)⚠️  Please restart your shell or run: source ~/.bashrc (or ~/.zshrc)$(NC)\n"
+	@printf "$(YELLOW)⚠️  Then re-run make$(NC)\n"
+	@exit 1
+else
+	@printf "$(GREEN)✅ UV is already installed: $(UV)$(NC)\n"
+endif
+
+# Create .venv using UV if it doesn't exist
+$(VENV):
+	@printf "$(BLUE)🏗️  Creating virtual environment with UV...$(NC)\n"
+	@$(UV) venv
+	@printf "$(GREEN)✅ Virtual environment created at $(VENV)$(NC)\n"
+
+# Setup: ensure UV is installed and create venv
+setup: install-uv $(VENV)
+	@printf "$(GREEN)✅ Setup complete!$(NC)\n"
+
+# Initialize UV environment (creates .venv and syncs all dependencies)
+init: setup
+	@printf "$(BLUE)🚀 Initializing UV workspace...$(NC)\n"
+	@$(UV) sync --all-extras --group dev
+	@printf "$(GREEN)✅ Workspace initialized with all features + dev dependencies$(NC)\n"
+
+# Lock dependencies
+lock: install-uv
+	@printf "$(BLUE)🔒 Locking dependencies...$(NC)\n"
+	@$(UV) lock
+	@printf "$(GREEN)✅ Dependencies locked (uv.lock updated)$(NC)\n"
+
+# Sync local development environment
+sync: setup
+	@printf "$(BLUE)📦 Syncing development environment...$(NC)\n"
+	@$(UV) sync --all-extras --group dev
+	@printf "$(GREEN)✅ Environment synced with all features$(NC)\n"
+
+# Update dependencies
+update: setup
+	@printf "$(BLUE)⬆️  Updating dependencies...$(NC)\n"
+	@$(UV) lock --upgrade
+	@printf "$(GREEN)✅ Dependencies updated$(NC)\n"
+
+# Update specific package
+update-package: setup
+ifndef PKG
+	@printf "$(RED)❌ ERROR: PKG variable not set$(NC)\n"
+	@echo "Usage: make update-package PKG=boto3"
+	@exit 1
+endif
+	@printf "$(BLUE)⬆️  Updating $(PKG)...$(NC)\n"
+	@$(UV) lock --upgrade-package $(PKG)
+	@printf "$(GREEN)✅ $(PKG) updated$(NC)\n"
+
+# Build idp_common package
+build-idp-common: setup
+	@printf "$(BLUE)🔨 Building idp_common package...$(NC)\n"
+	@cd lib/idp_common_pkg && $(UV) build
+	@printf "$(GREEN)✅ idp_common built$(NC)\n"
+
+# Build idp_cli package
+build-idp-cli: setup
+	@printf "$(BLUE)🔨 Building idp_cli package...$(NC)\n"
+	@cd idp_cli && $(UV) build
+	@printf "$(GREEN)✅ idp_cli built$(NC)\n"
+
+# Build all Python packages
+build-packages: build-idp-common build-idp-cli
+	@printf "$(GREEN)✅ All packages built$(NC)\n"
+
+# Run tests in idp_common_pkg and idp_cli directories
+test: setup
+	@printf "$(BLUE)🧪 Running tests...$(NC)\n"
+	@cd lib/idp_common_pkg && $(UV) run --all-extras --group dev pytest -m "unit"
+	@cd idp_cli && $(UV) run --group dev pytest -v
+	@printf "$(GREEN)✅ All tests passed$(NC)\n"
 
 # Run linting checks and fix issues automatically
-ruff-lint:
-	ruff check --fix
+ruff-lint: setup
+	@printf "$(BLUE)🔍 Running ruff linting...$(NC)\n"
+	@$(UV) run --group dev ruff check --fix
+	@printf "$(GREEN)✅ Linting complete$(NC)\n"
 
 # Format code according to project standards
-format:
-	ruff format
+format: setup
+	@printf "$(BLUE)✨ Formatting code...$(NC)\n"
+	@$(UV) run --group dev ruff format
+	@printf "$(GREEN)✅ Formatting complete$(NC)\n"
+
+# Run both linting and formatting in one command
+lint: ruff-lint format check-arn-partitions
 
 # CI/CD version of lint that only checks but doesn't modify files
 # Used in CI pipelines to verify code quality without making changes
-lint-cicd:
-	@echo "Running code quality checks..."
-	@if ! ruff check; then \
-		echo -e "$(RED)ERROR: Ruff linting failed!$(NC)"; \
-		echo -e "$(YELLOW)Please run 'make ruff-lint' locally to fix these issues.$(NC)"; \
+lint-cicd: setup
+	@printf "$(BLUE)Running code quality checks...$(NC)\n"
+	@if ! $(UV) run --group dev ruff check; then \
+		printf "$(RED)ERROR: Ruff linting failed!$(NC)\n"; \
+		printf "$(YELLOW)Please run 'make ruff-lint' locally to fix these issues.$(NC)\n"; \
 		exit 1; \
 	fi
-	@if ! ruff format --check; then \
-		echo -e "$(RED)ERROR: Code formatting check failed!$(NC)"; \
-		echo -e "$(YELLOW)Please run 'make format' locally to fix these issues.$(NC)"; \
+	@if ! $(UV) run --group dev ruff format --check; then \
+		printf "$(RED)ERROR: Code formatting check failed!$(NC)\n"; \
+		printf "$(YELLOW)Please run 'make format' locally to fix these issues.$(NC)\n"; \
 		exit 1; \
 	fi
-	@echo -e "$(GREEN)All code quality checks passed!$(NC)"
+	@printf "$(GREEN)All code quality checks passed!$(NC)\n"
 
 # Check CloudFormation templates for hardcoded AWS partition ARNs and service principals
 check-arn-partitions:
-	@echo "Checking CloudFormation templates for hardcoded ARN partitions and service principals..."
+	@printf "$(BLUE)Checking CloudFormation templates for hardcoded ARN partitions and service principals...$(NC)\n"
 	@FOUND_ISSUES=0; \
 	for template in template.yaml patterns/*/template.yaml patterns/*/sagemaker_classifier_endpoint.yaml options/*/template.yaml; do \
 		if [ -f "$$template" ]; then \
@@ -72,10 +157,66 @@ check-arn-partitions:
 		exit 1; \
 	fi
 
+# Clean up build artifacts and caches
+clean:
+	@printf "$(BLUE)🧹 Cleaning build artifacts...$(NC)\n"
+	@rm -rf .venv
+	@rm -rf lib/idp_common_pkg/dist lib/idp_common_pkg/build lib/idp_common_pkg/*.egg-info
+	@rm -rf idp_cli/dist idp_cli/build idp_cli/*.egg-info
+	@find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+	@find . -type d -name ".pytest_cache" -exec rm -rf {} + 2>/dev/null || true
+	@find . -type d -name ".ruff_cache" -exec rm -rf {} + 2>/dev/null || true
+	@find . -type f -name "*.pyc" -delete 2>/dev/null || true
+	@printf "$(GREEN)✅ Cleanup complete$(NC)\n"
+
+# Clean everything including uv.lock (use with caution)
+clean-all: clean
+	@printf "$(YELLOW)⚠️  Removing uv.lock...$(NC)\n"
+	@rm -f uv.lock
+	@printf "$(GREEN)✅ Deep cleanup complete$(NC)\n"
+
+# Show help
+help:
+	@printf "$(BLUE)IDP Accelerator Makefile - UV-based Python Development$(NC)\n"
+	@echo ""
+	@printf "$(YELLOW)Setup Targets:$(NC)"
+	@echo "  make setup          - Install UV and create .venv (automatically done by other targets)"
+	@echo "  make init           - Initialize workspace with dev dependencies"
+	@echo "  make sync           - Sync development environment"
+	@echo ""
+	@printf "$(YELLOW)Development Targets:$(NC)"
+	@echo "  make lint           - Run linting and formatting"
+	@echo "  make ruff-lint      - Run ruff linting with auto-fix"
+	@echo "  make format         - Format code with ruff"
+	@echo "  make test           - Run all tests"
+	@echo ""
+	@printf "$(YELLOW)Dependency Management:$(NC)"
+	@echo "  make lock           - Lock dependencies (update uv.lock)"
+	@echo "  make update         - Update all dependencies"
+	@echo "  make update-package PKG=<name>  - Update specific package"
+	@echo ""
+	@printf "$(YELLOW)Build Targets:$(NC)"
+	@echo "  make build-idp-common  - Build idp_common package"
+	@echo "  make build-idp-cli     - Build idp_cli package"
+	@echo "  make build-packages    - Build all packages"
+	@echo ""
+	@printf "$(YELLOW)Cleanup Targets:$(NC)"
+	@echo "  make clean          - Remove .venv and build artifacts"
+	@echo "  make clean-all      - Remove .venv, build artifacts, and uv.lock"
+	@echo ""
+	@printf "$(YELLOW)Other Targets:$(NC)"
+	@echo "  make all            - Run setup, lint, and test"
+	@echo "  make check-arn-partitions  - Check CFN templates for GovCloud compatibility"
+	@echo "  make help           - Show this help message"
+
 # A convenience Makefile target that runs 
 commit: lint test
 	$(info Generating commit message...)
 	export COMMIT_MESSAGE="$(shell q chat --no-interactive --trust-all-tools "Understand pending local git change and changes to be committed, then infer a commit message. Return this commit message only" | tail -n 1 | sed 's/\x1b\[[0-9;]*m//g')" && \
 	git add . && \
 	git commit -am "$${COMMIT_MESSAGE}" && \
 	git push
+
+.PHONY: all setup install-uv init lock sync update update-package \
+        build-idp-common build-idp-cli build-packages test ruff-lint format \
+        lint lint-cicd check-arn-partitions clean clean-all help commit