diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..13ed276 --- /dev/null +++ b/Makefile @@ -0,0 +1,71 @@ +.PHONY: help test test-cov lint format typecheck check build push publish deploy clean pre-commit + +IMAGE_NAME := ghcr.io/eopf-explorer/data-pipeline +TAG := v0 + +help: ## Show this help message + @echo "๐Ÿš€ EOPF GeoZarr Data Pipeline" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' + +setup: ## Install dependencies and pre-commit hooks + @echo "๐Ÿ“ฆ Installing dependencies..." + uv sync --all-extras + @echo "๐Ÿ”ง Installing pre-commit hooks..." + uv run pre-commit install + +test: ## Run tests with pytest + @echo "๐Ÿงช Running tests..." + uv run pytest -v + +test-cov: ## Run tests with coverage report + @echo "๐Ÿงช Running tests with coverage..." + uv run pytest --cov=scripts --cov-report=html --cov-report=term + @echo "๐Ÿ“Š Coverage report: htmlcov/index.html" + +lint: ## Check code style with ruff + @echo "๐Ÿ” Linting with ruff..." + uv run ruff check . + +format: ## Auto-format code with ruff + @echo "โœจ Formatting with ruff..." + uv run ruff format . + +typecheck: ## Type check with mypy + @echo "๐Ÿ” Type checking with mypy..." + uv run mypy scripts/ + +pre-commit: ## Run all pre-commit hooks + @echo "๐Ÿ”ง Running pre-commit hooks..." + uv run pre-commit run --all-files + +check: lint typecheck test ## Run all checks (lint + typecheck + test) + @echo "โœ… All checks passed!" + +build: ## Build Docker image + @echo "Building $(IMAGE_NAME):$(TAG) ..." + docker build --platform linux/amd64 \ + -f docker/Dockerfile \ + -t $(IMAGE_NAME):$(TAG) \ + -t $(IMAGE_NAME):latest \ + . + +push: + @echo "Pushing $(IMAGE_NAME):$(TAG) ..." + docker push $(IMAGE_NAME):$(TAG) + docker push $(IMAGE_NAME):latest + +publish: build push + @echo "Published $(IMAGE_NAME):$(TAG)" + +deploy: + kubectl apply -f workflows/template.yaml + kubectl apply -f workflows/eventsource.yaml + kubectl apply -f workflows/sensor.yaml + +clean: + @echo "Cleaning generated files..." + find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + find . -type f -name '*.pyc' -delete 2>/dev/null || true + rm -rf .pytest_cache .mypy_cache .ruff_cache htmlcov .coverage + @echo "โœ“ Clean complete" diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..5f9e705 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,36 @@ +FROM python:3.11-slim + +# System dependencies (including GDAL for rasterio) +RUN apt-get update && apt-get install -y \ + git \ + curl \ + ca-certificates \ + build-essential \ + libgdal-dev \ + gdal-bin \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install uv for fast dependency resolution +RUN pip install -U pip uv + +# Cachebust for data-model installation (change timestamp to force fresh install) +ARG CACHEBUST=2025-10-06T11:00:00Z + +# Install eopf-geozarr from minimal fix branch +# Includes critical set_spatial_dims() fix before write_crs() calls +RUN uv pip install --system --no-cache \ + git+https://github.com/EOPF-Explorer/data-model.git@fix/spatial-dims-minimal \ + pystac>=1.10.0 \ + httpx>=0.27.0 \ + boto3>=1.34.0 + +# Force fresh copy of scripts (invalidate cache) +ARG SCRIPTS_VERSION=2025-10-06T02:05:00Z + +# Copy scripts +COPY scripts/ /app/scripts/ +RUN chmod +x /app/scripts/*.py + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test new file mode 100644 index 0000000..21a23c7 --- /dev/null +++ b/docker/Dockerfile.test @@ -0,0 +1,31 @@ +FROM --platform=linux/amd64 python:3.11-slim + +# System dependencies +RUN apt-get update && apt-get install -y \ + git \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install uv for fast dependency resolution +RUN pip install -U pip uv + +# Install eopf-geozarr from TEST BRANCH + dependencies +# uv handles GDAL and geospatial deps better than pip (installs from binary wheels when available) +RUN uv pip install --system --no-cache \ + git+https://github.com/EOPF-Explorer/data-model.git@test/spatial-ref-fix \ + pystac>=1.10.0 \ + httpx>=0.27.0 \ + boto3>=1.34.0 + +# Copy scripts +COPY scripts/ /app/scripts/ +RUN chmod +x /app/scripts/*.py + +# ARG for invalidating cache - update this timestamp to force rebuild +# 2025-10-05T21:30:00Z - Add projection metadata extraction to STAC assets (TiTiler fix) +ARG CACHEBUST=20251005213000 + +CMD ["/bin/bash"]