feat: argo workflow orchestration and dependencies

.gitignore

@@ -0,0 +1,63 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Linting
+.ruff_cache/
+
+# Environment
+.env
+.env.local
+.env.*.local
+.venv
+env/
+venv/
+ENV/
+*.local.yaml
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Jupyter
+.ipynb_checkpoints/
+
+# Kubernetes
+.work/
+kubeconfig*
+.kube/
+
+# Temporary files
+*.tmp
+*.log
+runs/
+generated/
+.archive/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project-specific
+*.zarr
+out/

.pre-commit-config.yaml

@@ -0,0 +1,33 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.4
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.2
+    hooks:
+      - id: mypy
+        exclude: ^(tests/.*|examples/.*|notebooks/.*)$
+        additional_dependencies:
+          - types-boto3
+          - pystac>=1.10.0
+          - httpx>=0.27.0
+
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: v0.24.1
+    hooks:
+      - id: validate-pyproject
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+        args: ["--unsafe"]  # Allow custom tags in Argo YAML
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: mixed-line-ending

mypy.ini

@@ -0,0 +1,17 @@
+[mypy]
+python_version = 3.11
+warn_return_any = True
+warn_unused_configs = True
+disallow_untyped_defs = False
+check_untyped_defs = True
+
+# Ignore missing library stubs (these are external dependencies)
+[mypy-rasterio.*]
+ignore_missing_imports = True
+
+[mypy-s3fs.*]
+ignore_missing_imports = True
+
+# Zarr has complex types that confuse mypy - we trust runtime behavior
+[mypy-zarr.*]
+ignore_errors = True

pyproject.toml

@@ -0,0 +1,156 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "data-pipeline"
+version = "1.0.0"
+description = "GeoZarr conversion and STAC registration pipeline for Sentinel satellite data"
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "EOPF Explorer", email = "[email protected]" }]
+keywords = ["geozarr", "stac", "sentinel", "earth-observation", "zarr"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: GIS",
+]
+
+dependencies = [
+    "pystac>=1.10.0",
+    "httpx>=0.27.0",
+    "boto3>=1.34.0",
+    "xarray>=2024.0.0",
+    "zarr>=2.18.0",
+    "s3fs>=2024.0.0",
+    "click>=8.1.0",
+    "pika>=1.3.0",
+    "tenacity>=8.0.0",
+    "requests>=2.31.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-cov>=4.1.0",
+    "pytest-mock>=3.12.0",
+    "ruff>=0.8.0",
+    "mypy>=1.11.0",
+    "pre-commit>=3.7.0",
+    "types-boto3>=1.0.2",
+]
+
+notebooks = [
+    "ipykernel>=6.29.0",
+    "ipywidgets>=8.1.0",
+    "ipyleaflet>=0.18.0",
+    "pystac-client>=0.7.0",
+    "python-dotenv>=1.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/EOPF-Explorer/data-pipeline"
+Repository = "https://github.com/EOPF-Explorer/data-pipeline"
+Documentation = "https://github.com/EOPF-Explorer/data-pipeline#readme"
+
+[tool.hatch.build.targets.wheel]
+packages = ["scripts"]
+
+[tool.pytest.ini_options]
+minversion = "8.0"
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--strict-markers",
+    "--tb=short",
+    "--cov=scripts",
+    "--cov-report=term-missing",
+    "--cov-report=html",
+]
+markers = [
+    "unit: Unit tests (fast, no external dependencies)",
+    "integration: Integration tests (may use mocked external services)",
+]
+
+[tool.coverage.run]
+source = ["scripts"]
+omit = ["*/tests/*", "*/__pycache__/*", "*/.*"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@abstractmethod",
+]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+indent-width = 4
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+    "ARG", # flake8-unused-arguments
+    "SIM", # flake8-simplify
+]
+ignore = [
+    "E501",   # line too long (handled by formatter)
+    "B008",   # do not perform function calls in argument defaults
+    "ARG001", # unused function argument (common in event handlers)
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "ARG",  # Unused function args in fixtures
+    "S101", # Use of assert
+]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+line-ending = "auto"
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+strict_equality = true
+exclude = ["examples/"]
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+
+[[tool.mypy.overrides]]
+module = "notebooks.*"
+ignore_errors = true
+
+[[tool.mypy.overrides]]
+module = ["boto3.*", "botocore.*"]
+ignore_missing_imports = true