chore(ci): Use make targets in CI for consistency

.github/hooks/commit-msg

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# Allows for standard conventional commits OR git's auto-squash prefixes.
+commit_regex="^((fixup|squash|amend)!|(feat|fix|docs|style|refactor|perf|test|chore)!?(\(.+\))?:) .+$"
+
+if ! grep -iqE "$commit_regex" "$1"; then
+  echo "Aborting commit." >&2
+  echo "Commit messages should follow the conventional commit format or use a git autosquash prefix." >&2
+  echo "" >&2
+  echo "Conventional format: 'type(scope): message'" >&2
+  echo "Allowed types: feat, fix, docs, style, refactor, perf, test, chore" >&2
+  echo "See https://www.conventionalcommits.org/en/v1.0.0/#summary" >&2
+  echo "" >&2
+  echo "Autosquash prefixes: 'fixup!', 'squash!', 'amend!'" >&2
+  echo "Example: 'fixup! feat(user): add new login button'" >&2
+  exit 1
+fi

.github/hooks/pre-push

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+echo "Running pre-push checks..."
+
+# Ensure in repo root
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd "$REPO_ROOT" || exit 1
+
+echo " * Checking linting..."
+# Run linting checks (exit code 0 indicates nothing to fix)
+make lint.dryrun &> /dev/null
+if [ ! $? -eq 0 ]; then
+	  make lint &> /dev/null
+		echo "ERROR: Linting/typechecking failed. Files may have been reformatted."
+		echo "       Please stage and commit these changes before pushing, and ensure typesafety."
+		echo "       (hint: 'git commit -am \"chore(repo): Code formatting\"')"
+		exit 1
+fi
+

.github/pull-request-template.md

@@ -0,0 +1,16 @@
+### Contribution Checklist
+
+- [ ] Have you followed the Open Climate Fix [Contribution Guidelines](https://github.com/openclimatefix#github-contributions)?
+- [ ] Have you referenced the [Issue](https://github.com/openclimatefix/satellite-consumer/issues) this PR addresses, where applicable?
+- [ ] Have you checked to ensure there aren't other open [Pull Requests](https://github.com/openclimatefix/satellite-consumer/pulls) for the same change?
+- [ ] Have you added a summary of the changes?
+- [ ] Have you written new tests for your changes, where applicable?
+- [ ] Have you successfully run `make lint` with your changes locally?
+- [ ] Have you successfully run `make test` with your changes locally?
+
+> [!Warning]
+> PRs may be closed if all the above boxes are not checked.
+
+
+### Changes in this Pull Request
+

.github/workflows/build.yml

@@ -0,0 +1,58 @@
+name: Build Containers
+
+on:
+  push:
+    branches-ignore: ['main']
+    paths-ignore: ['README.md', '.github/workflows/test.yml']
+    tags: ['v*.*.*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-container:
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Extract metadata (tags, labels) for Container
+        id: meta
+        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+
+      - name: Build and push Docker images
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # 6.18.0
+        with:
+          context: .
+          file: Dockerfile
+          platforms: linux/amd64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+        env:
+          DOCKER_BUILD_RECORD_RETENTION_DAYS: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && '0' || '7' }}
+

.github/workflows/tag.yml

@@ -0,0 +1,38 @@
+name: Bump Tag
+
+on:
+  pull_request:
+    types: ["closed"]
+    branches: ["main"]
+
+jobs:
+
+  # Define an autotagger job that creates tags on changes to master
+  bump-tag:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
+          fetch-depth: 0
+          token: ${{ secrets.OCF_BOT_PAT_TOKEN }}
+
+      # Bump tag according to conventional commit rules
+      # See https://www.conventionalcommits.org/en/v1.0.0
+      - name: Bump version and push tag
+        uses: anothrNick/github-tag-action@a2c70ae13a881faf2b4953baaa9e49731997ab36 # v1.67.0
+        id: tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.OCF_BOT_PAT_TOKEN }}
+          RELEASE_BRANCHES: main
+          WITH_V: true
+          GIT_API_TAGGING: false
+          MAJOR_STRING_TOKEN: "BREAKING CHANGE"
+          MINOR_STRING_TOKEN: "feat("
+          PATCH_STRING_TOKEN: "fix("
+          DEFAULT_BUMP: none

.github/workflows/test.yml

@@ -0,0 +1,33 @@
+name: Unit Test
+
+on:  
+  push:
+    branches-ignore: [ "main" ]
+    paths-ignore: ['README.md']
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Set up UV
+        uses: astral-sh/setup-uv@e92bafb6253dcd438e0484186d7669ea7a8ca1cc # v6.4.3
+        with:
+          enable-cache: true
+
+      - name: Check linting / typechecking / formatting
+        run: make lint.dryrun
+
+      - name: Run unit tests
+        run: make test
+
+      - name: Test Summary
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
+        with:
+          paths: "unit-tests.xml"
+          output: "test-summary.md"
+        if: always()
+

.gitignore

@@ -50,6 +50,7 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+unit-tests.xml
 *.cover
 *.py,cover
 .hypothesis/

Makefile

@@ -1,16 +1,23 @@
-.PHONY: test
-test:
-	uv run python -m unittest discover -s src/satellite_consumer -p "test_*.py"
+.PHONY: init
+init:
+	@git config --local core.hooksPath .github/hooks
+	@uv --version &> /dev/null || (echo "uv is not installed. See https://docs.astral.sh/uv/getting-started/installation/" && exit 1)
+	@uv sync
 
-.PHONY: test-cov
-test-cov:
-	uv run python -m xmlrunner discover -s src/satellite_consumer -p "test_*.py"
+.PHONY: lint.dryrun
+lint.dryrun:
+	@uv run ruff check .
+	@uv run ruff format --check .
+	@uv run mypy .
 
 .PHONY: lint
 lint:
-	uv run ruff check --fix .
+	@uv run ruff check --fix .
+	@uv run ruff format .
+	@uv run mypy .
+
+.PHONY: test
+test:
+	@uv run python -m xmlrunner discover -s src/satellite_consumer -p "test_*.py" --output-file="unit-tests.xml" --outsuffix=""
 
-.PHONY: typecheck
-typecheck:
-	uv run mypy .
 

pyproject.toml

@@ -129,7 +129,7 @@ select = [
 fixable = ["ALL"]
 ignore = [
     "D203", "D213", "D215", "D400", "D401", "D404", "D406",
-    "D407", "D408", "D409", "D413",
+    "D407", "D408", "D409", "D413", "COM812", # handled by formatter
 ]
 
 [tool.ruff.lint.per-file-ignores]

src/satellite_consumer/cmd/main.py

@@ -75,37 +75,39 @@ def main() -> None:
         dt_range[1].isoformat(),
     )
 
-    asyncio.run(consume.consume_to_store(
-        dt_range=dt_range,
-        cadence_mins=conf.get_int(f"satellites.{sat}.cadence_mins"),
-        product_id=conf.get_string(f"satellites.{sat}.product_id"),
-        filter_regex=conf.get_string(f"satellites.{sat}.file_filter_regex"),
-        raw_zarr_paths=(
-            conf.get_string("consumer.raw_path"),
-            conf.get_string("consumer.zarr_path"),
+    asyncio.run(
+        consume.consume_to_store(
+            dt_range=dt_range,
+            cadence_mins=conf.get_int(f"satellites.{sat}.cadence_mins"),
+            product_id=conf.get_string(f"satellites.{sat}.product_id"),
+            filter_regex=conf.get_string(f"satellites.{sat}.file_filter_regex"),
+            raw_zarr_paths=(
+                conf.get_string("consumer.raw_path"),
+                conf.get_string("consumer.zarr_path"),
+            ),
+            channels=channels,
+            resolution_meters=resolution,
+            crop_region_lonlat=crop_region_lonlat,
+            eumetsat_credentials=(
+                conf.get_string("credentials.eumetsat.key"),
+                conf.get_string("credentials.eumetsat.secret"),
+            ),
+            use_icechunk=conf.get_bool("consumer.use_icechunk"),
+            aws_credentials=(
+                conf.get_string("credentials.aws.access_key_id", None),
+                conf.get_string("credentials.aws.secret_access_key", None),
+                conf.get_string("credentials.aws.endpoint_url", None),
+                conf.get_string("credentials.aws.region", None),
+            ),
+            gcs_credentials=conf.get_string("credentials.gcs.application_credentials", None),
+            dims_chunks_shards=(
+                conf.get_list(f"satellites.{sat}.dimensions"),
+                conf.get_list(f"satellites.{sat}.chunks"),
+                conf.get_list(f"satellites.{sat}.shards"),
+            ),
+            buffer_size=conf.get_int("consumer.buffer_size"),
         ),
-        channels=channels,
-        resolution_meters=resolution,
-        crop_region_lonlat=crop_region_lonlat,
-        eumetsat_credentials=(
-            conf.get_string("credentials.eumetsat.key"),
-            conf.get_string("credentials.eumetsat.secret"),
-        ),
-        use_icechunk=conf.get_bool("consumer.use_icechunk"),
-        aws_credentials=(
-            conf.get_string("credentials.aws.access_key_id", None),
-            conf.get_string("credentials.aws.secret_access_key", None),
-            conf.get_string("credentials.aws.endpoint_url", None),
-            conf.get_string("credentials.aws.region", None),
-        ),
-        gcs_credentials=conf.get_string("credentials.gcs.application_credentials", None),
-        dims_chunks_shards=(
-            conf.get_list(f"satellites.{sat}.dimensions"),
-            conf.get_list(f"satellites.{sat}.chunks"),
-            conf.get_list(f"satellites.{sat}.shards"),
-        ),
-        buffer_size=conf.get_int("consumer.buffer_size"),
-    ))
+    )
 
     log.info(f"sat consumer finished in {time.time() - prog_start!s}")
 

src/satellite_consumer/consume.py

@@ -30,8 +30,10 @@
 log = logging.getLogger(__name__)
 
 
-T = TypeVar("T") # Type of the input
-R = TypeVar("R") # Type of the return
+T = TypeVar("T")  # Type of the input
+R = TypeVar("R")  # Type of the return
+
+
 async def _buffered_apply(
     item_iter: Iterator[T],
     func: Callable[[T], R],