set up package for dev with uv (#8589)

* set up package for dev with uv

* update CI to use uv

* use uv to download python in CI

* cache python

* do not cache python

* update earthfile

* update some other stuff

* update poetry run to uv run

* Apply suggestions from code review

Co-authored-by: Copilot <[email protected]>

* update dev container

* update dev container

---------

Co-authored-by: Copilot <[email protected]>

Author: VIKTORVAV99

.devcontainer/devcontainer.json

@@ -1,23 +1,16 @@
 {
   "name": "Electricity Maps Dev",
-  "image": "mcr.microsoft.com/devcontainers/universal:2",
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.10-bookworm",
   "features": {
-    "ghcr.io/devcontainers/features/python:1": {
-      "version": "3.10"
-    },
-    "ghcr.io/devcontainers/features/node:1": {
-      "version": "lts"
-    },
-    "ghcr.io/devcontainers-contrib/features/poetry:2": {},
-    "ghcr.io/devcontainers-contrib/features/pnpm:2": {}
+    "ghcr.io/devcontainer-community/devcontainer-features/astral.sh-uv:1": {}
   },
+  "postCreateCommand": "uv sync --group parser --group dev --group scripts",
   "customizations": {
     "vscode": {
       "extensions": [
         "ms-python.vscode-pylance",
         "ms-python.python",
         "esbenp.prettier-vscode",
-        "dbaeumer.vscode-eslint",
         "redhat.vscode-yaml",
         "charliermarsh.ruff"
       ]

.github/pull_request_template.md

@@ -12,5 +12,5 @@
 
 ### Double check
 
-- [ ] I have tested my parser changes locally with `poetry run test_parser "zone_key"`
-- [ ] I have run `pnpx prettier@2 --write .` and `poetry run format` in the top level directory to format my changes.
+- [ ] I have tested my parser changes locally with `uv run test_parser "zone_key"`
+- [ ] I have run `pnpx prettier@2 --write .` and `uv run format` in the top level directory to format my changes.

.github/workflows/CI.yml

@@ -495,34 +495,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v6
-      - name: Install poetry
-        run: |
-          pipx install "poetry==2.*"
-      - name: Get poetry version
-        id: get-poetry-version
-        run: |
-          echo "$(poetry --version | awk '{print "poetry-version="$3}' | tr -d '()')" >> $GITHUB_OUTPUT
-      - name: Setup Python
-        uses: actions/setup-python@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
-          python-version-file: '.python-version'
-      - name: Restore cache
-        id: cache
-        uses: actions/cache@v5
-        with:
-          path: .venv
-          key: ${{ runner.os }}-venv-poetry_${{steps.get-poetry-version.outputs.poetry-version}}-${{ hashFiles('poetry.lock') }}
-          restore-keys: ${{ runner.os }}-venv-poetry_${{steps.get-poetry-version.outputs.poetry-version}}
+          enable-cache: true
+          cache-suffix: 'formatting'
+          python-version: '3.10'
       - name: Install Dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          poetry install -E parsers --sync
+          uv sync --group parsers
       - name: Check formatting
         run: |
-          poetry run ruff format . --check
+          uv run ruff format . --check
       - name: Check linting
         run: |
-          poetry run ruff check . --output-format=github
+          uv run ruff check . --output-format=github
 
   python_tests:
     name: Python / Tests
@@ -539,35 +526,22 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v6
-      - name: Install poetry
-        run: |
-          pipx install "poetry==2.*"
-      - name: Get poetry version
-        id: get-poetry-version
-        run: |
-          echo "$(poetry --version | awk '{print "poetry-version="$3}' | tr -d '()')" >> $GITHUB_OUTPUT
-      - name: Setup Python
-        uses: actions/setup-python@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
+          enable-cache: true
+          cache-suffix: 'tests'
           python-version: ${{ matrix.python-version }}
-      - name: Restore cache
-        id: cache
-        uses: actions/cache@v5
-        with:
-          path: .venv
-          key: ${{ runner.os }}-venv-python_${{ matrix.python-version }}-poetry_${{steps.get-poetry-version.outputs.poetry-version}}-${{ hashFiles('poetry.lock') }}
-          restore-keys: ${{ runner.os }}-venv-python_${{ matrix.python-version }}-poetry_${{steps.get-poetry-version.outputs.poetry-version}}
       - name: Install libxml2-dev and tesseract-ocr
         run: |
           sudo apt-get update
           sudo apt-get install libxml2-dev tesseract-ocr tesseract-ocr-eng
-      - name: Install Poetry dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Install Dependencies
         run: |
-          poetry install -E parsers --sync
+          uv sync --group parsers --group dev
       - name: Run tests
         run: |
-          poetry run test
+          uv run test
 
   # Config checks
   validate_config_filenames:

.github/workflows/create-release.yml

@@ -11,6 +11,7 @@ on:
           - patch
           - minor
           - major
+          - dev
 jobs:
   create-release:
     runs-on: ubuntu-latest
@@ -23,7 +24,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v6
         with:
-          ref: master
           token: ${{ secrets.BOT_GITHUB_TOKEN }}
       - name: Configure git
         run: |
@@ -42,13 +42,17 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: '3.10.13'
-      - name: Install Poetry
-        run: |
-          python -m pip install --upgrade pip
-          pip install poetry==2.1.1
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
       - name: Bump version
         run: |
-          poetry version ${{ inputs.type }}
+          if [[ "${{ inputs.type }}" == "dev" ]]; then
+            # Bump patch first to ensure we are moving forward, then add dev suffix
+            # This handles 1.2.1 -> 1.2.2.dev0
+            uv version --bump patch --bump dev
+          else
+            uv version --bump ${{ inputs.type }}
+          fi
       - name: Commit changes
         run: |
           git add pyproject.toml
@@ -57,13 +61,11 @@ jobs:
         uses: ad-m/[email protected]
         with:
           github_token: ${{ secrets.BOT_GITHUB_TOKEN }}
-          branch: master
+          branch: ${{ github.ref_name }}
       - name: Build package
         run: |
-          poetry build
+          uv build
       - name: Publish package
         run: |
           export ACCESS_TOKEN=$(gcloud auth print-access-token)
-          poetry config keyring.enabled false
-          poetry config repositories.internal https://europe-west1-python.pkg.dev/tmrow-152415/libs/
-          poetry publish --repository=internal --skip-existing --username=oauth2accesstoken --password "$ACCESS_TOKEN"
+          uv publish --publish-url https://europe-west1-python.pkg.dev/tmrow-152415/libs/ --username oauth2accesstoken --password "$ACCESS_TOKEN"

CONTRIBUTING.md

@@ -46,13 +46,13 @@ The zone configurations hold information such as the installed capacity, which p
 
 To get started with editing the parsers use the following steps:
 
-1. Run `poetry install -E parsers` to install all needed dependencies.
-2. Use `poetry run test_parser ZONE_KEY` to test any parser changes.
+1. Run `uv sync --group parsers` to install all needed dependencies.
+2. Use `uv run test_parser ZONE_KEY` to test any parser changes.
 
-Note: This requires you to have [Python 3.10][python homepage] and [Poetry][poetry homepage] installed, you can see their respective installation guides here:
+Note: This requires you to have [Python 3.10][python homepage] and [uv][uv homepage] installed, you can see their respective installation guides here:
 
 - [Downloading Python][python install guide]
-- [poetry installation][poetry install guide]
+- [uv installation][uv install guide]
 
 ### Parser information
 
@@ -66,7 +66,7 @@ For an example of how a parser can look we have an example here: </br> [electric
 
 We use [black][black homepage] and [isort][isort homepage] as code formatters for python which is automatically checked in the CI job `Python / Formatting`.
 
-If this jobs fails and you need to manually format the code you can run `poetry run format` in the top level of the repository.
+If this jobs fails and you need to manually format the code you can run `uv run format` in the top level of the repository.
 
 Check the [wiki page][wiki python code formatting] for more details and tips.
 
@@ -181,10 +181,10 @@ If it is frontend changes it will be deployed to our staging environment at http
 
 <!-- Link definitions to keep the text clean -->
 
-[poetry homepage]: https://python-poetry.org/
+[uv homepage]: https://github.com/astral-sh/uv
 [python homepage]: https://www.python.org/
 [python install guide]: https://wiki.python.org/moin/BeginnersGuide/Download
-[poetry install guide]: https://python-poetry.org/docs/#installation
+[uv install guide]: https://docs.astral.sh/uv/getting-started/installation/
 [example parser]: https://github.com/electricitymaps/electricitymaps-contrib/blob/master/electricitymap/contrib/parsers/examples/example_parser.py
 [black homepage]: https://github.com/psf/black
 [isort homepage]: https://pycqa.github.io/isort/

Earthfile

@@ -1,5 +1,5 @@
 VERSION 0.8
-FROM python:3.10
+FROM astral/uv:python3.10-bookworm-slim
 WORKDIR /contrib
 
 linting-files:
@@ -33,17 +33,15 @@ api-files:
   COPY config/zone_names.json ./config/zone_names.json
   SAVE ARTIFACT .
 
-poetry-lock:
-  COPY poetry.lock .
+uv-lock:
+  COPY uv.lock .
   SAVE ARTIFACT .
 
 prepare:
   FROM +src-files-with-parsers
-  RUN pip install "poetry==2.*"
   RUN apt-get update && apt-get install -y python3-opencv tesseract-ocr tesseract-ocr-jpn tesseract-ocr-eng libgl1
-  RUN poetry config virtualenvs.create false
-  COPY poetry.lock .
-  RUN poetry install --compile -E parsers
+  COPY uv.lock .
+  RUN uv sync --frozen --group parsers --group dev --compile-bytecode
 
 build:
   FROM +prepare
@@ -53,7 +51,7 @@ test:
   COPY tests ./tests
   COPY web/src/utils/constants.ts ./web/src/utils/constants.ts # TODO: python tests should not depend on this js file
   COPY web/geo/world.geojson ./web/geo/world.geojson
-  RUN poetry run check
+  RUN uv run check
 
 # includes both test target and build target here to make sure both can work
 # we can split into two later if required

capacity_update.py

@@ -1,5 +1,5 @@
 """
-Usage: poetry run update_capacity --zone FR --target_datetime "2022-01-01"
+Usage: uv run update_capacity --zone FR --target_datetime "2022-01-01"
 """
 
 import logging
@@ -33,8 +33,8 @@ def capacity_update(
     \n
     Examples
     -------
-    >>> poetry run capacity_update --zone FR --target_datetime "2022-01-01"
-    >>> poetry run capacity_update --source ENTSOE --target_datetime "2022-01-01"
+    >>> uv run capacity_update --zone FR --target_datetime "2022-01-01"
+    >>> uv run capacity_update --source ENTSOE --target_datetime "2022-01-01"
     """
     logging.basicConfig(
         level=logging.DEBUG,

electricitymap/contrib/capacity_parsers/README.md

@@ -84,7 +84,7 @@ Before opening a PR to update capacity data, you should check the following:
 
 For some zones, we have developed capacity parsers which collect the data automatically.
 
-The update of capacity configurations can be done in the `contrib` repo using `poetry run capacity_update`.
+The update of capacity configurations can be done in the `contrib` repo using `uv run capacity_update`.
 
 The `capacity_update` function has the following arguments:
 | Argument | Description |
@@ -97,11 +97,11 @@ The `capacity_update` function has the following arguments:
 Here is a list of examples:
 
 ```{python}
-poetry run capacity_update --zone DK-DK1 --target_datetime "2023-01-01 --update_aggregate True"
+uv run capacity_update --zone DK-DK1 --target_datetime "2023-01-01 --update_aggregate True"
 ```
 
 ```{python}
-poetry run capacity_update --source EIA --target_datetime "2023-06-01"
+uv run capacity_update --source EIA --target_datetime "2023-06-01"
 ```
 
 The following zones can be updated with a parser are listed on our wiki page [Capacity update process](https://github.com/electricitymaps/electricitymaps-contrib/wiki/Capacity-update-process)

parsers.dockerfile

@@ -3,10 +3,9 @@
 # Run with  "$ docker run parsers US-CAL-CISO production"
 # If needed, pass any token env var using "docker run -e TOKEN=<token>"
 
-FROM python:3.10
+FROM astral/uv:python3.10-bookworm-slim
 WORKDIR /workspace
 RUN apt-get update && apt-get install -y python3-opencv tesseract-ocr tesseract-ocr-jpn tesseract-ocr-eng libgl1
-RUN pip install "poetry==2.*"
 COPY . .
-RUN poetry install -E parsers
-ENTRYPOINT ["poetry", "run", "test_parser"]
+RUN uv sync --frozen --group parsers --group dev --compile-bytecode
+ENTRYPOINT ["uv", "run", "test_parser"]