Merge pull request #850 from CenterForOpenScience/feature/share-cleanupgrade-2025

[project][ENG-7225][ENG-8319] share clean(up)grade 2025 (milestone 2: upgrade)

Author: aaxelb

.docker-compose.env

@@ -1,7 +1,6 @@
 CELERY_BROKER_URL=amqp://guest:guest@rabbitmq:5672
 DATABASE_HOST=postgres
 ELASTICSEARCH8_URL=https://elastic8:9200/
-# ELASTICSEARCH5_URL=http://elasticsearch:9200/
 LOGIN_REDIRECT_URL=http://localhost:8003/
 OSF_API_URL=http://localhost:8000
 RABBITMQ_HOST=rabbitmq

.dockerignore

@@ -1,28 +1,8 @@
 **/.git/
 **/*.pyc
-README.md
 Dockerfile
 docker-compose.yml
 .dockerignore
 .gitignore
 celerybeat.pid
-/static/
-
-/au.*/
-/be.*/
-/br.*/
-/ca.*/
-/ch.*/
-/com.*/
-/edu.*/
-/et.*/
-/eu.*/
-/gov.*/
-/info.*/
-/io.*/
-/org.*/
-/pt.*/
-/ru.*/
-/tr.*/
-/uk.*/
-/za.*/
+.venv/

.github/workflows/run_tests.yml

@@ -9,12 +9,35 @@ permissions:
   checks: write  # for coveralls
 
 jobs:
+  lint_and_type:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+          cache: 'poetry'
+
+      - name: install despondencies
+        run: poetry install --with dev
+
+      - name: flake it
+        run: poetry run flake8 .
+
+      - name: type-check
+        run: poetry run mypy trove
+
   run_tests:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.10']  # TODO: 3.11, 3.12
-        postgres-version: ['15', '17']
+        python-version: ['3.13']
+        postgres-version: ['17']
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -30,19 +53,13 @@ jobs:
         ports:
           - 5432:5432
       elasticsearch8:
-        image: elasticsearch:8.7.0
+        image: elasticsearch:8.18.1
         env:
           xpack.security.enabled: false
           node.name: singlenode
           cluster.initial_master_nodes: singlenode
         ports:
           - 9208:9200
-      elasticsearch5:
-        image: elasticsearch:5.4
-        env:
-          ES_JAVA_OPTS: "-Xms512m -Xmx512m"
-        ports:
-          - 9205:9200
       rabbitmq:
         image: rabbitmq:management
         ports:
@@ -54,32 +71,25 @@ jobs:
       - name: install non-py dependencies
         run: sudo apt-get update && sudo apt-get install -y libxml2-dev libxslt1-dev libpq-dev git gcc
 
-      - name: set up python${{ matrix.python-version }}
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: setup python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: pip
-          cache-dependency-path: |
-            requirements.txt
-            dev-requirements.txt
-
-      - name: install py dependencies
-        run: pip install -r dev-requirements.txt
+          cache: 'poetry'
 
-      - name: install share
-        run: python setup.py develop
-
-      - name: flake it
-        run: flake8 .
+      - name: install despondencies
+        run: poetry install --with dev
 
       - name: run tests
         run: |
-          coverage run -m pytest --create-db -x
-          coverage xml -o _shtrove_coverage.xml
+          poetry run coverage run -m pytest --create-db -x
+          poetry run coverage xml -o _shtrove_coverage.xml
         env:
           DATABASE_PASSWORD: postgres
           ELASTICSEARCH8_URL: http://localhost:9208/
-          # ELASTICSEARCH5_URL: http://localhost:9205/
 
       - name: coveralls
         uses: coverallsapp/github-action@v2

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim-bullseye as app
+FROM python:3.13-slim-bullseye AS app
 
 RUN apt-get update \
     && apt-get install -y \
@@ -22,38 +22,58 @@ RUN apt-get update \
 
 RUN update-ca-certificates
 
+# set working directory once, use relative paths from "./"
 RUN mkdir -p /code
 WORKDIR /code
 
-RUN pip install -U pip
-RUN pip install uwsgi==2.0.21
+###
+# python dependencies
 
-COPY ./requirements.txt /code/requirements.txt
-COPY ./constraints.txt /code/constraints.txt
+# note: installs dependencies on the system, roundabouts `/usr/local/lib/python3.13/site-packages/`
 
-RUN pip install --no-cache-dir -c /code/constraints.txt -r /code/requirements.txt
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_OPTIONS_ALWAYS_COPY=1 \
+    POETRY_VIRTUALENVS_CREATE=0 \
+    POETRY_VIRTUALENVS_IN_PROJECT=0 \
+    POETRY_CACHE_DIR=/tmp/poetry-cache \
+    POETRY_HOME=/tmp/poetry-venv
 
-RUN apt-get remove -y \
-    gcc \
-    zlib1g-dev
+RUN python -m venv $POETRY_HOME
+
+RUN $POETRY_HOME/bin/pip install poetry==2.1.3
+
+COPY pyproject.toml poetry.lock ./
+
+RUN $POETRY_HOME/bin/poetry install --compile --no-root
+
+COPY ./ ./
 
-COPY ./ /code/
+RUN $POETRY_HOME/bin/poetry install --compile --only-root
 
 RUN python manage.py collectstatic --noinput
 
 ARG GIT_TAG=
 ARG GIT_COMMIT=
-ENV VERSION ${GIT_TAG}
-ENV GIT_COMMIT ${GIT_COMMIT}
-
-RUN python setup.py develop
+ENV VERSION=${GIT_TAG}
+ENV GIT_COMMIT=${GIT_COMMIT}
 
 CMD ["python", "manage.py", "--help"]
 
+### Dev
+FROM app AS dev
+
+RUN $POETRY_HOME/bin/poetry install --compile --only dev
+
 ### Dist
 FROM app AS dist
 
-### Dev
-FROM app AS dev
+RUN $POETRY_HOME/bin/poetry install --compile --only deploy
 
-RUN pip install --no-cache-dir -c /code/constraints.txt -r /code/dev-requirements.txt
+# remove packages needed only for install
+RUN apt-get remove -y \
+        gcc \
+        zlib1g-dev \
+    && apt-get clean \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /tmp/poetry-*

dev-requirements.txt

@@ -32,7 +32,7 @@ services:
   ####################
 
   elastic8_setup:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3
     volumes:
       - elastic8_cert_vol:/usr/share/elasticsearch/config/certs
     user: "0"
@@ -89,7 +89,7 @@ services:
     depends_on:
       elastic8_setup:
         condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3
     ports:
       - 9208:9200
     volumes:
@@ -126,21 +126,12 @@ services:
       - share_network
 
   postgres:
-    image: postgres:10
-    command:
-      - /bin/bash
-      - -c
-      - echo "$$POSTGRES_INITDB" > /docker-entrypoint-initdb.d/commands.sh &&
-        chmod +x /docker-entrypoint-initdb.d/commands.sh &&
-        /docker-entrypoint.sh postgres
+    image: postgres:15.4
     ports:
       - 5433:5432
     environment:
       POSTGRES_DB: share
       POSTGRES_HOST_AUTH_METHOD: trust
-      POSTGRES_INITDB: |
-        sed -i -e 's/max_connections.*/max_connections = 5000/' /var/lib/postgresql/data/postgresql.conf
-        sed -i -e 's/#log_min_duration_statement = .*/log_min_duration_statement = 0/' /var/lib/postgresql/data/postgresql.conf
     volumes:
       - "${POSTGRES_DATA_VOL:-postgres_data_vol}:/var/lib/postgresql/data/"
     stdin_open: true
@@ -152,27 +143,28 @@ services:
   ##################
 
   requirements:
-    image: quay.io/centerforopenscience/share:develop
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: dev
     command:
       - /bin/bash
       - -c
-      - apt-get update &&
-        apt-get install -y gcc &&
-        pip install -r requirements.txt -r dev-requirements.txt &&
-        (python3 -m compileall /usr/local/lib/python3.10 || true) &&
-        rm -Rf /python3.10/* &&
-        apt-get remove -y gcc &&
-        cp -Rf -p /usr/local/lib/python3.10 / &&
-        python3 setup.py develop
+      - $$POETRY_HOME/bin/poetry install --compile --with=dev &&
+        rm -Rf /python3.13/* &&
+        cp -Rf -p /usr/local/lib/python3.13 /
     restart: 'no'
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/python3.10
+      - share_requirements_vol:/python3.13
     networks:
       - share_network
 
   worker:
-    image: quay.io/centerforopenscience/share:develop
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: dev
     command:
       /bin/bash -c 'cp -r /elastic8_certs /elastic_certs && chown -R daemon:daemon /elastic_certs/ && /usr/local/bin/celery --app project worker --uid daemon -l INFO'
     depends_on:
@@ -183,7 +175,7 @@ services:
       - indexer
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/usr/local/lib/python3.10
+      - share_requirements_vol:/usr/local/lib/python3.13
       - elastic8_cert_vol:/elastic8_certs
     env_file:
       - .docker-compose.env
@@ -195,7 +187,10 @@ services:
       - share_network
 
   web:
-    image: quay.io/centerforopenscience/share:develop
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: dev
     command: python manage.py runserver 0.0.0.0:8000
     ports:
       - 8003:8000
@@ -205,7 +200,7 @@ services:
       - elastic8
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/usr/local/lib/python3.10
+      - share_requirements_vol:/usr/local/lib/python3.13
       - elastic8_cert_vol:/elastic8_certs
     env_file:
       - .docker-compose.env
@@ -217,15 +212,18 @@ services:
       - share_network
 
   indexer:
-    image: quay.io/centerforopenscience/share:develop
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: dev
     command: python manage.py shtrove_indexer_run
     depends_on:
       - postgres
       - rabbitmq
       - elastic8
     volumes:
       - ./:/code:cached
-      - share_requirements_vol:/usr/local/lib/python3.10
+      - share_requirements_vol:/usr/local/lib/python3.13
       - elastic8_cert_vol:/elastic8_certs
     env_file:
       - .docker-compose.env