Skip to content

Commit 7f19889

Browse files
patrick-austinpatrick-austin
authored
Merge pull request #122 from ral-facilities/75_remove_nox
Containerisation best practices
2 parents 4d814b5 + 598a5fb commit 7f19889

File tree

9 files changed

+219
-225
lines changed

9 files changed

+219
-225
lines changed

‎.github/requirements.txt‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
1-
poetry==2.1.2
2-
nox==2024.3.2
31
codecarbon==3.0.1

‎.github/workflows/ci.yaml‎

Lines changed: 117 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ jobs:
2323
strategy:
2424
fail-fast: false
2525
runs-on: ubuntu-22.04
26-
env:
27-
CACHE_DIR: $HOME/.cache/docker
2826

2927
name: Tests
3028
steps:
@@ -37,38 +35,19 @@ jobs:
3735
path: ~/.cache/pip
3836
key: pip-${{ hashFiles('.github/requirements.txt') }}
3937

40-
- name: Install Poetry & Nox
38+
- name: Install runner requirements
4139
run: pip install -r .github/requirements.txt
4240

4341
# Let CodeCarbon run in the background while we do everything else
4442
- name: Start CodeCarbon
45-
run: codecarbon monitor &
43+
run: codecarbon monitor --no-api > emissions.out 2>&1 &
4644

4745
# Do this early to minimise how long we wait for the health check later on
4846
- name: Docker compose up
49-
run: docker compose -f tests/docker-compose.yaml up -d
50-
51-
- name: Install python-ldap dependencies
52-
run: |
53-
sudo apt-get update
54-
sudo apt-get install -y libsasl2-dev python3.11-dev libldap2-dev libssl-dev
55-
56-
- name: Setup Python
57-
uses: actions/setup-python@v5
58-
with:
59-
python-version: "3.11"
60-
61-
- name: Load Poetry cache
62-
uses: actions/cache@v4
63-
with:
64-
path: ~/.cache/pypoetry/virtualenvs
65-
key: poetry-${{ hashFiles('poetry.lock') }}
47+
run: docker compose -f docker-compose.yaml --profile dependencies up -d
6648

67-
- name: Load Poetry cache for Nox tests session
68-
uses: actions/cache@v4
69-
with:
70-
path: /home/runner/work/datastore-api/datastore-api/.nox/tests*
71-
key: nox-tests-${{ hashFiles('poetry.lock') }}
49+
- name: Touch coverage.xml
50+
run: touch coverage.xml
7251

7352
- name: Set X509 certificate
7453
run: |
@@ -96,135 +75,163 @@ jobs:
9675
- name: Cache Docker layers
9776
uses: actions/cache@v4
9877
with:
99-
path: ${{ env.CACHE_DIR }}
78+
path: ${{ runner.temp }}/.buildx-cache
10079
key: ${{ runner.os }}-buildx-${{ hashFiles('poetry.lock') }}
10180
restore-keys: |
10281
${{ runner.os }}-buildx-
10382
10483
- name: Build Docker image
84+
uses: docker/build-push-action@v5
85+
with:
86+
context: .
87+
push: false
88+
tags: datastore-api:dev
89+
target: dev
90+
load: true
91+
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache
92+
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache-new,mode=max
93+
94+
- name: Move Cache
10595
run: |
106-
mkdir -p "${CACHE_DIR}"
107-
docker buildx build -t datastore-api:test --target test --load \
108-
--cache-from=type=local,src="${CACHE_DIR}" \
109-
--cache-to=type=local,dest="${CACHE_DIR}",mode=max \
110-
-f /home/runner/work/datastore-api/datastore-api/Dockerfile .
96+
rm -rf ${{ runner.temp }}/.buildx-cache
97+
mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache
11198
112-
- name: Run tests via Docker container
99+
- name: Run tests
113100
run: |
114-
docker run --rm \
115-
--network tests_datastore_network \
116-
-v /home/runner/work/datastore-api/datastore-api//hostkey.pem:/app/hostkey.pem \
117-
-v /home/runner/work/datastore-api/datastore-api//hostcert.pem:/app/hostcert.pem \
118-
-w /app \
101+
docker compose -f docker-compose.yaml --profile tests run \
119102
-e FTS3__STORAGE_ENDPOINTS__ECHO__ACCESS_KEY=${{ secrets.ECHO_S3_ACCESS_KEY }} \
120103
-e FTS3__STORAGE_ENDPOINTS__ECHO__SECRET_KEY=${{ secrets.ECHO_S3_SECRET_KEY }} \
121-
datastore-api:test
104+
tests
122105
123106
- name: Docker compose down
124-
run: docker compose -f tests/docker-compose.yaml down
107+
run: docker compose -f docker-compose.yaml down
125108

126109
- name: Upload code coverage report
127110
uses: codecov/codecov-action@v4
128111
with:
129112
token: ${{ secrets.CODECOV_TOKEN }}
130-
131-
# CodeCarbon needs the keyboard interrupt signal to gracefully stop
132-
- name: Stop CodeCarbon
133-
run: pkill -2 codecarbon
134113

135114
- name: Print emissions
136-
run: cat emissions.csv
115+
if: always()
116+
run: cat emissions.out
137117

138118

139119
linting:
140120
runs-on: ubuntu-22.04
141121
name: Code Linting
142122
steps:
143-
- name: Setup Python
144-
uses: actions/setup-python@v5
145-
with:
146-
python-version: "3.11"
147-
148123
- name: Checkout
149124
uses: actions/checkout@v4
150125

151-
- name: Load Pip cache
152-
uses: actions/cache@v4
153-
with:
154-
path: ~/.cache/pip
155-
key: pip-${{ hashFiles('.github/requirements.txt') }}
156-
157-
- name: Install Poetry & Nox
158-
run: pip install -r .github/requirements.txt
126+
- name: Setup Docker Buildx
127+
uses: docker/setup-buildx-action@v3
159128

160-
- name: Load Poetry cache for Nox lint session
129+
# Cache image will only be updated if poetry.lock changes
130+
- name: Cache Docker layers
161131
uses: actions/cache@v4
162132
with:
163-
path: /home/runner/work/datastore-api/datastore-api/.nox/lint*
164-
key: nox-lint-${{ hashFiles('poetry.lock') }}
133+
path: ${{ runner.temp }}/.buildx-cache
134+
key: ${{ runner.os }}-buildx-${{ hashFiles('poetry.lock') }}
135+
restore-keys: |
136+
${{ runner.os }}-buildx-
137+
138+
- name: Build Docker image
139+
uses: docker/build-push-action@v5
140+
with:
141+
context: .
142+
push: false
143+
tags: datastore-api:dev
144+
target: dev
145+
load: true
146+
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache
147+
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache-new,mode=max
148+
149+
- name: Move Cache
150+
run: |
151+
rm -rf ${{ runner.temp }}/.buildx-cache
152+
mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache
165153
166-
- name: Run Nox lint session
167-
run: nox -s lint
154+
- name: Run linting
155+
run: docker compose -f docker-compose.yaml run lint
168156

169157

170158
formatting:
171159
runs-on: ubuntu-22.04
172160
name: Code Formatting
173161
steps:
174-
- name: Setup Python
175-
uses: actions/setup-python@v5
176-
with:
177-
python-version: "3.11"
178162
- name: Checkout
179163
uses: actions/checkout@v4
180164

181-
- name: Load Pip cache
182-
uses: actions/cache@v4
183-
with:
184-
path: ~/.cache/pip
185-
key: pip-${{ hashFiles('.github/requirements.txt') }}
186-
- name: Install Poetry & Nox
187-
run: pip install -r .github/requirements.txt
165+
- name: Setup Docker Buildx
166+
uses: docker/setup-buildx-action@v3
188167

189-
- name: Load Poetry cache for Nox black session
168+
# Cache image will only be updated if poetry.lock changes
169+
- name: Cache Docker layers
190170
uses: actions/cache@v4
191171
with:
192-
path: /home/runner/work/datastore-api/datastore-api/.nox/black*
193-
key: nox-black-${{ hashFiles('poetry.lock') }}
172+
path: ${{ runner.temp }}/.buildx-cache
173+
key: ${{ runner.os }}-buildx-${{ hashFiles('poetry.lock') }}
174+
restore-keys: |
175+
${{ runner.os }}-buildx-
176+
177+
- name: Build Docker image
178+
uses: docker/build-push-action@v5
179+
with:
180+
context: .
181+
push: false
182+
tags: datastore-api:dev
183+
target: dev
184+
load: true
185+
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache
186+
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache-new,mode=max
187+
188+
- name: Move Cache
189+
run: |
190+
rm -rf ${{ runner.temp }}/.buildx-cache
191+
mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache
194192
195-
- name: Run Nox black session
196-
run: nox -s black -- --check datastore_api tests noxfile.py
193+
- name: Run formatting
194+
run: docker compose -f docker-compose.yaml run format
197195

198196

199197
safety:
200198
runs-on: ubuntu-22.04
201199
name: Dependency Safety
202200
steps:
203-
- name: Setup Python
204-
uses: actions/setup-python@v5
205-
with:
206-
python-version: "3.11"
207-
208201
- name: Checkout
209202
uses: actions/checkout@v4
210203

211-
- name: Load Pip cache
212-
uses: actions/cache@v4
213-
with:
214-
path: ~/.cache/pip
215-
key: pip-${{ hashFiles('.github/requirements.txt') }}
216-
217-
- name: Install Poetry & Nox
218-
run: pip install -r .github/requirements.txt
204+
- name: Setup Docker Buildx
205+
uses: docker/setup-buildx-action@v3
219206

220-
- name: Load Poetry cache for Nox safety session
207+
# Cache image will only be updated if poetry.lock changes
208+
- name: Cache Docker layers
221209
uses: actions/cache@v4
222210
with:
223-
path: /home/runner/work/datastore-api/datastore-api/.nox/safety*
224-
key: nox-safety-${{ hashFiles('poetry.lock') }}
211+
path: ${{ runner.temp }}/.buildx-cache
212+
key: ${{ runner.os }}-buildx-${{ hashFiles('poetry.lock') }}
213+
restore-keys: |
214+
${{ runner.os }}-buildx-
215+
216+
- name: Build Docker image
217+
uses: docker/build-push-action@v5
218+
with:
219+
context: .
220+
push: false
221+
tags: datastore-api:dev
222+
target: dev
223+
load: true
224+
cache-from: type=local,src=${{ runner.temp }}/.buildx-cache
225+
cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache-new,mode=max
226+
227+
- name: Move Cache
228+
run: |
229+
rm -rf ${{ runner.temp }}/.buildx-cache
230+
mv ${{ runner.temp }}/.buildx-cache-new ${{ runner.temp }}/.buildx-cache
231+
232+
- name: Run safety
233+
run: docker compose -f docker-compose.yaml run safety
225234

226-
- name: Run Nox safety session
227-
run: nox -s safety
228235

229236
build:
230237
needs: [ tests, linting, formatting, safety]
@@ -237,7 +244,19 @@ jobs:
237244
uses: actions/checkout@v2
238245

239246
- name: Docker compose up
240-
run: docker compose -f tests/docker-compose.yaml up -d
247+
run: docker compose -f docker-compose.yaml up -d
248+
249+
- name: Setup Docker Buildx
250+
uses: docker/setup-buildx-action@v3
251+
252+
# Cache image will only be updated if poetry.lock changes
253+
- name: Cache Docker layers
254+
uses: actions/cache@v4
255+
with:
256+
path: ${{ runner.temp }}/.buildx-cache
257+
key: ${{ runner.os }}-buildx-${{ hashFiles('poetry.lock') }}
258+
restore-keys: |
259+
${{ runner.os }}-buildx-
241260
242261
- name: Docker meta
243262
id: meta

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ venv/
88
config.yaml
99
!config.yaml.example
1010
.vscode/
11-
.nox/
1211
.python-version
1312
.coverage
1413
coverage.xml

‎Dockerfile‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM python:3.11-slim AS base
1+
FROM python:3.13-slim AS base
22

33
# Set environment variables
44
ENV PYTHONUNBUFFERED=1
@@ -74,6 +74,7 @@ RUN poetry install --with dev
7474
# Copy the project files to the container and install
7575
COPY config.yaml.example logging.ini.example /app/
7676
COPY pytest.ini.docker /app/pytest.ini
77+
COPY .flake8 /app/.flake8
7778
COPY tests/ /app/tests/
7879

7980
RUN touch hostkey.pem && \
@@ -90,19 +91,9 @@ CMD ["fastapi","run", "--host=0.0.0.0", "--port=8000", "--reload" ,"/app/datast
9091

9192

9293

93-
# ~~~Test stage: ~~~#
94-
#Set up testing environment
95-
FROM dev AS test
96-
97-
# Run tests
98-
CMD ["pytest", "--config-file", "pytest.ini"]
99-
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ #
100-
101-
102-
10394
# ~~~Production stage: ~~~#
10495
# Set up production environment
105-
FROM python:3.11-slim AS prod
96+
FROM python:3.13-slim AS prod
10697
ENV PYTHONUNBUFFERED=1
10798
ENV PATH="/root/.local/bin:$PATH"
10899
WORKDIR /app
@@ -115,14 +106,18 @@ RUN apt-get update && \
115106
libcurl4 \
116107
curl && \
117108
apt-get clean && \
118-
rm -rf /var/lib/apt/lists/*
109+
rm -rf /var/lib/apt/lists/* \
110+
addgroup -g 500 -S datastore; \
111+
adduser -S -D -G datastore -H -u 500 -h /app datastore
119112

120113
# Copy installed Python deps and source code
121114
COPY --from=builder /usr/local /usr/local
122115
COPY pyproject.toml poetry.lock /app/
123116
COPY datastore_api/ /app/datastore_api/
124117
RUN python -m pip install .
125118

119+
USER datastore
120+
126121
# Expose the port the app will run on
127122
EXPOSE 8000
128123

0 commit comments

Comments
 (0)