Skip to content

Commit 7af5b26

Browse files
Add the components to test the postgres database on github actions (#465)
1 parent a64c369 commit 7af5b26

File tree

5 files changed

+203
-12
lines changed

5 files changed

+203
-12
lines changed

.github/workflows/ci.yml

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,16 @@ jobs:
2929
- 3306:3306
3030
options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
3131

32+
postgres:
33+
image: postgres:latest
34+
env:
35+
POSTGRES_DB: murfey_test_db
36+
POSTGRES_PASSWORD: psql_pwd
37+
POSTGRES_USER: psql_user
38+
ports:
39+
- 5432:5432
40+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
41+
3242
steps:
3343
- uses: actions/checkout@v4
3444
- name: Use Python ${{ matrix.python-version }}
@@ -57,7 +67,7 @@ jobs:
5767
docker run --detach --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 test-rabbitmq
5868
docker container list -a
5969
60-
- name: Get database
70+
- name: Get ispyb database
6171
uses: actions/download-artifact@v4
6272
with:
6373
name: database
@@ -74,7 +84,7 @@ jobs:
7484
mysql-version: "11.3"
7585
auto-start: false
7686

77-
- name: Set up test database
87+
- name: Set up test ipsyb database
7888
run: |
7989
set -eu
8090
cp ".github/workflows/config/my.cnf" .my.cnf
@@ -103,6 +113,12 @@ jobs:
103113
run: wget -t 10 -w 1 http://127.0.0.1:15672 -O -
104114

105115
- name: Run tests
116+
env:
117+
POSTGRES_HOST: localhost
118+
POSTGRES_PORT: 5432
119+
POSTGRES_DB: murfey_test_db
120+
POSTGRES_PASSWORD: psql_pwd
121+
POSTGRES_USER: psql_user
106122
run: |
107123
export ISPYB_CREDENTIALS=".github/workflows/config/ispyb.cfg"
108124
PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch

src/murfey/workflows/spa/flush_spa_preprocess.py

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
from PIL import Image
66
from sqlalchemy.exc import NoResultFound
77
from sqlmodel import Session, select
8-
from werkzeug.utils import secure_filename
98

109
from murfey.server import _murfey_id, _transport_object, sanitise
1110
from murfey.server.api.auth import MurfeySessionID
11+
from murfey.util import secure_path
1212
from murfey.util.config import get_machine_config, get_microscope
1313
from murfey.util.db import (
1414
AutoProcProgram,
@@ -72,11 +72,8 @@ def register_grid_square(
7272
else:
7373
# mock up response so that below still works
7474
gs_ispyb_response = {"success": False, "return_value": None}
75-
secured_grid_square_image_path = secure_filename(grid_square_params.image)
76-
if (
77-
secured_grid_square_image_path
78-
and Path(secured_grid_square_image_path).is_file()
79-
):
75+
secured_grid_square_image_path = secure_path(Path(grid_square_params.image))
76+
if secured_grid_square_image_path and secured_grid_square_image_path.is_file():
8077
jpeg_size = Image.open(secured_grid_square_image_path).size
8178
else:
8279
jpeg_size = (0, 0)
@@ -98,7 +95,7 @@ def register_grid_square(
9895
thumbnail_size_x=grid_square_params.thumbnail_size_x or jpeg_size[0],
9996
thumbnail_size_y=grid_square_params.thumbnail_size_y or jpeg_size[1],
10097
pixel_size=grid_square_params.pixel_size,
101-
image=secured_grid_square_image_path,
98+
image=str(secured_grid_square_image_path),
10299
)
103100
murfey_db.add(grid_square)
104101
murfey_db.commit()
@@ -124,8 +121,8 @@ def register_foil_hole(
124121
f"Foil hole {sanitise(str(foil_hole_params.name))} could not be registered as grid square {sanitise(str(gs_name))} was not found"
125122
)
126123
return None
127-
secured_foil_hole_image_path = secure_filename(foil_hole_params.image)
128-
if foil_hole_params.image and Path(secured_foil_hole_image_path).is_file():
124+
secured_foil_hole_image_path = secure_path(Path(foil_hole_params.image))
125+
if foil_hole_params.image and secured_foil_hole_image_path.is_file():
129126
jpeg_size = Image.open(secured_foil_hole_image_path).size
130127
else:
131128
jpeg_size = (0, 0)
@@ -188,7 +185,7 @@ def register_foil_hole(
188185
thumbnail_size_x=foil_hole_params.thumbnail_size_x or jpeg_size[0],
189186
thumbnail_size_y=foil_hole_params.thumbnail_size_y or jpeg_size[1],
190187
pixel_size=foil_hole_params.pixel_size,
191-
image=secured_foil_hole_image_path,
188+
image=str(secured_foil_hole_image_path),
192189
)
193190
murfey_db.add(foil_hole)
194191
murfey_db.commit()

tests/__init__.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import os
2+
3+
from sqlmodel import create_engine
4+
5+
url = (
6+
f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}"
7+
f"@{os.environ['POSTGRES_HOST']}:{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_DB']}"
8+
)
9+
10+
engine = create_engine(url)

tests/conftest.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import pytest
2+
from sqlmodel import Session
3+
4+
from murfey.util.db import Session as MurfeySession
5+
from murfey.util.db import clear, setup
6+
from tests import engine, url
7+
8+
9+
@pytest.fixture
10+
def start_postgres():
11+
clear(url)
12+
setup(url)
13+
14+
murfey_session = MurfeySession(id=2, name="cm12345-6")
15+
with Session(engine) as murfey_db:
16+
murfey_db.add(murfey_session)
17+
murfey_db.commit()
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
from unittest import mock
2+
3+
from sqlmodel import Session, select
4+
5+
from murfey.util.db import DataCollectionGroup, GridSquare
6+
from murfey.util.models import GridSquareParameters
7+
from murfey.workflows.spa import flush_spa_preprocess
8+
from tests import engine
9+
10+
11+
@mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object")
12+
def test_register_grid_square_update_add_locations(mock_transport, start_postgres):
13+
"""Test the updating of an existing grid square"""
14+
# Create a grid square to update
15+
grid_square = GridSquare(
16+
id=1,
17+
name=101,
18+
session_id=2,
19+
tag="session_tag",
20+
)
21+
with Session(engine) as murfey_db:
22+
murfey_db.add(grid_square)
23+
murfey_db.commit()
24+
25+
# Parameters to update with
26+
new_parameters = GridSquareParameters(
27+
tag="session_tag",
28+
x_location=1.1,
29+
y_location=1.2,
30+
x_stage_position=1.3,
31+
y_stage_position=1.4,
32+
)
33+
34+
# Run the registration
35+
with Session(engine) as murfey_db:
36+
flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db)
37+
38+
# Check this would have updated ispyb
39+
mock_transport.do_update_grid_square.assert_called_with(1, new_parameters)
40+
41+
# Confirm the database was updated
42+
with Session(engine) as murfey_db:
43+
grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one()
44+
assert grid_square_final_parameters.x_location == new_parameters.x_location
45+
assert grid_square_final_parameters.y_location == new_parameters.y_location
46+
assert (
47+
grid_square_final_parameters.x_stage_position == new_parameters.x_stage_position
48+
)
49+
assert (
50+
grid_square_final_parameters.y_stage_position == new_parameters.y_stage_position
51+
)
52+
53+
54+
@mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object")
55+
def test_register_grid_square_update_add_nothing(mock_transport, start_postgres):
56+
"""Test the updating of an existing grid square, but with nothing to update with"""
57+
# Create a grid square to update
58+
grid_square = GridSquare(
59+
id=1,
60+
name=101,
61+
session_id=2,
62+
tag="session_tag",
63+
x_location=0.1,
64+
y_location=0.2,
65+
x_stage_position=0.3,
66+
y_stage_position=0.4,
67+
)
68+
with Session(engine) as murfey_db:
69+
murfey_db.add(grid_square)
70+
murfey_db.commit()
71+
72+
# Parameters to update with
73+
new_parameters = GridSquareParameters(tag="session_tag")
74+
75+
# Run the registration
76+
with Session(engine) as murfey_db:
77+
flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db)
78+
79+
# Check this would have updated ispyb
80+
mock_transport.do_update_grid_square.assert_called_with(1, new_parameters)
81+
82+
# Confirm the database was not updated
83+
with Session(engine) as murfey_db:
84+
grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one()
85+
assert grid_square_final_parameters.x_location == 0.1
86+
assert grid_square_final_parameters.y_location == 0.2
87+
assert grid_square_final_parameters.x_stage_position == 0.3
88+
assert grid_square_final_parameters.y_stage_position == 0.4
89+
90+
91+
@mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object")
92+
def test_register_grid_square_insert_with_ispyb(
93+
mock_transport, start_postgres, tmp_path
94+
):
95+
# Create a data collection group for lookups
96+
grid_square = DataCollectionGroup(
97+
id=1,
98+
session_id=2,
99+
tag="session_tag",
100+
atlas_id=90,
101+
)
102+
with Session(engine) as murfey_db:
103+
murfey_db.add(grid_square)
104+
murfey_db.commit()
105+
106+
# Set the ispyb return
107+
mock_transport.do_insert_grid_square.return_value = {
108+
"return_value": 1,
109+
"success": True,
110+
}
111+
112+
# Parameters to update with
113+
new_parameters = GridSquareParameters(
114+
tag="session_tag",
115+
x_location=1.1,
116+
y_location=1.2,
117+
x_stage_position=1.3,
118+
y_stage_position=1.4,
119+
readout_area_x=2048,
120+
readout_area_y=1024,
121+
thumbnail_size_x=512,
122+
thumbnail_size_y=256,
123+
pixel_size=1.02,
124+
image=f"{tmp_path}/image_path",
125+
angle=12.5,
126+
)
127+
128+
# Run the registration
129+
with Session(engine) as murfey_db:
130+
flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db)
131+
132+
# Check this would have updated ispyb
133+
mock_transport.do_insert_grid_square.assert_called_with(90, 101, new_parameters)
134+
135+
# Confirm the database entry was made
136+
with Session(engine) as murfey_db:
137+
grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one()
138+
assert grid_square_final_parameters.id == 1
139+
assert grid_square_final_parameters.name == 101
140+
assert grid_square_final_parameters.session_id == 2
141+
assert grid_square_final_parameters.tag == "session_tag"
142+
assert grid_square_final_parameters.x_location == 1.1
143+
assert grid_square_final_parameters.y_location == 1.2
144+
assert grid_square_final_parameters.x_stage_position == 1.3
145+
assert grid_square_final_parameters.y_stage_position == 1.4
146+
assert grid_square_final_parameters.readout_area_x == 2048
147+
assert grid_square_final_parameters.readout_area_y == 1024
148+
assert grid_square_final_parameters.thumbnail_size_x == 512
149+
assert grid_square_final_parameters.thumbnail_size_y == 256
150+
assert grid_square_final_parameters.pixel_size == 1.02
151+
assert grid_square_final_parameters.image == f"{tmp_path}/image_path"

0 commit comments

Comments
 (0)