Skip to content

Commit 76e7326

Browse files
feba-rajanfeba-rajan
committed
Merge remote-tracking branch 'origin/main' into fix/lint_failure
2 parents c2e2456 + fefaeb8 commit 76e7326

File tree

5 files changed

+275
-8
lines changed

5 files changed

+275
-8
lines changed

‎.github/workflows/test.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Test
2+
3+
on:
4+
push:
5+
branches-ignore: [main]
6+
workflow_dispatch:
7+
8+
jobs:
9+
test-request-api:
10+
runs-on: ubuntu-22.04
11+
steps:
12+
- uses: actions/checkout@v4
13+
14+
- name: Install test dependencies with pip
15+
working-directory: ./request-api
16+
run: |
17+
pip3 install -r requirements/test_requirements.txt
18+
19+
- name: Run Lint and Tests
20+
working-directory: ./request-api
21+
run: make test
22+
23+
24+
test-request-processor:
25+
runs-on: ubuntu-22.04
26+
steps:
27+
- name: Install GDAL binary package
28+
run: |
29+
sudo add-apt-repository ppa:ubuntugis/ppa
30+
sudo apt-get update
31+
sudo apt-get install gdal-bin
32+
gdalinfo --version
33+
34+
- uses: actions/checkout@v4
35+
36+
- name: Install test dependencies with pip
37+
working-directory: ./request-processor
38+
run: |
39+
pip3 install -r requirements/test_requirements.txt
40+
41+
- name: Download specification files
42+
working-directory: ./request-processor
43+
run: make specification
44+
45+
- name: Lint, Test, Coverage
46+
working-directory: ./request-processor
47+
run: make all

‎request-api/makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,13 @@ black:
1111
flake8:
1212
flake8 .
1313

14-
test-coverage:: coverage-unit coverage-integration
14+
test-coverage:: coverage-unit coverage-integration coverage-acceptance
1515

1616
coverage-unit:
1717
pytest --cov=src tests/unit/
1818

1919
coverage-integration:
2020
pytest --cov=src --cov-append --cov-fail-under=90 tests/integration/
21+
22+
coverage-acceptance:
23+
pytest --cov=src --cov-append --cov-fail-under=90 tests/acceptance/

‎request-api/tests/acceptance/test_main.py‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
import datetime
2+
from urllib.parse import quote
3+
from fastapi.testclient import TestClient
4+
import pytest
5+
6+
import database
7+
from main import app
8+
from request_model import models, schemas
9+
10+
client = TestClient(app)
11+
12+
expected_json = [
13+
{
14+
"line": 1,
15+
"issue_logs": [
16+
{
17+
"field": "organisation",
18+
"issue-type": "invalid organisation",
19+
"severity": "warning",
20+
}
21+
],
22+
},
23+
{
24+
"line": 2,
25+
"issue_logs": [
26+
{
27+
"field": "geometry",
28+
"issue-type": "invalid geometry",
29+
"severity": "error",
30+
}
31+
],
32+
},
33+
{
34+
"line": 3,
35+
"issue_logs": [
36+
{
37+
"field": "organisation",
38+
"issue-type": "invalid organisation",
39+
"severity": "warning",
40+
}
41+
],
42+
},
43+
]
44+
45+
46+
@pytest.mark.parametrize(
47+
"jsonpath, expected_json, expected_total, expected_pglimit",
48+
[
49+
(
50+
'$.issue_logs[*]."severity"=="warning"',
51+
[expected_json[0], expected_json[2]],
52+
"2",
53+
"50",
54+
),
55+
(
56+
'$.issue_logs[*]."severity"=="error" && $.issue_logs[*]."field"=="geometry"',
57+
[expected_json[1]],
58+
"1",
59+
"50",
60+
),
61+
(
62+
'$.issue_logs[*]."issue-type"=="invalid organisation" && $.issue_logs[*]."field"=="organisation"',
63+
[expected_json[0], expected_json[2]],
64+
"2",
65+
"50",
66+
),
67+
],
68+
)
69+
def test_read_response_details_jsonpath_filters(
70+
db,
71+
helpers,
72+
create_test_request,
73+
jsonpath,
74+
expected_json,
75+
expected_total,
76+
expected_pglimit,
77+
):
78+
jsonpath = quote(jsonpath)
79+
response = client.get(
80+
f"/requests/{create_test_request.id}/response-details?jsonpath={jsonpath}"
81+
)
82+
assert response.status_code == 200
83+
assert response.json() == expected_json
84+
assert response.headers["X-Pagination-Total-Results"] == expected_total
85+
assert response.headers["X-Pagination-Limit"] == str(expected_pglimit)
86+
87+
88+
@pytest.fixture(scope="module")
89+
def create_test_request():
90+
test_request_model = models.Request(
91+
type=schemas.RequestTypeEnum.check_file,
92+
created=datetime.datetime.now(),
93+
modified=datetime.datetime.now(),
94+
status="COMPLETE",
95+
params=schemas.CheckFileParams(
96+
collection="conservation-area",
97+
dataset="conservation-area",
98+
original_filename="conservation-area.csv",
99+
uploaded_filename="492f15d8-45e4-427e-bde0-f60d69889f40",
100+
).model_dump(),
101+
response=models.Response(
102+
data='{ "some_key": "some_value" }',
103+
details=[
104+
models.ResponseDetails(
105+
detail={
106+
"line": 1,
107+
"issue_logs": [
108+
{
109+
"field": "organisation",
110+
"issue-type": "invalid organisation",
111+
"severity": "warning",
112+
}
113+
],
114+
}
115+
),
116+
models.ResponseDetails(
117+
detail={
118+
"line": 2,
119+
"issue_logs": [
120+
{
121+
"field": "geometry",
122+
"issue-type": "invalid geometry",
123+
"severity": "error",
124+
}
125+
],
126+
}
127+
),
128+
models.ResponseDetails(
129+
detail={
130+
"line": 3,
131+
"issue_logs": [
132+
{
133+
"field": "organisation",
134+
"issue-type": "invalid organisation",
135+
"severity": "warning",
136+
}
137+
],
138+
}
139+
),
140+
],
141+
),
142+
)
143+
db_session = database.session_maker()
144+
with db_session() as session:
145+
session.add(test_request_model)
146+
session.commit()
147+
session.refresh(test_request_model)
148+
return test_request_model

‎request-api/tests/integration/test_main.py‎

Lines changed: 76 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from fastapi.testclient import TestClient
55
from pydantic_core import ValidationError
66

7+
from crud import get_response_details
78
import database
89
from main import app
910
from request_model import schemas, models
@@ -38,8 +39,28 @@ def test_read_request(db, sqs_queue, helpers):
3839

3940

4041
expected_jsondata = [
41-
{"line": 1, "issue_logs": [{"severity": "warning"}, {"severity": "error"}]},
42-
{"line": 2, "issue_logs": [{"severity": "warning"}, {"severity": "error"}]},
42+
{
43+
"line": 1,
44+
"issue_logs": [
45+
{
46+
"severity": "warning",
47+
"field": "organisation",
48+
"issue-type": "invalid geometry - fixed",
49+
},
50+
{"severity": "error"},
51+
],
52+
},
53+
{
54+
"line": 2,
55+
"issue_logs": [
56+
{
57+
"severity": "error",
58+
"field": "geometry",
59+
"issue-type": "invalid organisation",
60+
},
61+
{"severity": "error"},
62+
],
63+
},
4364
{"line": 3, "issue_logs": [{"severity": "warning"}]},
4465
]
4566

@@ -128,6 +149,43 @@ def test_read_unknown_request(db):
128149
assert response.status_code == 404
129150

130151

152+
@pytest.fixture(scope="module")
153+
def db_session(db):
154+
session = database.session_maker()
155+
session = session()
156+
157+
try:
158+
yield session
159+
finally:
160+
session.rollback()
161+
session.close()
162+
163+
164+
@pytest.mark.parametrize(
165+
"jsonpath, expected_lines",
166+
[
167+
(
168+
'$.issue_logs[*]."severity"=="error" && $.issue_logs[*]."field"=="geometry"',
169+
[2],
170+
),
171+
(
172+
'$.issue_logs[*]."severity"=="warning" && $.issue_logs[*]."issue-type"=="invalid geometry - fixed"',
173+
[1],
174+
),
175+
('$.issue_logs[*]."severity"=="warning"', [1, 3]),
176+
],
177+
)
178+
def test_get_response_details(db_session, test_request, jsonpath, expected_lines):
179+
# Direct query test
180+
result = get_response_details(db_session, test_request.id, jsonpath)
181+
182+
# Extract line numbers from the result
183+
result_lines = [detail.detail["line"] for detail in result.data]
184+
185+
assert sorted(result_lines) == sorted(expected_lines)
186+
assert result.total_results_available >= len(expected_lines)
187+
188+
131189
@pytest.fixture(scope="module")
132190
def test_request():
133191
request_model = models.Request(
@@ -147,13 +205,27 @@ def test_request():
147205
models.ResponseDetails(
148206
detail={
149207
"line": 1,
150-
"issue_logs": [{"severity": "warning"}, {"severity": "error"}],
208+
"issue_logs": [
209+
{
210+
"field": "organisation",
211+
"issue-type": "invalid geometry - fixed",
212+
"severity": "warning",
213+
},
214+
{"severity": "error"},
215+
],
151216
}
152217
),
153218
models.ResponseDetails(
154219
detail={
155220
"line": 2,
156-
"issue_logs": [{"severity": "warning"}, {"severity": "error"}],
221+
"issue_logs": [
222+
{
223+
"field": "geometry",
224+
"issue-type": "invalid organisation",
225+
"severity": "error",
226+
},
227+
{"severity": "error"},
228+
],
157229
}
158230
),
159231
models.ResponseDetails(

‎request-processor/src/application/core/pipeline.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,6 @@ def fetch_response_data(
114114
organisation_path=os.path.join(cache_dir, "organisation.csv"),
115115
save_harmonised=False,
116116
organisations=[organisation],
117-
custom_temp_dir=os.path.join(cache_dir),
118117
converted_dir=converted_dir,
119118
)
120119
except Exception as err:
@@ -136,7 +135,6 @@ def pipeline_run(
136135
save_harmonised=False,
137136
column_field_dir=None,
138137
dataset_resource_dir=None,
139-
custom_temp_dir=None, # TBD: rename to "tmpdir"
140138
endpoints=[],
141139
entry_date="",
142140
):
@@ -171,7 +169,6 @@ def pipeline_run(
171169
ConvertPhase(
172170
path=input_path,
173171
dataset_resource_log=dataset_resource_log,
174-
custom_temp_dir=custom_temp_dir,
175172
output_path=os.path.join(converted_dir, request_id, f"{resource}.csv"),
176173
),
177174
NormalisePhase(skip_patterns=skip_patterns, null_path=null_path),

0 commit comments

Comments
 (0)