Skip to content

More integration tests #213

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DanielElisenberg merged 5 commits into from
Nov 25, 2025
Merged

More integration tests #213

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
01debe9
more_integration_tests: new datastore resouce dirs
DanielElisenberg Nov 25, 2025
56c7011
more_integration_tests: add to common functions
DanielElisenberg Nov 25, 2025
db2ee3a
more_integration_tests: add rollback tests
DanielElisenberg Nov 25, 2025
5f4fc58
more_integration_tests: fix
DanielElisenberg Nov 25, 2025
8bb3f18
more_integration_tests: fix
DanielElisenberg Nov 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions tests/integration/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,22 @@ def recover_resources_from_backup():


def prepare_datastore(datastore_dir: str, *, package_to_input: bool = False):
"""
Prepare a datastore directory for tests by:
- Expanding the metadata files using the template.
- Optionally package datasets into the input directory with a newly
generated set of keys in the datastore's vault.
"""
if package_to_input:
_package_to_input(datastore_dir)
_render_metadata_files(Path(f"{datastore_dir}_working"))
_render_metadata_all(
Path(f"{datastore_dir}/datastore/metadata_all__1_0_0.json")
)
_render_metadata_all(
Path(f"{datastore_dir}/datastore/metadata_all__DRAFT.json")
)
metadata_dir = f"{datastore_dir}/datastore"
for filename in os.listdir(metadata_dir):
if "metadata_all" in filename:
_render_metadata_all(Path(f"{metadata_dir}/{filename}"))
tmp_dir = f"{datastore_dir}/datastore/tmp"
if not os.path.exists(tmp_dir):
return
for filename in os.listdir(tmp_dir):
if "metadata_all" in filename:
_render_metadata_all(Path(f"{tmp_dir}/{filename}"))
0 ...tores/FIRST_BUMP_DATASTORE_input/.gitkeep → ...E/data/DRAFT_ADD/DRAFT_ADD__DRAFT.parquet
View file
Open in desktop
File renamed without changes.
7 changes: 7 additions & 0 deletions ...s/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/datastore_versions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"versions": []
}

15 changes: 15 additions & 0 deletions tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/draft_version.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"version": "0.0.0.1234567891",
"description": "Draft",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "DRAFT_ADD",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "DRAFT"
}
],
"updateType": null
}
14 changes: 14 additions & 0 deletions .../integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"DRAFT_ADD"
]
}
0 ...ST_BUMP_ROLLBACK_DATASTORE_input/.gitkeep → ...tores/FIRST_BUMP_DATASTORE/vault/.gitkeep
View file
Open in desktop
File renamed without changes.
0 ...astores/ROLLBACK_DATASTORE_input/.gitkeep → ...ORE/data/DRAFT_ADD/DRAFT_ADD__1_0.parquet
View file
Open in desktop
File renamed without changes.
3 changes: 3 additions & 0 deletions ...tion/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/data_versions__1_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"RELEASED_DATASET": "RELEASED_DATASET__1_0.parquet",
}
23 changes: 23 additions & 0 deletions ...tion/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/datastore_versions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"versions": [
{
"version": "1.0.0.0",
"description": "Første versjon",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "DRAFT_ADD",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "RELEASED"
}
],
"updateType": "MAJOR"
}
]
}

8 changes: 8 additions & 0 deletions ...tegration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/draft_version.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"version": "0.0.0.1234567891",
"description": "Draft",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [],
"updateType": null
}
14 changes: 14 additions & 0 deletions ...ion/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__1_0_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"DRAFT_ADD"
]
}
14 changes: 14 additions & 0 deletions ...ion/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__DRAFT.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"DRAFT_ADD"
]
}
6 changes: 6 additions & 0 deletions .../resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/datastore_versions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"versions": []
}
15 changes: 15 additions & 0 deletions ...ation/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/draft_version.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"version": "0.0.0.1234567891",
"description": "Draft",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "DRAFT_ADD",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "PENDING_RELEASE"
}
],
"updateType": "MAJOR"
}
14 changes: 14 additions & 0 deletions ...resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/metadata_all__DRAFT.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"DRAFT_ADD"
]
}
Empty file added tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/vault/.gitkeep
View file
Open in desktop
Empty file.
Empty file added ...ration/resources/datastores/ROLLBACK_BUMP_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__2_0.parquet
View file
Open in desktop
Empty file.
Empty file added ...es/datastores/ROLLBACK_BUMP_DATASTORE/data/RELEASED_DATASET/RELEASED_DATASET__1_0.parquet
View file
Open in desktop
Empty file.
3 changes: 3 additions & 0 deletions ...ntegration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__1_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"DRAFT_ADD": "DRAFT_ADD__2_0.parquet"
}
Empty file added ...ntegration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__2_0.json
View file
Open in desktop
Empty file.
44 changes: 44 additions & 0 deletions ...ntegration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/datastore_versions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"versions": [
{
"version": "0.0.0.1234567891",
"description": "Draft",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "DRAFT_ADD",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "RELEASED"
},
{
"name": "RELEASED_DATASET",
"description": "Avpublisering",
"operation": "REMOVE",
"releaseStatus": "DELETED"
}
],
"updateType": "MAJOR"
},
{
"version": "1.0.0.0",
"description": "Første release",
"releaseTime": 1635299291,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "RELEASED_DATASET",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "RELEASED"
}
],
"updateType": "MAJOR"
}
]
}

8 changes: 8 additions & 0 deletions tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/draft_version.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"version": "0.0.0.1234567891",
"description": "Draft",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [],
"updateType": null
}
15 changes: 15 additions & 0 deletions ...tegration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__1_0_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"RELEASED_DATASET"
]
}

14 changes: 14 additions & 0 deletions ...tegration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__2_0_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"DRAFT_ADD"
]
}
14 changes: 14 additions & 0 deletions ...tegration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"DRAFT_ADD"
]
}
23 changes: 23 additions & 0 deletions ...ration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/datastore_versions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"versions": [
{
"version": "1.0.0.0",
"description": "Første release",
"releaseTime": 1635299291,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "RELEASED_DATASET",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "RELEASED"
}
],
"updateType": "MAJOR"
}
]
}

21 changes: 21 additions & 0 deletions ...integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/draft_version.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"version": "0.0.0.1234567891",
"description": "Draft",
"releaseTime": 1234567891,
"languageCode": "no",
"dataStructureUpdates": [
{
"name": "DRAFT_ADD",
"description": "Første publisering",
"operation": "ADD",
"releaseStatus": "PENDING_RELEASE"
},
{
"name": "RELEASED_DATASET",
"description": "Avpublisering",
"operation": "REMOVE",
"releaseStatus": "PENDING_DELETE"
}
],
"updateType": "MAJOR"
}
15 changes: 15 additions & 0 deletions ...ation/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/metadata_all__DRAFT.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"dataStore": {
"name": "no.ssb.dev",
"label": "Integration testing datastore",
"description": "Mock data for testing job-executor",
"languageCode": "no"
},
"languages": [
{"code": "no", "label": "Norsk"}
],
"dataStructures": [
"RELEASED_DATASET",
"DRAFT_ADD"
]
}
Empty file added tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/vault/.gitkeep
View file
Open in desktop
Empty file.
Loading
Loading