diff --git a/tests/integration/common.py b/tests/integration/common.py index d435739f..9d5b7ea0 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -91,12 +91,22 @@ def recover_resources_from_backup(): def prepare_datastore(datastore_dir: str, *, package_to_input: bool = False): + """ + Prepare a datastore directory for tests by: + - Expanding the metadata files using the template. + - Optionally package datasets into the input directory with a newly + generated set of keys in the datastore's vault. + """ if package_to_input: _package_to_input(datastore_dir) _render_metadata_files(Path(f"{datastore_dir}_working")) - _render_metadata_all( - Path(f"{datastore_dir}/datastore/metadata_all__1_0_0.json") - ) - _render_metadata_all( - Path(f"{datastore_dir}/datastore/metadata_all__DRAFT.json") - ) + metadata_dir = f"{datastore_dir}/datastore" + for filename in os.listdir(metadata_dir): + if "metadata_all" in filename: + _render_metadata_all(Path(f"{metadata_dir}/{filename}")) + tmp_dir = f"{datastore_dir}/datastore/tmp" + if not os.path.exists(tmp_dir): + return + for filename in os.listdir(tmp_dir): + if "metadata_all" in filename: + _render_metadata_all(Path(f"{tmp_dir}/{filename}")) diff --git a/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE_input/.gitkeep b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__DRAFT.parquet similarity index 100% rename from tests/integration/resources/datastores/FIRST_BUMP_DATASTORE_input/.gitkeep rename to tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__DRAFT.parquet diff --git a/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/datastore_versions.json b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/datastore_versions.json new file mode 100644 index 00000000..61493343 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/datastore_versions.json @@ -0,0 +1,7 @@ +{ + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "versions": [] +} + diff --git a/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/draft_version.json b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/draft_version.json new file mode 100644 index 00000000..f2293e44 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/draft_version.json @@ -0,0 +1,15 @@ +{ + "version": "0.0.0.1234567891", + "description": "Draft", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "DRAFT_ADD", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "DRAFT" + } + ], + "updateType": null +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json new file mode 100644 index 00000000..4a6f98c1 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json @@ -0,0 +1,14 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE_input/.gitkeep b/tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/vault/.gitkeep similarity index 100% rename from tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE_input/.gitkeep rename to tests/integration/resources/datastores/FIRST_BUMP_DATASTORE/vault/.gitkeep diff --git a/tests/integration/resources/datastores/ROLLBACK_DATASTORE_input/.gitkeep b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__1_0.parquet similarity index 100% rename from tests/integration/resources/datastores/ROLLBACK_DATASTORE_input/.gitkeep rename to tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__1_0.parquet diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/data_versions__1_0.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/data_versions__1_0.json new file mode 100644 index 00000000..964bb697 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/data_versions__1_0.json @@ -0,0 +1,3 @@ +{ + "RELEASED_DATASET": "RELEASED_DATASET__1_0.parquet", +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/datastore_versions.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/datastore_versions.json new file mode 100644 index 00000000..07f93823 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/datastore_versions.json @@ -0,0 +1,23 @@ +{ + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "versions": [ + { + "version": "1.0.0.0", + "description": "Første versjon", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "DRAFT_ADD", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "RELEASED" + } + ], + "updateType": "MAJOR" + } + ] +} + diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/draft_version.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/draft_version.json new file mode 100644 index 00000000..9dcf3a35 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/draft_version.json @@ -0,0 +1,8 @@ +{ + "version": "0.0.0.1234567891", + "description": "Draft", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [], + "updateType": null +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__1_0_0.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__1_0_0.json new file mode 100644 index 00000000..4a6f98c1 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__1_0_0.json @@ -0,0 +1,14 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__DRAFT.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__DRAFT.json new file mode 100644 index 00000000..4a6f98c1 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/metadata_all__DRAFT.json @@ -0,0 +1,14 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/datastore_versions.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/datastore_versions.json new file mode 100644 index 00000000..23de9a2a --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/datastore_versions.json @@ -0,0 +1,6 @@ +{ + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "versions": [] +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/draft_version.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/draft_version.json new file mode 100644 index 00000000..550bc4d8 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/draft_version.json @@ -0,0 +1,15 @@ +{ + "version": "0.0.0.1234567891", + "description": "Draft", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "DRAFT_ADD", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "PENDING_RELEASE" + } + ], + "updateType": "MAJOR" +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/metadata_all__DRAFT.json b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/metadata_all__DRAFT.json new file mode 100644 index 00000000..4a6f98c1 --- /dev/null +++ b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/datastore/tmp/metadata_all__DRAFT.json @@ -0,0 +1,14 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/vault/.gitkeep b/tests/integration/resources/datastores/FIRST_BUMP_ROLLBACK_DATASTORE/vault/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__2_0.parquet b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/data/DRAFT_ADD/DRAFT_ADD__2_0.parquet new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/data/RELEASED_DATASET/RELEASED_DATASET__1_0.parquet b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/data/RELEASED_DATASET/RELEASED_DATASET__1_0.parquet new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__1_0.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__1_0.json new file mode 100644 index 00000000..da74d799 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__1_0.json @@ -0,0 +1,3 @@ +{ + "DRAFT_ADD": "DRAFT_ADD__2_0.parquet" +} diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__2_0.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/data_versions__2_0.json new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/datastore_versions.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/datastore_versions.json new file mode 100644 index 00000000..dfdca40a --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/datastore_versions.json @@ -0,0 +1,44 @@ +{ + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "versions": [ + { + "version": "0.0.0.1234567891", + "description": "Draft", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "DRAFT_ADD", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "RELEASED" + }, + { + "name": "RELEASED_DATASET", + "description": "Avpublisering", + "operation": "REMOVE", + "releaseStatus": "DELETED" + } + ], + "updateType": "MAJOR" + }, + { + "version": "1.0.0.0", + "description": "Første release", + "releaseTime": 1635299291, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "RELEASED_DATASET", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "RELEASED" + } + ], + "updateType": "MAJOR" + } + ] +} + diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/draft_version.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/draft_version.json new file mode 100644 index 00000000..9dcf3a35 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/draft_version.json @@ -0,0 +1,8 @@ +{ + "version": "0.0.0.1234567891", + "description": "Draft", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [], + "updateType": null +} diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__1_0_0.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__1_0_0.json new file mode 100644 index 00000000..5761e099 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__1_0_0.json @@ -0,0 +1,15 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "RELEASED_DATASET" + ] +} + diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__2_0_0.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__2_0_0.json new file mode 100644 index 00000000..4a6f98c1 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__2_0_0.json @@ -0,0 +1,14 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json new file mode 100644 index 00000000..4a6f98c1 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/metadata_all__DRAFT.json @@ -0,0 +1,14 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/datastore_versions.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/datastore_versions.json new file mode 100644 index 00000000..f9e331d1 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/datastore_versions.json @@ -0,0 +1,23 @@ +{ + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "versions": [ + { + "version": "1.0.0.0", + "description": "Første release", + "releaseTime": 1635299291, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "RELEASED_DATASET", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "RELEASED" + } + ], + "updateType": "MAJOR" + } + ] +} + diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/draft_version.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/draft_version.json new file mode 100644 index 00000000..194bd7a8 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/draft_version.json @@ -0,0 +1,21 @@ +{ + "version": "0.0.0.1234567891", + "description": "Draft", + "releaseTime": 1234567891, + "languageCode": "no", + "dataStructureUpdates": [ + { + "name": "DRAFT_ADD", + "description": "Første publisering", + "operation": "ADD", + "releaseStatus": "PENDING_RELEASE" + }, + { + "name": "RELEASED_DATASET", + "description": "Avpublisering", + "operation": "REMOVE", + "releaseStatus": "PENDING_DELETE" + } + ], + "updateType": "MAJOR" +} diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/metadata_all__DRAFT.json b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/metadata_all__DRAFT.json new file mode 100644 index 00000000..376553e8 --- /dev/null +++ b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/datastore/tmp/metadata_all__DRAFT.json @@ -0,0 +1,15 @@ +{ + "dataStore": { + "name": "no.ssb.dev", + "label": "Integration testing datastore", + "description": "Mock data for testing job-executor", + "languageCode": "no" + }, + "languages": [ + {"code": "no", "label": "Norsk"} + ], + "dataStructures": [ + "RELEASED_DATASET", + "DRAFT_ADD" + ] +} diff --git a/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/vault/.gitkeep b/tests/integration/resources/datastores/ROLLBACK_BUMP_DATASTORE/vault/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_rollback.py b/tests/integration/test_rollback.py index a817b7b3..d94e83e1 100644 --- a/tests/integration/test_rollback.py +++ b/tests/integration/test_rollback.py @@ -1,7 +1,184 @@ -def test_rollback(): ... +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from unittest.mock import MagicMock +import pytest +from microdata_tools.packaging import os -def test_rollback_first_bump(): ... +from job_executor.adapter.datastore_api.models import ( + Job, + JobParameters, + JobStatus, + Operation, + UserInfo, +) +from job_executor.adapter.fs.models.datastore_versions import DatastoreVersion +from job_executor.domain import rollback +from tests.integration.common import ( + backup_resources, + prepare_datastore, + recover_resources_from_backup, +) +RESOURCES_DIR = Path("tests/integration/resources") +DATASTORE_DIR = RESOURCES_DIR / "datastores/TEST_DATASTORE" +WORKING_DIR = RESOURCES_DIR / "TEST_DATASTORE_working" +INPUT_DIR = RESOURCES_DIR / "TEST_DATASTORE_input" -def test_rollback_import_jobs(): ... +user_info = UserInfo( + user_id="1", + first_name="Test", + last_name="Testersen", +) + + +@dataclass +class MockedDatastoreApi: + update_job_status: MagicMock + get_datastore_directory: MagicMock + + +@pytest.fixture(autouse=True) +def mocked_datastore_api(mocker) -> MockedDatastoreApi: + return MockedDatastoreApi( + update_job_status=mocker.patch( + "job_executor.adapter.datastore_api.update_job_status", + return_value=None, + ), + get_datastore_directory=mocker.patch( + "job_executor.adapter.datastore_api.get_datastore_directory", + return_value=DATASTORE_DIR, + ), + ) + + +@pytest.fixture(autouse=True) +def selected_datastore(request): + return request.param + + +@pytest.fixture(autouse=True) +def set_up_resources(selected_datastore): + assert selected_datastore + backup_resources() + if selected_datastore != DATASTORE_DIR: + shutil.rmtree(DATASTORE_DIR) + shutil.move(selected_datastore, DATASTORE_DIR) + prepare_datastore(str(DATASTORE_DIR)) + yield + recover_resources_from_backup() + + +@pytest.mark.parametrize( + "selected_datastore", + [RESOURCES_DIR / "datastores/ROLLBACK_BUMP_DATASTORE"], + indirect=True, +) +def test_rollback_bump(mocked_datastore_api: MockedDatastoreApi): + with open( + DATASTORE_DIR / "datastore" / "tmp" / "draft_version.json", "r" + ) as f: + bump_manifesto = DatastoreVersion.model_validate(json.load(f)) + job = Job( + job_id="job_id", + datastore_rdn="TEST_DATASTORE", + status=JobStatus.INITIATED, + created_at="2022-10-26T12:00:00Z", + created_by=user_info, + parameters=JobParameters( + operation=Operation.BUMP, + target="DATASTORE", + bump_manifesto=bump_manifesto, + description="some description", + bump_from_version="1.0.0", + bump_to_version="2.0.0", + ), + ) + rollback.fix_interrupted_job(job) + assert mocked_datastore_api.update_job_status.call_count == 1 + mocked_datastore_api.update_job_status.assert_called_with( + job.job_id, + JobStatus.FAILED, + "Bump operation was interrupted and rolled back.", + ) + metadata_dir = DATASTORE_DIR / "datastore" + assert not os.path.exists(metadata_dir / "tmp") + assert not os.path.exists(metadata_dir / "metadata_all__2_0_0.json") + assert not os.path.exists(metadata_dir / "data_versions__2_0.json") + + +@pytest.mark.parametrize( + "selected_datastore", + [RESOURCES_DIR / "datastores/FIRST_BUMP_ROLLBACK_DATASTORE"], + indirect=True, +) +def test_rollback_first_bump(mocked_datastore_api: MockedDatastoreApi): + with open( + DATASTORE_DIR / "datastore" / "tmp" / "draft_version.json", "r" + ) as f: + bump_manifesto = DatastoreVersion.model_validate(json.load(f)) + job = Job( + job_id="job_id", + datastore_rdn="TEST_DATASTORE", + status=JobStatus.INITIATED, + created_at="2022-10-26T12:00:00Z", + created_by=user_info, + parameters=JobParameters( + operation=Operation.BUMP, + target="DATASTORE", + bump_manifesto=bump_manifesto, + description="some description", + bump_from_version="1.0.0", + bump_to_version="2.0.0", + ), + ) + rollback.fix_interrupted_job(job) + assert mocked_datastore_api.update_job_status.call_count == 1 + mocked_datastore_api.update_job_status.assert_called_with( + job.job_id, + JobStatus.FAILED, + "Bump operation was interrupted and rolled back.", + ) + metadata_dir = DATASTORE_DIR / "datastore" + assert not os.path.exists(metadata_dir / "tmp") + assert all( + file + in [ + "metadata_all__DRAFT.json", + "draft_version.json", + "datastore_versions.json", + ] + for file in os.listdir(metadata_dir) + ) + + +@pytest.mark.parametrize( + "selected_datastore", + [DATASTORE_DIR], + indirect=True, +) +def test_rollback_import_worker_jobs(mocked_datastore_api: MockedDatastoreApi): + working_dir = Path(f"{str(DATASTORE_DIR)}_working") + for dataset_name in ["BUILT_ADD", "BUILT_CHANGE", "BUILT_PATCH_METADATA"]: + interrupted_add = Job( + job_id="job_id", + datastore_rdn="TEST_DATASTORE", + status=JobStatus.PSEUDONYMIZING, + created_at="2022-10-26T12:00:00Z", + created_by=user_info, + parameters=JobParameters( + operation=Operation.ADD, target=dataset_name + ), + ) + rollback.fix_interrupted_job(interrupted_add) + mocked_datastore_api.update_job_status.assert_called_with( + interrupted_add.job_id, + JobStatus.FAILED, + "Job was failed due to an unexpected interruption", + ) + assert not os.path.exists( + working_dir / f"{dataset_name}__DRAFT.parquet" + ) + assert not os.path.exists(working_dir / f"{dataset_name}.json")