diff --git a/tests/integration/resources/README.md b/.test.env similarity index 100% rename from tests/integration/resources/README.md rename to .test.env diff --git a/Dockerfile b/Dockerfile index e575219f..8145cf6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/astral-sh/uv:bookworm-slim as builder +FROM ghcr.io/astral-sh/uv:trixie-slim as builder WORKDIR /app COPY pyproject.toml uv.lock /app/ diff --git a/README.md b/README.md index 5535107c..d311aa16 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ uv run pytest ### Build docker image ``` -docker build --tag job_executor . +docker buildx build --tag job-executor:test-local . ``` ## Built with diff --git a/job_executor/adapter/datastore_api/__init__.py b/job_executor/adapter/datastore_api/__init__.py index a0731ad2..1e7ad6b3 100644 --- a/job_executor/adapter/datastore_api/__init__.py +++ b/job_executor/adapter/datastore_api/__init__.py @@ -16,10 +16,11 @@ Operation, ) from job_executor.common.exceptions import HttpRequestError, HttpResponseError -from job_executor.config import environment +from job_executor.config import environment, secrets DATASTORE_API_URL = environment.datastore_api_url DEFAULT_REQUESTS_TIMEOUT = (10, 60) # (read timeout, connect timeout) +DATASTORE_API_SERVICE_KEY = secrets.datastore_api_service_key logger = logging.getLogger() @@ -117,6 +118,25 @@ def get_datastore_directory(rdn: str) -> Path: return Path(DatastoreResponse.model_validate(response.json()).directory) +def post_public_key(datastore_rdn: str, public_key_pem: bytes) -> None: + """ + Post the public RSA key to the datastore-api. + + :param datastore_rdn: The RDN of the datastore + :param public_key_pem: The public key in PEM format as bytes + """ + request_url = f"{DATASTORE_API_URL}/datastores/{datastore_rdn}/public-key" + execute_request( + "POST", + request_url, + data=public_key_pem, + headers={ + "Content-Type": "application/x-pem-file", + "X-API-Key": DATASTORE_API_SERVICE_KEY, + }, + ) + + def query_for_jobs() -> JobQueryResult: """ Retrieves different types of jobs based on the system's state @@ -147,6 +167,7 @@ def query_for_jobs() -> JobQueryResult: Operation.REMOVE, Operation.ROLLBACK_REMOVE, Operation.DELETE_ARCHIVE, + Operation.GENERATE_RSA_KEYS, ], ), queued_worker_jobs=get_jobs( diff --git a/job_executor/adapter/datastore_api/models.py b/job_executor/adapter/datastore_api/models.py index 26551eb7..74b8b959 100644 --- a/job_executor/adapter/datastore_api/models.py +++ b/job_executor/adapter/datastore_api/models.py @@ -50,6 +50,7 @@ class Operation(StrEnum): REMOVE = "REMOVE" ROLLBACK_REMOVE = "ROLLBACK_REMOVE" DELETE_ARCHIVE = "DELETE_ARCHIVE" + GENERATE_RSA_KEYS = "GENERATE_RSA_KEYS" class ReleaseStatus(StrEnum): diff --git a/job_executor/adapter/fs/__init__.py b/job_executor/adapter/fs/__init__.py index f45bf19d..73a35030 100644 --- a/job_executor/adapter/fs/__init__.py +++ b/job_executor/adapter/fs/__init__.py @@ -4,7 +4,9 @@ from job_executor.adapter.fs.datastore_files import DatastoreDirectory from job_executor.adapter.fs.input_files import InputDirectory +from job_executor.adapter.fs.private_keys_directory import PrivateKeysDirectory from job_executor.adapter.fs.working_files import WorkingDirectory +from job_executor.config import environment class FileSystemAdapter(Protocol): @@ -21,13 +23,17 @@ class LocalStorageAdapter: datastore_dir: DatastoreDirectory working_dir: WorkingDirectory input_dir: InputDirectory + private_keys_dir: PrivateKeysDirectory - def __init__(self, datastore_dir_path: Path) -> None: + def __init__(self, datastore_dir_path: Path, datastore_rdn: str) -> None: self.datastore_dir = DatastoreDirectory(datastore_dir_path) self.working_dir = WorkingDirectory( Path(f"{datastore_dir_path}_working") ) self.input_dir = InputDirectory(Path(f"{datastore_dir_path}_input")) + self.private_keys_dir = PrivateKeysDirectory( + Path(environment.private_keys_dir) / datastore_rdn + ) def move_working_dir_parquet_to_datastore(self, dataset_name: str) -> None: """ diff --git a/job_executor/adapter/fs/datastore_files.py b/job_executor/adapter/fs/datastore_files.py index 4783b12e..05bc4579 100644 --- a/job_executor/adapter/fs/datastore_files.py +++ b/job_executor/adapter/fs/datastore_files.py @@ -19,7 +19,6 @@ class DatastoreDirectory: root_dir: Path - vault_dir: Path data_dir: Path metadata_dir: Path draft_metadata_all_path: Path @@ -31,7 +30,6 @@ def __init__(self, root_dir: Path) -> None: self.root_dir = root_dir self.data_dir = root_dir / "data" self.metadata_dir = root_dir / "datastore" - self.vault_dir = root_dir / "vault" self.draft_version_path = self.metadata_dir / "draft_version.json" self.archive_dir = self.root_dir / "archive" self.draft_metadata_all_path = ( diff --git a/job_executor/adapter/fs/private_keys_directory.py b/job_executor/adapter/fs/private_keys_directory.py new file mode 100644 index 00000000..05ada123 --- /dev/null +++ b/job_executor/adapter/fs/private_keys_directory.py @@ -0,0 +1,27 @@ +import os +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class PrivateKeysDirectory: + path_with_rdn: Path + + def create(self) -> bool: + if not self.path_with_rdn.exists(): + os.makedirs(self.path_with_rdn) + return True + return False + + def save_private_key(self, microdata_private_key_pem: bytes) -> None: + with open(self._get_private_key_location(), "wb") as file: + file.write(microdata_private_key_pem) + + def clean_up(self) -> bool: + if self._get_private_key_location().exists(): + os.remove(self._get_private_key_location()) + return True + return False + + def _get_private_key_location(self) -> Path: + return self.path_with_rdn / "microdata_private_key.pem" diff --git a/job_executor/app.py b/job_executor/app.py index 33e47fae..f9b7ccf7 100644 --- a/job_executor/app.py +++ b/job_executor/app.py @@ -24,7 +24,7 @@ def initialize_app() -> Manager: rollback.fix_interrupted_jobs() for rdn in datastore_api.get_datastores(): local_storage = LocalStorageAdapter( - datastore_api.get_datastore_directory(rdn) + datastore_api.get_datastore_directory(rdn), rdn ) if local_storage.datastore_dir.temporary_backup_exists(): raise StartupException(f"tmp directory exists for {rdn}") diff --git a/job_executor/config/__init__.py b/job_executor/config/__init__.py index cae60c15..d17eb3a3 100644 --- a/job_executor/config/__init__.py +++ b/job_executor/config/__init__.py @@ -14,6 +14,7 @@ class Environment: docker_host_name: str commit_id: str max_gb_all_workers: int + private_keys_dir: str def _initialize_environment() -> Environment: @@ -27,6 +28,7 @@ def _initialize_environment() -> Environment: docker_host_name=os.environ["DOCKER_HOST_NAME"], commit_id=os.environ["COMMIT_ID"], max_gb_all_workers=int(os.environ["MAX_GB_ALL_WORKERS"]), + private_keys_dir=os.environ["PRIVATE_KEYS_DIR"], ) @@ -36,13 +38,15 @@ def _initialize_environment() -> Environment: @dataclass class Secrets: pseudonym_service_api_key: str + datastore_api_service_key: str def _initialize_secrets() -> Secrets: with open(environment.secrets_file, encoding="utf-8") as f: secrets_file = json.load(f) return Secrets( - pseudonym_service_api_key=secrets_file["PSEUDONYM_SERVICE_API_KEY"] + pseudonym_service_api_key=secrets_file["PSEUDONYM_SERVICE_API_KEY"], + datastore_api_service_key=secrets_file["DATASTORE_API_SERVICE_KEY"], ) diff --git a/job_executor/domain/datastores.py b/job_executor/domain/datastores.py index 10c8c8cd..4cea277f 100644 --- a/job_executor/domain/datastores.py +++ b/job_executor/domain/datastores.py @@ -24,6 +24,7 @@ rollback_bump, rollback_manager_phase_import_job, ) +from job_executor.domain.rsa_keys import generate_rsa_key_pair logger = logging.getLogger() @@ -586,3 +587,51 @@ def delete_archived_input(job_context: JobContext) -> None: logger.error(f"{job_id}: An unexpected error occured") logger.exception(f"{job_id}: {str(e)}", exc_info=e) datastore_api.update_job_status(job_id, JobStatus.FAILED) + + +def generate_rsa_keys( + job_context: JobContext, +) -> None: + """ + Generate RSA key pair for a datastore. + Stores the private key in /private_keys/{rdn}/microdata_private_key.pem + and posts the public key to the datastore-api. + """ + job_id = job_context.job.job_id + datastore_rdn = job_context.job.datastore_rdn + private_keys_dir = job_context.local_storage.private_keys_dir + try: + logger.info(f"{job_id}: initiated") + datastore_api.update_job_status(job_id, JobStatus.INITIATED) + + logger.info( + f"{job_id}: Checking private keys directory at " + f"{private_keys_dir.path_with_rdn}" + ) + if private_keys_dir.create(): + logger.info(f"{job_id}: Private keys directory created") + + logger.info(f"{job_id}: Generating RSA key pair") + private_key_pem, public_key_pem = generate_rsa_key_pair() + + private_keys_dir.save_private_key(private_key_pem) + logger.info(f"{job_id}: Saved private key") + + try: + logger.info(f"{job_id}: Posting public key to datastore-api") + datastore_api.post_public_key(datastore_rdn, public_key_pem) + except Exception as post_error: + logger.error( + f"{job_id}: Failed to post public key to datastore-api, " + "cleaning up saved private key" + ) + if private_keys_dir.clean_up(): + logger.info(f"{job_id}: Deleted private key due to error") + raise post_error + + logger.info(f"{job_id}: completed") + datastore_api.update_job_status(job_id, JobStatus.COMPLETED) + except Exception as e: + logger.error(f"{job_id}: Failed to generate RSA keys") + logger.exception(f"{job_id}: {str(e)}", exc_info=e) + datastore_api.update_job_status(job_id, JobStatus.FAILED, str(e)) diff --git a/job_executor/domain/manager/__init__.py b/job_executor/domain/manager/__init__.py index 2d921a11..6e0f51ab 100644 --- a/job_executor/domain/manager/__init__.py +++ b/job_executor/domain/manager/__init__.py @@ -167,6 +167,8 @@ def _handle_manager_job(self, job_context: JobContext) -> None: datastores.delete_draft(job_context) elif operation == Operation.DELETE_ARCHIVE: datastores.delete_archived_input(job_context) + elif operation == Operation.GENERATE_RSA_KEYS: + datastores.generate_rsa_keys(job_context) else: datastore_api.update_job_status( job_context.job.job_id, diff --git a/job_executor/domain/models.py b/job_executor/domain/models.py index fc118394..7784f72a 100644 --- a/job_executor/domain/models.py +++ b/job_executor/domain/models.py @@ -18,7 +18,8 @@ class JobContext: def build_job_context(job: Job, handler: handler_type) -> JobContext: local_storage = LocalStorageAdapter( - datastore_api.get_datastore_directory(job.datastore_rdn) + datastore_api.get_datastore_directory(job.datastore_rdn), + job.datastore_rdn, ) job_size = ( local_storage.input_dir.get_importable_tar_size_in_bytes( diff --git a/job_executor/domain/rollback.py b/job_executor/domain/rollback.py index 04e8ff03..544db724 100644 --- a/job_executor/domain/rollback.py +++ b/job_executor/domain/rollback.py @@ -20,6 +20,7 @@ RollbackException, StartupException, ) +from job_executor.config import environment logger = logging.getLogger() @@ -27,7 +28,8 @@ def rollback_bump(job: Job, bump_manifesto: DatastoreVersion) -> None: job_id = job.job_id local_storage = LocalStorageAdapter( - datastore_api.get_datastore_directory(job.datastore_rdn) + datastore_api.get_datastore_directory(job.datastore_rdn), + job.datastore_rdn, ) try: logger.info(f"{job_id}: Restoring files from temporary backup") @@ -137,7 +139,8 @@ def rollback_worker_phase_import_job( ) -> None: job_id = job.job_id local_storage = LocalStorageAdapter( - datastore_api.get_datastore_directory(job.datastore_rdn) + datastore_api.get_datastore_directory(job.datastore_rdn), + job.datastore_rdn, ) logger.warning( f"{job_id}: Rolling back worker job " @@ -193,7 +196,8 @@ def rollback_manager_phase_import_job( """ job_id = job.job_id local_storage = LocalStorageAdapter( - datastore_api.get_datastore_directory(job.datastore_rdn) + datastore_api.get_datastore_directory(job.datastore_rdn), + job.datastore_rdn, ) logger.warning( f"{job_id}: Rolling back import job " @@ -290,9 +294,31 @@ def fix_interrupted_job(job: Job) -> None: JobStatus.FAILED, "Bump operation was interrupted and rolled back.", ) + elif job_operation == "GENERATE_RSA_KEYS": + rollback_generate_rsa_keys_job(job) + logger.info( + f'{job.job_id}: Setting status to "failed" for interrupted job' + ) + datastore_api.update_job_status( + job.job_id, + JobStatus.FAILED, + "Job was failed due to an unexpected interruption", + ) else: log_message = ( f"Unrecognized job operation {job_operation} for job {job.job_id}" ) logger.error(log_message) raise RollbackException(log_message) + + +def rollback_generate_rsa_keys_job(job: Job) -> None: + rdn = job.datastore_rdn + target_dir = Path(environment.private_keys_dir) / rdn + private_key_location = target_dir / "microdata_private_key.pem" + if private_key_location.exists(): + os.remove(private_key_location) + logger.info( + f"{job.job_id}: Rollback: deleted private key at " + f"{private_key_location}" + ) diff --git a/job_executor/domain/rsa_keys.py b/job_executor/domain/rsa_keys.py new file mode 100644 index 00000000..f6b9de68 --- /dev/null +++ b/job_executor/domain/rsa_keys.py @@ -0,0 +1,27 @@ +import logging + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +logger = logging.getLogger() + + +def generate_rsa_key_pair() -> tuple[bytes, bytes]: + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + public_key = private_key.public_key() + + private_key_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + public_key_pem = public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + + return private_key_pem, public_key_pem diff --git a/job_executor/domain/worker/build_dataset_worker.py b/job_executor/domain/worker/build_dataset_worker.py index 936a04b9..65b888c4 100644 --- a/job_executor/domain/worker/build_dataset_worker.py +++ b/job_executor/domain/worker/build_dataset_worker.py @@ -1,12 +1,14 @@ import logging import os from multiprocessing import Queue +from pathlib import Path from time import perf_counter from job_executor.adapter import datastore_api from job_executor.adapter.datastore_api.models import JobStatus from job_executor.adapter.fs import LocalStorageAdapter from job_executor.common.exceptions import BuilderStepError, HttpResponseError +from job_executor.config import environment from job_executor.config.log import configure_worker_logger from job_executor.domain.models import JobContext from job_executor.domain.worker.steps import ( @@ -48,6 +50,7 @@ def run_worker(job_context: JobContext, logging_queue: Queue) -> None: job_id = job_context.job.job_id local_storage = job_context.local_storage dataset_name = job_context.job.parameters.target + datastore_rdn = job_context.job.datastore_rdn try: configure_worker_logger(logging_queue, job_id) logger.info( @@ -60,7 +63,7 @@ def run_worker(job_context: JobContext, logging_queue: Queue) -> None: dataset_name, local_storage.input_dir.path, local_storage.working_dir.path, - local_storage.datastore_dir.vault_dir, + Path(environment.private_keys_dir) / datastore_rdn, ) datastore_api.update_job_status(job_id, JobStatus.VALIDATING) (data_file_name, _) = dataset_validator.run_for_dataset( diff --git a/job_executor/domain/worker/build_metadata_worker.py b/job_executor/domain/worker/build_metadata_worker.py index 9ae7680d..adb2c93f 100644 --- a/job_executor/domain/worker/build_metadata_worker.py +++ b/job_executor/domain/worker/build_metadata_worker.py @@ -1,11 +1,13 @@ import logging from multiprocessing import Queue +from pathlib import Path from time import perf_counter from job_executor.adapter import datastore_api from job_executor.adapter.datastore_api.models import JobStatus from job_executor.adapter.fs import LocalStorageAdapter from job_executor.common.exceptions import BuilderStepError, HttpResponseError +from job_executor.config import environment from job_executor.config.log import configure_worker_logger from job_executor.domain.models import JobContext from job_executor.domain.worker.steps import ( @@ -29,6 +31,7 @@ def run_worker(job_context: JobContext, logging_queue: Queue) -> None: job_id = job_context.job.job_id local_storage = job_context.local_storage dataset_name = job_context.job.parameters.target + datastore_rdn = job_context.job.datastore_rdn try: configure_worker_logger(logging_queue, job_id) logger.info( @@ -41,7 +44,7 @@ def run_worker(job_context: JobContext, logging_queue: Queue) -> None: dataset_name, local_storage.input_dir.path, local_storage.working_dir.path, - local_storage.datastore_dir.vault_dir, + Path(environment.private_keys_dir) / datastore_rdn, ) datastore_api.update_job_status(job_id, JobStatus.VALIDATING) dataset_validator.run_for_metadata( diff --git a/tests/conftest.py b/tests/conftest.py index eaebce75..e188b89b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,3 +9,4 @@ os.environ["DOCKER_HOST_NAME"] = "localhost" os.environ["COMMIT_ID"] = "abc123" os.environ["MAX_GB_ALL_WORKERS"] = "50" +os.environ["PRIVATE_KEYS_DIR"] = "tests/integration/resources/private_keys" diff --git a/tests/integration/common.py b/tests/integration/common.py index 9e68043f..9c9ee864 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -8,11 +8,10 @@ from cryptography.hazmat.primitives.asymmetric import rsa from microdata_tools import package_dataset +PRIVATE_KEYS_DIR = Path("tests/integration/resources/private_keys") -def _create_key_pair(vault_dir: Path): - if not vault_dir.exists(): - os.makedirs(vault_dir) +def _create_key_pair(public_key_dir: Path, private_key_dir: Path): private_key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend() ) @@ -21,11 +20,11 @@ def _create_key_pair(vault_dir: Path): encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo, ) - public_key_location = vault_dir / "microdata_public_key.pem" + public_key_location = public_key_dir / "microdata_public_key.pem" with open(public_key_location, "wb") as file: file.write(microdata_public_key_pem) - with open(vault_dir / "microdata_private_key.pem", "wb") as file: + with open(private_key_dir / "microdata_private_key.pem", "wb") as file: file.write( private_key.private_bytes( encoding=serialization.Encoding.PEM, @@ -74,11 +73,12 @@ def _render_metadata_all(metadata_all_file: Path) -> None: def _package_to_input(datastore_dir: str): package_dir = Path("tests/integration/resources/input_datasets") input_dir = Path(f"{datastore_dir}_input") - vault_dir = Path(f"{datastore_dir}/vault") - _create_key_pair(vault_dir) + public_key_dir = Path(f"{datastore_dir}/vault") + private_key_dir = PRIVATE_KEYS_DIR / Path(datastore_dir).name + _create_key_pair(public_key_dir, private_key_dir) for dataset in os.listdir(package_dir): package_dataset( - rsa_keys_dir=vault_dir, + rsa_keys_dir=public_key_dir, dataset_dir=Path(package_dir / dataset), output_dir=Path(input_dir), ) diff --git a/tests/integration/resources/private_keys/TEST_DATASTORE/.gitkeep b/tests/integration/resources/private_keys/TEST_DATASTORE/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_datastore.py b/tests/integration/test_datastore.py index 649cc36e..9f474d8e 100644 --- a/tests/integration/test_datastore.py +++ b/tests/integration/test_datastore.py @@ -17,9 +17,11 @@ from job_executor.adapter.fs import LocalStorageAdapter from job_executor.adapter.fs.models.datastore_versions import DatastoreVersion from job_executor.adapter.fs.models.metadata import Metadata +from job_executor.common.exceptions import HttpResponseError from job_executor.domain import datastores from job_executor.domain.models import JobContext from tests.integration.common import ( + PRIVATE_KEYS_DIR, backup_resources, prepare_datastore, recover_resources_from_backup, @@ -34,6 +36,7 @@ @dataclass class MockedDatastoreApi: update_job_status: MagicMock + post_public_key: MagicMock @pytest.fixture(autouse=True) @@ -42,7 +45,11 @@ def mocked_datastore_api(mocker) -> MockedDatastoreApi: update_job_status=mocker.patch( "job_executor.adapter.datastore_api.update_job_status", return_value=None, - ) + ), + post_public_key=mocker.patch( + "job_executor.adapter.datastore_api.post_public_key", + return_value=None, + ), ) @@ -96,7 +103,7 @@ def generate_job_parameters( target=target, release_status=release_status, ) - case Operation.DELETE_DRAFT: + case Operation.DELETE_DRAFT | Operation.GENERATE_RSA_KEYS: return JobParameters( operation=operation, target=target, @@ -105,7 +112,7 @@ def generate_job_parameters( return JobContext( handler="worker", - local_storage=LocalStorageAdapter(DATASTORE_DIR), + local_storage=LocalStorageAdapter(DATASTORE_DIR, "TEST_DATASTORE"), job_size=100, job=Job( job_id="1", @@ -294,3 +301,47 @@ def test_delete_draft(mocked_datastore_api: MockedDatastoreApi): ) assert draft_version.get_dataset_release_status(DATASET_NAME) is None assert mocked_datastore_api.update_job_status.call_count == 2 + + +def test_generate_rsa_keys(mocked_datastore_api: MockedDatastoreApi): + generate_rsa_keys_job_context = generate_job_context( + operation=Operation.GENERATE_RSA_KEYS, + target="DATASTORE", + ) + datastores.generate_rsa_keys(generate_rsa_keys_job_context) + + assert mocked_datastore_api.post_public_key.call_count == 1 + assert mocked_datastore_api.update_job_status.call_count == 2 + + private_key_path = ( + PRIVATE_KEYS_DIR + / generate_rsa_keys_job_context.job.datastore_rdn + / "microdata_private_key.pem" + ) + assert private_key_path.exists() + with open(private_key_path, "rb") as f: + private_key_content = f.read() + assert b"BEGIN PRIVATE KEY" in private_key_content + assert b"END PRIVATE KEY" in private_key_content + + +def test_generate_rsa_keys_cleanup_on_post_public_key_failure( + mocked_datastore_api: MockedDatastoreApi, +): + mocked_datastore_api.post_public_key.side_effect = HttpResponseError( + "500: Internal Server Error" + ) + generate_rsa_keys_job_context = generate_job_context( + operation=Operation.GENERATE_RSA_KEYS, + target="DATASTORE", + ) + datastores.generate_rsa_keys(generate_rsa_keys_job_context) + + assert mocked_datastore_api.update_job_status.call_count == 2 + + private_key_path = ( + PRIVATE_KEYS_DIR + / generate_rsa_keys_job_context.job.datastore_rdn + / "microdata_private_key.pem" + ) + assert not private_key_path.exists() diff --git a/tests/integration/test_import.py b/tests/integration/test_import.py index 9f6761d5..749eb792 100644 --- a/tests/integration/test_import.py +++ b/tests/integration/test_import.py @@ -78,15 +78,17 @@ def mocked_pseudonym_service(mocker) -> MockedPseudonymService: @pytest.fixture(autouse=True) def set_up_resources(): backup_resources() - prepare_datastore(str(DATASTORE_DIR), package_to_input=True) - yield - recover_resources_from_backup() + try: + prepare_datastore(str(DATASTORE_DIR), package_to_input=True) + yield + finally: + recover_resources_from_backup() def generate_job_context(operation: Operation, target: str) -> JobContext: return JobContext( handler="worker", - local_storage=LocalStorageAdapter(DATASTORE_DIR), + local_storage=LocalStorageAdapter(DATASTORE_DIR, "TEST_DATASTORE"), job_size=100, job=Job( job_id="1", diff --git a/tests/resources/secrets/secrets.json b/tests/resources/secrets/secrets.json index e5798917..11acb812 100644 --- a/tests/resources/secrets/secrets.json +++ b/tests/resources/secrets/secrets.json @@ -1 +1,4 @@ -{"PSEUDONYM_SERVICE_API_KEY": "abc123"} +{ + "PSEUDONYM_SERVICE_API_KEY": "123", + "DATASTORE_API_SERVICE_KEY": "abc123" +} \ No newline at end of file diff --git a/tests/unit/adapter/fs/model/test_datastore_versions.py b/tests/unit/adapter/fs/model/test_datastore_versions.py index 86e9ff7e..5477ab21 100644 --- a/tests/unit/adapter/fs/model/test_datastore_versions.py +++ b/tests/unit/adapter/fs/model/test_datastore_versions.py @@ -16,7 +16,7 @@ def load_json(file_path): DATASTORE_DIR = "tests/unit/resources/adapter/fs/TEST_DATASTORE" METADATA_DIR = f"{DATASTORE_DIR}/datastore" -local_storage = LocalStorageAdapter(Path(DATASTORE_DIR)) +local_storage = LocalStorageAdapter(Path(DATASTORE_DIR), "TEST_DATASTORE") DATASTORE_VERSIONS_PATH = f"{METADATA_DIR}/datastore_versions.json" diff --git a/tests/unit/adapter/fs/test_local_storage.py b/tests/unit/adapter/fs/test_local_storage.py index 394dafce..d54c5baa 100644 --- a/tests/unit/adapter/fs/test_local_storage.py +++ b/tests/unit/adapter/fs/test_local_storage.py @@ -19,7 +19,7 @@ WORKING_DIR = DATASTORE_DIR + "_working" DATASTORE_DATA_DIR = f"{DATASTORE_DIR}/data" -local_storage = LocalStorageAdapter(Path(DATASTORE_DIR)) +local_storage = LocalStorageAdapter(Path(DATASTORE_DIR), "TEST_DATASTORE") DATASTORE_VERSIONS_PATH = f"{DATASTORE_DIR}/datastore/datastore_versions.json" DRAFT_METADATA_ALL_PATH = f"{DATASTORE_DIR}/datastore/metadata_all__draft.json" diff --git a/tests/unit/adapter/test_datastore_api.py b/tests/unit/adapter/test_datastore_api.py index d1fda422..e06f7685 100644 --- a/tests/unit/adapter/test_datastore_api.py +++ b/tests/unit/adapter/test_datastore_api.py @@ -179,3 +179,35 @@ def mock_get_jobs(job_status=None, operations=None): else: assert result.queued_manager_jobs == JOB_LIST assert result.queued_worker_jobs == JOB_LIST + + +def test_post_public_key(requests_mock: RequestsMocker): + public_key_pem = ( + b"-----BEGIN PUBLIC KEY-----\n" + b"test_key_content\n" + b"-----END PUBLIC KEY-----" + ) + requests_mock.post( + f"{DATASTORE_API_URL}/datastores/{DATASTORE_RDN}/public-key", + json={"message": "OK"}, + ) + datastore_api.post_public_key(DATASTORE_RDN, public_key_pem) + request_history = requests_mock.request_history + assert len(request_history) == 1 + assert request_history[0].body == public_key_pem + + +def test_post_public_key_error(requests_mock: RequestsMocker): + public_key_pem = ( + b"-----BEGIN PUBLIC KEY-----\n" + b"test_key_content\n" + b"-----END PUBLIC KEY-----" + ) + requests_mock.post( + f"{DATASTORE_API_URL}/datastores/{DATASTORE_RDN}/public-key", + status_code=500, + text=ERROR_RESPONSE, + ) + with pytest.raises(HttpResponseError) as e: + datastore_api.post_public_key(DATASTORE_RDN, public_key_pem) + assert ERROR_RESPONSE in str(e)