|
24 | 24 | import tempfile |
25 | 25 | from pathlib import Path |
26 | 26 | from venv import logger |
| 27 | + |
27 | 28 | import environ |
28 | | -from scanpipe.archiving import LocalFilesystemProvider, S3LikeProvider, SftpProvider |
29 | 29 |
|
| 30 | +from scanpipe.archiving import LocalFilesystemProvider |
| 31 | +from scanpipe.archiving import S3LikeProvider |
| 32 | +from scanpipe.archiving import SftpProvider |
30 | 33 |
|
31 | 34 | PROJECT_DIR = environ.Path(__file__) - 1 |
32 | 35 | ROOT_DIR = PROJECT_DIR - 1 |
|
378 | 381 | ENABLE_DOWNLOAD_ARCHIVING = env.bool("ENABLE_DOWNLOAD_ARCHIVING", default=False) |
379 | 382 |
|
380 | 383 | # localstorage, s3, sftp |
381 | | -DOWNLOAD_ARCHIVING_PROVIDER = env.str("DOWNLOAD_ARCHIVING_PROVIDER", default="localstorage") |
| 384 | +DOWNLOAD_ARCHIVING_PROVIDER = env.str( |
| 385 | + "DOWNLOAD_ARCHIVING_PROVIDER", default="localstorage" |
| 386 | +) |
382 | 387 |
|
383 | 388 | # For local storage, we would store the root path in that setting |
384 | | -DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION = env.dict("DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION", default=None) |
| 389 | +DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION = env.dict( |
| 390 | + "DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION", default=None |
| 391 | +) |
385 | 392 |
|
386 | 393 | # Initialize the DownloadStore based on provider |
387 | 394 |
|
|
398 | 405 | config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {} |
399 | 406 | required_keys = ["bucket_name", "aws_userid", "aws_apikey"] |
400 | 407 | if not all(key in config for key in required_keys): |
401 | | - logger.error(f"S3 provider requires {required_keys} in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION") |
| 408 | + logger.error( |
| 409 | + f"S3 provider requires {required_keys}" |
| 410 | + "in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION" |
| 411 | + ) |
402 | 412 | else: |
403 | 413 | try: |
404 | 414 | download_store = S3LikeProvider( |
|
413 | 423 | config = DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION or {} |
414 | 424 | required_keys = ["host", "root_path", "ssh_credentials"] |
415 | 425 | if not all(key in config for key in required_keys): |
416 | | - logger.error(f"SFTP provider requires {required_keys} in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION") |
| 426 | + logger.error( |
| 427 | + f"SFTP provider requires {required_keys}" |
| 428 | + "in DOWNLOAD_ARCHIVING_PROVIDER_CONFIGURATION" |
| 429 | + ) |
417 | 430 | else: |
418 | 431 | try: |
419 | 432 | download_store = SftpProvider( |
|
424 | 437 | except Exception as e: |
425 | 438 | logger.error(f"Failed to initialize SftpProvider: {e}") |
426 | 439 | else: |
427 | | - logger.error(f"Unknown DOWNLOAD_ARCHIVING_PROVIDER: {DOWNLOAD_ARCHIVING_PROVIDER}") |
| 440 | + logger.error( |
| 441 | + f"Unknown DOWNLOAD_ARCHIVING_PROVIDER: {DOWNLOAD_ARCHIVING_PROVIDER}" |
| 442 | + ) |
428 | 443 |
|
429 | 444 | # Job Queue |
430 | 445 |
|
|
0 commit comments