diff --git a/.gitignore b/.gitignore index 2f739edb..a8e1f81b 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,13 @@ cython_debug/ #.idea/ .vscode + +# pyenv version +.python-version + +# vim backup files +*~ +repo_time_tester.py +reset.sh +seed_data_to_ds.py +docker_test/scripts/license.json diff --git a/GIT_WORKFLOW.md b/GIT_WORKFLOW.md new file mode 100644 index 00000000..0c8e8bab --- /dev/null +++ b/GIT_WORKFLOW.md @@ -0,0 +1,78 @@ +# Git Workflow for RC Development + +## Context +After submitting PR from RC1 branch to upstream, continuing development that builds upon RC1 changes. + +## Recommended Approach: Branch from RC1 + +### Create new branch from RC1 +```bash +git checkout -b RC2 RC1 +``` + +### Benefits +- Continue developing immediately with the RC1 foundation +- Keep RC1 frozen for the PR review process +- Maintain flexibility for PR changes + +### Workflow + +1. **Develop new features on RC2** + ```bash + git checkout RC2 + # Make changes, commit as normal + ``` + +2. **If RC1 needs changes from PR review:** + ```bash + # Switch back to RC1 + git checkout RC1 + + # Make requested changes + # Commit changes + + # Push updates to PR + git push origin RC1 + ``` + +3. **Sync RC2 with updated RC1:** + ```bash + git checkout RC2 + git rebase RC1 + ``` + +4. **After RC1 is merged upstream:** + ```bash + # Sync master with upstream + git checkout master + git fetch upstream + git merge upstream/master + git push origin master + + # Rebase RC2 onto master + git checkout RC2 + git rebase master + + # Clean up merged RC1 branch + git branch -d RC1 + git push origin --delete RC1 + ``` + +## Alternative Naming +Instead of RC2, consider more descriptive names: +- `deepfreeze-phase2` +- `feature/deepfreeze-enhancements` +- `RC2-deepfreeze-completion` + +## Other Approaches Considered + +### New branch from master (for independent work) +```bash +git checkout master +git checkout -b feature/new-feature +``` +**Use when:** Next work is independent of RC1 changes + +### Wait for PR merge (most conservative) +Wait until PR is accepted, sync with upstream, then branch +**Use when:** No urgency and want clean linear history diff --git a/README.rst b/README.rst index ca72b6fc..3def93e6 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,8 @@ .. _readme: +**THIS FORK OF ELASTIC/CURATOR REPRESENTS A WORK-IN-PROGRESS AND SHOULD NOT BE CONSIDERED "RUNNABLE". IT IS STILL IN DEVELOPMENT.** + +**HERE THERE BE TYGERS.** Curator ======= diff --git a/curator/actions/__init__.py b/curator/actions/__init__.py index a8c365f1..e547bf86 100644 --- a/curator/actions/__init__.py +++ b/curator/actions/__init__.py @@ -1,10 +1,12 @@ """Use __init__ to make these not need to be nested under lowercase.Capital""" + from curator.actions.alias import Alias from curator.actions.allocation import Allocation from curator.actions.close import Close from curator.actions.cluster_routing import ClusterRouting from curator.actions.cold2frozen import Cold2Frozen from curator.actions.create_index import CreateIndex +from curator.actions.deepfreeze import Cleanup, Deepfreeze, Refreeze, Rotate, Setup, Status, Thaw from curator.actions.delete_indices import DeleteIndices from curator.actions.forcemerge import ForceMerge from curator.actions.index_settings import IndexSettings @@ -13,24 +15,31 @@ from curator.actions.replicas import Replicas from curator.actions.rollover import Rollover from curator.actions.shrink import Shrink -from curator.actions.snapshot import Snapshot, DeleteSnapshots, Restore +from curator.actions.snapshot import DeleteSnapshots, Restore, Snapshot CLASS_MAP = { - 'alias' : Alias, - 'allocation' : Allocation, - 'close' : Close, - 'cluster_routing' : ClusterRouting, - 'cold2frozen': Cold2Frozen, - 'create_index' : CreateIndex, - 'delete_indices' : DeleteIndices, - 'delete_snapshots' : DeleteSnapshots, - 'forcemerge' : ForceMerge, - 'index_settings' : IndexSettings, - 'open' : Open, - 'reindex' : Reindex, - 'replicas' : Replicas, - 'restore' : Restore, - 'rollover' : Rollover, - 'snapshot' : Snapshot, - 'shrink' : Shrink, + "alias": Alias, + "allocation": Allocation, + "cleanup": Cleanup, + "close": Close, + "cluster_routing": ClusterRouting, + "cold2frozen": Cold2Frozen, + "create_index": CreateIndex, + "deepfreeze": Deepfreeze, + "delete_indices": DeleteIndices, + "delete_snapshots": DeleteSnapshots, + "forcemerge": ForceMerge, + "index_settings": IndexSettings, + "open": Open, + "refreeze": Refreeze, + "reindex": Reindex, + "replicas": Replicas, + "restore": Restore, + "rollover": Rollover, + "shrink": Shrink, + "snapshot": Snapshot, + "setup": Setup, + "rotate": Rotate, + "status": Status, + "thaw": Thaw, } diff --git a/curator/actions/deepfreeze/README.md b/curator/actions/deepfreeze/README.md new file mode 100644 index 00000000..4b5d9051 --- /dev/null +++ b/curator/actions/deepfreeze/README.md @@ -0,0 +1,15 @@ +# Deepfreeze Module + +## To Do +- [ ] Fix generation of Repository using utility method instead of constructor +- [ ] Ensure dry_run is respected throughout +- [ ] Ensure Repository updates in the STATUS_INDEX are happening properly and reliably + + +## To Fix + + +## Author + +Deepfreeze was written by Bret Wortman (bret.wortman@elastic.co) but it's built on +the foundation of Curator, which is the work of Aaron Mildenstein and many others. diff --git a/curator/actions/deepfreeze/__init__.py b/curator/actions/deepfreeze/__init__.py new file mode 100644 index 00000000..4babc6b2 --- /dev/null +++ b/curator/actions/deepfreeze/__init__.py @@ -0,0 +1,51 @@ +"""Deepfreeze actions module""" + +from .constants import PROVIDERS, SETTINGS_ID, STATUS_INDEX +from .exceptions import ( + ActionException, + DeepfreezeException, + MissingIndexError, + MissingSettingsError, +) +from .cleanup import Cleanup +from .helpers import Deepfreeze, Repository, Settings +from .refreeze import Refreeze +from .rotate import Rotate +from .setup import Setup +from .status import Status +from .thaw import Thaw +from .utilities import ( + check_restore_status, + create_repo, + decode_date, + ensure_settings_index, + find_repos_by_date_range, + get_all_indices_in_repo, + get_all_repos, + get_matching_repo_names, + get_matching_repos, + get_next_suffix, + get_repositories_by_names, + get_settings, + get_thaw_request, + get_timestamp_range, + list_thaw_requests, + mount_repo, + push_to_glacier, + save_settings, + save_thaw_request, + unmount_repo, + update_repository_date_range, +) + +CLASS_MAP = { + "cleanup": Cleanup, + "deepfreeze": Deepfreeze, + "refreeze": Refreeze, + "repository": Repository, + "settings": Settings, + "setup": Setup, + "rotate": Rotate, + "status": Status, + "thaw": Thaw, +} diff --git a/curator/actions/deepfreeze/cleanup.py b/curator/actions/deepfreeze/cleanup.py new file mode 100644 index 00000000..5ef6af57 --- /dev/null +++ b/curator/actions/deepfreeze/cleanup.py @@ -0,0 +1,845 @@ +"""Cleanup action for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging +from datetime import datetime, timezone + +from elasticsearch8 import Elasticsearch + +from curator.actions.deepfreeze.utilities import ( + check_restore_status, + get_all_indices_in_repo, + get_matching_repos, + get_repositories_by_names, + get_settings, + list_thaw_requests, +) +from curator.s3client import s3_client_factory + + +class Cleanup: + """ + The Cleanup action checks thawed repositories and unmounts them if their S3 objects + have reverted to Glacier storage. It also deletes indices whose snapshots are only + in the repositories being cleaned up. + + When objects are restored from Glacier, they're temporarily available in Standard tier + for a specified duration. After that duration expires, they revert to Glacier storage. + This action: + 1. Detects thawed repositories that have passed their expires_at timestamp and marks them as expired + 2. Unmounts expired repositories and resets them to frozen state + 3. Deletes indices whose snapshots are only in expired repositories + 4. Cleans up old thaw requests based on status and retention settings + 5. Cleans up orphaned thawed ILM policies + + :param client: A client connection object + :type client: Elasticsearch + + :methods: + do_action: Perform the cleanup operation (detect expired repos, unmount, delete indices). + do_dry_run: Perform a dry-run of the cleanup operation. + do_singleton_action: Entry point for singleton CLI execution. + """ + + def __init__(self, client: Elasticsearch) -> None: + self.loggit = logging.getLogger("curator.actions.deepfreeze") + self.loggit.debug("Initializing Deepfreeze Cleanup") + + self.client = client + self.settings = get_settings(client) + self.s3 = s3_client_factory(self.settings.provider) + + self.loggit.info("Deepfreeze Cleanup initialized") + + def _get_indices_to_delete(self, repos_to_cleanup: list) -> list[str]: + """ + Find indices that should be deleted because they only have snapshots + in repositories being cleaned up. + + :param repos_to_cleanup: List of Repository objects being cleaned up + :type repos_to_cleanup: list[Repository] + + :return: List of index names to delete + :rtype: list[str] + """ + self.loggit.debug("Finding indices to delete from repositories being cleaned up") + + # Get all repository names being cleaned up + cleanup_repo_names = {repo.name for repo in repos_to_cleanup} + self.loggit.debug("Repositories being cleaned up: %s", cleanup_repo_names) + + # Collect all indices from snapshots in repositories being cleaned up + indices_in_cleanup_repos = set() + for repo in repos_to_cleanup: + try: + indices = get_all_indices_in_repo(self.client, repo.name) + indices_in_cleanup_repos.update(indices) + self.loggit.debug( + "Repository %s contains %d indices in its snapshots", + repo.name, + len(indices) + ) + except Exception as e: + self.loggit.warning( + "Could not get indices from repository %s: %s", repo.name, e + ) + continue + + if not indices_in_cleanup_repos: + self.loggit.debug("No indices found in repositories being cleaned up") + return [] + + self.loggit.debug( + "Found %d total indices in repositories being cleaned up", + len(indices_in_cleanup_repos) + ) + + # Get all repositories in the cluster + try: + all_repos = self.client.snapshot.get_repository() + all_repo_names = set(all_repos.keys()) + except Exception as e: + self.loggit.error("Failed to get repository list: %s", e) + return [] + + # Repositories NOT being cleaned up + other_repos = all_repo_names - cleanup_repo_names + self.loggit.debug("Other repositories in cluster: %s", other_repos) + + # Check which indices exist only in repositories being cleaned up + indices_to_delete = [] + for index in indices_in_cleanup_repos: + # Check if this index exists in Elasticsearch + if not self.client.indices.exists(index=index): + self.loggit.debug( + "Index %s does not exist in cluster, skipping", index + ) + continue + + # Check if this index has snapshots in other repositories + has_snapshots_elsewhere = False + for repo_name in other_repos: + try: + indices_in_repo = get_all_indices_in_repo(self.client, repo_name) + if index in indices_in_repo: + self.loggit.debug( + "Index %s has snapshots in repository %s, will not delete", + index, + repo_name + ) + has_snapshots_elsewhere = True + break + except Exception as e: + self.loggit.warning( + "Could not check repository %s for index %s: %s", + repo_name, + index, + e + ) + continue + + # Only delete if index has no snapshots in other repositories + if not has_snapshots_elsewhere: + indices_to_delete.append(index) + self.loggit.debug( + "Index %s will be deleted (only exists in repositories being cleaned up)", + index + ) + + self.loggit.info("Found %d indices to delete", len(indices_to_delete)) + return indices_to_delete + + def _detect_and_mark_expired_repos(self) -> int: + """ + Detect repositories whose S3 restore has expired and mark them as expired. + + Checks repositories in two ways: + 1. Thawed repos with expires_at timestamp that has passed + 2. Mounted repos (regardless of state) by checking S3 restore status directly + + :return: Count of repositories marked as expired + :rtype: int + """ + self.loggit.debug("Detecting expired repositories") + + from curator.actions.deepfreeze.constants import THAW_STATE_THAWED + all_repos = get_matching_repos(self.client, self.settings.repo_name_prefix) + + # Get thawed repos for timestamp-based checking + thawed_repos = [repo for repo in all_repos if repo.thaw_state == THAW_STATE_THAWED] + + # Get mounted repos for S3-based checking (may overlap with thawed_repos) + mounted_repos = [repo for repo in all_repos if repo.is_mounted] + + self.loggit.debug( + "Found %d thawed repositories and %d mounted repositories to check", + len(thawed_repos), + len(mounted_repos) + ) + + now = datetime.now(timezone.utc) + expired_count = 0 + checked_repos = set() # Track repos we've already processed + + # METHOD 1: Check thawed repos with expires_at timestamp + for repo in thawed_repos: + if repo.name in checked_repos: + continue + + if repo.expires_at: + expires_at = repo.expires_at + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + + if now >= expires_at: + self.loggit.info( + "Repository %s has expired based on timestamp (expired at %s)", + repo.name, + expires_at.isoformat() + ) + repo.mark_expired() + try: + repo.persist(self.client) + self.loggit.info("Marked repository %s as expired", repo.name) + expired_count += 1 + checked_repos.add(repo.name) + except Exception as e: + self.loggit.error( + "Failed to mark repository %s as expired: %s", + repo.name, + e + ) + else: + checked_repos.add(repo.name) + else: + self.loggit.warning( + "Repository %s is in thawed state but has no expires_at timestamp", + repo.name + ) + + # METHOD 2: Check mounted repos by querying S3 restore status + self.loggit.debug("Checking S3 restore status for mounted repositories") + for repo in mounted_repos: + if repo.name in checked_repos: + continue + + try: + # Check actual S3 restore status + self.loggit.debug( + "Checking S3 restore status for repository %s (bucket: %s, path: %s)", + repo.name, + repo.bucket, + repo.base_path + ) + + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + + # If all objects are back in Glacier (not restored), mark as expired + if status["not_restored"] > 0 and status["restored"] == 0 and status["in_progress"] == 0: + self.loggit.info( + "Repository %s has expired based on S3 status: %d/%d objects not restored", + repo.name, + status["not_restored"], + status["total"] + ) + repo.mark_expired() + try: + repo.persist(self.client) + self.loggit.info("Marked repository %s as expired", repo.name) + expired_count += 1 + checked_repos.add(repo.name) + except Exception as e: + self.loggit.error( + "Failed to mark repository %s as expired: %s", + repo.name, + e + ) + elif status["restored"] > 0 or status["in_progress"] > 0: + self.loggit.debug( + "Repository %s still has restored objects: %d restored, %d in progress", + repo.name, + status["restored"], + status["in_progress"] + ) + checked_repos.add(repo.name) + + except Exception as e: + self.loggit.error( + "Failed to check S3 restore status for repository %s: %s", + repo.name, + e + ) + continue + + if expired_count > 0: + self.loggit.info("Marked %d repositories as expired", expired_count) + + return expired_count + + def _cleanup_old_thaw_requests(self) -> tuple[list[str], list[str]]: + """ + Clean up old thaw requests based on status and age. + + Deletes: + - Completed requests older than retention period + - Failed requests older than retention period + - Refrozen requests older than retention period (35 days by default) + - Stale in-progress requests where all referenced repos are no longer thawed + + :return: Tuple of (deleted_request_ids, skipped_request_ids) + :rtype: tuple[list[str], list[str]] + """ + self.loggit.debug("Cleaning up old thaw requests") + + # Get all thaw requests + try: + requests = list_thaw_requests(self.client) + except Exception as e: + self.loggit.error("Failed to list thaw requests: %s", e) + return [], [] + + if not requests: + self.loggit.debug("No thaw requests found") + return [], [] + + self.loggit.info("Found %d thaw requests to evaluate for cleanup", len(requests)) + + now = datetime.now(timezone.utc) + deleted = [] + skipped = [] + + # Get retention settings + retention_completed = self.settings.thaw_request_retention_days_completed + retention_failed = self.settings.thaw_request_retention_days_failed + retention_refrozen = self.settings.thaw_request_retention_days_refrozen + + for request in requests: + request_id = request.get("id") + if not request_id: + self.loggit.warning("Thaw request missing id, skipping") + continue + + status = request.get("status", "unknown") + created_at_str = request.get("created_at") + repos = request.get("repos", []) + + if not created_at_str: + self.loggit.warning("Thaw request %s missing created_at timestamp, skipping", request_id) + continue + + try: + created_at = datetime.fromisoformat(created_at_str) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + age_days = (now - created_at).days + + should_delete = False + reason = "" + + if status == "completed" and age_days > retention_completed: + should_delete = True + reason = f"completed request older than {retention_completed} days (age: {age_days} days)" + + elif status == "failed" and age_days > retention_failed: + should_delete = True + reason = f"failed request older than {retention_failed} days (age: {age_days} days)" + + elif status == "refrozen" and age_days > retention_refrozen: + should_delete = True + reason = f"refrozen request older than {retention_refrozen} days (age: {age_days} days)" + + elif status == "in_progress": + # Check if all referenced repos are no longer in thawing/thawed state + if repos: + try: + from curator.actions.deepfreeze.constants import THAW_STATE_THAWING, THAW_STATE_THAWED + repo_objects = get_repositories_by_names(self.client, repos) + # Check if any repos are still in thawing or thawed state + any_active = any( + repo.thaw_state in [THAW_STATE_THAWING, THAW_STATE_THAWED] + for repo in repo_objects + ) + + if not any_active: + should_delete = True + reason = "in-progress request with no active repos (all repos have been cleaned up)" + except Exception as e: + self.loggit.warning( + "Could not check repos for request %s: %s", request_id, e + ) + skipped.append(request_id) + continue + + if should_delete: + try: + from curator.actions.deepfreeze.constants import STATUS_INDEX + self.client.delete(index=STATUS_INDEX, id=request_id) + self.loggit.info( + "Deleted thaw request %s (%s)", request_id, reason + ) + deleted.append(request_id) + except Exception as e: + self.loggit.error( + "Failed to delete thaw request %s: %s", request_id, e + ) + skipped.append(request_id) + else: + self.loggit.debug( + "Keeping thaw request %s (status: %s, age: %d days)", + request_id, + status, + age_days + ) + + except Exception as e: + self.loggit.error( + "Error processing thaw request %s: %s", request_id, e + ) + skipped.append(request_id) + + self.loggit.info( + "Thaw request cleanup complete: %d deleted, %d skipped", + len(deleted), + len(skipped) + ) + return deleted, skipped + + def do_action(self) -> None: + """ + Check thawed repositories and unmount them if their S3 objects have reverted to Glacier. + Also delete indices whose snapshots are only in the repositories being cleaned up. + + :return: None + :rtype: None + """ + self.loggit.debug("Checking for expired thawed repositories") + + # First, detect and mark any thawed repositories that have passed their expiration time + self.loggit.info("Detecting expired thawed repositories based on expires_at timestamp") + try: + newly_expired = self._detect_and_mark_expired_repos() + if newly_expired > 0: + self.loggit.info("Detected and marked %d newly expired repositories", newly_expired) + except Exception as e: + self.loggit.error("Error detecting expired repositories: %s", e) + + # Get all repositories and filter for expired ones + from curator.actions.deepfreeze.constants import THAW_STATE_EXPIRED + all_repos = get_matching_repos(self.client, self.settings.repo_name_prefix) + expired_repos = [repo for repo in all_repos if repo.thaw_state == THAW_STATE_EXPIRED] + + if not expired_repos: + self.loggit.info("No expired repositories found to clean up") + return + + self.loggit.info("Found %d expired repositories to clean up", len(expired_repos)) + + # Track repositories that were successfully cleaned up + repos_to_cleanup = [] + + for repo in expired_repos: + self.loggit.info("Cleaning up expired repository %s", repo.name) + + try: + # CRITICAL FIX: Verify repository mount status from Elasticsearch + # The in-memory flag may be out of sync with actual cluster state + is_actually_mounted = False + try: + existing_repos = self.client.snapshot.get_repository(name=repo.name) + is_actually_mounted = repo.name in existing_repos + if is_actually_mounted: + self.loggit.debug("Repository %s is mounted in Elasticsearch", repo.name) + else: + self.loggit.debug("Repository %s is not mounted in Elasticsearch", repo.name) + except Exception as e: + self.loggit.warning( + "Could not verify mount status for repository %s: %s", + repo.name, + e + ) + is_actually_mounted = False + + # Unmount if actually mounted + if is_actually_mounted: + try: + self.loggit.info( + "Unmounting repository %s (state: %s, expires_at: %s)", + repo.name, + repo.thaw_state, + repo.expires_at + ) + self.client.snapshot.delete_repository(name=repo.name) + self.loggit.info("Repository %s unmounted successfully", repo.name) + except Exception as e: + self.loggit.error( + "Failed to unmount repository %s: %s (type: %s)", + repo.name, + str(e), + type(e).__name__ + ) + # Don't add to cleanup list if unmount failed + continue + elif repo.is_mounted: + # In-memory flag says mounted, but ES says not mounted + self.loggit.info( + "Repository %s marked as mounted but not found in Elasticsearch (likely already unmounted)", + repo.name + ) + else: + self.loggit.debug("Repository %s was not mounted", repo.name) + + # Reset repository to frozen state + repo.reset_to_frozen() + repo.persist(self.client) + self.loggit.info("Repository %s reset to frozen state", repo.name) + + # Add to cleanup list for index deletion + repos_to_cleanup.append(repo) + + except Exception as e: + self.loggit.error( + "Error cleaning up repository %s: %s", repo.name, e + ) + + # Delete indices whose snapshots are only in repositories being cleaned up + if repos_to_cleanup: + self.loggit.info("Checking for indices to delete from cleaned up repositories") + try: + indices_to_delete = self._get_indices_to_delete(repos_to_cleanup) + + if indices_to_delete: + self.loggit.info( + "Deleting %d indices whose snapshots are only in cleaned up repositories", + len(indices_to_delete) + ) + for index in indices_to_delete: + try: + # CRITICAL FIX: Validate index exists and get its status before deletion + if not self.client.indices.exists(index=index): + self.loggit.warning("Index %s no longer exists, skipping deletion", index) + continue + + # Get index health before deletion for audit trail + try: + health = self.client.cluster.health(index=index, level='indices') + index_health = health.get('indices', {}).get(index, {}) + status = index_health.get('status', 'unknown') + active_shards = index_health.get('active_shards', 'unknown') + active_primary_shards = index_health.get('active_primary_shards', 'unknown') + + self.loggit.info( + "Preparing to delete index %s (health: %s, primary_shards: %s, total_shards: %s)", + index, + status, + active_primary_shards, + active_shards + ) + except Exception as health_error: + # Log but don't fail deletion if health check fails + self.loggit.debug("Could not get health for index %s: %s", index, health_error) + + # Perform deletion + self.client.indices.delete(index=index) + self.loggit.info("Successfully deleted index %s", index) + + except Exception as e: + self.loggit.error( + "Failed to delete index %s: %s (type: %s)", + index, + str(e), + type(e).__name__ + ) + else: + self.loggit.info("No indices need to be deleted") + except Exception as e: + self.loggit.error("Error deleting indices: %s", e) + + # Clean up old thaw requests + self.loggit.info("Cleaning up old thaw requests") + try: + deleted, skipped = self._cleanup_old_thaw_requests() + if deleted: + self.loggit.info("Deleted %d old thaw requests (%d skipped)", len(deleted), len(skipped)) + except Exception as e: + self.loggit.error("Error cleaning up thaw requests: %s", e) + + # Clean up orphaned thawed ILM policies + self.loggit.info("Cleaning up orphaned thawed ILM policies") + try: + deleted_policies = self._cleanup_orphaned_thawed_policies() + if deleted_policies: + self.loggit.info("Deleted %d orphaned thawed ILM policies", len(deleted_policies)) + except Exception as e: + self.loggit.error("Error cleaning up orphaned ILM policies: %s", e) + + def _cleanup_orphaned_thawed_policies(self) -> list[str]: + """ + Delete thawed ILM policies that no longer have any indices assigned to them. + + Thawed ILM policies are named {repo_name}-thawed (e.g., deepfreeze-000010-thawed). + When all indices using a thawed policy have been deleted, the policy should be + removed to prevent accumulation. + + :return: List of deleted policy names + :rtype: list[str] + """ + self.loggit.debug("Searching for orphaned thawed ILM policies") + + deleted_policies = [] + + try: + # Get all ILM policies + all_policies = self.client.ilm.get_lifecycle() + + # Filter for thawed policies (ending with -thawed) + thawed_policies = { + name: data for name, data in all_policies.items() + if name.endswith("-thawed") and name.startswith(self.settings.repo_name_prefix) + } + + if not thawed_policies: + self.loggit.debug("No thawed ILM policies found") + return deleted_policies + + self.loggit.debug("Found %d thawed ILM policies to check", len(thawed_policies)) + + for policy_name, policy_data in thawed_policies.items(): + try: + # Check if policy has any indices assigned + in_use_by = policy_data.get("in_use_by", {}) + indices = in_use_by.get("indices", []) + datastreams = in_use_by.get("data_streams", []) + + if not indices and not datastreams: + # Policy has no indices or datastreams, safe to delete + self.loggit.info( + "Deleting orphaned thawed ILM policy %s (no indices assigned)", + policy_name + ) + self.client.ilm.delete_lifecycle(name=policy_name) + deleted_policies.append(policy_name) + self.loggit.info("Successfully deleted ILM policy %s", policy_name) + else: + self.loggit.debug( + "Keeping ILM policy %s (%d indices, %d datastreams)", + policy_name, + len(indices), + len(datastreams) + ) + + except Exception as e: + self.loggit.error( + "Failed to check/delete ILM policy %s: %s", policy_name, e + ) + + except Exception as e: + self.loggit.error("Error listing ILM policies: %s", e) + + return deleted_policies + + def do_dry_run(self) -> None: + """ + Perform a dry-run of the cleanup operation. + Shows which repositories would be unmounted and which indices would be deleted. + + :return: None + :rtype: None + """ + self.loggit.info("DRY-RUN MODE. No changes will be made.") + + # First, show which thawed repositories would be detected as expired + self.loggit.info("DRY-RUN: Checking for thawed repositories that have passed expiration time") + from curator.actions.deepfreeze.constants import THAW_STATE_THAWED + all_repos = get_matching_repos(self.client, self.settings.repo_name_prefix) + thawed_repos = [repo for repo in all_repos if repo.thaw_state == THAW_STATE_THAWED] + + if thawed_repos: + now = datetime.now(timezone.utc) + would_expire = [] + + for repo in thawed_repos: + if repo.expires_at: + expires_at = repo.expires_at + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=timezone.utc) + + if now >= expires_at: + time_expired = now - expires_at + would_expire.append((repo.name, expires_at, time_expired)) + else: + time_remaining = expires_at - now + self.loggit.debug( + "DRY-RUN: Repository %s not yet expired (expires in %s)", + repo.name, + time_remaining + ) + else: + self.loggit.warning( + "DRY-RUN: Repository %s is thawed but has no expires_at timestamp", + repo.name + ) + + if would_expire: + self.loggit.info( + "DRY-RUN: Would mark %d repositories as expired:", + len(would_expire) + ) + for name, expired_at, time_ago in would_expire: + self.loggit.info( + "DRY-RUN: - %s (expired %s ago at %s)", + name, + time_ago, + expired_at.isoformat() + ) + else: + self.loggit.info("DRY-RUN: No thawed repositories have passed expiration time") + else: + self.loggit.info("DRY-RUN: No thawed repositories found to check") + + # Get all repositories and filter for expired ones + from curator.actions.deepfreeze.constants import THAW_STATE_EXPIRED + all_repos = get_matching_repos(self.client, self.settings.repo_name_prefix) + expired_repos = [repo for repo in all_repos if repo.thaw_state == THAW_STATE_EXPIRED] + + if not expired_repos: + self.loggit.info("DRY-RUN: No expired repositories found to clean up") + return + + self.loggit.info("DRY-RUN: Found %d expired repositories to clean up", len(expired_repos)) + + # Track repositories that would be cleaned up + repos_to_cleanup = [] + + for repo in expired_repos: + action = "unmount and reset to frozen" if repo.is_mounted else "reset to frozen" + self.loggit.info( + "DRY-RUN: Would %s repository %s (state: %s)", + action, + repo.name, + repo.thaw_state + ) + repos_to_cleanup.append(repo) + + # Show which indices would be deleted + if repos_to_cleanup: + self.loggit.info( + "DRY-RUN: Checking for indices that would be deleted from cleaned up repositories" + ) + try: + indices_to_delete = self._get_indices_to_delete(repos_to_cleanup) + + if indices_to_delete: + self.loggit.info( + "DRY-RUN: Would delete %d indices whose snapshots are only in cleaned up repositories:", + len(indices_to_delete) + ) + for index in indices_to_delete: + self.loggit.info("DRY-RUN: - %s", index) + else: + self.loggit.info("DRY-RUN: No indices would be deleted") + except Exception as e: + self.loggit.error("DRY-RUN: Error finding indices to delete: %s", e) + + # Show which thaw requests would be cleaned up + self.loggit.info("DRY-RUN: Checking for old thaw requests that would be deleted") + try: + requests = list_thaw_requests(self.client) + + if not requests: + self.loggit.info("DRY-RUN: No thaw requests found") + else: + now = datetime.now(timezone.utc) + retention_completed = self.settings.thaw_request_retention_days_completed + retention_failed = self.settings.thaw_request_retention_days_failed + retention_refrozen = self.settings.thaw_request_retention_days_refrozen + + would_delete = [] + + for request in requests: + request_id = request.get("id") + if not request_id: + self.loggit.warning("Thaw request missing id, skipping") + continue + + status = request.get("status", "unknown") + created_at_str = request.get("created_at") + repos = request.get("repos", []) + + if not created_at_str: + self.loggit.warning("Thaw request %s missing created_at timestamp, skipping", request_id) + continue + + try: + created_at = datetime.fromisoformat(created_at_str) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + age_days = (now - created_at).days + + should_delete = False + reason = "" + + if status == "completed" and age_days > retention_completed: + should_delete = True + reason = f"completed request older than {retention_completed} days (age: {age_days} days)" + + elif status == "failed" and age_days > retention_failed: + should_delete = True + reason = f"failed request older than {retention_failed} days (age: {age_days} days)" + + elif status == "refrozen" and age_days > retention_refrozen: + should_delete = True + reason = f"refrozen request older than {retention_refrozen} days (age: {age_days} days)" + + elif status == "in_progress" and repos: + try: + from curator.actions.deepfreeze.constants import THAW_STATE_THAWING, THAW_STATE_THAWED + repo_objects = get_repositories_by_names(self.client, repos) + any_active = any( + repo.thaw_state in [THAW_STATE_THAWING, THAW_STATE_THAWED] + for repo in repo_objects + ) + + if not any_active: + should_delete = True + reason = "in-progress request with no active repos (all repos have been cleaned up)" + except Exception as e: + self.loggit.warning( + "DRY-RUN: Could not check repos for request %s: %s", request_id, e + ) + + if should_delete: + would_delete.append((request_id, reason)) + + except Exception as e: + self.loggit.error( + "DRY-RUN: Error processing thaw request %s: %s", request_id, e + ) + + if would_delete: + self.loggit.info( + "DRY-RUN: Would delete %d old thaw requests:", + len(would_delete) + ) + for request_id, reason in would_delete: + self.loggit.info("DRY-RUN: - %s (%s)", request_id, reason) + else: + self.loggit.info("DRY-RUN: No thaw requests would be deleted") + + except Exception as e: + self.loggit.error("DRY-RUN: Error checking thaw requests: %s", e) + + def do_singleton_action(self, dry_run: bool = False) -> None: + """ + Entry point for singleton CLI execution. + + :param dry_run: If True, perform a dry-run without making changes + :type dry_run: bool + + :return: None + :rtype: None + """ + if dry_run: + self.do_dry_run() + else: + self.do_action() diff --git a/curator/actions/deepfreeze/constants.py b/curator/actions/deepfreeze/constants.py new file mode 100644 index 00000000..915abe25 --- /dev/null +++ b/curator/actions/deepfreeze/constants.py @@ -0,0 +1,35 @@ +"""Constants for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +STATUS_INDEX = "deepfreeze-status" +SETTINGS_ID = "1" +PROVIDERS = ["aws"] + +# Repository thaw lifecycle states +THAW_STATE_ACTIVE = "active" # Active repository, never been through thaw lifecycle +THAW_STATE_FROZEN = "frozen" # In cold storage (Glacier), not currently accessible +THAW_STATE_THAWING = "thawing" # S3 restore in progress, waiting for retrieval +THAW_STATE_THAWED = "thawed" # S3 restore complete, mounted and in use +THAW_STATE_EXPIRED = "expired" # S3 restore expired, reverted to Glacier, ready for cleanup + +THAW_STATES = [ + THAW_STATE_ACTIVE, + THAW_STATE_FROZEN, + THAW_STATE_THAWING, + THAW_STATE_THAWED, + THAW_STATE_EXPIRED, +] + +# Thaw request status lifecycle +THAW_STATUS_IN_PROGRESS = "in_progress" # Thaw operation is actively running +THAW_STATUS_COMPLETED = "completed" # Thaw completed, data available and mounted +THAW_STATUS_FAILED = "failed" # Thaw operation failed +THAW_STATUS_REFROZEN = "refrozen" # Thaw was completed but has been refrozen (cleaned up) + +THAW_REQUEST_STATUSES = [ + THAW_STATUS_IN_PROGRESS, + THAW_STATUS_COMPLETED, + THAW_STATUS_FAILED, + THAW_STATUS_REFROZEN, +] diff --git a/curator/actions/deepfreeze/exceptions.py b/curator/actions/deepfreeze/exceptions.py new file mode 100644 index 00000000..9ca43d79 --- /dev/null +++ b/curator/actions/deepfreeze/exceptions.py @@ -0,0 +1,38 @@ +"""Deepfreeze Exceptions""" + + +class DeepfreezeException(Exception): + """ + Base class for all exceptions raised by Deepfreeze which are not Elasticsearch + exceptions. + """ + + +class MissingIndexError(DeepfreezeException): + """ + Exception raised when the status index is missing + """ + + +class MissingSettingsError(DeepfreezeException): + """ + Exception raised when the status index exists, but the settings document is missing + """ + + +class ActionException(DeepfreezeException): + """ + Generic class for unexpected coneditions during DF actions + """ + + +class PreconditionError(DeepfreezeException): + """ + Exception raised when preconditions are not met for a deepfreeze action + """ + + +class RepositoryException(DeepfreezeException): + """ + Exception raised when a probley with a repository occurs + """ diff --git a/curator/actions/deepfreeze/helpers.py b/curator/actions/deepfreeze/helpers.py new file mode 100644 index 00000000..ae468c06 --- /dev/null +++ b/curator/actions/deepfreeze/helpers.py @@ -0,0 +1,378 @@ +"""Helper classes for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import json +import logging +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +from elasticsearch8 import Elasticsearch + +from .constants import STATUS_INDEX + + +class Deepfreeze: + """ + Allows nesting of actions under the deepfreeze command + """ + + +@dataclass +class Repository: + """ + Data class for repository. Given a name, it will retrieve the repository from the + status index. If given other parameters, it will create a new repository object. + + Attributes: + name (str): The name of the repository. + bucket (str): The name of the bucket. + base_path (str): The base path of the repository. + start (datetime): The start date of the repository. + end (datetime): The end date of the repository. + is_thawed (bool): Whether the repository is thawed (DEPRECATED - use thaw_state). + is_mounted (bool): Whether the repository is mounted. + thaw_state (str): Lifecycle state - "active", "frozen", "thawing", "thawed", "expired" + thawed_at (datetime): When S3 restore completed (thawing -> thawed transition). + expires_at (datetime): When S3 restore will/did expire. + doctype (str): The document type of the repository. + id [str]: The ID of the repository in Elasticsearch. + + Lifecycle States: + active: Active repository, never been through thaw lifecycle + frozen: In cold storage (Glacier), not currently accessible + thawing: S3 restore in progress, waiting for retrieval + thawed: S3 restore complete, mounted and in use + expired: S3 restore expired, reverted to Glacier, ready for cleanup + + State Transitions: + active -> frozen: When repository is moved to cold storage (future feature) + frozen -> thawing: When thaw request initiated + thawing -> thawed: When S3 restore completes and repo is mounted + thawed -> expired: When S3 restore expiry time passes + expired -> frozen: When cleanup runs (refreeze operation) + + Methods: + to_dict() -> dict: + Convert the Repository object to a dictionary. + + to_json() -> str: + Convert the Repository object to a JSON string. + + __lt__(other) -> bool: + Less than comparison based on the repository name. + + persist(es: Elasticsearch) -> None: + Persist the repository to the status index. + + Example: + repo = Repository({name="repo1", bucket="bucket1", base_path="path1", start=datetime.now(), end=datetime.now()}) + repo = Repository(name="deepfreeze-000032") + repo_dict = repo.to_dict() + repo_json = repo.to_json() + """ + + name: str = None + bucket: str = None + base_path: str = None + start: datetime = None + end: datetime = None + is_thawed: bool = False # DEPRECATED - use thaw_state instead + is_mounted: bool = True + thaw_state: str = "active" # active, frozen, thawing, thawed, expired + thawed_at: datetime = None # When restore completed + expires_at: datetime = None # When restore will/did expire + doctype: str = "repository" + docid: str = None + + def __post_init__(self): + """Convert string dates from Elasticsearch to datetime objects""" + if isinstance(self.start, str): + self.start = datetime.fromisoformat(self.start) + if isinstance(self.end, str): + self.end = datetime.fromisoformat(self.end) + if isinstance(self.thawed_at, str): + self.thawed_at = datetime.fromisoformat(self.thawed_at) + if isinstance(self.expires_at, str): + self.expires_at = datetime.fromisoformat(self.expires_at) + + + @classmethod + def from_elasticsearch( + cls, client: Elasticsearch, name: str, index: str = STATUS_INDEX + ) -> Optional['Repository']: + """ + Fetch a document from Elasticsearch by name and create a Repository instance. + + Args: + name: The name of the repository to fetch + client: Elasticsearch client instance + index: The Elasticsearch index to query (default: 'repositories') + + Returns: + Repository instance or None if not found + """ + try: + # Query Elasticsearch for a document matching the name + logging.debug(f"Fetching Repository from Elasticsearch: {name}") + response = client.search( + index=index, + query={"match": {"name.keyword": name}}, # Use .keyword for exact match + size=1, + ) + + # Check if we got any hits + hits = response['hits']['hits'] + if not hits: + return None + + # Extract the document source + doc = hits[0]['_source'] + id = hits[0]['_id'] + + logging.debug(f"Document fetched: {doc}") + + # Create and return a new Repository instance + return cls(**doc, docid=id) + + except Exception as e: + # CRITICAL FIX: Use logger instead of print() + logging.error( + "Error fetching Repository from Elasticsearch: %s (type: %s)", + e, + type(e).__name__, + exc_info=True + ) + return None + + def to_dict(self) -> dict: + """ + Convert the Repository object to a dictionary. + Convert datetime to ISO 8601 string format for JSON compatibility. + + Params: + None + + Returns: + dict: A dictionary representation of the Repository object. + """ + logging.debug("Converting Repository to dict") + logging.debug(f"Repository start: {self.start}") + logging.debug(f"Repository end: {self.end}") + # Convert datetime objects to ISO strings for proper storage + start_str = self.start.isoformat() if isinstance(self.start, datetime) else self.start + end_str = self.end.isoformat() if isinstance(self.end, datetime) else self.end + thawed_at_str = self.thawed_at.isoformat() if isinstance(self.thawed_at, datetime) else self.thawed_at + expires_at_str = self.expires_at.isoformat() if isinstance(self.expires_at, datetime) else self.expires_at + + return { + "name": self.name, + "bucket": self.bucket, + "base_path": self.base_path, + "start": start_str, + "end": end_str, + "is_thawed": self.is_thawed, # Keep for backward compatibility + "is_mounted": self.is_mounted, + "thaw_state": self.thaw_state, + "thawed_at": thawed_at_str, + "expires_at": expires_at_str, + "doctype": self.doctype, + } + + def unmount(self) -> None: + """ + Unmount the repository by setting is_mounted to False. + + Params: + None + + Returns: + None + """ + self.is_mounted = False + + def start_thawing(self, expires_at: datetime) -> None: + """ + Transition repository to 'thawing' state when S3 restore is initiated. + + Params: + expires_at (datetime): When the S3 restore will expire + + Returns: + None + """ + from .constants import THAW_STATE_THAWING + self.thaw_state = THAW_STATE_THAWING + self.expires_at = expires_at + self.is_thawed = True # Maintain backward compatibility + + def mark_thawed(self) -> None: + """ + Transition repository to 'thawed' state when S3 restore completes and repo is mounted. + + Params: + None + + Returns: + None + """ + from .constants import THAW_STATE_THAWED + from datetime import datetime, timezone + self.thaw_state = THAW_STATE_THAWED + self.thawed_at = datetime.now(timezone.utc) + self.is_thawed = True # Maintain backward compatibility + self.is_mounted = True + + def mark_expired(self) -> None: + """ + Transition repository to 'expired' state when S3 restore has expired. + + Params: + None + + Returns: + None + """ + from .constants import THAW_STATE_EXPIRED + self.thaw_state = THAW_STATE_EXPIRED + # Keep thawed_at and expires_at for historical tracking + + def reset_to_frozen(self) -> None: + """ + Transition repository back to 'frozen' state after cleanup. + + Params: + None + + Returns: + None + """ + from .constants import THAW_STATE_FROZEN + self.thaw_state = THAW_STATE_FROZEN + self.is_thawed = False # Maintain backward compatibility + self.is_mounted = False + self.thawed_at = None + self.expires_at = None + + def to_json(self) -> str: + """ + Convert the Repository object to a JSON string. + + Params: + None + + Returns: + str: A JSON string representation of the Repository object. + """ + return json.dumps(self.to_dict(), indent=4) + + def __lt__(self, other): + """ + Less than comparison based on the repository name. + + Params: + other (Repository): Another Repository object to compare with. + + Returns: + bool: True if this repository's name is less than the other repository's name, False otherwise. + """ + return self.name < other.name + + def persist(self, es: Elasticsearch) -> None: + """ + Persist the repository to the status index. + + Params: + es (Elasticsearch): The Elasticsearch client. + + Returns: + None + """ + logging.debug("Persisting Repository to Elasticsearch") + logging.debug(f"Repository name: {self.name}") + logging.debug(f"Repository id: {self.docid}") + logging.debug(f"Repository body: {self.to_dict()}") + es.update(index=STATUS_INDEX, id=self.docid, body={"doc": self.to_dict()}) + + +@dataclass +class Settings: + """ + Data class for settings. Can be instantiated from a dictionary or from individual + parameters. + + Attributes: + doctype (str): The document type of the settings. + repo_name_prefix (str): The prefix for repository names. + bucket_name_prefix (str): The prefix for bucket names. + base_path_prefix (str): The base path prefix. + canned_acl (str): The canned ACL. + storage_class (str): The storage class. + provider (str): The provider. + rotate_by (str): The rotation style. + style (str): The style of the settings. + last_suffix (str): The last suffix. + thaw_request_retention_days_completed (int): Days to retain completed thaw requests. + thaw_request_retention_days_failed (int): Days to retain failed thaw requests. + thaw_request_retention_days_refrozen (int): Days to retain refrozen thaw requests. + + """ + + doctype: str = "settings" + repo_name_prefix: str = "deepfreeze" + bucket_name_prefix: str = "deepfreeze" + base_path_prefix: str = "snapshots" + canned_acl: str = "private" + storage_class: str = "intelligent_tiering" + provider: str = "aws" + rotate_by: str = "path" + style: str = "oneup" + last_suffix: str = None + thaw_request_retention_days_completed: int = 7 + thaw_request_retention_days_failed: int = 30 + thaw_request_retention_days_refrozen: int = 35 + + def __init__( + self, + settings_hash: dict[str, str] = None, + repo_name_prefix: str = "deepfreeze", + bucket_name_prefix: str = "deepfreeze", + base_path_prefix: str = "snapshots", + canned_acl: str = "private", + storage_class: str = "intelligent_tiering", + provider: str = "aws", + rotate_by: str = "path", + style: str = "oneup", + last_suffix: str = None, + thaw_request_retention_days_completed: int = 7, + thaw_request_retention_days_failed: int = 30, + thaw_request_retention_days_refrozen: int = 35, + ) -> None: + if settings_hash is not None: + for key, value in settings_hash.items(): + setattr(self, key, value) + if repo_name_prefix: + self.repo_name_prefix = repo_name_prefix + if bucket_name_prefix: + self.bucket_name_prefix = bucket_name_prefix + if base_path_prefix: + self.base_path_prefix = base_path_prefix + if canned_acl: + self.canned_acl = canned_acl + if storage_class: + self.storage_class = storage_class + if provider: + self.provider = provider + if rotate_by: + self.rotate_by = rotate_by + if style: + self.style = style + if last_suffix: + self.last_suffix = last_suffix + if thaw_request_retention_days_completed: + self.thaw_request_retention_days_completed = thaw_request_retention_days_completed + if thaw_request_retention_days_failed: + self.thaw_request_retention_days_failed = thaw_request_retention_days_failed + if thaw_request_retention_days_refrozen: + self.thaw_request_retention_days_refrozen = thaw_request_retention_days_refrozen diff --git a/curator/actions/deepfreeze/refreeze.py b/curator/actions/deepfreeze/refreeze.py new file mode 100644 index 00000000..4d937bb2 --- /dev/null +++ b/curator/actions/deepfreeze/refreeze.py @@ -0,0 +1,670 @@ +"""Refreeze action for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging + +from elasticsearch8 import Elasticsearch +from rich import print as rprint + +from curator.actions.deepfreeze.constants import STATUS_INDEX, THAW_STATUS_REFROZEN +from curator.actions.deepfreeze.exceptions import MissingIndexError +from curator.actions.deepfreeze.utilities import ( + get_all_indices_in_repo, + get_repositories_by_names, + get_settings, + get_thaw_request, + list_thaw_requests, +) +from curator.exceptions import ActionError + + +class Refreeze: + """ + The Refreeze action is a user-initiated operation to signal "I'm done with this thaw." + It unmounts repositories that were previously thawed and resets them back to frozen state. + + Unlike the automatic Cleanup action which processes expired repositories on a schedule, + Refreeze is explicitly invoked by users when they're finished accessing thawed data, + even if the S3 restore hasn't expired yet. + + When you thaw from AWS Glacier, you get a temporary restored copy that exists for a + specified duration. After that expires, AWS automatically removes the temporary copy - + the original Glacier object never moved. Refreeze doesn't push anything back; it's about + unmounting the repositories and resetting state. + + :param client: A client connection object + :type client: Elasticsearch + :param thaw_request_id: The ID of the thaw request to refreeze (optional - if None, all open requests) + :type thaw_request_id: str + + :methods: + do_action: Perform the refreeze operation. + do_dry_run: Perform a dry-run of the refreeze operation. + do_singleton_action: Entry point for singleton CLI execution. + """ + + def __init__(self, client: Elasticsearch, thaw_request_id: str = None, porcelain: bool = False) -> None: + self.loggit = logging.getLogger("curator.actions.deepfreeze") + self.loggit.debug("Initializing Deepfreeze Refreeze") + + self.client = client + self.thaw_request_id = thaw_request_id + self.porcelain = porcelain + + # CRITICAL FIX: Validate that settings exist before proceeding + try: + self.settings = get_settings(client) + if self.settings is None: + error_msg = ( + f"Deepfreeze settings not found in status index {STATUS_INDEX}. " + f"Run 'curator_cli deepfreeze setup' first to initialize deepfreeze." + ) + self.loggit.error(error_msg) + if self.porcelain: + rprint(f"ERROR\tmissing_settings\t{error_msg}") + raise ActionError(error_msg) + self.loggit.debug("Settings loaded successfully") + except MissingIndexError as e: + error_msg = f"Status index {STATUS_INDEX} does not exist. Run 'curator_cli deepfreeze setup' first." + self.loggit.error(error_msg) + if self.porcelain: + rprint(f"ERROR\tmissing_index\t{error_msg}") + raise ActionError(error_msg) from e + + if thaw_request_id: + self.loggit.info("Deepfreeze Refreeze initialized for request %s", thaw_request_id) + else: + self.loggit.info("Deepfreeze Refreeze initialized for all open requests") + + def _get_open_thaw_requests(self) -> list: + """ + Get all open (in_progress) thaw requests. + + :return: List of thaw request dicts + :rtype: list + """ + all_requests = list_thaw_requests(self.client) + return [req for req in all_requests if req.get("status") == "in_progress"] + + def _confirm_bulk_refreeze(self, requests: list) -> bool: + """ + Display a list of thaw requests and get user confirmation to proceed. + In porcelain mode, automatically returns True (no interactive confirmation). + + :param requests: List of thaw request dicts + :type requests: list + + :return: True if user confirms (or in porcelain mode), False otherwise + :rtype: bool + """ + # In porcelain mode, skip confirmation and just proceed + if self.porcelain: + return True + + rprint(f"\n[bold yellow]WARNING: This will refreeze {len(requests)} open thaw request(s)[/bold yellow]\n") + + # Show the requests + for req in requests: + request_id = req.get("id") + repo_count = len(req.get("repos", [])) + created_at = req.get("created_at", "Unknown") + start_date = req.get("start_date", "--") + end_date = req.get("end_date", "--") + + rprint(f" [cyan]• {request_id}[/cyan]") + rprint(f" [dim]Created: {created_at}[/dim]") + rprint(f" [dim]Date Range: {start_date} to {end_date}[/dim]") + rprint(f" [dim]Repositories: {repo_count}[/dim]\n") + + # Get confirmation + try: + response = input("Do you want to proceed with refreezing all these requests? [y/N]: ").strip().lower() + return response in ['y', 'yes'] + except (EOFError, KeyboardInterrupt): + rprint("\n[yellow]Operation cancelled by user[/yellow]") + return False + + def _delete_mounted_indices_for_repo(self, repo_name: str) -> tuple[int, list[str]]: + """ + Delete all mounted indices from a repository. + + Searchable snapshot indices can exist with multiple name variations: + - Original name (e.g., .ds-df-test-2024.01.01-000001) + - partial- prefix (e.g., partial-.ds-df-test-2024.01.01-000001) + - restored- prefix (e.g., restored-.ds-df-test-2024.01.01-000001) + + :param repo_name: The repository name + :type repo_name: str + + :return: Tuple of (deleted_count, failed_indices) + :rtype: tuple[int, list[str]] + """ + deleted_count = 0 + failed_indices = [] + + try: + # Get all indices from repository snapshots + snapshot_indices = get_all_indices_in_repo(self.client, repo_name) + self.loggit.debug( + "Found %d indices in repository %s snapshots", + len(snapshot_indices), + repo_name + ) + + # Check for each index with all possible name variations + for base_index in snapshot_indices: + # Try all possible index name variations + possible_names = [ + base_index, # Original name + f"partial-{base_index}", # Searchable snapshot + f"restored-{base_index}", # Fully restored + ] + + for index_name in possible_names: + try: + if self.client.indices.exists(index=index_name): + self.loggit.info("Deleting index %s from repository %s", index_name, repo_name) + self.client.indices.delete(index=index_name) + deleted_count += 1 + self.loggit.debug("Successfully deleted index %s", index_name) + # Only try one variation - if we found and deleted it, stop + break + except Exception as e: + self.loggit.error( + "Failed to delete index %s: %s (type: %s)", + index_name, + e, + type(e).__name__, + exc_info=True + ) + failed_indices.append(index_name) + + except Exception as e: + self.loggit.error( + "Failed to get indices from repository %s: %s", + repo_name, + e, + exc_info=True + ) + return 0, [] + + return deleted_count, failed_indices + + def _delete_thawed_ilm_policy(self, repo_name: str) -> bool: + """ + Delete the per-repository thawed ILM policy. + + Policy name format: {repo_name}-thawed (e.g., deepfreeze-000010-thawed) + + Before deleting the policy, removes it from any indices still using it to avoid + "policy in use" errors. + + :param repo_name: The repository name + :type repo_name: str + + :return: True if deleted successfully, False otherwise + :rtype: bool + """ + policy_name = f"{repo_name}-thawed" + + try: + # Check if policy exists first + self.client.ilm.get_lifecycle(name=policy_name) + + # Before deleting, remove the policy from any indices still using it + self.loggit.debug("Checking for indices using ILM policy %s", policy_name) + try: + # Get all indices using this policy + ilm_explain = self.client.ilm.explain_lifecycle(index="*") + indices_using_policy = [ + idx for idx, info in ilm_explain.get("indices", {}).items() + if info.get("policy") == policy_name + ] + + if indices_using_policy: + self.loggit.info( + "Found %d indices still using policy %s, removing policy from them", + len(indices_using_policy), + policy_name + ) + for idx in indices_using_policy: + try: + self.loggit.debug("Removing ILM policy from index %s", idx) + self.client.ilm.remove_policy(index=idx) + except Exception as idx_err: + self.loggit.warning( + "Failed to remove ILM policy from index %s: %s", + idx, + idx_err + ) + except Exception as check_err: + self.loggit.warning( + "Failed to check for indices using policy %s: %s", + policy_name, + check_err + ) + + # Policy exists and indices have been cleaned up, delete it + self.loggit.info("Deleting thawed ILM policy %s", policy_name) + self.client.ilm.delete_lifecycle(name=policy_name) + self.loggit.debug("Successfully deleted ILM policy %s", policy_name) + return True + + except Exception as e: + # If policy doesn't exist (404), that's okay - might be pre-ILM implementation + if "404" in str(e) or "resource_not_found" in str(e).lower(): + self.loggit.debug("ILM policy %s does not exist, skipping deletion", policy_name) + return True + else: + self.loggit.warning( + "Failed to delete ILM policy %s: %s (type: %s)", + policy_name, + e, + type(e).__name__, + exc_info=True + ) + return False + + def _refreeze_single_request(self, request_id: str) -> dict: + """ + Refreeze a single thaw request. + + Operations performed for each repository: + 1. Delete all mounted indices from the repository + 2. Unmount the repository from Elasticsearch + 3. Delete the per-repository thawed ILM policy + 4. Reset repository state to frozen + 5. Persist state changes + + :param request_id: The thaw request ID + :type request_id: str + + :return: Dict with unmounted_repos, failed_repos, deleted_indices, deleted_policies + :rtype: dict + """ + self.loggit.info("Refreezing thaw request %s", request_id) + + # Get the thaw request + try: + request = get_thaw_request(self.client, request_id) + except Exception as e: + self.loggit.error("Failed to get thaw request %s: %s", request_id, e) + if self.porcelain: + print(f"ERROR\trequest_not_found\t{request_id}\t{str(e)}") + else: + rprint(f"[red]Error: Could not find thaw request '{request_id}'[/red]") + return {"unmounted_repos": [], "failed_repos": [], "deleted_indices": 0, "deleted_policies": 0} + + # Get the repositories from the request + repo_names = request.get("repos", []) + if not repo_names: + self.loggit.warning("No repositories found in thaw request %s", request_id) + return {"unmounted_repos": [], "failed_repos": [], "deleted_indices": 0, "deleted_policies": 0} + + self.loggit.info("Found %d repositories to refreeze", len(repo_names)) + + # Get the repository objects + try: + repos = get_repositories_by_names(self.client, repo_names) + except Exception as e: + self.loggit.error("Failed to get repositories: %s", e) + return {"unmounted_repos": [], "failed_repos": [], "deleted_indices": 0, "deleted_policies": 0} + + if not repos: + self.loggit.warning("No repository objects found for names: %s", repo_names) + return {"unmounted_repos": [], "failed_repos": [], "deleted_indices": 0, "deleted_policies": 0} + + # Track success/failure and statistics + unmounted = [] + failed = [] + total_deleted_indices = 0 + total_deleted_policies = 0 + + # Process each repository + for repo in repos: + # ENHANCED LOGGING: Add detailed repository state information + self.loggit.info( + "Processing repository %s - current state: mounted=%s, thaw_state=%s, bucket=%s, base_path=%s", + repo.name, + repo.is_mounted, + repo.thaw_state, + repo.bucket, + repo.base_path + ) + + try: + # STEP 1: Delete mounted indices BEFORE unmounting repository + self.loggit.info("Deleting mounted indices for repository %s", repo.name) + deleted_count, failed_indices = self._delete_mounted_indices_for_repo(repo.name) + total_deleted_indices += deleted_count + if deleted_count > 0: + self.loggit.info( + "Deleted %d indices from repository %s", + deleted_count, + repo.name + ) + if failed_indices: + self.loggit.warning( + "Failed to delete %d indices from repository %s", + len(failed_indices), + repo.name + ) + + # STEP 2: Unmount repository if still mounted + if repo.is_mounted: + try: + self.loggit.info("Unmounting repository %s", repo.name) + self.client.snapshot.delete_repository(name=repo.name) + self.loggit.info("Successfully unmounted repository %s", repo.name) + unmounted.append(repo.name) + except Exception as e: + # If it's already unmounted, that's okay + if "repository_missing_exception" in str(e).lower(): + self.loggit.debug("Repository %s was already unmounted", repo.name) + else: + self.loggit.warning( + "Failed to unmount repository %s: %s (type: %s)", + repo.name, + e, + type(e).__name__, + exc_info=True + ) + # Continue anyway to update the state + else: + self.loggit.debug("Repository %s was not mounted, skipping unmount", repo.name) + + # STEP 3: Delete per-repository thawed ILM policy + if self._delete_thawed_ilm_policy(repo.name): + total_deleted_policies += 1 + self.loggit.debug("Deleted ILM policy for repository %s", repo.name) + + # STEP 4: Reset to frozen state + self.loggit.debug( + "Resetting repository %s to frozen state (old state: %s)", + repo.name, + repo.thaw_state + ) + repo.reset_to_frozen() + + # Persist the state change + self.loggit.debug("Persisting state change for repository %s", repo.name) + repo.persist(self.client) + self.loggit.info( + "Repository %s successfully reset to frozen state and persisted", + repo.name + ) + + except Exception as e: + self.loggit.error( + "Error processing repository %s: %s (type: %s)", + repo.name, + e, + type(e).__name__, + exc_info=True + ) + failed.append(repo.name) + + # STEP 5: Update the thaw request status to refrozen + # (Cleanup action will remove old refrozen requests based on retention settings) + try: + self.client.update( + index=STATUS_INDEX, + id=request_id, + body={"doc": {"status": THAW_STATUS_REFROZEN}} + ) + self.loggit.info("Thaw request %s marked as refrozen", request_id) + except Exception as e: + self.loggit.error("Failed to update thaw request status: %s", e) + + return { + "unmounted_repos": unmounted, + "failed_repos": failed, + "deleted_indices": total_deleted_indices, + "deleted_policies": total_deleted_policies, + } + + def do_action(self) -> None: + """ + Unmount repositories from thaw request(s) and reset them to frozen state. + + If thaw_request_id is provided, refreeze that specific request. + If thaw_request_id is None, refreeze all open requests (with confirmation). + + :return: None + :rtype: None + """ + # Determine which requests to process + if self.thaw_request_id: + # Single request mode + request_ids = [self.thaw_request_id] + else: + # Bulk mode - get all open requests + open_requests = self._get_open_thaw_requests() + + if not open_requests: + if not self.porcelain: + rprint("[yellow]No open thaw requests found to refreeze[/yellow]") + return + + # Get confirmation + if not self._confirm_bulk_refreeze(open_requests): + if not self.porcelain: + rprint("[yellow]Refreeze operation cancelled[/yellow]") + return + + request_ids = [req.get("id") for req in open_requests] + + # Process each request + total_unmounted = [] + total_failed = [] + total_deleted_indices = 0 + total_deleted_policies = 0 + + for request_id in request_ids: + result = self._refreeze_single_request(request_id) + total_unmounted.extend(result["unmounted_repos"]) + total_failed.extend(result["failed_repos"]) + total_deleted_indices += result["deleted_indices"] + total_deleted_policies += result["deleted_policies"] + + # Report results + if self.porcelain: + # Machine-readable output: tab-separated values + for repo_name in total_unmounted: + print(f"UNMOUNTED\t{repo_name}") + for repo_name in total_failed: + print(f"FAILED\t{repo_name}") + print(f"SUMMARY\t{len(total_unmounted)}\t{len(total_failed)}\t{total_deleted_indices}\t{total_deleted_policies}\t{len(request_ids)}") + else: + if len(request_ids) == 1: + rprint(f"\n[green]Refreeze completed for thaw request '{request_ids[0]}'[/green]") + else: + rprint(f"\n[green]Refreeze completed for {len(request_ids)} thaw requests[/green]") + + rprint(f"[cyan]Unmounted {len(total_unmounted)} repositories[/cyan]") + rprint(f"[cyan]Deleted {total_deleted_indices} indices[/cyan]") + rprint(f"[cyan]Deleted {total_deleted_policies} ILM policies[/cyan]") + if total_failed: + rprint(f"[red]Failed to process {len(total_failed)} repositories: {', '.join(total_failed)}[/red]") + + def do_dry_run(self) -> None: + """ + Perform a dry-run of the refreeze operation. + Shows which repositories would be unmounted and reset. + + If thaw_request_id is provided, show dry-run for that specific request. + If thaw_request_id is None, show dry-run for all open requests. + + :return: None + :rtype: None + """ + # Determine which requests to process + if self.thaw_request_id: + # Single request mode + request_ids = [self.thaw_request_id] + if not self.porcelain: + rprint(f"\n[cyan]DRY-RUN: Would refreeze thaw request '{self.thaw_request_id}'[/cyan]\n") + else: + # Bulk mode - get all open requests + open_requests = self._get_open_thaw_requests() + + if not open_requests: + if not self.porcelain: + rprint("[yellow]DRY-RUN: No open thaw requests found to refreeze[/yellow]") + return + + if not self.porcelain: + rprint(f"\n[cyan]DRY-RUN: Would refreeze {len(open_requests)} open thaw requests:[/cyan]\n") + + # Show the requests + for req in open_requests: + request_id = req.get("id") + repo_count = len(req.get("repos", [])) + created_at = req.get("created_at", "Unknown") + start_date = req.get("start_date", "--") + end_date = req.get("end_date", "--") + + rprint(f" [cyan]• {request_id}[/cyan]") + rprint(f" [dim]Created: {created_at}[/dim]") + rprint(f" [dim]Date Range: {start_date} to {end_date}[/dim]") + rprint(f" [dim]Repositories: {repo_count}[/dim]\n") + + request_ids = [req.get("id") for req in open_requests] + + # Process each request in dry-run mode + total_repos = 0 + total_indices = 0 + total_policies = 0 + + for request_id in request_ids: + try: + request = get_thaw_request(self.client, request_id) + except Exception as e: + self.loggit.error("DRY-RUN: Failed to get thaw request %s: %s", request_id, e) + if self.porcelain: + print(f"ERROR\tdry_run_request_not_found\t{request_id}\t{str(e)}") + else: + rprint(f"[red]DRY-RUN: Could not find thaw request '{request_id}'[/red]") + continue + + repo_names = request.get("repos", []) + if not repo_names: + continue + + try: + repos = get_repositories_by_names(self.client, repo_names) + except Exception as e: + self.loggit.error("DRY-RUN: Failed to get repositories for request %s: %s", request_id, e) + continue + + if not repos: + continue + + # Count indices and policies that would be deleted + for repo in repos: + # Count indices + try: + snapshot_indices = get_all_indices_in_repo(self.client, repo.name) + for base_index in snapshot_indices: + # Check if any variation exists + possible_names = [ + base_index, + f"partial-{base_index}", + f"restored-{base_index}", + ] + for index_name in possible_names: + if self.client.indices.exists(index=index_name): + total_indices += 1 + break + except Exception: + pass + + # Count policies + policy_name = f"{repo.name}-thawed" + try: + self.client.ilm.get_lifecycle(name=policy_name) + total_policies += 1 + except Exception: + pass + + # Show details if single request, or summary if bulk + if self.porcelain: + # Machine-readable output + for repo in repos: + action = "unmount_and_reset" if repo.is_mounted else "reset" + # Count indices for this repo + repo_index_count = 0 + try: + snapshot_indices = get_all_indices_in_repo(self.client, repo.name) + for base_index in snapshot_indices: + possible_names = [base_index, f"partial-{base_index}", f"restored-{base_index}"] + for index_name in possible_names: + if self.client.indices.exists(index=index_name): + repo_index_count += 1 + break + except Exception: + pass + + # Check if policy exists + policy_exists = False + try: + self.client.ilm.get_lifecycle(name=f"{repo.name}-thawed") + policy_exists = True + except Exception: + pass + + print(f"DRY_RUN\t{repo.name}\t{repo.thaw_state}\t{repo.is_mounted}\t{action}\t{repo_index_count}\t{policy_exists}") + else: + if len(request_ids) == 1: + rprint(f"[cyan]Would process {len(repos)} repositories:[/cyan]\n") + for repo in repos: + action = "unmount and reset to frozen" if repo.is_mounted else "reset to frozen" + rprint(f" [cyan]- {repo.name}[/cyan] (state: {repo.thaw_state}, mounted: {repo.is_mounted})") + rprint(f" [dim]Would {action}[/dim]") + + # Show indices that would be deleted + try: + snapshot_indices = get_all_indices_in_repo(self.client, repo.name) + repo_index_count = 0 + for base_index in snapshot_indices: + possible_names = [base_index, f"partial-{base_index}", f"restored-{base_index}"] + for index_name in possible_names: + if self.client.indices.exists(index=index_name): + repo_index_count += 1 + break + if repo_index_count > 0: + rprint(f" [dim]Would delete {repo_index_count} mounted indices[/dim]") + except Exception: + pass + + # Show ILM policy that would be deleted + policy_name = f"{repo.name}-thawed" + try: + self.client.ilm.get_lifecycle(name=policy_name) + rprint(f" [dim]Would delete ILM policy {policy_name}[/dim]") + except Exception: + pass + + rprint(f"\n[cyan]DRY-RUN: Would mark thaw request '{request_id}' as completed[/cyan]\n") + + total_repos += len(repos) + + # Summary for bulk mode + if len(request_ids) > 1 and not self.porcelain: + rprint(f"[cyan]DRY-RUN: Would process {total_repos} total repositories across {len(request_ids)} thaw requests[/cyan]") + rprint(f"[cyan]DRY-RUN: Would delete {total_indices} indices and {total_policies} ILM policies[/cyan]") + rprint(f"[cyan]DRY-RUN: Would mark {len(request_ids)} thaw requests as completed[/cyan]\n") + + # Porcelain mode summary + if self.porcelain: + print(f"DRY_RUN_SUMMARY\t{total_repos}\t{total_indices}\t{total_policies}\t{len(request_ids)}") + + def do_singleton_action(self) -> None: + """ + Entry point for singleton CLI execution. + + :return: None + :rtype: None + """ + self.do_action() diff --git a/curator/actions/deepfreeze/rotate.py b/curator/actions/deepfreeze/rotate.py new file mode 100644 index 00000000..10fb442c --- /dev/null +++ b/curator/actions/deepfreeze/rotate.py @@ -0,0 +1,605 @@ +"""Rotate action for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging +import sys + +from elasticsearch8 import Elasticsearch + +from curator.actions.deepfreeze.cleanup import Cleanup +from curator.actions.deepfreeze.constants import STATUS_INDEX +from curator.actions.deepfreeze.helpers import Repository +from curator.actions.deepfreeze.utilities import ( + create_repo, + create_versioned_ilm_policy, + ensure_settings_index, + get_composable_templates, + get_index_templates, + get_matching_repo_names, + get_matching_repos, + get_next_suffix, + get_policies_by_suffix, + get_policies_for_repo, + get_settings, + is_policy_safe_to_delete, + push_to_glacier, + save_settings, + unmount_repo, + update_repository_date_range, + update_template_ilm_policy, +) +from curator.exceptions import RepositoryException +from curator.s3client import s3_client_factory + + +class Rotate: + """ + The Deepfreeze is responsible for managing the repository rotation given + a config file of user-managed options and settings. + + :param client: A client connection object + :type client: Elasticsearch + :param keep: How many repositories to retain, defaults to 6 + :type keep: str + :param year: Optional year to override current year + :type year: int + :param month: Optional month to override current month + :type month: int + + :raises RepositoryException: If a repository with the given prefix already exists + + :methods: + update_ilm_policies: Update ILM policies to use the new repository. + unmount_oldest_repos: Unmount the oldest repositories. + is_thawed: Check if a repository is thawed. + """ + + def __init__( + self, + client: Elasticsearch, + keep: str = "6", + year: int = None, # type: ignore + month: int = None, # type: ignore + ) -> None: + self.loggit = logging.getLogger("curator.actions.deepfreeze") + self.loggit.debug("Initializing Deepfreeze Rotate") + + self.settings = get_settings(client) # type: ignore + self.loggit.debug("Settings: %s", str(self.settings)) + + self.client = client + self.keep = int(keep) + self.year = year + self.month = month + self.base_path = "" + self.suffix = get_next_suffix( + self.settings.style, self.settings.last_suffix, year, month + ) + self.settings.last_suffix = self.suffix + + self.s3 = s3_client_factory(self.settings.provider) + + self.new_repo_name = f"{self.settings.repo_name_prefix}-{self.suffix}" + if self.settings.rotate_by == "bucket": + self.new_bucket_name = f"{self.settings.bucket_name_prefix}-{self.suffix}" + self.base_path = f"{self.settings.base_path_prefix}" + else: + self.new_bucket_name = f"{self.settings.bucket_name_prefix}" + self.base_path = f"{self.settings.base_path_prefix}-{self.suffix}" + + self.loggit.debug("Getting repo list") + self.repo_list = get_matching_repo_names( + self.client, self.settings.repo_name_prefix # type: ignore + ) + self.repo_list.sort(reverse=True) + self.loggit.debug("Repo list: %s", self.repo_list) + self.latest_repo = "" + try: + self.latest_repo = self.repo_list[0] + self.loggit.debug("Latest repo: %s", self.latest_repo) + except IndexError: + raise RepositoryException( + f"no repositories match {self.settings.repo_name_prefix}" + ) + if self.new_repo_name in self.repo_list: + raise RepositoryException(f"repository {self.new_repo_name} already exists") + if not self.client.indices.exists(index=STATUS_INDEX): + self.client.indices.create(index=STATUS_INDEX) + self.loggit.warning("Created index %s", STATUS_INDEX) + + # Validate that ILM policies exist for the current repository + # This must be checked during initialization to fail fast + self.loggit.debug("Checking for ILM policies that reference %s", self.latest_repo) + policies_for_repo = get_policies_for_repo(self.client, self.latest_repo) # type: ignore + if not policies_for_repo: + raise RepositoryException( + f"No ILM policies found that reference repository {self.latest_repo}. " + f"Rotation requires existing ILM policies to create versioned copies. " + f"Please create ILM policies that use searchable_snapshot actions " + f"with snapshot_repository: {self.latest_repo}, or run setup with " + f"--create-sample-ilm-policy to create a default policy." + ) + self.loggit.info( + "Found %d ILM policies referencing %s", + len(policies_for_repo), + self.latest_repo + ) + + self.loggit.info("Deepfreeze initialized") + + def update_repo_date_range(self, dry_run=False): + """ + Update the date ranges for all repositories in the status index. + + :return: None + :rtype: None + + """ + self.loggit.debug("Updating repo date ranges") + # Get ALL repo objects (not just mounted) which match our prefix + # We need to update date ranges for all repos to avoid gaps in coverage + repos = get_matching_repos( + self.client, self.settings.repo_name_prefix, mounted=None # type: ignore + ) + self.loggit.debug("Found %s matching repos", len(repos)) + + # Update date range for each repository + for repo in repos: + self.loggit.debug("Updating date range for %s (mounted: %s)", repo.name, repo.is_mounted) + + if dry_run: + self.loggit.info("DRY-RUN: Would update date range for %s", repo.name) + continue + + # Use the shared utility function to update dates + # It handles multiple index naming patterns and persists automatically + updated = update_repository_date_range(self.client, repo) # type: ignore + + if updated: + self.loggit.debug("Successfully updated date range for %s", repo.name) + else: + self.loggit.debug("No date range update for %s", repo.name) + + def update_ilm_policies(self, dry_run=False) -> None: + """ + Create versioned ILM policies for the new repository and update index templates. + + Instead of modifying existing policies, this creates NEW versioned policies + (e.g., my-policy-000005) that reference the new repository. Index templates + are then updated to use the new versioned policies, ensuring new indices use + the new repository while existing indices keep their old policies. + + :param dry_run: If True, do not actually create policies or update templates + :type dry_run: bool + + :return: None + :rtype: None + + :raises Exception: If policies or templates cannot be updated + """ + self.loggit.debug("Creating versioned ILM policies for new repository") + + if self.latest_repo == self.new_repo_name: + self.loggit.info("Already on the latest repo") + sys.exit(0) + + self.loggit.info( + "Creating versioned policies for transition from %s to %s", + self.latest_repo, + self.new_repo_name, + ) + + # Find all policies that reference the latest repository + # Note: We already validated policies exist during __init__, so this should always succeed + self.loggit.debug("Searching for policies that reference %s", self.latest_repo) + policies_to_version = get_policies_for_repo(self.client, self.latest_repo) # type: ignore + + self.loggit.info( + "Found %d policies to create versioned copies for: %s", + len(policies_to_version), + ", ".join(policies_to_version.keys()), + ) + + # Track policy name mappings (old -> new) for template updates + policy_mappings = {} + + # Create versioned copies of each policy + for policy_name, policy_data in policies_to_version.items(): + policy_body = policy_data.get("policy", {}) + + # Strip old suffix from policy name if it exists + # This handles subsequent rotations where policy might be "my-policy-000002" + # We want base name "my-policy" to create "my-policy-000003" + base_policy_name = policy_name + if "-" in policy_name: + parts = policy_name.rsplit("-", 1) + # Check if last part looks like a suffix (all digits or date format) + potential_suffix = parts[1] + if potential_suffix.isdigit() or ( + "." in potential_suffix + and all(p.isdigit() for p in potential_suffix.split(".")) + ): + base_policy_name = parts[0] + self.loggit.debug( + "Stripped suffix from %s, using base name: %s", + policy_name, + base_policy_name, + ) + + # Check for delete_searchable_snapshot setting and warn if True + for phase_name, phase_config in policy_body.get("phases", {}).items(): + delete_action = phase_config.get("actions", {}).get("delete", {}) + if delete_action.get("delete_searchable_snapshot", False): + self.loggit.warning( + "Policy %s has delete_searchable_snapshot=true in %s phase. " + "Snapshots may be deleted when indices transition!", + policy_name, + phase_name, + ) + + if not dry_run: + try: + new_policy_name = create_versioned_ilm_policy( + self.client, # type: ignore + base_policy_name, # Use base name, not full name + policy_body, + self.new_repo_name, + self.suffix, + ) + policy_mappings[policy_name] = new_policy_name + self.loggit.info( + "Created versioned policy: %s -> %s", + policy_name, + new_policy_name, + ) + except Exception as e: + self.loggit.error( + "Failed to create versioned policy for %s: %s", policy_name, e + ) + raise + else: + new_policy_name = f"{base_policy_name}-{self.suffix}" + policy_mappings[policy_name] = new_policy_name + self.loggit.info( + "DRY-RUN: Would create policy %s -> %s", + policy_name, + new_policy_name, + ) + + # Update index templates to use the new versioned policies + self.loggit.info("Updating index templates to use new versioned policies") + templates_updated = 0 + + # Update composable templates + try: + composable_templates = get_composable_templates(self.client) # type: ignore + for template_name in composable_templates.get("index_templates", []): + template_name = template_name["name"] + for old_policy, new_policy in policy_mappings.items(): + if not dry_run: + try: + if update_template_ilm_policy( + self.client, template_name, old_policy, new_policy, is_composable=True # type: ignore + ): + templates_updated += 1 + self.loggit.info( + "Updated composable template %s: %s -> %s", + template_name, + old_policy, + new_policy, + ) + except Exception as e: + self.loggit.debug( + "Could not update template %s: %s", template_name, e + ) + else: + self.loggit.info( + "DRY-RUN: Would update composable template %s if it uses policy %s", + template_name, + old_policy, + ) + except Exception as e: + self.loggit.warning("Could not get composable templates: %s", e) + + # Update legacy templates + try: + legacy_templates = get_index_templates(self.client) # type: ignore + for template_name in legacy_templates.keys(): + for old_policy, new_policy in policy_mappings.items(): + if not dry_run: + try: + if update_template_ilm_policy( + self.client, template_name, old_policy, new_policy, is_composable=False # type: ignore + ): + templates_updated += 1 + self.loggit.info( + "Updated legacy template %s: %s -> %s", + template_name, + old_policy, + new_policy, + ) + except Exception as e: + self.loggit.debug( + "Could not update template %s: %s", template_name, e + ) + else: + self.loggit.info( + "DRY-RUN: Would update legacy template %s if it uses policy %s", + template_name, + old_policy, + ) + except Exception as e: + self.loggit.warning("Could not get legacy templates: %s", e) + + if templates_updated > 0: + self.loggit.info("Updated %d index templates", templates_updated) + else: + self.loggit.warning("No index templates were updated") + + self.loggit.info("Finished ILM policy versioning and template updates") + + def cleanup_policies_for_repo(self, repo_name: str, dry_run=False) -> None: + """ + Clean up ILM policies associated with an unmounted repository. + + Finds all policies with the same suffix as the repository and deletes them + if they are not in use by any indices, data streams, or templates. + + :param repo_name: The repository name (e.g., "deepfreeze-000003") + :type repo_name: str + :param dry_run: If True, do not actually delete policies + :type dry_run: bool + + :return: None + :rtype: None + """ + self.loggit.debug("Cleaning up policies for repository %s", repo_name) + + # Extract suffix from repository name + # Repository format: {prefix}-{suffix} + try: + suffix = repo_name.split("-")[-1] + self.loggit.debug( + "Extracted suffix %s from repository %s", suffix, repo_name + ) + except Exception as e: + self.loggit.error( + "Could not extract suffix from repository %s: %s", repo_name, e + ) + return + + # Find all policies with this suffix + policies_with_suffix = get_policies_by_suffix(self.client, suffix) # type: ignore + + if not policies_with_suffix: + self.loggit.info("No policies found with suffix -%s", suffix) + return + + self.loggit.info( + "Found %d policies with suffix -%s to evaluate for deletion", + len(policies_with_suffix), + suffix, + ) + + deleted_count = 0 + skipped_count = 0 + + for policy_name in policies_with_suffix.keys(): + # Check if the policy is safe to delete + if is_policy_safe_to_delete(self.client, policy_name): # type: ignore + if not dry_run: + try: + self.client.ilm.delete_lifecycle(name=policy_name) + deleted_count += 1 + self.loggit.info( + "Deleted policy %s (no longer in use)", policy_name + ) + except Exception as e: + self.loggit.error( + "Failed to delete policy %s: %s", policy_name, e + ) + skipped_count += 1 + else: + self.loggit.info("DRY-RUN: Would delete policy %s", policy_name) + deleted_count += 1 + else: + skipped_count += 1 + self.loggit.info( + "Skipping policy %s (still in use by indices/datastreams/templates)", + policy_name, + ) + + self.loggit.info( + "Policy cleanup complete: %d deleted, %d skipped", + deleted_count, + skipped_count, + ) + + def is_thawed(self, repo: str) -> bool: + """ + Check if a repository is thawed by querying the STATUS_INDEX. + + :param repo: The name of the repository + :returns: True if the repository is thawed, False otherwise + """ + self.loggit.debug("Checking if %s is thawed", repo) + try: + repository = Repository.from_elasticsearch(self.client, repo, STATUS_INDEX) + if repository is None: + self.loggit.warning( + "Repository %s not found in STATUS_INDEX, assuming not thawed", repo + ) + return False + + is_thawed = repository.is_thawed + self.loggit.debug( + "Repository %s thawed status: %s (mounted: %s)", + repo, + is_thawed, + repository.is_mounted, + ) + return is_thawed + except Exception as e: + self.loggit.error("Error checking thawed status for %s: %s", repo, e) + # If we can't determine the status, err on the side of caution and assume it's thawed + # This prevents accidentally unmounting a thawed repo if there's a database issue + return True + + def unmount_oldest_repos(self, dry_run=False) -> None: + """ + Take the oldest repos from the list and remove them, only retaining + the number chosen in the config under "keep". + + :param dry_run: If True, do not actually remove the repositories + :type dry_run: bool + + :return: None + :rtype: None + + :raises Exception: If the repository cannot be removed + """ + # TODO: Use a list of Repositories, not a list of names. Be consistent and always use Repositories. + self.loggit.debug("Total list: %s", self.repo_list) + s = self.repo_list[self.keep :] + self.loggit.debug("Repos to remove: %s", s) + for repo in s: + if self.is_thawed(repo): + self.loggit.warning("Skipping thawed repo %s", repo) + continue + self.loggit.info("Removing repo %s", repo) + if not dry_run: + # ? Do I want to check for existence of snapshots still mounted from + # ? the repo here or in unmount_repo? + unmounted_repo = unmount_repo(self.client, repo) # type: ignore + push_to_glacier(self.s3, unmounted_repo) + try: + self.loggit.debug("Fetching repo %s doc", repo) + repository = Repository.from_elasticsearch( + self.client, repo, STATUS_INDEX + ) + self.loggit.debug("Looking for %s, found %s", repo, repository) + repository.unmount() # type: ignore + self.loggit.debug("preparing to persist %s", repo) + repository.persist(self.client) # type: ignore + self.loggit.info( + "Updated status to unmounted for repo %s", repository.name # type: ignore + ) + + # Clean up ILM policies associated with this repository + self.loggit.info( + "Cleaning up ILM policies associated with repository %s", repo + ) + self.cleanup_policies_for_repo(repo, dry_run=False) + + except Exception as e: + self.loggit.error( + "Failed to update doc unmounting repo %s: %s", repo, str(e) + ) + raise + else: + self.loggit.info("DRY-RUN: Would clean up policies for repo %s", repo) + self.cleanup_policies_for_repo(repo, dry_run=True) + + def do_dry_run(self) -> None: + """ + Perform a dry-run of the rotation process. + + :return: None + :rtype: None + + :raises Exception: If the repository cannot be created + :raises Exception: If the repository already exists + """ + self.loggit.info("DRY-RUN MODE. No changes will be made.") + msg = ( + f"DRY-RUN: deepfreeze {self.latest_repo} will be rotated out" + f" and {self.new_repo_name} will be added & made active." + ) + self.loggit.info(msg) + self.loggit.info("DRY-RUN: Creating bucket %s", self.new_bucket_name) + create_repo( + self.client, # type: ignore + self.new_repo_name, + self.new_bucket_name, + self.base_path, + self.settings.canned_acl, + self.settings.storage_class, + dry_run=True, + ) + self.update_ilm_policies(dry_run=True) + self.unmount_oldest_repos(dry_run=True) + self.update_repo_date_range(dry_run=True) + # Clean up any thawed repositories that have expired + cleanup = Cleanup(self.client) + cleanup.do_dry_run() + + def do_action(self) -> None: + """ + Perform high-level repo rotation steps in sequence. + + :return: None + :rtype: None + + :raises Exception: If the repository cannot be created + :raises Exception: If the repository already exists + """ + ensure_settings_index(self.client) # type: ignore + self.loggit.debug("Saving settings") + save_settings(self.client, self.settings) # type: ignore + + # HIGH PRIORITY FIX: Add validation and logging for bucket/repo creation + # Create the new bucket and repo, but only if rotate_by is bucket + if self.settings.rotate_by == "bucket": + self.loggit.info("Checking if bucket %s exists before creation", self.new_bucket_name) + try: + # create_bucket already checks bucket_exists internally + self.s3.create_bucket(self.new_bucket_name) + except Exception as e: + self.loggit.error( + "Failed to create bucket %s: %s. Check S3 permissions and bucket naming rules.", + self.new_bucket_name, + e, + exc_info=True + ) + raise + + # Verify repository doesn't already exist before creation + self.loggit.info( + "Creating repository %s with bucket=%s, base_path=%s, storage_class=%s", + self.new_repo_name, + self.new_bucket_name, + self.base_path, + self.settings.storage_class + ) + try: + existing_repos = self.client.snapshot.get_repository() + if self.new_repo_name in existing_repos: + error_msg = f"Repository {self.new_repo_name} already exists in Elasticsearch" + self.loggit.error(error_msg) + raise ActionError(error_msg) + + create_repo( + self.client, # type: ignore + self.new_repo_name, + self.new_bucket_name, + self.base_path, + self.settings.canned_acl, + self.settings.storage_class, + ) + self.loggit.info("Successfully created repository %s", self.new_repo_name) + except Exception as e: + self.loggit.error( + "Failed to create repository %s: %s", + self.new_repo_name, + e, + exc_info=True + ) + raise + # Go through mounted repos and make sure the date ranges are up-to-date + self.update_repo_date_range() + self.update_ilm_policies() + self.unmount_oldest_repos() + # Clean up any thawed repositories that have expired + cleanup = Cleanup(self.client) + cleanup.do_action() diff --git a/curator/actions/deepfreeze/setup.py b/curator/actions/deepfreeze/setup.py new file mode 100644 index 00000000..aaa31ba7 --- /dev/null +++ b/curator/actions/deepfreeze/setup.py @@ -0,0 +1,489 @@ +"""Setup action for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging +import sys + +from elasticsearch8 import Elasticsearch +from rich.console import Console +from rich.panel import Panel +from rich import print as rprint +from rich.markup import escape + +from curator.s3client import s3_client_factory + +from .constants import STATUS_INDEX +from .exceptions import PreconditionError, RepositoryException +from .helpers import Settings +from .utilities import ( + create_ilm_policy, + create_repo, + ensure_settings_index, + get_matching_repo_names, + save_settings, +) + + +class Setup: + """ + Setup is responsible for creating the initial repository and bucket for + deepfreeze operations. + + :param client: A client connection object + :param repo_name_prefix: A prefix for repository names, defaults to `deepfreeze` + :param bucket_name_prefix: A prefix for bucket names, defaults to `deepfreeze` + :param base_path_prefix: Path within a bucket where snapshots are stored, defaults to `snapshots` + :param canned_acl: One of the AWS canned ACL values (see + ``), + defaults to `private` + :param storage_class: AWS Storage class (see ``), + defaults to `intelligent_tiering` + :param provider: The provider to use (AWS only for now), defaults to `aws`, and will be saved + to the deepfreeze status index for later reference. + :param rotate_by: Rotate by bucket or path within a bucket?, defaults to `path` + + :raises RepositoryException: If a repository with the given prefix already exists + + :methods: + do_dry_run: Perform a dry-run of the setup process. + do_action: Perform create initial bucket and repository. + + :example: + >>> from curator.actions.deepfreeze import Setup + >>> setup = Setup(client, repo_name_prefix="deepfreeze", bucket_name_prefix="deepfreeze", base_path_prefix="snapshots", canned_acl="private", storage_class="intelligent_tiering", provider="aws", rotate_by="path") + >>> setup.do_dry_run() + >>> setup.do_action() + """ + + def __init__( + self, + client: Elasticsearch, + year: int = None, + month: int = None, + repo_name_prefix: str = "deepfreeze", + bucket_name_prefix: str = "deepfreeze", + base_path_prefix: str = "snapshots", + canned_acl: str = "private", + storage_class: str = "intelligent_tiering", + provider: str = "aws", + rotate_by: str = "path", + style: str = "oneup", + create_sample_ilm_policy: bool = False, + ilm_policy_name: str = "deepfreeze-sample-policy", + porcelain: bool = False, + ) -> None: + self.loggit = logging.getLogger("curator.actions.deepfreeze") + self.loggit.debug("Initializing Deepfreeze Setup") + + # Console for STDERR output + self.console = Console(stderr=True) + + self.client = client + self.porcelain = porcelain + self.year = year + self.month = month + self.settings = Settings( + repo_name_prefix=repo_name_prefix, + bucket_name_prefix=bucket_name_prefix, + base_path_prefix=base_path_prefix, + canned_acl=canned_acl, + storage_class=storage_class, + provider=provider, + rotate_by=rotate_by, + style=style, + ) + self.create_sample_ilm_policy = create_sample_ilm_policy + self.ilm_policy_name = ilm_policy_name + self.base_path = self.settings.base_path_prefix + + self.s3 = s3_client_factory(self.settings.provider) + + self.suffix = "000001" + if self.settings.style != "oneup": + self.suffix = f"{self.year:04}.{self.month:02}" + self.settings.last_suffix = self.suffix + + self.new_repo_name = f"{self.settings.repo_name_prefix}-{self.suffix}" + if self.settings.rotate_by == "bucket": + self.new_bucket_name = f"{self.settings.bucket_name_prefix}-{self.suffix}" + self.base_path = f"{self.settings.base_path_prefix}" + else: + self.new_bucket_name = f"{self.settings.bucket_name_prefix}" + self.base_path = f"{self.base_path}-{self.suffix}" + + self.loggit.debug("Deepfreeze Setup initialized") + + def _check_preconditions(self) -> None: + """ + Check preconditions before performing setup. Raise exceptions if any + preconditions are not met. If this completes without raising an exception, + the setup can proceed. + + :raises PreconditionError: If any preconditions are not met. + + :return: None + :rtype: None + """ + errors = [] + + # First, make sure the status index does not exist yet + self.loggit.debug("Checking if status index %s exists", STATUS_INDEX) + if self.client.indices.exists(index=STATUS_INDEX): + errors.append({ + "issue": f"Status index [cyan]{STATUS_INDEX}[/cyan] already exists", + "solution": f"Delete the existing index before running setup:\n" + f" [yellow]curator_cli --host DELETE index --name {STATUS_INDEX}[/yellow]\n" + f" or use the Elasticsearch API:\n" + f" [yellow]curl -X DELETE 'http://:9200/{STATUS_INDEX}'[/yellow]" + }) + + # Second, see if any existing repositories match the prefix + self.loggit.debug( + "Checking if any existing repositories match %s", + self.settings.repo_name_prefix, + ) + repos = self.client.snapshot.get_repository(name="_all") + self.loggit.debug("Existing repositories: %s", repos) + matching_repos = [repo for repo in repos.keys() if repo.startswith(self.settings.repo_name_prefix)] + + if matching_repos: + repo_list = "\n ".join([f"[cyan]{repo}[/cyan]" for repo in matching_repos]) + errors.append({ + "issue": f"Found {len(matching_repos)} existing repositor{'y' if len(matching_repos) == 1 else 'ies'} matching prefix [cyan]{self.settings.repo_name_prefix}[/cyan]:\n {repo_list}", + "solution": "Delete the existing repositories before running setup:\n" + f" [yellow]curator_cli deepfreeze cleanup[/yellow]\n" + " or manually delete each repository:\n" + f" [yellow]curl -X DELETE 'http://:9200/_snapshot/'[/yellow]\n" + "\n[bold]WARNING:[/bold] Ensure you have backups before deleting repositories!" + }) + + # Third, check if the bucket already exists + self.loggit.debug("Checking if bucket %s exists", self.new_bucket_name) + if self.s3.bucket_exists(self.new_bucket_name): + errors.append({ + "issue": f"S3 bucket [cyan]{self.new_bucket_name}[/cyan] already exists", + "solution": f"Delete the existing bucket before running setup:\n" + f" [yellow]aws s3 rb s3://{self.new_bucket_name} --force[/yellow]\n" + "\n[bold]WARNING:[/bold] This will delete all data in the bucket!\n" + "Or use a different bucket_name_prefix in your configuration." + }) + + # HIGH PRIORITY FIX: Check for S3 repository plugin (only for ES 7.x and below) + # NOTE: Elasticsearch 8.x+ has built-in S3 repository support, no plugin needed + self.loggit.debug("Checking S3 repository support") + try: + # Get Elasticsearch version + cluster_info = self.client.info() + es_version = cluster_info.get("version", {}).get("number", "0.0.0") + major_version = int(es_version.split(".")[0]) + + if major_version < 8: + # ES 7.x and below require the repository-s3 plugin + self.loggit.debug( + "Elasticsearch %s detected - checking for S3 repository plugin", + es_version + ) + + # Get cluster plugins + nodes_info = self.client.nodes.info(node_id="_all", metric="plugins") + + # Check if any node has the S3 plugin + has_s3_plugin = False + for node_id, node_data in nodes_info.get("nodes", {}).items(): + plugins = node_data.get("plugins", []) + for plugin in plugins: + if plugin.get("name") == "repository-s3": + has_s3_plugin = True + self.loggit.debug("Found S3 plugin on node %s", node_id) + break + if has_s3_plugin: + break + + if not has_s3_plugin: + errors.append({ + "issue": "Elasticsearch S3 repository plugin is not installed", + "solution": "Install the S3 repository plugin on all Elasticsearch nodes:\n" + " [yellow]bin/elasticsearch-plugin install repository-s3[/yellow]\n" + " Then restart all Elasticsearch nodes.\n" + " See: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html" + }) + else: + self.loggit.debug("S3 repository plugin is installed") + else: + # ES 8.x+ has built-in S3 support + self.loggit.debug( + "Elasticsearch %s detected - S3 repository support is built-in", + es_version + ) + except Exception as e: + self.loggit.warning("Could not verify S3 repository support: %s", e) + # Don't add to errors - this is a soft check that may fail due to permissions + + # If any errors were found, display them all and raise exception + if errors: + if self.porcelain: + # Machine-readable output: tab-separated values + for error in errors: + # Extract clean text from rich markup + issue_text = error['issue'].replace('[cyan]', '').replace('[/cyan]', '').replace('[yellow]', '').replace('[/yellow]', '').replace('[bold]', '').replace('[/bold]', '').replace('\n', ' ') + print(f"ERROR\tprecondition\t{issue_text}") + else: + self.console.print("\n[bold red]Setup Preconditions Failed[/bold red]\n", style="bold") + + for i, error in enumerate(errors, 1): + self.console.print(Panel( + f"[bold]Issue:[/bold]\n{error['issue']}\n\n" + f"[bold]Solution:[/bold]\n{error['solution']}", + title=f"[bold red]Error {i} of {len(errors)}[/bold red]", + border_style="red", + expand=False + )) + self.console.print() # Add spacing between panels + + # Create summary error message + summary = f"Found {len(errors)} precondition error{'s' if len(errors) > 1 else ''} that must be resolved before setup can proceed." + self.console.print(Panel( + f"[bold]{summary}[/bold]\n\n" + "Deepfreeze setup requires a clean environment. Please resolve the issues above and try again.", + title="[bold red]Setup Cannot Continue[/bold red]", + border_style="red", + expand=False + )) + + summary = f"Found {len(errors)} precondition error{'s' if len(errors) > 1 else ''} that must be resolved before setup can proceed." + raise PreconditionError(summary) + + def do_dry_run(self) -> None: + """ + Perform a dry-run of the setup process. + + :return: None + :rtype: None + """ + self.loggit.info("DRY-RUN MODE. No changes will be made.") + msg = f"DRY-RUN: deepfreeze setup of {self.new_repo_name} backed by {self.new_bucket_name}, with base path {self.base_path}." + self.loggit.info(msg) + self._check_preconditions() + + self.loggit.info("DRY-RUN: Creating bucket %s", self.new_bucket_name) + create_repo( + self.client, + self.new_repo_name, + self.new_bucket_name, + self.base_path, + self.settings.canned_acl, + self.settings.storage_class, + dry_run=True, + ) + + def do_action(self) -> None: + """ + Perform setup steps to create initial bucket and repository and save settings. + + :return: None + :rtype: None + """ + self.loggit.debug("Starting Setup action") + + try: + # Check preconditions + self._check_preconditions() + + # Create settings index and save settings + self.loggit.info("Creating settings index and saving configuration") + try: + ensure_settings_index(self.client, create_if_missing=True) + save_settings(self.client, self.settings) + except Exception as e: + if self.porcelain: + print(f"ERROR\tsettings_index\t{str(e)}") + else: + self.console.print(Panel( + f"[bold]Failed to create settings index or save configuration[/bold]\n\n" + f"Error: {escape(str(e))}\n\n" + f"[bold]Possible Solutions:[/bold]\n" + f" • Check Elasticsearch connection and permissions\n" + f" • Verify the cluster is healthy and has capacity\n" + f" • Check Elasticsearch logs for details", + title="[bold red]Settings Index Error[/bold red]", + border_style="red", + expand=False + )) + raise + + # Create S3 bucket + # ENHANCED LOGGING: Log bucket creation parameters + self.loggit.info( + "Creating S3 bucket %s with ACL=%s, storage_class=%s", + self.new_bucket_name, + self.settings.canned_acl, + self.settings.storage_class + ) + self.loggit.debug( + "Full bucket creation parameters: bucket=%s, ACL=%s, storage_class=%s, provider=%s", + self.new_bucket_name, + self.settings.canned_acl, + self.settings.storage_class, + self.settings.provider + ) + try: + self.s3.create_bucket(self.new_bucket_name) + self.loggit.info("Successfully created S3 bucket %s", self.new_bucket_name) + except Exception as e: + if self.porcelain: + print(f"ERROR\ts3_bucket\t{self.new_bucket_name}\t{str(e)}") + else: + self.console.print(Panel( + f"[bold]Failed to create S3 bucket [cyan]{self.new_bucket_name}[/cyan][/bold]\n\n" + f"Error: {escape(str(e))}\n\n" + f"[bold]Possible Solutions:[/bold]\n" + f" • Check AWS credentials and permissions\n" + f" • Verify IAM policy allows s3:CreateBucket\n" + f" • Check if bucket name is globally unique\n" + f" • Verify AWS region settings\n" + f" • Check AWS account limits for S3 buckets", + title="[bold red]S3 Bucket Creation Error[/bold red]", + border_style="red", + expand=False + )) + raise + + # Create repository + # ENHANCED LOGGING: Log repository configuration + self.loggit.info("Creating repository %s", self.new_repo_name) + self.loggit.debug( + "Repository configuration: name=%s, bucket=%s, base_path=%s, ACL=%s, storage_class=%s", + self.new_repo_name, + self.new_bucket_name, + self.base_path, + self.settings.canned_acl, + self.settings.storage_class + ) + try: + create_repo( + self.client, + self.new_repo_name, + self.new_bucket_name, + self.base_path, + self.settings.canned_acl, + self.settings.storage_class, + ) + self.loggit.info("Successfully created repository %s", self.new_repo_name) + except Exception as e: + if self.porcelain: + print(f"ERROR\trepository\t{self.new_repo_name}\t{str(e)}") + else: + self.console.print(Panel( + f"[bold]Failed to create repository [cyan]{self.new_repo_name}[/cyan][/bold]\n\n" + f"Error: {escape(str(e))}\n\n" + f"[bold]Possible Solutions:[/bold]\n" + f" • Verify Elasticsearch has S3 plugin installed\n" + f" • Check AWS credentials are configured in Elasticsearch keystore\n" + f" • Verify S3 bucket [cyan]{self.new_bucket_name}[/cyan] is accessible\n" + f" • Check repository settings (ACL, storage class, etc.)\n" + f" • Review Elasticsearch logs for detailed error messages", + title="[bold red]Repository Creation Error[/bold red]", + border_style="red", + expand=False + )) + raise + + # Optionally create sample ILM policy + if self.create_sample_ilm_policy: + policy_name = self.ilm_policy_name + policy_body = { + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": {"max_size": "45gb", "max_age": "7d"} + }, + }, + "frozen": { + "min_age": "14d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": self.new_repo_name + } + }, + }, + "delete": { + "min_age": "365d", + "actions": { + "delete": {"delete_searchable_snapshot": False} + }, + }, + } + } + } + self.loggit.info("Creating ILM policy %s", policy_name) + self.loggit.debug("ILM policy body: %s", policy_body) + try: + create_ilm_policy( + client=self.client, policy_name=policy_name, policy_body=policy_body + ) + except Exception as e: + # ILM policy creation is optional, so just warn but don't fail + if self.porcelain: + print(f"WARNING\tilm_policy\t{policy_name}\t{str(e)}") + else: + self.console.print(Panel( + f"[bold yellow]Warning: Failed to create sample ILM policy[/bold yellow]\n\n" + f"Error: {escape(str(e))}\n\n" + f"Setup will continue, but you'll need to create the ILM policy manually.\n" + f"This is not a critical error.", + title="[bold yellow]ILM Policy Warning[/bold yellow]", + border_style="yellow", + expand=False + )) + self.loggit.warning("Failed to create sample ILM policy: %s", e) + + # Success! + if self.porcelain: + # Machine-readable output: tab-separated values + # Format: SUCCESS\t{repo_name}\t{bucket_name}\t{base_path} + print(f"SUCCESS\t{self.new_repo_name}\t{self.new_bucket_name}\t{self.base_path}") + if self.create_sample_ilm_policy: + print(f"ILM_POLICY\t{self.ilm_policy_name}\tcreated") + else: + self.console.print(Panel( + f"[bold green]Setup completed successfully![/bold green]\n\n" + f"Repository: [cyan]{self.new_repo_name}[/cyan]\n" + f"S3 Bucket: [cyan]{self.new_bucket_name}[/cyan]\n" + f"Base Path: [cyan]{escape(self.base_path)}[/cyan]\n\n" + f"[bold]Next Steps:[/bold]\n" + f" 1. Update your ILM policies to use repository [cyan]{self.new_repo_name}[/cyan]\n" + f" 2. Ensure all ILM policies have [yellow]delete_searchable_snapshot: false[/yellow]\n" + f" 3. Thawed indices will automatically get per-repository ILM policies\n" + f" 4. See: https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-delete.html", + title="[bold green]Deepfreeze Setup Complete[/bold green]", + border_style="green", + expand=False + )) + + self.loggit.info("Setup complete. Repository %s is ready to use.", self.new_repo_name) + + except PreconditionError: + # Precondition errors are already formatted and displayed, just re-raise + raise + except Exception as e: + # Catch any unexpected errors + if self.porcelain: + print(f"ERROR\tunexpected\t{str(e)}") + else: + self.console.print(Panel( + f"[bold]An unexpected error occurred during setup[/bold]\n\n" + f"Error: {escape(str(e))}\n\n" + f"[bold]What to do:[/bold]\n" + f" • Check the logs for detailed error information\n" + f" • Verify all prerequisites are met (AWS credentials, ES connection, etc.)\n" + f" • You may need to manually clean up any partially created resources\n" + f" • Run [yellow]curator_cli deepfreeze cleanup[/yellow] to remove any partial state", + title="[bold red]Unexpected Setup Error[/bold red]", + border_style="red", + expand=False + )) + self.loggit.error("Unexpected error during setup: %s", e, exc_info=True) + raise diff --git a/curator/actions/deepfreeze/status.py b/curator/actions/deepfreeze/status.py new file mode 100644 index 00000000..03cd42e0 --- /dev/null +++ b/curator/actions/deepfreeze/status.py @@ -0,0 +1,452 @@ +"""Status action for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging +from datetime import datetime + +from elasticsearch8 import Elasticsearch +from rich import print +from rich.console import Console +from rich.table import Table + +from curator.actions.deepfreeze.utilities import get_all_repos, get_settings, check_restore_status, list_thaw_requests +from curator.s3client import s3_client_factory + + +class Status: + """ + Get the status of the deepfreeze components. No dry_run for this action makes + sense as it changes nothing, so the do_singleton_action method simply runs the + do_action method directly. + + :param client: A client connection object + :type client: Elasticsearch + :param limit: Number of most recent repositories to show (None = show all) + :type limit: int + :param show_repos: Show repositories section + :type show_repos: bool + :param show_thawed: Show thawed repositories section + :type show_thawed: bool + :param show_buckets: Show buckets section + :type show_buckets: bool + :param show_ilm: Show ILM policies section + :type show_ilm: bool + :param show_config: Show configuration section + :type show_config: bool + :param porcelain: Output plain text without rich formatting + :type porcelain: bool + + :methods: + do_action: Perform high-level status steps in sequence. + do_singleton_action: Perform high-level status steps in sequence. + get_cluster_name: Get the name of the cluster. + do_repositories: Get the status of the repositories. + do_buckets: Get the status of the buckets. + do_ilm_policies: Get the status of the ILM policies. + do_thawsets: Get the status of the thawsets. + do_config: Get the status of the configuration. + """ + + def __init__( + self, + client: Elasticsearch, + limit: int = None, + show_repos: bool = False, + show_thawed: bool = False, + show_buckets: bool = False, + show_ilm: bool = False, + show_config: bool = False, + porcelain: bool = False, + ) -> None: + self.loggit = logging.getLogger("curator.actions.deepfreeze") + self.loggit.debug("Initializing Deepfreeze Status") + self.settings = get_settings(client) + self.client = client + self.limit = limit + + # If no specific sections are requested, show all + self.show_all = not (show_repos or show_thawed or show_buckets or show_ilm or show_config) + self.show_repos = show_repos or self.show_all + self.show_thawed = show_thawed or self.show_all + self.show_buckets = show_buckets or self.show_all + self.show_ilm = show_ilm or self.show_all + self.show_config = show_config or self.show_all + self.porcelain = porcelain + + self.console = Console() + if not porcelain: + self.console.clear() + # Initialize S3 client for checking restore status + self.s3 = s3_client_factory(self.settings.provider) + + def get_cluster_name(self) -> str: + """ + Connects to the Elasticsearch cluster and returns its name. + + :param es_host: The URL of the Elasticsearch instance (default: "http://localhost:9200"). + :type es_host: str + :return: The name of the Elasticsearch cluster. + :rtype: str + """ + try: + cluster_info = self.client.cluster.health() + return cluster_info.get("cluster_name", "Unknown Cluster") + except Exception as e: + return f"Error: {e}" + + def do_action(self) -> None: + """ + Perform the status action + + :return: None + :rtype: None + """ + self.loggit.info("Getting status") + if not self.porcelain: + print() + + if self.show_thawed: + self.do_thawed_repositories() + if self.show_repos: + self.do_repositories() + if self.show_buckets: + self.do_buckets() + if self.show_ilm: + self.do_ilm_policies() + if self.show_config: + self.do_config() + + def do_config(self): + """ + Print the configuration settings + + :return: None + :rtype: None + """ + config_items = [ + ("Repo Prefix", self.settings.repo_name_prefix), + ("Bucket Prefix", self.settings.bucket_name_prefix), + ("Base Path Prefix", self.settings.base_path_prefix), + ("Canned ACL", self.settings.canned_acl), + ("Storage Class", self.settings.storage_class), + ("Provider", self.settings.provider), + ("Rotate By", self.settings.rotate_by), + ("Style", self.settings.style), + ("Last Suffix", self.settings.last_suffix), + ("Cluster Name", self.get_cluster_name()), + ] + + if self.porcelain: + # Output tab-separated key-value pairs for scripting + for setting, value in config_items: + print(f"{setting}\t{value}") + else: + table = Table(title="Configuration") + table.add_column("Setting", style="cyan", no_wrap=False, overflow="fold") + table.add_column("Value", style="magenta", no_wrap=False, overflow="fold") + + for setting, value in config_items: + table.add_row(setting, value) + + self.console.print(table) + + def do_ilm_policies(self): + """ + Print the ILM policies affected by deepfreeze + + :return: None + :rtype: None + """ + table = Table(title="ILM Policies") + table.add_column("Policy", style="cyan", no_wrap=False, overflow="fold") + table.add_column("Repository", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Indices", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Datastreams", style="magenta", no_wrap=False, overflow="fold") + + current_repo = f"{self.settings.repo_name_prefix}-{self.settings.last_suffix}" + policies = self.client.ilm.get_lifecycle() + + for policy in policies: + for phase in policies[policy]["policy"]["phases"]: + if ( + "searchable_snapshot" + in policies[policy]["policy"]["phases"][phase]["actions"] + ): + repo_name = policies[policy]["policy"]["phases"][phase]["actions"][ + "searchable_snapshot" + ]["snapshot_repository"] + + # Check if repository starts with our prefix + if repo_name.startswith(self.settings.repo_name_prefix): + # Mark current repo with asterisk + repo_display = repo_name if repo_name != current_repo else f"{repo_name}*" + + num_indices = len(policies[policy]["in_use_by"]["indices"]) + num_datastreams = len(policies[policy]["in_use_by"]["data_streams"]) + + if self.porcelain: + # Output tab-separated values for scripting + print(f"{policy}\t{repo_display}\t{num_indices}\t{num_datastreams}") + else: + table.add_row(policy, repo_display, str(num_indices), str(num_datastreams)) + break + + if not self.porcelain: + self.console.print(table) + + def do_buckets(self): + """ + Print the buckets in use by deepfreeze + + :return: None + :rtype: None + """ + self.loggit.debug("Showing buckets") + + # Get all repositories with our prefix + all_repos = get_all_repos(self.client) + matching_repos = [ + repo for repo in all_repos + if repo.name.startswith(self.settings.repo_name_prefix) + ] + + # Extract unique bucket/base_path combinations + bucket_info = {} + for repo in matching_repos: + if repo.bucket and repo.base_path is not None: + key = (repo.bucket, repo.base_path) + if key not in bucket_info: + bucket_info[key] = repo.name + + # Sort by bucket/base_path + sorted_buckets = sorted(bucket_info.keys()) + total_buckets = len(sorted_buckets) + + # Apply limit if specified + if self.limit is not None and self.limit > 0: + sorted_buckets = sorted_buckets[-self.limit:] + self.loggit.debug("Limiting display to last %s buckets", self.limit) + + # Determine current bucket/base_path + if self.settings.rotate_by == "bucket": + current_bucket = f"{self.settings.bucket_name_prefix}-{self.settings.last_suffix}" + current_base_path = self.settings.base_path_prefix + else: + current_bucket = self.settings.bucket_name_prefix + current_base_path = f"{self.settings.base_path_prefix}-{self.settings.last_suffix}" + + # Set up the table with appropriate title + if self.limit is not None and self.limit > 0 and total_buckets > self.limit: + table_title = f"Buckets (showing last {len(sorted_buckets)} of {total_buckets})" + else: + table_title = "Buckets" + + table = Table(title=table_title) + table.add_column("Provider", style="cyan", no_wrap=False, overflow="fold") + table.add_column("Bucket", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Base_path", style="magenta", no_wrap=False, overflow="fold") + + for bucket, base_path in sorted_buckets: + # Mark current bucket/base_path with asterisk + if bucket == current_bucket and base_path == current_base_path: + bucket_display = f"{bucket}*" + else: + bucket_display = bucket + + if self.porcelain: + # Output tab-separated values for scripting + print(f"{self.settings.provider}\t{bucket_display}\t{base_path}") + else: + table.add_row(self.settings.provider, bucket_display, base_path) + + if not self.porcelain: + self.console.print(table) + + def do_thawed_repositories(self): + """ + Print thawed and thawing repositories in a separate section + + :return: None + :rtype: None + """ + self.loggit.debug("Showing thawed repositories") + + # Get all repositories + all_repos = get_all_repos(self.client) + all_repos.sort() + + # Get active thaw requests to track which repos are being thawed + active_thaw_requests = [] + repos_being_thawed = set() + try: + all_thaw_requests = list_thaw_requests(self.client) + active_thaw_requests = [req for req in all_thaw_requests if req.get("status") == "in_progress"] + for req in active_thaw_requests: + repos_being_thawed.update(req.get("repos", [])) + except Exception as e: + self.loggit.warning("Could not retrieve thaw requests: %s", e) + + # Filter to only thawed/thawing repos + thawed_repos = [] + for repo in all_repos: + if repo.name in repos_being_thawed or repo.is_thawed: + thawed_repos.append(repo) + + # If no thawed repos, don't show the section + if not thawed_repos: + return + + # Create the table + table = Table(title="Thawed Repositories") + table.add_column("Repository", style="cyan", no_wrap=False, overflow="fold") + table.add_column("State", style="yellow", no_wrap=False, overflow="fold") + table.add_column("Mounted", style="green", no_wrap=False, overflow="fold") + table.add_column("Snapshots", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Expires", style="red", no_wrap=False, overflow="fold") + table.add_column("Start", style="magenta", no_wrap=False, overflow="fold") + table.add_column("End", style="magenta", no_wrap=False, overflow="fold") + + for repo in thawed_repos: + # Determine mounted status + mounted_status = "yes" if repo.is_mounted else "no" + + # Get snapshot count + count = "--" + if repo.is_mounted: + try: + snapshots = self.client.snapshot.get( + repository=repo.name, snapshot="_all" + ) + count = len(snapshots.get("snapshots", [])) + except Exception as e: + self.loggit.warning("Repository %s not mounted: %s", repo.name, e) + + # Format dates for display + start_str = ( + repo.start.isoformat() if isinstance(repo.start, datetime) + else repo.start if repo.start + else "N/A" + ) + end_str = ( + repo.end.isoformat() if isinstance(repo.end, datetime) + else repo.end if repo.end + else "N/A" + ) + + # Format expiry time + expires_str = ( + repo.expires_at.isoformat() if isinstance(repo.expires_at, datetime) + else repo.expires_at if repo.expires_at + else "N/A" + ) + + if self.porcelain: + print(f"{repo.name}\t{repo.thaw_state}\t{mounted_status}\t{count}\t{expires_str}\t{start_str}\t{end_str}") + else: + table.add_row(repo.name, repo.thaw_state, mounted_status, str(count), expires_str, start_str, end_str) + + if not self.porcelain: + self.console.print(table) + + def do_repositories(self): + """ + Print the repositories in use by deepfreeze + + :return: None + :rtype: None + """ + self.loggit.debug("Showing repositories") + + # Get and sort all repositories + active_repo = f"{self.settings.repo_name_prefix}-{self.settings.last_suffix}" + self.loggit.debug("Getting repositories") + all_repos = get_all_repos(self.client) + all_repos.sort() + total_repos = len(all_repos) + self.loggit.debug("Got %s repositories", total_repos) + + # Get active thaw requests to track which repos are being thawed + active_thaw_requests = [] + repos_being_thawed = set() + try: + all_thaw_requests = list_thaw_requests(self.client) + active_thaw_requests = [req for req in all_thaw_requests if req.get("status") == "in_progress"] + for req in active_thaw_requests: + repos_being_thawed.update(req.get("repos", [])) + self.loggit.debug("Found %d active thaw requests covering %d repos", + len(active_thaw_requests), len(repos_being_thawed)) + except Exception as e: + self.loggit.warning("Could not retrieve thaw requests: %s", e) + + # Apply limit to all repos equally + if self.limit is not None and self.limit > 0: + repos_to_display = all_repos[-self.limit:] + self.loggit.debug("Limiting display to last %s repositories", self.limit) + else: + repos_to_display = all_repos + + # Set up the table with appropriate title + if self.limit is not None and self.limit > 0: + table_title = f"Repositories (showing last {len(repos_to_display)} of {total_repos})" + else: + table_title = "Repositories" + + table = Table(title=table_title) + table.add_column("Repository", style="cyan", no_wrap=False, overflow="fold") + table.add_column("State", style="yellow", no_wrap=False, overflow="fold") + table.add_column("Mounted", style="green", no_wrap=False, overflow="fold") + table.add_column("Snapshots", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Start", style="magenta", no_wrap=False, overflow="fold") + table.add_column("End", style="magenta", no_wrap=False, overflow="fold") + + for repo in repos_to_display: + # Mark active repository with asterisk + repo_name = f"{repo.name}*" if repo.name == active_repo else repo.name + + # Determine mounted status + mounted_status = "yes" if repo.is_mounted else "no" + + # Get snapshot count + count = "--" + self.loggit.debug(f"Checking mount status for {repo.name}") + if repo.is_mounted: + try: + snapshots = self.client.snapshot.get( + repository=repo.name, snapshot="_all" + ) + count = len(snapshots.get("snapshots", [])) + self.loggit.debug(f"Got {count} snapshots for {repo.name}") + except Exception as e: + self.loggit.warning("Repository %s not mounted: %s", repo.name, e) + repo.unmount() + + # Format dates for display + start_str = ( + repo.start.isoformat() if isinstance(repo.start, datetime) + else repo.start if repo.start + else "N/A" + ) + end_str = ( + repo.end.isoformat() if isinstance(repo.end, datetime) + else repo.end if repo.end + else "N/A" + ) + + if self.porcelain: + # Output tab-separated values for scripting + print(f"{repo_name}\t{repo.thaw_state}\t{mounted_status}\t{count}\t{start_str}\t{end_str}") + else: + table.add_row(repo_name, repo.thaw_state, mounted_status, str(count), start_str, end_str) + + if not self.porcelain: + self.console.print(table) + + def do_singleton_action(self) -> None: + """ + Dry run makes no sense here, so we're just going to do this either way. + + :return: None + :rtype: None + """ + self.do_action() diff --git a/curator/actions/deepfreeze/thaw.py b/curator/actions/deepfreeze/thaw.py new file mode 100644 index 00000000..6d735b83 --- /dev/null +++ b/curator/actions/deepfreeze/thaw.py @@ -0,0 +1,1256 @@ +"""Thaw action for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging +import time +import uuid +from datetime import datetime +from typing import Optional + +from elasticsearch8 import Elasticsearch +from rich import print as rprint +from rich.console import Console +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn +from rich.table import Table + +from curator.actions.deepfreeze.utilities import ( + check_restore_status, + decode_date, + find_and_mount_indices_in_date_range, + find_repos_by_date_range, + get_repositories_by_names, + get_settings, + get_thaw_request, + list_thaw_requests, + mount_repo, + save_thaw_request, + update_repository_date_range, + update_thaw_request, +) +from curator.s3client import s3_client_factory + + +class Thaw: + """ + The Thaw action restores repositories from Glacier storage to instant-access tiers + for a specified date range, or checks status of existing thaw requests. + + :param client: A client connection object + :type client: Elasticsearch + :param start_date: Start of date range (ISO 8601 format) - required for new thaw + :type start_date: str + :param end_date: End of date range (ISO 8601 format) - required for new thaw + :type end_date: str + :param sync: Wait for restore and mount (True) or return immediately (False) + :type sync: bool + :param duration: Number of days to keep objects restored from Glacier + :type duration: int + :param retrieval_tier: AWS retrieval tier (Standard/Expedited/Bulk) + :type retrieval_tier: str + :param check_status: Thaw request ID to check status and mount if ready + :type check_status: str + :param list_requests: List all thaw requests + :type list_requests: bool + :param include_completed: Include completed requests when listing (default: exclude) + :type include_completed: bool + :param porcelain: Output plain text without rich formatting + :type porcelain: bool + + :methods: + do_action: Perform the thaw operation or route to appropriate mode. + do_dry_run: Perform a dry-run of the thaw operation. + do_check_status: Check status of a thaw request and mount if ready. + do_list_requests: Display all thaw requests in a table. + _display_thaw_status: Display detailed status of a thaw request. + _parse_date: Parse and validate date inputs. + _thaw_repository: Thaw a single repository. + _wait_for_restore: Wait for restoration to complete. + _update_repo_dates: Update repository date ranges after mounting. + """ + + def __init__( + self, + client: Elasticsearch, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + sync: bool = False, + duration: int = 7, + retrieval_tier: str = "Standard", + check_status: Optional[str] = None, + list_requests: bool = False, + include_completed: bool = False, + porcelain: bool = False, + ) -> None: + self.loggit = logging.getLogger("curator.actions.deepfreeze") + self.loggit.debug("Initializing Deepfreeze Thaw") + + self.client = client + self.sync = sync + self.duration = duration + self.retrieval_tier = retrieval_tier + self.check_status = check_status + self.list_requests = list_requests + self.include_completed = include_completed + self.porcelain = porcelain + self.console = Console() + + # Determine operation mode + if list_requests: + self.mode = "list" + elif check_status is not None: + # check_status can be "" (check all) or a specific request ID + if check_status == "": + self.mode = "check_all_status" + else: + self.mode = "check_status" + else: + self.mode = "create" + # Parse and validate dates for create mode + if not start_date or not end_date: + raise ValueError( + "start_date and end_date are required when creating a new thaw request" + ) + self.start_date = self._parse_date(start_date, "start_date") + self.end_date = self._parse_date(end_date, "end_date") + + if self.start_date > self.end_date: + raise ValueError("start_date must be before or equal to end_date") + + # Get settings and initialize S3 client (not needed for list mode) + if self.mode not in ["list"]: + self.settings = get_settings(client) + self.s3 = s3_client_factory(self.settings.provider) + + # Generate request ID for async create operations + if self.mode == "create": + self.request_id = str(uuid.uuid4()) + + self.loggit.info("Deepfreeze Thaw initialized in %s mode", self.mode) + + def _parse_date(self, date_str: str, field_name: str) -> datetime: + """ + Parse a date string in ISO 8601 format. + + :param date_str: The date string to parse + :type date_str: str + :param field_name: The name of the field (for error messages) + :type field_name: str + + :returns: The parsed datetime object + :rtype: datetime + + :raises ValueError: If the date string is invalid + """ + try: + dt = decode_date(date_str) + self.loggit.debug("Parsed %s: %s", field_name, dt.isoformat()) + return dt + except Exception as e: + raise ValueError( + f"Invalid {field_name}: {date_str}. " + f"Expected ISO 8601 format (e.g., '2025-01-15T00:00:00Z'). Error: {e}" + ) + + def _thaw_repository(self, repo) -> bool: + """ + Thaw a single repository by restoring its objects from Glacier. + + :param repo: The repository to thaw + :type repo: Repository + + :returns: True if successful, False otherwise + :rtype: bool + """ + self.loggit.info("Thawing repository %s", repo.name) + + # Check if repository is already thawed + if repo.is_thawed and repo.is_mounted: + self.loggit.info("Repository %s is already thawed and mounted", repo.name) + return True + + # Get the list of object keys to restore + self.loggit.debug( + "Listing objects in s3://%s/%s", repo.bucket, repo.base_path + ) + object_keys = self.s3.list_objects(repo.bucket, repo.base_path) + + self.loggit.info( + "Found %d objects to restore in repository %s", len(object_keys), repo.name + ) + + # Restore objects from Glacier + try: + self.s3.thaw( + bucket_name=repo.bucket, + base_path=repo.base_path, + object_keys=object_keys, + restore_days=self.duration, + retrieval_tier=self.retrieval_tier, + ) + self.loggit.info( + "Successfully initiated restore for repository %s", repo.name + ) + + # Update repository state to 'thawing' + from datetime import timedelta, timezone + expires_at = datetime.now(timezone.utc) + timedelta(days=self.duration) + repo.start_thawing(expires_at) + repo.persist(self.client) + self.loggit.debug( + "Repository %s marked as 'thawing', expires at %s", + repo.name, + expires_at.isoformat() + ) + + return True + except Exception as e: + self.loggit.error("Failed to thaw repository %s: %s", repo.name, e) + return False + + def _wait_for_restore(self, repo, poll_interval: int = 30, show_progress: bool = False) -> bool: + """ + Wait for restoration to complete by polling S3. + + :param repo: The repository to check + :type repo: Repository + :param poll_interval: Seconds between status checks + :type poll_interval: int + :param show_progress: Whether to show rich progress bar (for sync mode) + :type show_progress: bool + + :returns: True if restoration completed, False if timeout or error + :rtype: bool + """ + self.loggit.info("Waiting for restoration of repository %s", repo.name) + + max_attempts = 1200 # 10 hours with 30-second polls + attempt = 0 + + # Initial status check to get total objects + initial_status = check_restore_status(self.s3, repo.bucket, repo.base_path) + total_objects = initial_status["total"] + + if show_progress and total_objects > 0: + # Use rich progress bar for sync mode + with Progress( + SpinnerColumn(), + TextColumn("[bold blue]{task.description}"), + BarColumn(), + TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), + TextColumn("({task.completed}/{task.total} objects)"), + TimeElapsedColumn(), + console=self.console, + ) as progress: + task = progress.add_task( + f"Restoring {repo.name}", + total=total_objects, + completed=initial_status["restored"] + ) + + while attempt < max_attempts: + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + + # Update progress bar + progress.update(task, completed=status["restored"]) + + if status["complete"]: + progress.update(task, completed=total_objects) + self.loggit.info("Restoration complete for repository %s", repo.name) + return True + + attempt += 1 + if attempt < max_attempts: + time.sleep(poll_interval) + + self.loggit.warning( + "Restoration timed out for repository %s after %d checks", + repo.name, + max_attempts, + ) + return False + else: + # Non-progress mode (async or no objects) + while attempt < max_attempts: + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + + self.loggit.debug( + "Restore status for %s: %d/%d objects restored, %d in progress", + repo.name, + status["restored"], + status["total"], + status["in_progress"], + ) + + if status["complete"]: + self.loggit.info("Restoration complete for repository %s", repo.name) + return True + + attempt += 1 + if attempt < max_attempts: + self.loggit.debug( + "Waiting %d seconds before next status check...", poll_interval + ) + time.sleep(poll_interval) + + self.loggit.warning( + "Restoration timed out for repository %s after %d checks", + repo.name, + max_attempts, + ) + return False + + def _update_repo_dates(self, repo) -> None: + """ + Update repository date ranges after mounting. + + :param repo: The repository to update + :type repo: Repository + + :return: None + :rtype: None + """ + self.loggit.debug("Updating date range for repository %s", repo.name) + + try: + updated = update_repository_date_range(self.client, repo) + if updated: + self.loggit.info( + "Updated date range for %s: %s to %s", + repo.name, + repo.start.isoformat() if repo.start else "None", + repo.end.isoformat() if repo.end else "None" + ) + else: + self.loggit.debug( + "No date range update needed for %s", repo.name + ) + except Exception as e: + self.loggit.warning( + "Failed to update date range for %s: %s", repo.name, e + ) + + def do_check_status(self) -> None: + """ + Check the status of a thaw request and mount repositories if restoration is complete. + Also mounts indices in the date range if all repositories are ready. + + IMPORTANT: Mounting happens BEFORE status display so users see current state. + NOTE: Skips refrozen requests as they have been cleaned up and are no longer active. + + :return: None + :rtype: None + """ + # Type guard: check_status must be a non-empty string in this mode + if not self.check_status: + raise ValueError("check_status must be provided for single status check") + + self.loggit.info("Checking status of thaw request %s", self.check_status) + + # Retrieve the thaw request + request = get_thaw_request(self.client, self.check_status) + + # Skip refrozen requests - they have been cleaned up and are no longer active + if request.get("status") == "refrozen": + self.loggit.info("Thaw request %s has been refrozen, skipping status check", self.check_status) + if not self.porcelain: + rprint(f"\n[yellow]Thaw request {self.check_status} has been refrozen and is no longer active.[/yellow]\n") + return + + # Get the repository objects + repos = get_repositories_by_names(self.client, request["repos"]) + + if not repos: + self.loggit.warning("No repositories found for thaw request") + return + + # STEP 1: Check restoration status and mount repositories if ready + # This happens BEFORE displaying status so users see current state + all_complete = True + mounted_count = 0 + newly_mounted_repos = [] + + # Show progress if mounting will happen + repos_to_check = [repo for repo in repos if not repo.is_mounted] + + if repos_to_check and not self.porcelain: + rprint("[cyan]Checking restoration status...[/cyan]") + + for repo in repos: + if repo.is_mounted: + self.loggit.debug("Repository %s is already mounted", repo.name) + continue + + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + + if status["complete"]: + self.loggit.info("Restoration complete for %s, mounting...", repo.name) + if not self.porcelain: + rprint(f" [cyan]→[/cyan] Mounting [bold]{repo.name}[/bold]...") + mount_repo(self.client, repo) + self._update_repo_dates(repo) + mounted_count += 1 + newly_mounted_repos.append(repo) + if not self.porcelain: + rprint(" [green]✓[/green] Mounted successfully") + else: + self.loggit.info( + "Restoration in progress for %s: %d/%d objects restored", + repo.name, + status["restored"], + status["total"], + ) + all_complete = False + + # STEP 2: Mount indices if all repositories are complete + # Parse date range from the thaw request + start_date_str = request.get("start_date") + end_date_str = request.get("end_date") + + # Check if we should mount indices: + # - All repos are complete (restoration finished) + # - At least one repo is mounted + # - We have date range info + should_mount_indices = ( + all_complete + and start_date_str + and end_date_str + and any(repo.is_mounted for repo in repos) + ) + + if should_mount_indices: + # Type guards: these must be strings if should_mount_indices is True + assert start_date_str is not None + assert end_date_str is not None + try: + start_date = decode_date(start_date_str) + end_date = decode_date(end_date_str) + + self.loggit.info( + "Mounting indices for date range %s to %s", + start_date.isoformat(), + end_date.isoformat(), + ) + + if not self.porcelain: + rprint("[cyan]Looking for indices to mount...[/cyan]") + + # Use all mounted repos, not just newly mounted ones + # This handles the case where repos were already mounted + mounted_repos = [repo for repo in repos if repo.is_mounted] + + mount_result = find_and_mount_indices_in_date_range( + self.client, mounted_repos, start_date, end_date + ) + + self.loggit.info( + "Mounted %d indices (%d skipped outside date range, %d failed, %d added to data streams)", + mount_result["mounted"], + mount_result["skipped"], + mount_result["failed"], + mount_result["datastream_successful"], + ) + + if not self.porcelain: + rprint( + f"[green]Mounted {mount_result['mounted']} indices " + f"({mount_result['skipped']} skipped outside date range, " + f"{mount_result['failed']} failed, " + f"{mount_result['datastream_successful']} added to data streams)[/green]" + ) + + except Exception as e: + self.loggit.warning("Failed to mount indices: %s", e) + if not self.porcelain: + rprint(f"[yellow]Warning: Failed to mount indices: {e}[/yellow]") + + # STEP 3: Update thaw request status if all repositories are ready + if all_complete: + update_thaw_request(self.client, self.check_status, status="completed") + self.loggit.info("All repositories restored and mounted. Thaw request completed.") + else: + if mounted_count > 0: + self.loggit.info( + "Mounted %d repositories. Some restorations still in progress.", + mounted_count, + ) + + # STEP 4: Display updated status AFTER mounting + # Now users see the current state including any newly mounted repos/indices + if not self.porcelain: + rprint() # Blank line before status display + self._display_thaw_status(request, repos) + + def do_check_all_status(self) -> None: + """ + Check the status of all thaw requests, mount repositories when ready, + and display grouped by request ID. + + IMPORTANT: Mounting happens BEFORE status display so users see current state. + NOTE: Skips refrozen requests as they have been cleaned up and are no longer active. + + :return: None + :rtype: None + """ + self.loggit.info("Checking status of all thaw requests") + + # Get all thaw requests + all_requests = list_thaw_requests(self.client) + + # Filter out refrozen requests - they have been cleaned up and are no longer active + requests = [req for req in all_requests if req.get("status") != "refrozen"] + filtered_count = len(all_requests) - len(requests) + if filtered_count > 0: + self.loggit.debug("Filtered %d refrozen requests", filtered_count) + + if not requests: + if not self.porcelain: + rprint("\n[yellow]No active thaw requests found.[/yellow]\n") + return + + # Process each request + for req in requests: + request_id = req["id"] + + # Get the full request data + try: + request = get_thaw_request(self.client, request_id) + except Exception as e: + self.loggit.warning("Failed to get thaw request %s: %s", request_id, e) + continue + + # Get the repository objects + repos = get_repositories_by_names(self.client, request.get("repos", [])) + + if not repos: + self.loggit.warning("No repositories found for thaw request %s", request_id) + continue + + # Get date range for display/output + start_date_str = request.get("start_date", "") + end_date_str = request.get("end_date", "") + + # STEP 1: Check restoration status and mount repositories if ready + # This happens BEFORE displaying status so users see current state + all_complete = True + mounted_count = 0 + newly_mounted_repos = [] + + # Show progress indicator if any repos need checking + repos_to_check = [repo for repo in repos if not repo.is_mounted] + if repos_to_check and not self.porcelain: + rprint(f"[cyan]Checking request {request_id}...[/cyan]") + + # Check each repository's status and mount if ready + for repo in repos: + # Check restore status if not mounted + if not repo.is_mounted: + try: + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + if status["complete"]: + # Mount the repository + self.loggit.info("Restoration complete for %s, mounting...", repo.name) + if not self.porcelain: + rprint(f" [cyan]→[/cyan] Mounting [bold]{repo.name}[/bold]...") + mount_repo(self.client, repo) + self._update_repo_dates(repo) + mounted_count += 1 + newly_mounted_repos.append(repo) + if not self.porcelain: + rprint(" [green]✓[/green] Mounted successfully") + else: + self.loggit.debug( + "Restoration in progress for %s: %d/%d objects restored", + repo.name, + status["restored"], + status["total"], + ) + all_complete = False + except Exception as e: + self.loggit.warning("Failed to check status for %s: %s", repo.name, e) + all_complete = False + + # STEP 2: Mount indices if all repositories are complete and mounted + # Check if we should mount indices: + # - All repos are complete (restoration finished) + # - We have date range info + # - At least one repo is mounted + should_mount_indices = ( + all_complete + and start_date_str + and end_date_str + and any(repo.is_mounted for repo in repos) + ) + + if should_mount_indices: + try: + start_date = decode_date(start_date_str) + end_date = decode_date(end_date_str) + + self.loggit.info( + "Mounting indices for date range %s to %s", + start_date.isoformat(), + end_date.isoformat(), + ) + + if not self.porcelain: + rprint("[cyan]Looking for indices to mount...[/cyan]") + + # Use all mounted repos, not just newly mounted ones + # This handles the case where repos were mounted in a previous check + mounted_repos = [repo for repo in repos if repo.is_mounted] + + mount_result = find_and_mount_indices_in_date_range( + self.client, mounted_repos, start_date, end_date + ) + + self.loggit.info( + "Mounted %d indices (%d skipped outside date range, %d failed, %d added to data streams)", + mount_result["mounted"], + mount_result["skipped"], + mount_result["failed"], + mount_result["datastream_successful"], + ) + + if not self.porcelain: + rprint( + f"[green]Mounted {mount_result['mounted']} indices " + f"({mount_result['skipped']} skipped outside date range, " + f"{mount_result['failed']} failed, " + f"{mount_result['datastream_successful']} added to data streams)[/green]" + ) + except Exception as e: + self.loggit.warning("Failed to mount indices: %s", e) + if not self.porcelain: + rprint(f"[yellow]Warning: Failed to mount indices: {e}[/yellow]") + + # STEP 3: Update thaw request status if all repositories are ready + if all_complete: + update_thaw_request(self.client, request_id, status="completed") + self.loggit.info("Thaw request %s completed", request_id) + + # STEP 4: Build repo data for display AFTER mounting + repo_data = [] + for repo in repos: + # Check restore status if not mounted + if not repo.is_mounted: + try: + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + if status["complete"]: + progress = "Complete" + else: + progress = f"{status['restored']}/{status['total']}" + except Exception as e: + self.loggit.warning("Failed to check status for %s: %s", repo.name, e) + progress = "Error" + else: + progress = "Complete" + + repo_data.append({ + "name": repo.name, + "bucket": repo.bucket if repo.bucket else "", + "path": repo.base_path if repo.base_path else "", + "state": repo.thaw_state, + "mounted": "yes" if repo.is_mounted else "no", + "progress": progress, + }) + + # STEP 5: Display updated status AFTER mounting + if self.porcelain: + # Machine-readable output: tab-separated values + # Format: REQUEST\t{request_id}\t{status}\t{created_at}\t{start_date}\t{end_date} + print(f"REQUEST\t{request['request_id']}\t{request['status']}\t{request['created_at']}\t{start_date_str}\t{end_date_str}") + + # Format: REPO\t{name}\t{bucket}\t{path}\t{state}\t{mounted}\t{progress} + for repo_info in repo_data: + print(f"REPO\t{repo_info['name']}\t{repo_info['bucket']}\t{repo_info['path']}\t{repo_info['state']}\t{repo_info['mounted']}\t{repo_info['progress']}") + else: + # Human-readable output: formatted display + # Format dates for display + if start_date_str and "T" in start_date_str: + start_date_display = start_date_str.replace("T", " ").split(".")[0] + else: + start_date_display = start_date_str if start_date_str else "--" + + if end_date_str and "T" in end_date_str: + end_date_display = end_date_str.replace("T", " ").split(".")[0] + else: + end_date_display = end_date_str if end_date_str else "--" + + # Display request info + rprint(f"\n[bold cyan]Thaw Request: {request['request_id']}[/bold cyan]") + rprint(f"[cyan]Status: {request['status']}[/cyan]") + rprint(f"[cyan]Created: {request['created_at']}[/cyan]") + rprint(f"[green]Date Range: {start_date_display} to {end_date_display}[/green]\n") + + # Create table for repository status + table = Table(title="Repository Status") + table.add_column("Repository", style="cyan", no_wrap=False, overflow="fold") + table.add_column("Bucket", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Path", style="magenta", no_wrap=False, overflow="fold") + table.add_column("State", style="yellow", no_wrap=False, overflow="fold") + table.add_column("Mounted", style="green", no_wrap=False, overflow="fold") + table.add_column("Restore Progress", style="magenta", no_wrap=False, overflow="fold") + + for repo_info in repo_data: + table.add_row( + repo_info['name'], + repo_info['bucket'] if repo_info['bucket'] else "--", + repo_info['path'] if repo_info['path'] else "--", + repo_info['state'], + repo_info['mounted'], + repo_info['progress'], + ) + + self.console.print(table) + + # Show completion/progress message + if all_complete: + rprint(f"[green]Request {request_id} completed[/green]") + elif mounted_count > 0: + rprint( + f"[yellow]Mounted {mounted_count} repositories. " + f"Some restorations still in progress.[/yellow]" + ) + + if not self.porcelain: + rprint() + + def do_list_requests(self) -> None: + """ + List thaw requests in a formatted table. + + By default, excludes completed and refrozen requests. Use include_completed=True to show all. + + :return: None + :rtype: None + """ + self.loggit.info("Listing thaw requests (include_completed=%s)", self.include_completed) + + all_requests = list_thaw_requests(self.client) + + # Filter completed and refrozen requests unless explicitly included + if not self.include_completed: + requests = [req for req in all_requests if req.get("status") not in ("completed", "refrozen")] + filtered_count = len(all_requests) - len(requests) + self.loggit.debug( + "Filtered %d completed/refrozen requests, %d remaining", + filtered_count, + len(requests) + ) + else: + requests = all_requests + + if not requests: + if not self.porcelain: + if self.include_completed: + rprint("\n[yellow]No thaw requests found.[/yellow]\n") + else: + rprint("\n[yellow]No active thaw requests found. Use --include-completed to see completed/refrozen requests.[/yellow]\n") + return + + if self.porcelain: + # Machine-readable output: tab-separated values + # Format: REQUEST\t{id}\t{status}\t{repo_count}\t{start_date}\t{end_date}\t{created_at} + for req in requests: + repo_count = str(len(req.get("repos", []))) + status = req.get("status", "unknown") + start_date = req.get("start_date", "") + end_date = req.get("end_date", "") + created_at = req.get("created_at", "") + + print(f"REQUEST\t{req['id']}\t{status}\t{repo_count}\t{start_date}\t{end_date}\t{created_at}") + else: + # Human-readable output: formatted table + # Create table + table = Table(title="Thaw Requests") + table.add_column("Request ID", style="cyan", no_wrap=False, overflow="fold") + table.add_column("St", style="magenta", no_wrap=False, overflow="fold") # Abbreviated Status + table.add_column("Repos", style="magenta", no_wrap=False, overflow="fold") # Abbreviated Repositories + table.add_column("Start Date", style="green", no_wrap=False, overflow="fold") + table.add_column("End Date", style="green", no_wrap=False, overflow="fold") + table.add_column("Created At", style="magenta", no_wrap=False, overflow="fold") + + # Add rows + for req in requests: + repo_count = str(len(req.get("repos", []))) + created_at = req.get("created_at", "Unknown") + # Format datetime if it's ISO format + if "T" in created_at: + created_at = created_at.replace("T", " ").split(".")[0] + + # Format date range + start_date = req.get("start_date", "") + end_date = req.get("end_date", "") + + # Format dates to show full datetime (same format as created_at) + if start_date and "T" in start_date: + start_date = start_date.replace("T", " ").split(".")[0] + if end_date and "T" in end_date: + end_date = end_date.replace("T", " ").split(".")[0] + + # Use "--" for missing dates + start_date = start_date if start_date else "--" + end_date = end_date if end_date else "--" + + # Abbreviate status for display + status = req.get("status", "unknown") + status_abbrev = { + "in_progress": "IP", + "completed": "C", + "failed": "F", + "refrozen": "R", + "unknown": "U", + }.get(status, status[:2].upper()) + + table.add_row( + req["id"], # Show full Request ID + status_abbrev, + repo_count, + start_date, + end_date, + created_at, + ) + + self.console.print(table) + rprint("[dim]Status: IP=In Progress, C=Completed, R=Refrozen, F=Failed, U=Unknown[/dim]") + + def _display_thaw_status(self, request: dict, repos: list) -> None: + """ + Display detailed status information for a thaw request. + + :param request: The thaw request document + :type request: dict + :param repos: List of Repository objects + :type repos: list + + :return: None + :rtype: None + """ + # Get date range for display/output + start_date_str = request.get("start_date", "") + end_date_str = request.get("end_date", "") + + # Build repo data with restore progress + repo_data = [] + for repo in repos: + # Check restore status if not mounted + if not repo.is_mounted: + try: + status = check_restore_status(self.s3, repo.bucket, repo.base_path) + if status["complete"]: + progress = "Complete" + else: + progress = f"{status['restored']}/{status['total']}" + except Exception as e: + self.loggit.warning("Failed to check status for %s: %s", repo.name, e) + progress = "Error" + else: + progress = "Complete" + + repo_data.append({ + "name": repo.name, + "bucket": repo.bucket if repo.bucket else "", + "path": repo.base_path if repo.base_path else "", + "state": repo.thaw_state, + "mounted": "yes" if repo.is_mounted else "no", + "progress": progress, + }) + + if self.porcelain: + # Machine-readable output: tab-separated values + # Format: REQUEST\t{request_id}\t{status}\t{created_at}\t{start_date}\t{end_date} + print(f"REQUEST\t{request['request_id']}\t{request['status']}\t{request['created_at']}\t{start_date_str}\t{end_date_str}") + + # Format: REPO\t{name}\t{bucket}\t{path}\t{state}\t{mounted}\t{progress} + for repo_info in repo_data: + print(f"REPO\t{repo_info['name']}\t{repo_info['bucket']}\t{repo_info['path']}\t{repo_info['state']}\t{repo_info['mounted']}\t{repo_info['progress']}") + else: + # Human-readable output: formatted display + # Format dates for display + if start_date_str and "T" in start_date_str: + start_date_display = start_date_str.replace("T", " ").split(".")[0] + else: + start_date_display = start_date_str if start_date_str else "--" + + if end_date_str and "T" in end_date_str: + end_date_display = end_date_str.replace("T", " ").split(".")[0] + else: + end_date_display = end_date_str if end_date_str else "--" + + rprint(f"\n[bold cyan]Thaw Request: {request['request_id']}[/bold cyan]") + rprint(f"[cyan]Status: {request['status']}[/cyan]") + rprint(f"[cyan]Created: {request['created_at']}[/cyan]") + rprint(f"[green]Date Range: {start_date_display} to {end_date_display}[/green]\n") + + # Create table for repository status + table = Table(title="Repository Status") + table.add_column("Repository", style="cyan", no_wrap=False, overflow="fold") + table.add_column("Bucket", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Path", style="magenta", no_wrap=False, overflow="fold") + table.add_column("State", style="yellow", no_wrap=False, overflow="fold") + table.add_column("Mounted", style="green", no_wrap=False, overflow="fold") + table.add_column("Restore Progress", style="magenta", no_wrap=False, overflow="fold") + + for repo_info in repo_data: + table.add_row( + repo_info['name'], + repo_info['bucket'] if repo_info['bucket'] else "--", + repo_info['path'] if repo_info['path'] else "--", + repo_info['state'], + repo_info['mounted'], + repo_info['progress'], + ) + + self.console.print(table) + rprint() + + def do_dry_run(self) -> None: + """ + Perform a dry-run of the thaw operation. + + :return: None + :rtype: None + """ + self.loggit.info("DRY-RUN MODE. No changes will be made.") + + if self.mode == "list": + self.loggit.info("DRY-RUN: Would list all thaw requests") + self.do_list_requests() + return + + if self.mode == "check_status": + # Type guard: check_status must be a non-empty string in this mode + if not self.check_status: + raise ValueError("check_status must be provided for single status check") + + self.loggit.info( + "DRY-RUN: Would check status of thaw request %s", self.check_status + ) + # Still show current status in dry-run + request = get_thaw_request(self.client, self.check_status) + repos = get_repositories_by_names(self.client, request["repos"]) + self._display_thaw_status(request, repos) + self.loggit.info("DRY-RUN: Would mount any repositories with completed restoration") + return + + if self.mode == "check_all_status": + self.loggit.info("DRY-RUN: Would check status of all thaw requests and mount any repositories with completed restoration") + return + + # Create mode + msg = ( + f"DRY-RUN: Thawing repositories with data between " + f"{self.start_date.isoformat()} and {self.end_date.isoformat()}" + ) + self.loggit.info(msg) + + # Find matching repositories + repos = find_repos_by_date_range(self.client, self.start_date, self.end_date) + + if not repos: + self.loggit.warning("DRY-RUN: No repositories found for date range") + return + + self.loggit.info("DRY-RUN: Found %d repositories to thaw:", len(repos)) + for repo in repos: + self.loggit.info( + " - %s (bucket: %s, path: %s, dates: %s to %s)", + repo.name, + repo.bucket, + repo.base_path, + repo.start, + repo.end, + ) + + if self.sync: + self.loggit.info("DRY-RUN: Would wait for restoration and mount repositories") + else: + self.loggit.info( + "DRY-RUN: Would return request ID: %s", self.request_id + ) + + def do_action(self) -> None: + """ + Perform the thaw operation (routes to appropriate handler based on mode). + + :return: None + :rtype: None + """ + if self.mode == "list": + self.do_list_requests() + return + + if self.mode == "check_status": + self.do_check_status() + return + + if self.mode == "check_all_status": + self.do_check_all_status() + return + + # Create mode - original thaw logic + self.loggit.info( + "Thawing repositories with data between %s and %s", + self.start_date.isoformat(), + self.end_date.isoformat(), + ) + + # Phase 1: Find matching repositories + if self.sync: + self.console.print(Panel( + f"[bold cyan]Phase 1: Finding Repositories[/bold cyan]\n\n" + f"Date Range: [yellow]{self.start_date.isoformat()}[/yellow] to " + f"[yellow]{self.end_date.isoformat()}[/yellow]", + border_style="cyan", + expand=False + )) + + repos = find_repos_by_date_range(self.client, self.start_date, self.end_date) + + if not repos: + self.loggit.warning("No repositories found for date range") + if self.sync: + self.console.print(Panel( + "[yellow]No repositories found matching the specified date range.[/yellow]", + title="[bold yellow]No Repositories Found[/bold yellow]", + border_style="yellow", + expand=False + )) + return + + self.loggit.info("Found %d repositories to thaw", len(repos)) + + if self.sync: + # Display found repositories + table = Table(title=f"Found {len(repos)} Repositories") + table.add_column("Repository", style="cyan", no_wrap=False, overflow="fold") + table.add_column("Bucket", style="magenta", no_wrap=False, overflow="fold") + table.add_column("Base Path", style="magenta", no_wrap=False, overflow="fold") + for repo in repos: + table.add_row(repo.name, repo.bucket or "--", repo.base_path or "--") + self.console.print(table) + self.console.print() + + # Phase 2: Initiate thaw for each repository + if self.sync: + self.console.print(Panel( + f"[bold cyan]Phase 2: Initiating Glacier Restore[/bold cyan]\n\n" + f"Retrieval Tier: [yellow]{self.retrieval_tier}[/yellow]\n" + f"Duration: [yellow]{self.duration} days[/yellow]", + border_style="cyan", + expand=False + )) + + thawed_repos = [] + for repo in repos: + if self.sync: + self.console.print(f" [cyan]→[/cyan] Initiating restore for [bold]{repo.name}[/bold]...") + if self._thaw_repository(repo): + thawed_repos.append(repo) + if self.sync: + self.console.print(" [green]✓[/green] Restore initiated successfully") + else: + if self.sync: + self.console.print(" [red]✗[/red] Failed to initiate restore") + + if not thawed_repos: + self.loggit.error("Failed to thaw any repositories") + if self.sync: + self.console.print(Panel( + "[red]Failed to initiate restore for any repositories.[/red]", + title="[bold red]Thaw Failed[/bold red]", + border_style="red", + expand=False + )) + return + + self.loggit.info("Successfully initiated thaw for %d repositories", len(thawed_repos)) + if self.sync: + self.console.print() + + # Handle sync vs async modes + if self.sync: + # Save thaw request for status tracking (will be marked completed when done) + save_thaw_request( + self.client, + self.request_id, + thawed_repos, + "in_progress", + self.start_date, + self.end_date, + ) + self.loggit.debug("Saved sync thaw request %s for status tracking", self.request_id) + + # Phase 3: Wait for restoration + self.console.print(Panel( + "[bold cyan]Phase 3: Waiting for Glacier Restoration[/bold cyan]\n\n" + "This may take several hours depending on the retrieval tier.\n" + "Progress will be updated as objects are restored.", + border_style="cyan", + expand=False + )) + + successfully_restored = [] + failed_restores = [] + + # Wait for each repository to be restored + for repo in thawed_repos: + if self._wait_for_restore(repo, show_progress=True): + successfully_restored.append(repo) + else: + failed_restores.append(repo) + self.loggit.warning( + "Skipping mount for %s due to restoration timeout", repo.name + ) + + if not successfully_restored: + self.console.print(Panel( + "[red]No repositories were successfully restored.[/red]", + title="[bold red]Restoration Failed[/bold red]", + border_style="red", + expand=False + )) + return + + self.console.print() + + # Phase 4: Mount repositories + self.console.print(Panel( + f"[bold cyan]Phase 4: Mounting Repositories[/bold cyan]\n\n" + f"Mounting {len(successfully_restored)} restored " + f"repositor{'y' if len(successfully_restored) == 1 else 'ies'}.", + border_style="cyan", + expand=False + )) + + mounted_count = 0 + for repo in successfully_restored: + self.console.print(f" [cyan]→[/cyan] Mounting [bold]{repo.name}[/bold]...") + try: + mount_repo(self.client, repo) + self.console.print(" [green]✓[/green] Mounted successfully") + mounted_count += 1 + except Exception as e: + self.console.print(f" [red]✗[/red] Failed to mount: {e}") + self.loggit.error("Failed to mount %s: %s", repo.name, e) + + self.console.print() + + # Phase 5: Update date ranges + self.console.print(Panel( + "[bold cyan]Phase 5: Updating Repository Metadata[/bold cyan]", + border_style="cyan", + expand=False + )) + + for repo in successfully_restored: + self._update_repo_dates(repo) + + self.console.print() + + # Phase 6: Mount indices + self.console.print(Panel( + "[bold cyan]Phase 6: Mounting Indices[/bold cyan]\n\n" + "Finding and mounting indices within the requested date range.", + border_style="cyan", + expand=False + )) + + mount_result = find_and_mount_indices_in_date_range( + self.client, successfully_restored, self.start_date, self.end_date + ) + + self.console.print(f" [cyan]→[/cyan] Mounted [bold]{mount_result['mounted']}[/bold] indices") + if mount_result['skipped'] > 0: + self.console.print( + f" [dim]•[/dim] Skipped [dim]{mount_result['skipped']}[/dim] indices outside date range" + ) + if mount_result['failed'] > 0: + self.console.print( + f" [yellow]⚠[/yellow] Failed to mount [yellow]{mount_result['failed']}[/yellow] indices" + ) + if mount_result['datastream_successful'] > 0: + self.console.print( + f" [green]✓[/green] Added [bold]{mount_result['datastream_successful']}[/bold] indices to data streams" + ) + if mount_result['datastream_failed'] > 0: + self.console.print( + f" [yellow]⚠[/yellow] Failed to add [yellow]{mount_result['datastream_failed']}[/yellow] indices to data streams" + ) + + # Final summary + self.console.print() + summary_lines = [ + "[bold green]Thaw Operation Completed Successfully![/bold green]\n", + f"Repositories Processed: [cyan]{len(repos)}[/cyan]", + f"Restore Initiated: [cyan]{len(thawed_repos)}[/cyan]", + f"Successfully Restored: [cyan]{len(successfully_restored)}[/cyan]", + f"Successfully Mounted: [cyan]{mounted_count}[/cyan]", + f"Indices Mounted: [cyan]{mount_result['mounted']}[/cyan]", + ] + if failed_restores: + summary_lines.append(f"Failed Restores: [yellow]{len(failed_restores)}[/yellow]") + if mount_result['failed'] > 0: + summary_lines.append(f"Failed Index Mounts: [yellow]{mount_result['failed']}[/yellow]") + if mount_result['datastream_successful'] > 0: + summary_lines.append(f"Data Stream Indices Added: [cyan]{mount_result['datastream_successful']}[/cyan]") + + self.console.print(Panel( + "\n".join(summary_lines), + title="[bold green]Summary[/bold green]", + border_style="green", + expand=False + )) + + # Mark thaw request as completed + update_thaw_request(self.client, self.request_id, status="completed") + self.loggit.debug("Marked thaw request %s as completed", self.request_id) + + self.loggit.info("Thaw operation completed") + + else: + # Async mode - initiate restore and return immediately + self.loggit.info("Async mode: Saving thaw request...") + + # Save thaw request for later querying + save_thaw_request( + self.client, + self.request_id, + thawed_repos, + "in_progress", + self.start_date, + self.end_date, + ) + + self.loggit.info( + "Thaw request saved with ID: %s. " + "Use this ID to check status and mount when ready.", + self.request_id, + ) + + # Display the thaw ID prominently for the user + self.console.print() + self.console.print(Panel( + f"[bold green]Thaw Request Initiated[/bold green]\n\n" + f"Request ID: [cyan]{self.request_id}[/cyan]\n\n" + f"Glacier restore has been initiated for [cyan]{len(thawed_repos)}[/cyan] " + f"repositor{'y' if len(thawed_repos) == 1 else 'ies'}.\n" + f"Retrieval Tier: [yellow]{self.retrieval_tier}[/yellow]\n" + f"Duration: [yellow]{self.duration} days[/yellow]\n\n" + f"[dim]Check status with:[/dim]\n" + f"[yellow]curator_cli deepfreeze thaw --check-status {self.request_id}[/yellow]", + border_style="green", + expand=False + )) + self.console.print() + + def do_singleton_action(self) -> None: + """ + Entry point for singleton CLI execution. + + :return: None + :rtype: None + """ + self.do_action() diff --git a/curator/actions/deepfreeze/utilities.py b/curator/actions/deepfreeze/utilities.py new file mode 100644 index 00000000..0a5cfc4e --- /dev/null +++ b/curator/actions/deepfreeze/utilities.py @@ -0,0 +1,1933 @@ +"""Utility functions for deepfreeze""" + +# pylint: disable=too-many-arguments,too-many-instance-attributes, raise-missing-from + +import logging +import re +import time +from datetime import datetime, timezone + +import botocore +from elasticsearch8 import Elasticsearch, NotFoundError + +from curator.actions import CreateIndex +from curator.actions.deepfreeze.exceptions import MissingIndexError +from curator.exceptions import ActionError +from curator.s3client import S3Client + +from .constants import SETTINGS_ID, STATUS_INDEX +from .helpers import Repository, Settings + + +def push_to_glacier(s3: S3Client, repo: Repository) -> None: + """Push objects to Glacier storage + + :param s3: The S3 client object + :type s3: S3Client + :param repo: The repository to push to Glacier + :type repo: Repository + + :return: None + :rtype: None + + :raises Exception: If the object is not in the restoration process + """ + try: + # Normalize base_path: remove leading/trailing slashes, ensure it ends with / + base_path = repo.base_path.strip('/') + if base_path: + base_path += '/' + + # Initialize variables for pagination + success = True + object_count = 0 + + # List objects + objects = s3.list_objects(repo.bucket, base_path) + + # Process each object + for obj in objects: + key = obj['Key'] + current_storage_class = obj.get('StorageClass', 'STANDARD') + + # Log the object being processed + logging.info( + f"Processing object: s3://{repo.bucket}/{key} (Current: {current_storage_class})" + ) + + try: + # Copy object to itself with new storage class + copy_source = {'Bucket': repo.bucket, 'Key': key} + s3.copy_object( + Bucket=repo.bucket, + Key=key, + CopySource=copy_source, + StorageClass='GLACIER', + ) + + # Log success + logging.info(f"Successfully moved s3://{repo.bucket}/{key} to GLACIER") + object_count += 1 + + except botocore.exceptions.ClientError as e: + logging.error(f"Failed to move s3://{repo.bucket}/{key}: {e}") + success = False + continue + # Log summary + logging.info( + f"Processed {object_count} objects in s3://{repo.bucket}/{base_path}" + ) + if success: + logging.info("All objects successfully moved to GLACIER") + else: + logging.warning("Some objects failed to move to GLACIER") + + return success + + except botocore.exceptions.ClientError as e: + logging.error(f"Failed to process bucket s3://{repo.bucket}: {e}") + return False + + +def get_all_indices_in_repo(client: Elasticsearch, repository: str) -> list[str]: + """ + Retrieve all indices from snapshots in the given repository. + + :param client: A client connection object + :param repository: The name of the repository + :returns: A list of indices + :rtype: list[str] + + :raises Exception: If the repository does not exist + :raises Exception: If the repository is empty + :raises Exception: If the repository is not mounted + """ + indices = set() + + # TODO: Convert these three lines to use an existing Curator function? + snapshots = client.snapshot.get(repository=repository, snapshot="_all") + for snapshot in snapshots["snapshots"]: + indices.update(snapshot["indices"]) + + # logging.debug("Indices: %s", indices) + return list(indices) + + +def get_timestamp_range( + client: Elasticsearch, indices: list[str] +) -> tuple[datetime, datetime]: + """ + Retrieve the earliest and latest @timestamp values from the given indices. + + :param client: A client connection object + :param indices: A list of indices + :returns: A tuple containing the earliest and latest @timestamp values + :rtype: tuple[datetime, datetime] + + :raises Exception: If the indices list is empty + :raises Exception: If the indices do not exist + :raises Exception: If the indices are empty + + :example: + >>> get_timestamp_range(client, ["index1", "index2"]) + (datetime.datetime(2021, 1, 1, 0, 0), datetime.datetime(2021, 1, 2, 0, 0)) + """ + logging.debug("Determining timestamp range for indices: %s", indices) + if not indices: + return None, None + # TODO: Consider using Curator filters to accomplish this + query = { + "size": 0, + "aggs": { + "earliest": {"min": {"field": "@timestamp"}}, + "latest": {"max": {"field": "@timestamp"}}, + }, + } + logging.debug("starting with %s indices", len(indices)) + # Remove any indices that do not exist + filtered = [index for index in indices if client.indices.exists(index=index)] + logging.debug("after removing non-existent indices: %s", len(filtered)) + + try: + response = client.search( + index=",".join(filtered), body=query, allow_partial_search_results=True + ) + logging.debug("Response: %s", response) + except Exception as e: + logging.error("Error retrieving timestamp range: %s", e) + return None, None + + earliest = response["aggregations"]["earliest"]["value_as_string"] + latest = response["aggregations"]["latest"]["value_as_string"] + + logging.debug("BDW from query: Earliest: %s, Latest: %s", earliest, latest) + + logging.debug("Earliest: %s, Latest: %s", earliest, latest) + + return datetime.fromisoformat(earliest), datetime.fromisoformat(latest) + + +def ensure_settings_index( + client: Elasticsearch, create_if_missing: bool = False +) -> None: + """ + Ensure that the status index exists in Elasticsearch. + + :param client: A client connection object + :type client: Elasticsearch + + :return: None + :rtype: None + + :raises Exception: If the index cannot be created + :raises Exception: If the index already exists + :raises Exception: If the index cannot be retrieved + :raises Exception: If the index is not empty + + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + if create_if_missing: + if not client.indices.exists(index=STATUS_INDEX): + loggit.info("Creating index %s", STATUS_INDEX) + CreateIndex(client, STATUS_INDEX).do_action() + else: + if not client.indices.exists(index=STATUS_INDEX): + raise MissingIndexError( + f"Status index {STATUS_INDEX} is missing but should exist" + ) + + +def get_settings(client: Elasticsearch) -> Settings: + """ + Get the settings for the deepfreeze operation from the status index. + + :param client: A client connection object + :type client: Elasticsearch + + :returns: The settings + :rtype: dict + + :raises Exception: If the settings document does not exist + + :example: + >>> get_settings(client) + {'repo_name_prefix': 'deepfreeze', 'bucket_name_prefix': 'deepfreeze', 'base_path_prefix': 'snapshots', 'canned_acl': 'private', 'storage_class': 'intelligent_tiering', 'provider': 'aws', 'rotate_by': 'path', 'style': 'oneup', 'last_suffix': '000001'} + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + if not client.indices.exists(index=STATUS_INDEX): + raise MissingIndexError(f"Status index {STATUS_INDEX} is missing") + try: + doc = client.get(index=STATUS_INDEX, id=SETTINGS_ID) + loggit.info("Settings document found") + # Filter out doctype as it's not accepted by Settings constructor + source_data = doc["_source"].copy() + source_data.pop('doctype', None) + return Settings(**source_data) + except NotFoundError: + loggit.info("Settings document not found") + return None + + +def save_settings(client: Elasticsearch, settings: Settings) -> None: + """ + Save the settings for the deepfreeze operation to the status index. + + :param client: A client connection object + :type client: Elasticsearch + :param settings: The settings to save + :type settings: Settings + + :return: None + :rtype: None + + :raises Exception: If the settings document cannot be created + :raises Exception: If the settings document cannot be updated + :raises Exception: If the settings document cannot be retrieved + :raises Exception: If the settings document is not empty + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + try: + client.get(index=STATUS_INDEX, id=SETTINGS_ID) + loggit.info("Settings document already exists, updating it") + client.update(index=STATUS_INDEX, id=SETTINGS_ID, doc=settings.__dict__) + except NotFoundError: + loggit.info("Settings document does not exist, creating it") + client.create(index=STATUS_INDEX, id=SETTINGS_ID, document=settings.__dict__) + loggit.info("Settings saved") + + +def create_repo( + client: Elasticsearch, + repo_name: str, + bucket_name: str, + base_path: str, + canned_acl: str, + storage_class: str, + dry_run: bool = False, +) -> None: + """ + Creates a new repo using the previously-created bucket. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The name of the repository to create + :type repo_name: str + :param bucket_name: The name of the bucket to use for the repository + :type bucket_name: str + :param base_path_prefix: Path within a bucket where snapshots are stored + :type base_path_prefix: str + :param canned_acl: One of the AWS canned ACL values + :type canned_acl: str + :param storage_class: AWS Storage class + :type storage_class: str + :param dry_run: If True, do not actually create the repository + :type dry_run: bool + + :raises Exception: If the repository cannot be created + :raises Exception: If the repository already exists + :raises Exception: If the repository cannot be retrieved + :raises Exception: If the repository is not empty + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.info("Creating repo %s using bucket %s", repo_name, bucket_name) + if dry_run: + return + try: + client.snapshot.create_repository( + name=repo_name, + body={ + "type": "s3", + "settings": { + "bucket": bucket_name, + "base_path": base_path, + "canned_acl": canned_acl, + "storage_class": storage_class, + }, + }, + ) + except Exception as e: + loggit.error(e) + raise ActionError(e) + # Get and save a repository object for this repo + loggit.debug("Saving repo %s to status index", repo_name) + repository = get_repository(client, repo_name) + repository.bucket = bucket_name if not repository.bucket else repository.bucket + repository.base_path = ( + base_path if not repository.base_path else repository.base_path + ) + loggit.debug("Repo = %s", repository) + client.index(index=STATUS_INDEX, body=repository.to_dict()) + loggit.debug("Repo %s saved to status index", repo_name) + # + # TODO: Gather the reply and parse it to make sure this succeeded + + +def get_next_suffix(style: str, last_suffix: str, year: int, month: int) -> str: + """ + Gets the next suffix + + :param style: The style of the suffix + :type style: str + :param last_suffix: The last suffix + :type last_suffix: str + :param year: Optional year to override current year + :type year: int + :param month: Optional month to override current month + :type month: int + + :returns: The next suffix in the format YYYY.MM + :rtype: str + + :raises ValueError: If the style is not valid + """ + if style == "oneup": + return str(int(last_suffix) + 1).zfill(6) + elif style == "date": + current_year = year or datetime.now().year + current_month = month or datetime.now().month + return f"{current_year:04}.{current_month:02}" + else: + raise ValueError("Invalid style") + + +def get_repository(client: Elasticsearch, name: str) -> Repository: + """ + Get the repository object from the status index. + + :param client: A client connection object + :type client: Elasticsearch + :param name: The name of the repository + :type name: str + + :returns: The repository + :rtype: Repository + + :raises Exception: If the repository does not exist + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + logging.debug("Getting repository %s", name) + try: + doc = client.search( + index=STATUS_INDEX, body={"query": {"match": {"name": name}}} + ) + logging.debug("Got: %s", doc) + if doc["hits"]["total"]["value"] == 0: + logging.debug("Got no hits") + return Repository(name=name) + for n in range(len(doc["hits"]["hits"])): + if doc["hits"]["hits"][n]["_source"]["name"] == name: + logging.debug("Got a match") + return Repository( + **doc["hits"]["hits"][n]["_source"], + docid=doc["hits"]["hits"][n]["_id"], + ) + # If we get here, we have no match + logging.debug("No match found") + return Repository(name=name) + except NotFoundError: + loggit.warning("Repository document not found") + return Repository(name=name) + + +def get_all_repos(client: Elasticsearch) -> list[Repository]: + """ + Get the complete list of repos from our index and return a Repository object for each. + + :param client: A client connection object + :type client: Elasticsearch + + :returns: The unmounted repos. + :rtype: list[Repository] + + :raises Exception: If the repository does not exist + + """ + # logging.debug("Looking for unmounted repos") + # # Perform search in ES for all repos in the status index + # ! This will now include mounted and unmounted repos both! + query = {"query": {"match": {"doctype": "repository"}}, "size": 10000} + logging.debug("Searching for repos") + response = client.search(index=STATUS_INDEX, body=query) + logging.debug("Response: %s", response) + repos = response["hits"]["hits"] + logging.debug("Repos retrieved: %s", repos) + # return a Repository object for each + # TEMP: + rv = [] + for repo in repos: + logging.debug("Repo: %s", repo) + logging.debug("Repo ID: %s", repo["_id"]) + logging.debug("Repo Source: %s", repo["_source"]) + rv.append(Repository(**repo["_source"], docid=repo["_id"])) + logging.debug("Repo object: %s", rv[-1]) + return rv + + +# return [Repository(**repo["_source"], docid=response["_id"]) for repo in repos] + + +def get_matching_repo_names(client: Elasticsearch, repo_name_prefix: str) -> list[str]: + """ + Get the complete list of repos and return just the ones whose names + begin with the given prefix. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name_prefix: A prefix for repository names + :type repo_name_prefix: str + + :returns: The repos. + :rtype: list[object] + + :raises Exception: If the repository does not exist + """ + repos = client.snapshot.get_repository() + logging.debug("Repos retrieved: %s", repos) + pattern = re.compile(repo_name_prefix) + logging.debug("Looking for repos matching %s", repo_name_prefix) + return [repo for repo in repos if pattern.search(repo)] + + +def get_matching_repos( + client: Elasticsearch, repo_name_prefix: str, mounted: bool = False +) -> list[Repository]: + """ + Get the list of repos from our index and return a Repository object for each one + which matches the given prefix. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name_prefix: A prefix for repository names + :type repo_name_prefix: str + + :returns: The repos. + :rtype: list[Repository] + + :raises Exception: If the repository does not exist + """ + query = {"query": {"match": {"doctype": "repository"}}, "size": 10000} + response = client.search(index=STATUS_INDEX, body=query) + logging.debug("Response: %s", response) + repos = response["hits"]["hits"] + logging.debug("Repos retrieved: %s", repos) + repos = [ + repo for repo in repos if repo["_source"]["name"].startswith(repo_name_prefix) + ] + if mounted: + mounted_repos = [ + repo for repo in repos if repo["_source"]["is_mounted"] is True + ] + logging.debug("Mounted repos: %s", mounted_repos) + return [Repository(**repo["_source"]) for repo in mounted_repos] + # return a Repository object for each + return [Repository(**repo["_source"], docid=repo["_id"]) for repo in repos] + + +def unmount_repo(client: Elasticsearch, repo: str) -> Repository: + """ + Encapsulate the actions of deleting the repo and, at the same time, + doing any record-keeping we need. + + :param client: A client connection object + :type client: Elasticsearch + :param repo: The name of the repository to unmount + :type repo: str + + :returns: The repo. + :rtype: Repository + + :raises Exception: If the repository does not exist + :raises Exception: If the repository is not empty + :raises Exception: If the repository cannot be deleted + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + # Get repository info from Elasticsearch + repo_info = client.snapshot.get_repository(name=repo)[repo] + bucket = repo_info["settings"]["bucket"] + base_path = repo_info["settings"]["base_path"] + + # Get repository object from status index + repo_obj = get_repository(client, repo) + repo_obj.bucket = bucket if not repo_obj.bucket else repo_obj.bucket + repo_obj.base_path = base_path if not repo_obj.base_path else repo_obj.base_path + + # Try to update date ranges using the shared utility function + # This will fall back gracefully if indices aren't available + updated = update_repository_date_range(client, repo_obj) + if updated: + loggit.info("Successfully updated date range for %s before unmounting", repo) + else: + loggit.debug( + "Could not update date range for %s (keeping existing dates: %s to %s)", + repo, + repo_obj.start.isoformat() if repo_obj.start else "None", + repo_obj.end.isoformat() if repo_obj.end else "None" + ) + + # Mark repository as unmounted + repo_obj.unmount() + msg = f"Recording repository details as {repo_obj}" + loggit.debug(msg) + + # Remove the repository from Elasticsearch + loggit.debug("Removing repo %s", repo) + try: + client.snapshot.delete_repository(name=repo) + except Exception as e: + loggit.warning("Repository %s could not be unmounted due to %s", repo, e) + loggit.warning("Another attempt will be made when rotate runs next") + + # Update the status index with final repository state + loggit.debug("Updating repo: %s", repo_obj) + client.update(index=STATUS_INDEX, doc=repo_obj.to_dict(), id=repo_obj.docid) + loggit.debug("Repo %s removed", repo) + return repo_obj + + +def decode_date(date_in: str) -> datetime: + """ + Decode a date from a string or datetime object. + + :param date_in: The date to decode + :type date_in: str or datetime + + :returns: The decoded date + :rtype: datetime + + :raises ValueError: If the date is not valid + """ + if isinstance(date_in, datetime): + dt = date_in + elif isinstance(date_in, str): + logging.debug("Decoding date %s", date_in) + dt = datetime.fromisoformat(date_in) + else: + raise ValueError("Invalid date format") + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def create_ilm_policy( + client: Elasticsearch, policy_name: str, policy_body: str +) -> None: + """ + Create a sample ILM policy. + + :param client: A client connection object + :type client: Elasticsearch + :param policy_name: The name of the policy to create + :type policy_name: str + + :return: None + :rtype: None + + :raises Exception: If the policy cannot be created + :raises Exception: If the policy already exists + :raises Exception: If the policy cannot be retrieved + :raises Exception: If the policy is not empty + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.info("Creating ILM policy %s", policy_name) + try: + client.ilm.put_lifecycle(name=policy_name, body=policy_body) + except Exception as e: + loggit.error(e) + raise ActionError(e) + + +def create_thawed_ilm_policy(client: Elasticsearch, repo_name: str) -> str: + """ + Create an ILM policy for thawed indices from a specific repository. + + The policy is named {repo_name}-thawed and includes only a delete phase + since the indices are already mounted as searchable snapshots. + + NOTE: Thawed indices are ALREADY searchable snapshots mounted from the frozen + repository. They don't need a frozen phase - just a delete phase to clean up + after the thaw period expires. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The repository name (e.g., "deepfreeze-000010") + :type repo_name: str + + :returns: The created policy name + :rtype: str + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + + policy_name = f"{repo_name}-thawed" + policy_body = { + "policy": { + "phases": { + "delete": { + "min_age": "29d", + "actions": { + "delete": {"delete_searchable_snapshot": True} + }, + }, + } + } + } + + loggit.info("Creating thawed ILM policy %s for repository %s", policy_name, repo_name) + loggit.debug("Thawed ILM policy body: %s", policy_body) + + try: + # Check if policy already exists + try: + client.ilm.get_lifecycle(name=policy_name) + loggit.info("Thawed ILM policy %s already exists, skipping creation", policy_name) + return policy_name + except Exception: + # Policy doesn't exist, create it + pass + + client.ilm.put_lifecycle(name=policy_name, body=policy_body) + loggit.info("Successfully created thawed ILM policy %s", policy_name) + return policy_name + + except Exception as e: + loggit.error("Failed to create thawed ILM policy %s: %s", policy_name, e) + raise ActionError(f"Failed to create thawed ILM policy {policy_name}: {e}") + + +def update_repository_date_range(client: Elasticsearch, repo: Repository) -> bool: + """ + Update the date range for a repository by querying document @timestamp values. + + Gets the actual min/max @timestamp from all indices contained in the repository's + snapshots. The date range can only EXTEND (never shrink) as new data is added. + + For mounted repos: Queries mounted indices directly. + For unmounted repos: Attempts to mount snapshots temporarily to query, or skips update. + + :param client: A client connection object + :type client: Elasticsearch + :param repo: The repository to update + :type repo: Repository + + :returns: True if dates were updated, False otherwise + :rtype: bool + + :raises Exception: If the repository does not exist + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Updating date range for repository %s (mounted: %s)", repo.name, repo.is_mounted) + + # Store existing range to ensure we only extend, never shrink + existing_start = repo.start + existing_end = repo.end + + earliest = None + latest = None + + try: + # Get all indices from snapshots in this repository + snapshot_indices = get_all_indices_in_repo(client, repo.name) + loggit.debug("Found %d indices in repository snapshots", len(snapshot_indices)) + + if not snapshot_indices: + loggit.debug("No indices found in repository %s", repo.name) + return False + + # If repo is mounted, query the mounted indices + if repo.is_mounted: + # Find which indices are actually mounted (try multiple naming patterns) + mounted_indices = [] + for idx in snapshot_indices: + # Try original name + if client.indices.exists(index=idx): + mounted_indices.append(idx) + loggit.debug("Found mounted index: %s", idx) + # Try with partial- prefix (searchable snapshots) + elif client.indices.exists(index=f"partial-{idx}"): + mounted_indices.append(f"partial-{idx}") + loggit.debug("Found mounted searchable snapshot: partial-%s", idx) + # Try with restored- prefix (fully restored indices) + elif client.indices.exists(index=f"restored-{idx}"): + mounted_indices.append(f"restored-{idx}") + loggit.debug("Found restored index: restored-%s", idx) + + if mounted_indices: + loggit.debug("Found %d mounted indices, querying timestamp ranges", len(mounted_indices)) + # Query actual @timestamp ranges from mounted indices + earliest, latest = get_timestamp_range(client, mounted_indices) + else: + loggit.debug("Repo is mounted but no searchable snapshot indices found") + return False + else: + # Repo is not mounted - we cannot query @timestamp without mounting + # For unmounted repos, preserve existing date range or skip update + loggit.debug( + "Repository %s is not mounted, cannot query document timestamps. " + "Keeping existing date range: %s to %s", + repo.name, + existing_start.isoformat() if existing_start else "None", + existing_end.isoformat() if existing_end else "None" + ) + return False + + if not earliest or not latest: + loggit.warning("Could not determine timestamp range for repository %s", repo.name) + return False + + loggit.debug("Queried timestamp range: %s to %s", earliest, latest) + + # CRITICAL: Only EXTEND the date range, never shrink it + # This ensures we capture all data that has ever been in the repository + if existing_start and existing_end: + # We have existing dates - extend them + final_start = min(existing_start, earliest) + final_end = max(existing_end, latest) + + if final_start == existing_start and final_end == existing_end: + loggit.debug("Date range unchanged for %s", repo.name) + return False + + loggit.info( + "Extending date range for %s: (%s to %s) -> (%s to %s)", + repo.name, + existing_start.isoformat(), + existing_end.isoformat(), + final_start.isoformat(), + final_end.isoformat() + ) + else: + # No existing dates - use the queried range + final_start = earliest + final_end = latest + loggit.info( + "Setting initial date range for %s: %s to %s", + repo.name, + final_start.isoformat(), + final_end.isoformat() + ) + + # Update the repository object + repo.start = final_start + repo.end = final_end + + # Persist to status index + query = {"query": {"term": {"name.keyword": repo.name}}} + response = client.search(index=STATUS_INDEX, body=query) + + if response["hits"]["total"]["value"] > 0: + doc_id = response["hits"]["hits"][0]["_id"] + client.update( + index=STATUS_INDEX, + id=doc_id, + body={"doc": repo.to_dict()} + ) + else: + # Create new document if it doesn't exist + client.index(index=STATUS_INDEX, body=repo.to_dict()) + + return True + + except Exception as e: + loggit.error("Error updating date range for repository %s: %s", repo.name, e) + return False + + +def find_repos_by_date_range( + client: Elasticsearch, start: datetime, end: datetime +) -> list[Repository]: + """ + Find repositories that contain data overlapping with the given date range. + + :param client: A client connection object + :type client: Elasticsearch + :param start: The start of the date range + :type start: datetime + :param end: The end of the date range + :type end: datetime + + :returns: A list of repositories with overlapping date ranges + :rtype: list[Repository] + + :raises Exception: If the status index does not exist + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug( + "Finding repositories with data between %s and %s", + start.isoformat(), + end.isoformat(), + ) + + # Query for repositories where the date range overlaps with the requested range + # Overlap occurs if: repo.start <= end AND repo.end >= start + query = { + "query": { + "bool": { + "must": [ + {"term": {"doctype": "repository"}}, + {"range": {"start": {"lte": end.isoformat()}}}, + {"range": {"end": {"gte": start.isoformat()}}}, + ] + } + }, + "size": 10000 + } + + try: + response = client.search(index=STATUS_INDEX, body=query) + repos = response["hits"]["hits"] + loggit.debug("Found %d repositories matching date range", len(repos)) + return [Repository(**repo["_source"], docid=repo["_id"]) for repo in repos] + except NotFoundError: + loggit.warning("Status index not found") + return [] + + +def check_restore_status(s3: S3Client, bucket: str, base_path: str) -> dict: + """ + Check the restoration status of objects in an S3 bucket. + + Uses head_object to check the Restore metadata field, which is the only way + to determine if a Glacier object has been restored (storage class remains GLACIER + even after restoration). + + :param s3: The S3 client object + :type s3: S3Client + :param bucket: The bucket name + :type bucket: str + :param base_path: The base path in the bucket + :type base_path: str + + :returns: A dictionary with restoration status information + :rtype: dict + + :raises Exception: If the bucket or objects cannot be accessed + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Checking restore status for s3://%s/%s", bucket, base_path) + + # Normalize base_path + normalized_path = base_path.strip("/") + if normalized_path: + normalized_path += "/" + + objects = s3.list_objects(bucket, normalized_path) + + total_count = len(objects) + restored_count = 0 + in_progress_count = 0 + not_restored_count = 0 + + for obj in objects: + key = obj["Key"] + storage_class = obj.get("StorageClass", "STANDARD") + + # For objects in instant-access tiers, no need to check restore status + if storage_class in [ + "STANDARD", + "STANDARD_IA", + "ONEZONE_IA", + "INTELLIGENT_TIERING", + ]: + restored_count += 1 + continue + + # For Glacier objects, must use head_object to check Restore metadata + try: + metadata = s3.head_object(bucket, key) + restore_header = metadata.get("Restore") + + if restore_header: + # Restore header exists - parse it to check status + # Format: 'ongoing-request="true"' or 'ongoing-request="false", expiry-date="..."' + if 'ongoing-request="true"' in restore_header: + in_progress_count += 1 + loggit.debug("Object %s: restoration in progress", key) + else: + # ongoing-request="false" means restoration is complete + restored_count += 1 + loggit.debug("Object %s: restored (expiry in header)", key) + else: + # No Restore header means object is in Glacier and not being restored + not_restored_count += 1 + loggit.debug("Object %s: in %s, not restored", key, storage_class) + + except Exception as e: + loggit.warning("Failed to check restore status for %s: %s", key, e) + # Count as not restored if we can't determine status + not_restored_count += 1 + + status = { + "total": total_count, + "restored": restored_count, + "in_progress": in_progress_count, + "not_restored": not_restored_count, + "complete": (restored_count == total_count) if total_count > 0 else False, + } + + loggit.debug("Restore status: %s", status) + return status + + +def mount_repo(client: Elasticsearch, repo: Repository) -> None: + """ + Mount a repository by creating it in Elasticsearch and updating its status. + + :param client: A client connection object + :type client: Elasticsearch + :param repo: The repository to mount + :type repo: Repository + + :return: None + :rtype: None + + :raises Exception: If the repository cannot be created + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.info("Mounting repository %s", repo.name) + + # Get settings to retrieve canned_acl and storage_class + settings = get_settings(client) + + # Create the repository in Elasticsearch + try: + client.snapshot.create_repository( + name=repo.name, + body={ + "type": "s3", + "settings": { + "bucket": repo.bucket, + "base_path": repo.base_path, + "canned_acl": settings.canned_acl, + "storage_class": settings.storage_class, + }, + }, + ) + loggit.info("Repository %s created successfully", repo.name) + + # Mark repository as thawed (uses new state machine) + repo.mark_thawed() + repo.persist(client) + loggit.info("Repository %s status updated to 'thawed'", repo.name) + + except Exception as e: + loggit.error("Failed to mount repository %s: %s", repo.name, e) + raise ActionError(f"Failed to mount repository {repo.name}: {e}") + + +def save_thaw_request( + client: Elasticsearch, + request_id: str, + repos: list[Repository], + status: str, + start_date: datetime = None, + end_date: datetime = None, +) -> None: + """ + Save a thaw request to the status index for later querying. + + :param client: A client connection object + :type client: Elasticsearch + :param request_id: A unique identifier for this thaw request + :type request_id: str + :param repos: The list of repositories being thawed + :type repos: list[Repository] + :param status: The current status of the thaw request + :type status: str + :param start_date: Start of the date range for this thaw request + :type start_date: datetime + :param end_date: End of the date range for this thaw request + :type end_date: datetime + + :return: None + :rtype: None + + :raises Exception: If the request cannot be saved + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Saving thaw request %s", request_id) + + request_doc = { + "doctype": "thaw_request", + "request_id": request_id, + "repos": [repo.name for repo in repos], + "status": status, + "created_at": datetime.now(timezone.utc).isoformat(), + } + + # Add date range if provided + if start_date: + request_doc["start_date"] = start_date.isoformat() + if end_date: + request_doc["end_date"] = end_date.isoformat() + + try: + client.index(index=STATUS_INDEX, id=request_id, body=request_doc) + loggit.info("Thaw request %s saved successfully", request_id) + except Exception as e: + loggit.error("Failed to save thaw request %s: %s", request_id, e) + raise ActionError(f"Failed to save thaw request {request_id}: {e}") + + +def get_thaw_request(client: Elasticsearch, request_id: str) -> dict: + """ + Retrieve a thaw request from the status index by ID. + + :param client: A client connection object + :type client: Elasticsearch + :param request_id: The thaw request ID + :type request_id: str + + :returns: The thaw request document + :rtype: dict + + :raises Exception: If the request is not found + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Retrieving thaw request %s", request_id) + + try: + response = client.get(index=STATUS_INDEX, id=request_id) + return response["_source"] + except NotFoundError: + loggit.error("Thaw request %s not found", request_id) + raise ActionError(f"Thaw request {request_id} not found") + except Exception as e: + loggit.error("Failed to retrieve thaw request %s: %s", request_id, e) + raise ActionError(f"Failed to retrieve thaw request {request_id}: {e}") + + +def list_thaw_requests(client: Elasticsearch) -> list[dict]: + """ + List all thaw requests from the status index. + + :param client: A client connection object + :type client: Elasticsearch + + :returns: List of thaw request documents + :rtype: list[dict] + + :raises Exception: If the query fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Listing all thaw requests") + + query = {"query": {"term": {"doctype": "thaw_request"}}, "size": 10000} + + try: + response = client.search(index=STATUS_INDEX, body=query) + requests = response["hits"]["hits"] + loggit.debug("Found %d thaw requests", len(requests)) + return [{"id": req["_id"], **req["_source"]} for req in requests] + except NotFoundError: + loggit.warning("Status index not found") + return [] + except Exception as e: + loggit.error("Failed to list thaw requests: %s", e) + raise ActionError(f"Failed to list thaw requests: {e}") + + +def update_thaw_request( + client: Elasticsearch, request_id: str, status: str = None, **fields +) -> None: + """ + Update a thaw request in the status index. + + :param client: A client connection object + :type client: Elasticsearch + :param request_id: The thaw request ID + :type request_id: str + :param status: New status value (optional) + :type: str + :param fields: Additional fields to update + :type fields: dict + + :return: None + :rtype: None + + :raises Exception: If the update fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Updating thaw request %s", request_id) + + update_doc = {} + if status: + update_doc["status"] = status + update_doc.update(fields) + + try: + client.update(index=STATUS_INDEX, id=request_id, doc=update_doc) + loggit.info("Thaw request %s updated successfully", request_id) + except Exception as e: + loggit.error("Failed to update thaw request %s: %s", request_id, e) + raise ActionError(f"Failed to update thaw request {request_id}: {e}") + + +def get_repositories_by_names( + client: Elasticsearch, repo_names: list[str] +) -> list[Repository]: + """ + Get Repository objects by a list of repository names. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_names: List of repository names + :type repo_names: list[str] + + :returns: List of Repository objects + :rtype: list[Repository] + + :raises Exception: If the query fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Getting repositories by names: %s", repo_names) + + if not repo_names: + return [] + + query = { + "query": { + "bool": { + "must": [ + {"term": {"doctype": "repository"}}, + {"terms": {"name.keyword": repo_names}}, + ] + } + }, + "size": 10000 + } + + try: + response = client.search(index=STATUS_INDEX, body=query) + repos = response["hits"]["hits"] + loggit.debug("Found %d repositories", len(repos)) + return [Repository(**repo["_source"], docid=repo["_id"]) for repo in repos] + except NotFoundError: + loggit.warning("Status index not found") + return [] + except Exception as e: + loggit.error("Failed to get repositories: %s", e) + raise ActionError(f"Failed to get repositories: {e}") + + +def get_index_templates(client: Elasticsearch) -> dict: + """ + Get all legacy index templates. + + :param client: A client connection object + :type client: Elasticsearch + + :returns: Dictionary of legacy index templates + :rtype: dict + + :raises Exception: If the query fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Getting legacy index templates") + try: + return client.indices.get_template() + except Exception as e: + loggit.error("Failed to get legacy index templates: %s", e) + raise ActionError(f"Failed to get legacy index templates: {e}") + + +def get_composable_templates(client: Elasticsearch) -> dict: + """ + Get all composable index templates. + + :param client: A client connection object + :type client: Elasticsearch + + :returns: Dictionary of composable index templates + :rtype: dict + + :raises Exception: If the query fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Getting composable index templates") + try: + return client.indices.get_index_template() + except Exception as e: + loggit.error("Failed to get composable index templates: %s", e) + raise ActionError(f"Failed to get composable index templates: {e}") + + +def update_template_ilm_policy( + client: Elasticsearch, + template_name: str, + old_policy_name: str, + new_policy_name: str, + is_composable: bool = True, +) -> bool: + """ + Update an index template to use a new ILM policy. + + :param client: A client connection object + :type client: Elasticsearch + :param template_name: The name of the template to update + :type template_name: str + :param old_policy_name: The old policy name to replace + :type old_policy_name: str + :param new_policy_name: The new policy name + :type new_policy_name: str + :param is_composable: Whether this is a composable template + :type is_composable: bool + + :returns: True if template was updated, False otherwise + :rtype: bool + + :raises Exception: If the update fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug( + "Updating template %s from policy %s to %s", + template_name, + old_policy_name, + new_policy_name, + ) + + try: + if is_composable: + # Get composable template + templates = client.indices.get_index_template(name=template_name) + if not templates or "index_templates" not in templates: + loggit.warning("Template %s not found", template_name) + return False + + template = templates["index_templates"][0]["index_template"] + + # Check if template uses the old policy + ilm_policy = template.get("template", {}).get("settings", {}).get("index", {}).get("lifecycle", {}).get("name") + + if ilm_policy == old_policy_name: + # Update the policy name + if "template" not in template: + template["template"] = {} + if "settings" not in template["template"]: + template["template"]["settings"] = {} + if "index" not in template["template"]["settings"]: + template["template"]["settings"]["index"] = {} + if "lifecycle" not in template["template"]["settings"]["index"]: + template["template"]["settings"]["index"]["lifecycle"] = {} + + template["template"]["settings"]["index"]["lifecycle"]["name"] = new_policy_name + + # Put the updated template + client.indices.put_index_template(name=template_name, body=template) + loggit.info("Updated composable template %s to use policy %s", template_name, new_policy_name) + return True + else: + # Get legacy template + templates = client.indices.get_template(name=template_name) + if not templates or template_name not in templates: + loggit.warning("Template %s not found", template_name) + return False + + template = templates[template_name] + + # Check if template uses the old policy + ilm_policy = template.get("settings", {}).get("index", {}).get("lifecycle", {}).get("name") + + if ilm_policy == old_policy_name: + # Update the policy name + if "settings" not in template: + template["settings"] = {} + if "index" not in template["settings"]: + template["settings"]["index"] = {} + if "lifecycle" not in template["settings"]["index"]: + template["settings"]["index"]["lifecycle"] = {} + + template["settings"]["index"]["lifecycle"]["name"] = new_policy_name + + # Put the updated template + client.indices.put_template(name=template_name, body=template) + loggit.info("Updated legacy template %s to use policy %s", template_name, new_policy_name) + return True + + return False + except Exception as e: + loggit.error("Failed to update template %s: %s", template_name, e) + raise ActionError(f"Failed to update template {template_name}: {e}") + + +def create_versioned_ilm_policy( + client: Elasticsearch, + base_policy_name: str, + base_policy_body: dict, + new_repo_name: str, + suffix: str, +) -> str: + """ + Create a versioned ILM policy with updated repository reference. + + :param client: A client connection object + :type client: Elasticsearch + :param base_policy_name: The base policy name + :type base_policy_name: str + :param base_policy_body: The base policy body + :type base_policy_body: dict + :param new_repo_name: The new repository name + :type new_repo_name: str + :param suffix: The suffix to append to the policy name + :type suffix: str + + :returns: The new versioned policy name + :rtype: str + + :raises Exception: If policy creation fails + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + + # Create versioned policy name + new_policy_name = f"{base_policy_name}-{suffix}" + + loggit.debug( + "Creating versioned policy %s referencing repository %s", + new_policy_name, + new_repo_name, + ) + + # Deep copy the policy body to avoid modifying the original + import copy + new_policy_body = copy.deepcopy(base_policy_body) + + # Update all searchable_snapshot repository references + if "phases" in new_policy_body: + for phase_name, phase_config in new_policy_body["phases"].items(): + if "actions" in phase_config and "searchable_snapshot" in phase_config["actions"]: + phase_config["actions"]["searchable_snapshot"]["snapshot_repository"] = new_repo_name + loggit.debug( + "Updated %s phase to reference repository %s", + phase_name, + new_repo_name, + ) + + # Create the new policy + try: + client.ilm.put_lifecycle(name=new_policy_name, policy=new_policy_body) + loggit.info("Created versioned ILM policy %s", new_policy_name) + return new_policy_name + except Exception as e: + loggit.error("Failed to create policy %s: %s", new_policy_name, e) + raise ActionError(f"Failed to create policy {new_policy_name}: {e}") + + +def get_policies_for_repo(client: Elasticsearch, repo_name: str) -> dict: + """ + Find all ILM policies that reference a specific repository. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The repository name + :type repo_name: str + + :returns: Dictionary of policy names to policy bodies + :rtype: dict + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Finding policies that reference repository %s", repo_name) + + policies = client.ilm.get_lifecycle() + matching_policies = {} + + for policy_name, policy_data in policies.items(): + policy_body = policy_data.get("policy", {}) + phases = policy_body.get("phases", {}) + + for phase_name, phase_config in phases.items(): + actions = phase_config.get("actions", {}) + if "searchable_snapshot" in actions: + snapshot_repo = actions["searchable_snapshot"].get("snapshot_repository") + if snapshot_repo == repo_name: + matching_policies[policy_name] = policy_data + loggit.debug("Found policy %s referencing %s", policy_name, repo_name) + break + + loggit.info("Found %d policies referencing repository %s", len(matching_policies), repo_name) + return matching_policies + + +def get_policies_by_suffix(client: Elasticsearch, suffix: str) -> dict: + """ + Find all ILM policies that end with a specific suffix. + + :param client: A client connection object + :type client: Elasticsearch + :param suffix: The suffix to search for (e.g., "000003") + :type suffix: str + + :returns: Dictionary of policy names to policy bodies + :rtype: dict + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Finding policies ending with suffix -%s", suffix) + + policies = client.ilm.get_lifecycle() + matching_policies = {} + + suffix_pattern = f"-{suffix}" + + for policy_name, policy_data in policies.items(): + if policy_name.endswith(suffix_pattern): + matching_policies[policy_name] = policy_data + loggit.debug("Found policy %s with suffix %s", policy_name, suffix) + + loggit.info("Found %d policies with suffix -%s", len(matching_policies), suffix) + return matching_policies + + +def is_policy_safe_to_delete(client: Elasticsearch, policy_name: str) -> bool: + """ + Check if an ILM policy is safe to delete (not in use by any indices/datastreams/templates). + + :param client: A client connection object + :type client: Elasticsearch + :param policy_name: The policy name + :type policy_name: str + + :returns: True if safe to delete, False otherwise + :rtype: bool + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Checking if policy %s is safe to delete", policy_name) + + try: + policies = client.ilm.get_lifecycle(name=policy_name) + if policy_name not in policies: + loggit.warning("Policy %s not found", policy_name) + return False + + policy_data = policies[policy_name] + in_use_by = policy_data.get("in_use_by", {}) + + indices_count = len(in_use_by.get("indices", [])) + datastreams_count = len(in_use_by.get("data_streams", [])) + templates_count = len(in_use_by.get("composable_templates", [])) + + total_usage = indices_count + datastreams_count + templates_count + + if total_usage > 0: + loggit.info( + "Policy %s is in use by %d indices, %d data streams, %d templates", + policy_name, + indices_count, + datastreams_count, + templates_count, + ) + return False + + loggit.debug("Policy %s is safe to delete (not in use)", policy_name) + return True + except NotFoundError: + loggit.warning("Policy %s not found", policy_name) + return False + except Exception as e: + loggit.error("Error checking policy %s: %s", policy_name, e) + return False + + +def find_snapshots_for_index( + client: Elasticsearch, repo_name: str, index_name: str +) -> list[str]: + """ + Find all snapshots in a repository that contain a specific index. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The repository name + :type repo_name: str + :param index_name: The index name to search for + :type index_name: str + + :returns: List of snapshot names containing the index + :rtype: list[str] + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Finding snapshots containing index %s in repo %s", index_name, repo_name) + + try: + snapshots = client.snapshot.get(repository=repo_name, snapshot="_all") + matching_snapshots = [] + + for snapshot in snapshots["snapshots"]: + if index_name in snapshot["indices"]: + matching_snapshots.append(snapshot["snapshot"]) + loggit.debug( + "Found index %s in snapshot %s", index_name, snapshot["snapshot"] + ) + + loggit.info( + "Found %d snapshots containing index %s", len(matching_snapshots), index_name + ) + return matching_snapshots + + except Exception as e: + loggit.error("Failed to find snapshots for index %s: %s", index_name, e) + return [] + + +def mount_snapshot_index( + client: Elasticsearch, repo_name: str, snapshot_name: str, index_name: str, ilm_policy: str = None +) -> bool: + """ + Mount an index from a snapshot as a searchable snapshot. + + :param client: A client connection object + :type client: Elasticsearch + :param repo_name: The repository name + :type repo_name: str + :param snapshot_name: The snapshot name + :type snapshot_name: str + :param index_name: The index name to mount + :type index_name: str + :param ilm_policy: Optional ILM policy to assign to the index + :type ilm_policy: str + + :returns: True if successful, False otherwise + :rtype: bool + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.info( + "Mounting index %s from snapshot %s/%s", index_name, repo_name, snapshot_name + ) + + # Check if index is already mounted + already_mounted = client.indices.exists(index=index_name) + if already_mounted: + loggit.info("Index %s is already mounted", index_name) + # Still assign ILM policy if provided and not already mounted + if ilm_policy: + try: + client.indices.put_settings( + index=index_name, + body={"index.lifecycle.name": ilm_policy} + ) + loggit.info("Assigned ILM policy %s to already-mounted index %s", ilm_policy, index_name) + except Exception as e: + loggit.warning("Failed to assign ILM policy to already-mounted index %s: %s", index_name, e) + return True + + try: + client.searchable_snapshots.mount( + repository=repo_name, + snapshot=snapshot_name, + body={"index": index_name}, + ) + loggit.info("Successfully mounted index %s", index_name) + + # Assign ILM policy if provided + if ilm_policy: + try: + client.indices.put_settings( + index=index_name, + body={"index.lifecycle.name": ilm_policy} + ) + loggit.info("Assigned ILM policy %s to index %s", ilm_policy, index_name) + except Exception as e: + loggit.warning("Failed to assign ILM policy to index %s: %s", index_name, e) + + return True + + except Exception as e: + loggit.error("Failed to mount index %s: %s", index_name, e) + return False + + +def wait_for_index_ready( + client: Elasticsearch, index_name: str, max_wait_seconds: int = 30 +) -> bool: + """ + Wait for an index to become ready for search queries after mounting. + + Searchable snapshot indices need time for shards to allocate before + they can handle queries. This function waits for the index to have + at least one active shard. + + :param client: A client connection object + :type client: Elasticsearch + :param index_name: The index name to wait for + :type index_name: str + :param max_wait_seconds: Maximum time to wait in seconds + :type max_wait_seconds: int + + :returns: True if index is ready, False if timeout + :rtype: bool + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.debug("Waiting for index %s to be ready", index_name) + + start_time = time.time() + while time.time() - start_time < max_wait_seconds: + try: + # Check if at least one shard is active + health = client.cluster.health(index=index_name, wait_for_active_shards=1, timeout="5s") + if health.get("active_shards", 0) > 0: + loggit.debug("Index %s is ready (active shards: %d)", index_name, health["active_shards"]) + return True + except Exception as e: + loggit.debug("Index %s not ready yet: %s", index_name, e) + + # Wait a bit before retrying + time.sleep(2) + + loggit.warning("Index %s did not become ready within %d seconds", index_name, max_wait_seconds) + return False + + +def get_index_datastream_name(client: Elasticsearch, index_name: str) -> str: + """ + Get the data stream name for an index by checking its settings. + + Only returns a data stream name if the index has concrete metadata + indicating it was part of a data stream. + + :param client: A client connection object + :type client: Elasticsearch + :param index_name: The index name + :type index_name: str + + :returns: The data stream name if the index was part of one, None otherwise + :rtype: str + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + + try: + # Get index settings to check for data stream metadata + settings = client.indices.get_settings(index=index_name) + + if index_name in settings: + index_settings = settings[index_name].get("settings", {}) + index_metadata = index_settings.get("index", {}) + + # Check if this index has data stream metadata + # Data stream backing indices have a hidden setting indicating their data stream + datastream_name = index_metadata.get("provided_name") + + # Also check if index was created by a data stream + if datastream_name and datastream_name.startswith(".ds-"): + # Extract the actual data stream name from the backing index name + # Pattern: .ds-{name}-{date}-{number} + remaining = datastream_name[4:] + parts = remaining.rsplit("-", 2) + if len(parts) >= 3: + ds_name = parts[0] + loggit.debug("Index %s belongs to data stream %s (from metadata)", index_name, ds_name) + return ds_name + + # Fallback: check the actual index name itself + # When indices are remounted from snapshots, metadata might not be preserved + # but the index name pattern (.ds-{name}-{date}-{number}) is retained + if index_name.startswith(".ds-"): + loggit.debug("Checking index name %s for data stream pattern", index_name) + # Extract the actual data stream name from the backing index name + # Pattern: .ds-{name}-{date}-{number} + remaining = index_name[4:] + parts = remaining.rsplit("-", 2) + if len(parts) >= 3: + ds_name = parts[0] + loggit.debug("Index %s belongs to data stream %s (from index name)", index_name, ds_name) + return ds_name + + return None + + except Exception as e: + loggit.debug("Could not determine data stream for index %s: %s", index_name, e) + return None + + +def add_index_to_datastream( + client: Elasticsearch, datastream_name: str, index_name: str +) -> bool: + """ + Add a backing index back to its data stream. + + :param client: A client connection object + :type client: Elasticsearch + :param datastream_name: The data stream name + :type datastream_name: str + :param index_name: The backing index name + :type index_name: str + + :returns: True if successful, False otherwise + :rtype: bool + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.info("Adding index %s to data stream %s", index_name, datastream_name) + + try: + # First check if data stream exists + try: + client.indices.get_data_stream(name=datastream_name) + except NotFoundError: + loggit.warning("Data stream %s does not exist", datastream_name) + return False + + # Add the backing index to the data stream + client.indices.modify_data_stream( + body={ + "actions": [ + {"add_backing_index": {"data_stream": datastream_name, "index": index_name}} + ] + } + ) + loggit.info("Successfully added index %s to data stream %s", index_name, datastream_name) + return True + + except Exception as e: + loggit.error("Failed to add index %s to data stream %s: %s", index_name, datastream_name, e) + return False + + +def find_and_mount_indices_in_date_range( + client: Elasticsearch, repos: list[Repository], start_date: datetime, end_date: datetime, ilm_policy: str = None +) -> dict: + """ + Find and mount all indices within a date range from the given repositories. + + For each repository, creates a per-repo thawed ILM policy ({repo_name}-thawed) that: + - References the specific repository in the frozen phase + - Deletes indices after 29 days + + For each index found: + 1. Mount it as a searchable snapshot + 2. Wait for the index to become ready for queries + 3. Try to check if its @timestamp range overlaps with the requested date range + 4. If no overlap, unmount the index + 5. If overlap (or if timestamp check fails), keep mounted + 6. Assign the per-repo thawed ILM policy to the index + 7. For any kept index that's a data stream backing index, add it back to the data stream + + Note: Data stream reassignment happens for ALL mounted indices, even if the + timestamp query fails. This ensures indices are properly rejoined to their + data streams regardless of query errors. + + :param client: A client connection object + :type client: Elasticsearch + :param repos: List of repositories to search + :type repos: list[Repository] + :param start_date: Start of date range + :type start_date: datetime + :param end_date: End of date range + :type end_date: datetime + :param ilm_policy: Deprecated - per-repo policies are now created automatically + :type ilm_policy: str + + :returns: Dictionary with mounted, skipped, failed counts, and created policies + :rtype: dict + """ + loggit = logging.getLogger("curator.actions.deepfreeze") + loggit.info( + "Finding and mounting indices between %s and %s", + start_date.isoformat(), + end_date.isoformat(), + ) + + mounted_indices = [] + skipped_indices = [] + failed_indices = [] + datastream_adds = {"successful": [], "failed": []} + created_policies = [] + + for repo in repos: + # Create per-repo thawed ILM policy + try: + thawed_policy = create_thawed_ilm_policy(client, repo.name) + created_policies.append(thawed_policy) + loggit.info("Using thawed ILM policy %s for repository %s", thawed_policy, repo.name) + except Exception as e: + loggit.error("Failed to create thawed ILM policy for %s: %s", repo.name, e) + # Continue anyway - indices will still mount, just without ILM policy + thawed_policy = None + try: + # Get all indices from snapshots in this repository + all_indices = get_all_indices_in_repo(client, repo.name) + loggit.debug("Found %d indices in repository %s", len(all_indices), repo.name) + + # For each index, check if it overlaps with the date range + for index_name in all_indices: + # Find the snapshot containing this index (use the latest one) + snapshots = find_snapshots_for_index(client, repo.name, index_name) + if not snapshots: + loggit.warning("No snapshots found for index %s", index_name) + continue + + # Use the most recent snapshot + snapshot_name = snapshots[-1] + + # Check if index is already mounted - if so, skip the mount call + already_mounted = client.indices.exists(index=index_name) + if already_mounted: + loggit.debug("Index %s is already mounted, skipping mount operation", index_name) + # Still assign ILM policy if provided + if thawed_policy and not mount_snapshot_index(client, repo.name, snapshot_name, index_name, thawed_policy): + loggit.warning("Failed to assign ILM policy to already-mounted index %s", index_name) + else: + # Mount the index temporarily to check its date range + if not mount_snapshot_index(client, repo.name, snapshot_name, index_name, thawed_policy): + failed_indices.append(index_name) + continue + + # Wait for index to become ready for queries + if not wait_for_index_ready(client, index_name): + loggit.warning("Index %s did not become ready in time, may have query issues", index_name) + + # Track if index should stay mounted (default: yes, we're conservative) + keep_mounted = True + + # Try to check date range to see if we should keep it + try: + index_start, index_end = get_timestamp_range(client, [index_name]) + + if index_start and index_end: + # We have timestamps, check if index overlaps with requested range + # Overlap occurs if: index_start <= end_date AND index_end >= start_date + index_start_dt = decode_date(index_start) + index_end_dt = decode_date(index_end) + + if index_start_dt <= end_date and index_end_dt >= start_date: + loggit.info( + "Index %s overlaps date range (%s to %s), keeping mounted", + index_name, + index_start_dt.isoformat(), + index_end_dt.isoformat(), + ) + else: + # No overlap, unmount the index + loggit.info( + "Index %s does not overlap date range (%s to %s), unmounting", + index_name, + index_start_dt.isoformat(), + index_end_dt.isoformat(), + ) + keep_mounted = False + try: + client.indices.delete(index=index_name) + loggit.debug("Unmounted index %s", index_name) + except Exception as e: + loggit.warning("Failed to unmount index %s: %s", index_name, e) + skipped_indices.append(index_name) + else: + # Could not get timestamps, keep mounted since we can't determine overlap + loggit.warning( + "Could not determine date range for %s, keeping mounted", + index_name + ) + + except Exception as e: + # Error during date range check, keep mounted to be safe + loggit.warning( + "Error checking date range for index %s: %s, keeping mounted", + index_name, + e + ) + + # For any index that's still mounted, add to list and check for data stream + if keep_mounted: + mounted_indices.append(index_name) + + # Check if this index was part of a data stream and reassign it + # This happens regardless of whether timestamp query succeeded + datastream_name = get_index_datastream_name(client, index_name) + if datastream_name: + loggit.info( + "Index %s was part of data stream %s, attempting to re-add", + index_name, + datastream_name, + ) + if add_index_to_datastream(client, datastream_name, index_name): + datastream_adds["successful"].append( + {"index": index_name, "datastream": datastream_name} + ) + else: + datastream_adds["failed"].append( + {"index": index_name, "datastream": datastream_name} + ) + else: + loggit.debug( + "Index %s is not a data stream backing index, skipping data stream step", + index_name, + ) + + except Exception as e: + loggit.error("Error processing repository %s: %s", repo.name, e) + + result = { + "mounted": len(mounted_indices), + "skipped": len(skipped_indices), + "failed": len(failed_indices), + "mounted_indices": mounted_indices, + "skipped_indices": skipped_indices, + "failed_indices": failed_indices, + "datastream_successful": len(datastream_adds["successful"]), + "datastream_failed": len(datastream_adds["failed"]), + "datastream_details": datastream_adds, + "created_policies": created_policies, + } + + loggit.info( + "Mounted %d indices, skipped %d outside date range, failed %d. Added %d to data streams.", + result["mounted"], + result["skipped"], + result["failed"], + result["datastream_successful"], + ) + + return result diff --git a/curator/cli_singletons/__init__.py b/curator/cli_singletons/__init__.py index 567f1229..93aef249 100644 --- a/curator/cli_singletons/__init__.py +++ b/curator/cli_singletons/__init__.py @@ -1,7 +1,9 @@ """Use __init__ to make these not need to be nested under lowercase.Capital""" + from curator.cli_singletons.alias import alias from curator.cli_singletons.allocation import allocation from curator.cli_singletons.close import close +from curator.cli_singletons.deepfreeze import deepfreeze, rotate, setup, status from curator.cli_singletons.delete import delete_indices, delete_snapshots from curator.cli_singletons.forcemerge import forcemerge from curator.cli_singletons.open_indices import open_indices diff --git a/curator/cli_singletons/deepfreeze.py b/curator/cli_singletons/deepfreeze.py new file mode 100644 index 00000000..3dbc852d --- /dev/null +++ b/curator/cli_singletons/deepfreeze.py @@ -0,0 +1,586 @@ +"""Deepfreeze Singleton""" + +import logging +from datetime import datetime + +import click + +from curator.cli_singletons.object_class import CLIAction + +today = datetime.today() + + +@click.group() +def deepfreeze(): + """ + Deepfreeze command group + """ + + +@deepfreeze.command() +@click.option( + "-y", + "--year", + type=int, + default=today.year, + show_default=True, + help="Year for the new repo. Only used if style=date.", +) +@click.option( + "-m", + "--month", + type=int, + default=today.month, + show_default=True, + help="Month for the new repo. Only used if style=date.", +) +@click.option( + "-r", + "--repo_name_prefix", + type=str, + default="deepfreeze", + show_default=True, + help="prefix for naming rotating repositories", +) +@click.option( + "-b", + "--bucket_name_prefix", + type=str, + default="deepfreeze", + show_default=True, + help="prefix for naming buckets", +) +@click.option( + "-d", + "--base_path_prefix", + type=str, + default="snapshots", + show_default=True, + help="base path in the bucket to use for searchable snapshots", +) +@click.option( + "-a", + "--canned_acl", + type=click.Choice( + [ + "private", + "public-read", + "public-read-write", + "authenticated-read", + "log-delivery-write", + "bucket-owner-read", + "bucket-owner-full-control", + ] + ), + default="private", + show_default=True, + help="Canned ACL as defined by AWS", +) +@click.option( + "-s", + "--storage_class", + type=click.Choice( + [ + "standard", + "reduced_redundancy", + "standard_ia", + "intelligent_tiering", + "onezone_ia", + ] + ), + default="standard", + show_default=True, + help="What storage class to use, as defined by AWS", +) +@click.option( + "-o", + "--provider", + type=click.Choice( + [ + "aws", + # "gcp", + # "azure", + ] + ), + default="aws", + help="What provider to use (AWS only for now)", +) +@click.option( + "-t", + "--rotate_by", + type=click.Choice( + [ + # "bucket", + "path", + ] + ), + default="path", + help="Rotate by path. This is the only option available for now", + # help="Rotate by bucket or path within a bucket?", +) +@click.option( + "-n", + "--style", + type=click.Choice( + [ + # "date", + "oneup", + ] + ), + default="oneup", + help="How to number (suffix) the rotating repositories. Oneup is the only option available for now.", + # help="How to number (suffix) the rotating repositories", +) +@click.option( + "-c", + "--create_sample_ilm_policy", + is_flag=True, + default=False, + show_default=True, + help="Create a sample ILM policy", +) +@click.option( + "-i", + "--ilm_policy_name", + type=str, + show_default=True, + default="deepfreeze-sample-policy", + help="Name of the sample ILM policy", +) +@click.option( + "-p", + "--porcelain", + is_flag=True, + default=False, + help="Machine-readable output (tab-separated values, no formatting)", +) +@click.pass_context +def setup( + ctx, + year, + month, + repo_name_prefix, + bucket_name_prefix, + base_path_prefix, + canned_acl, + storage_class, + provider, + rotate_by, + style, + create_sample_ilm_policy, + ilm_policy_name, + porcelain, +): + """ + Set up a cluster for deepfreeze and save the configuration for all future actions. + + Setup can be tuned by setting the following options to override defaults. Note that + --year and --month are only used if style=date. If style=oneup, then year and month + are ignored. + + Depending on the S3 provider chosen, some options might not be available, or option + values may vary. + """ + logging.debug("setup") + manual_options = { + "year": year, + "month": month, + "repo_name_prefix": repo_name_prefix, + "bucket_name_prefix": bucket_name_prefix, + "base_path_prefix": base_path_prefix, + "canned_acl": canned_acl, + "storage_class": storage_class, + "provider": provider, + "rotate_by": rotate_by, + "style": style, + "create_sample_ilm_policy": create_sample_ilm_policy, + "ilm_policy_name": ilm_policy_name, + "porcelain": porcelain, + } + + action = CLIAction( + ctx.info_name, + ctx.obj["configdict"], + manual_options, + [], + True, + ) + action.do_singleton_action(dry_run=ctx.obj["dry_run"]) + + +@deepfreeze.command() +@click.option( + "-y", + "--year", + type=int, + default=today.year, + help="Year for the new repo (default is today)", +) +@click.option( + "-m", + "--month", + type=int, + default=today.month, + help="Month for the new repo (default is today)", +) +@click.option( + "-k", + "--keep", + type=int, + default=6, + help="How many repositories should remain mounted?", +) +@click.pass_context +def rotate( + ctx, + year, + month, + keep, +): + """ + Deepfreeze rotation (add a new repo and age oldest off) + """ + manual_options = { + "year": year, + "month": month, + "keep": keep, + } + action = CLIAction( + ctx.info_name, + ctx.obj["configdict"], + manual_options, + [], + True, + ) + action.do_singleton_action(dry_run=ctx.obj["dry_run"]) + + +@deepfreeze.command() +@click.option( + "-l", + "--limit", + type=int, + default=None, + help="Limit display to the last N repositories (default: show all)", +) +@click.option( + "-r", + "--repos", + is_flag=True, + default=False, + help="Show repositories section only", +) +@click.option( + "-t", + "--thawed", + is_flag=True, + default=False, + help="Show thawed repositories section only", +) +@click.option( + "-b", + "--buckets", + is_flag=True, + default=False, + help="Show buckets section only", +) +@click.option( + "-i", + "--ilm", + is_flag=True, + default=False, + help="Show ILM policies section only", +) +@click.option( + "-c", + "--config", + is_flag=True, + default=False, + help="Show configuration section only", +) +@click.option( + "-p", + "--porcelain", + is_flag=True, + default=False, + help="Output plain text without formatting (suitable for scripting)", +) +@click.pass_context +def status( + ctx, + limit, + repos, + thawed, + buckets, + ilm, + config, + porcelain, +): + """ + Show the status of deepfreeze + + By default, all sections are displayed. Use section flags (-r, -t, -b, -i, -c) to show specific sections only. + Multiple section flags can be combined. + """ + manual_options = { + "limit": limit, + "show_repos": repos, + "show_thawed": thawed, + "show_buckets": buckets, + "show_ilm": ilm, + "show_config": config, + "porcelain": porcelain, + } + action = CLIAction( + ctx.info_name, + ctx.obj["configdict"], + manual_options, + [], + True, + ) + action.do_singleton_action(dry_run=ctx.obj["dry_run"]) + + +@deepfreeze.command() +@click.pass_context +def cleanup( + ctx, +): + """ + Clean up expired thawed repositories + """ + manual_options = {} + action = CLIAction( + ctx.info_name, + ctx.obj["configdict"], + manual_options, + [], + True, + ) + action.do_singleton_action(dry_run=ctx.obj["dry_run"]) + + +@deepfreeze.command() +@click.option( + "-t", + "--thaw-request-id", + "thaw_request_id", + type=str, + default=None, + help="The ID of the thaw request to refreeze (optional - if not provided, all open requests)", +) +@click.option( + "-p", + "--porcelain", + is_flag=True, + default=False, + help="Machine-readable output (tab-separated values, no formatting)", +) +@click.pass_context +def refreeze( + ctx, + thaw_request_id, + porcelain, +): + """ + Unmount repositories from thaw request(s) and reset them to frozen state. + + This is a user-initiated operation to signal "I'm done with this thaw." + It unmounts all repositories associated with the thaw request(s) and resets + their state back to frozen, even if the S3 restore hasn't expired yet. + + \b + Two modes of operation: + 1. Specific request: Provide -t to refreeze one request + 2. All open requests: Omit -t to refreeze all open requests (requires confirmation) + + \b + Examples: + + # Refreeze a specific thaw request + + curator_cli deepfreeze refreeze -t + + # Refreeze all open thaw requests (with confirmation) + + curator_cli deepfreeze refreeze + """ + manual_options = { + "thaw_request_id": thaw_request_id, + "porcelain": porcelain, + } + action = CLIAction( + ctx.info_name, + ctx.obj["configdict"], + manual_options, + [], + True, + ) + action.do_singleton_action(dry_run=ctx.obj["dry_run"]) + + +@deepfreeze.command() +@click.option( + "-s", + "--start-date", + type=str, + default=None, + help="Start of date range in ISO 8601 format (e.g., 2025-01-15T00:00:00Z)", +) +@click.option( + "-e", + "--end-date", + type=str, + default=None, + help="End of date range in ISO 8601 format (e.g., 2025-01-31T23:59:59Z)", +) +@click.option( + "--sync/--async", + "sync", + default=False, + show_default=True, + help="Wait for restore and mount (sync) or return immediately (async)", +) +@click.option( + "-d", + "--duration", + type=int, + default=30, + show_default=True, + help="Number of days to keep objects restored from Glacier", +) +@click.option( + "-t", + "--retrieval-tier", + type=click.Choice(["Standard", "Expedited", "Bulk"]), + default="Standard", + show_default=True, + help="AWS Glacier retrieval tier", +) +@click.option( + "--check-status", + "check_status", + type=str, + is_flag=False, + flag_value="", # Empty string when used without a value + default=None, + help="Check status of thaw request(s). Provide ID for specific request, or no value to check all", +) +@click.option( + "--list", + "list_requests", + is_flag=True, + default=False, + help="List all active thaw requests", +) +@click.option( + "-c", + "--include-completed", + "include_completed", + is_flag=True, + default=False, + help="Include completed requests when listing (default: exclude completed)", +) +@click.option( + "-p", + "--porcelain", + is_flag=True, + default=False, + help="Machine-readable output (tab-separated values, no formatting)", +) +@click.pass_context +def thaw( + ctx, + start_date, + end_date, + sync, + duration, + retrieval_tier, + check_status, + list_requests, + include_completed, + porcelain, +): + """ + Thaw repositories from Glacier storage for a specified date range, + or check status of existing thaw requests. + + \b + Four modes of operation: + 1. Create new thaw: Requires --start-date and --end-date + 2. Check specific request: Use --check-status (mounts if ready) + 3. Check all requests: Use --check-status (without value, mounts if ready) + 4. List requests: Use --list (shows summary table) + + \b + Examples: + + # Create new thaw request (async) + + curator_cli deepfreeze thaw -s 2025-01-01T00:00:00Z -e 2025-01-15T23:59:59Z --async + + # Create new thaw request (sync - waits for completion) + + curator_cli deepfreeze thaw -s 2025-01-01T00:00:00Z -e 2025-01-15T23:59:59Z --sync + + # Check status of a specific request and mount if ready + + curator_cli deepfreeze thaw --check-status + + # Check status of ALL thaw requests and mount if ready + + curator_cli deepfreeze thaw --check-status + + # List active thaw requests (excludes completed by default) + + curator_cli deepfreeze thaw --list + + # List all thaw requests (including completed) + + curator_cli deepfreeze thaw --list --include-completed + curator_cli deepfreeze thaw --list -c + """ + # Validate mutual exclusivity + # Note: check_status can be None (not provided), "" (flag without value), or a string ID + modes_active = sum( + [bool(start_date or end_date), check_status is not None, bool(list_requests)] + ) + + if modes_active == 0: + click.echo( + "Error: Must specify one of: --start-date/--end-date, --check-status, or --list" + ) + ctx.exit(1) + + if modes_active > 1: + click.echo( + "Error: Cannot use --start-date/--end-date with --check-status or --list" + ) + ctx.exit(1) + + # Validate that create mode has both start and end dates + if (start_date or end_date) and not (start_date and end_date): + click.echo( + "Error: Both --start-date and --end-date are required for creating a new thaw request" + ) + ctx.exit(1) + + manual_options = { + "start_date": start_date, + "end_date": end_date, + "sync": sync, + "duration": duration, + "retrieval_tier": retrieval_tier, + "check_status": check_status, + "list_requests": list_requests, + "include_completed": include_completed, + "porcelain": porcelain, + } + action = CLIAction( + ctx.info_name, + ctx.obj["configdict"], + manual_options, + [], + True, + ) + action.do_singleton_action(dry_run=ctx.obj["dry_run"]) diff --git a/curator/cli_singletons/object_class.py b/curator/cli_singletons/object_class.py index cb854250..e395282c 100644 --- a/curator/cli_singletons/object_class.py +++ b/curator/cli_singletons/object_class.py @@ -4,30 +4,38 @@ import typing as t import logging import sys -from voluptuous import Schema + from es_client.builder import Builder from es_client.exceptions import FailedValidation from es_client.helpers.schemacheck import SchemaCheck from es_client.helpers.utils import prune_nones +from voluptuous import Schema + from curator import IndexList, SnapshotList from curator.debug import debug from curator.actions import ( Alias, Allocation, + Cleanup, Close, ClusterRouting, CreateIndex, DeleteIndices, + DeleteSnapshots, ForceMerge, IndexSettings, Open, + Refreeze, Reindex, Replicas, + Restore, Rollover, + Rotate, + Setup, Shrink, Snapshot, - DeleteSnapshots, - Restore, + Status, + Thaw, ) from curator.defaults.settings import VERSION_MAX, VERSION_MIN, snapshot_actions from curator.exceptions import ConfigurationError, NoIndices, NoSnapshots @@ -38,29 +46,35 @@ logger = logging.getLogger(__name__) CLASS_MAP = { - 'alias': Alias, - 'allocation': Allocation, - 'close': Close, - 'cluster_routing': ClusterRouting, - 'create_index': CreateIndex, - 'delete_indices': DeleteIndices, - 'delete_snapshots': DeleteSnapshots, - 'forcemerge': ForceMerge, - 'index_settings': IndexSettings, - 'open': Open, - 'reindex': Reindex, - 'replicas': Replicas, - 'restore': Restore, - 'rollover': Rollover, - 'shrink': Shrink, - 'snapshot': Snapshot, + "alias": Alias, + "allocation": Allocation, + "cleanup": Cleanup, + "close": Close, + "cluster_routing": ClusterRouting, + "create_index": CreateIndex, + "delete_indices": DeleteIndices, + "delete_snapshots": DeleteSnapshots, + "forcemerge": ForceMerge, + "index_settings": IndexSettings, + "open": Open, + "refreeze": Refreeze, + "reindex": Reindex, + "replicas": Replicas, + "restore": Restore, + "rollover": Rollover, + "shrink": Shrink, + "snapshot": Snapshot, + "rotate": Rotate, + "setup": Setup, + "status": Status, + "thaw": Thaw, } EXCLUDED_OPTIONS = [ - 'ignore_empty_list', - 'timeout_override', - 'continue_if_exception', - 'disable_action', + "ignore_empty_list", + "timeout_override", + "continue_if_exception", + "disable_action", ] @@ -109,30 +123,30 @@ def __init__( self.include_system = self.options.pop('include_system', False) # Extract allow_ilm_indices so it can be handled separately. - if 'allow_ilm_indices' in self.options: - self.allow_ilm = self.options.pop('allow_ilm_indices') + if "allow_ilm_indices" in self.options: + self.allow_ilm = self.options.pop("allow_ilm_indices") else: self.allow_ilm = False if action == 'alias': debug.lv5('ACTION = ALIAS') self.alias = { - 'name': option_dict['name'], - 'extra_settings': option_dict['extra_settings'], - 'wini': ( - kwargs['warn_if_no_indices'] - if 'warn_if_no_indices' in kwargs + "name": option_dict["name"], + "extra_settings": option_dict["extra_settings"], + "wini": ( + kwargs["warn_if_no_indices"] + if "warn_if_no_indices" in kwargs else False ), } - for k in ['add', 'remove']: + for k in ["add", "remove"]: if k in kwargs: self.alias[k] = {} - self.check_filters(kwargs[k], loc='alias singleton', key=k) - self.alias[k]['filters'] = self.filters + self.check_filters(kwargs[k], loc="alias singleton", key=k) + self.alias[k]["filters"] = self.filters if self.allow_ilm: - self.alias[k]['filters'].append({'filtertype': 'ilm'}) + self.alias[k]["filters"].append({"filtertype": "ilm"}) # No filters for these actions - elif action in ['cluster_routing', 'create_index', 'rollover']: + elif action in ["cleanup", "cluster_routing", "create_index", "refreeze", "rollover", "setup", "rotate", "status", "thaw"]: self.action_kwargs = {} if action == 'rollover': debug.lv5('rollover option_dict = %s', option_dict) @@ -146,7 +160,7 @@ def __init__( # pylint: disable=broad-except except Exception as exc: raise ConfigurationError( - f'Unable to connect to Elasticsearch as configured: {exc}' + f"Unable to connect to Elasticsearch as configured: {exc}" ) from exc # If we're here, we'll see the output from GET http(s)://hostname.tld:PORT debug.lv5('Connection result: %s', builder.client.info()) @@ -168,24 +182,23 @@ def check_options(self, option_dict): debug.lv5('Validating provided options: %s', option_dict) # Kludgy work-around to needing 'repository' in options for these actions # but only to pass the schema check. It's removed again below. - if self.action in ['delete_snapshots', 'restore']: - option_dict['repository'] = self.repository + if self.action in ["delete_snapshots", "restore"]: + option_dict["repository"] = self.repository _ = SchemaCheck( prune_nones(option_dict), options.get_schema(self.action), - 'options', + "options", f'{self.action} singleton action "options"', ).result() self.options = self.prune_excluded(_) - # Remove this after the schema check, as the action class won't need - # it as an arg - if self.action in ['delete_snapshots', 'restore']: - del self.options['repository'] + # Remove this after the schema check, as the action class won't need it as an arg + if self.action in ["delete_snapshots", "restore"]: + del self.options["repository"] except FailedValidation as exc: logger.critical('Unable to parse options: %s', exc) sys.exit(1) - def check_filters(self, filter_dict, loc='singleton', key='filters'): + def check_filters(self, filter_dict, loc="singleton", key="filters"): """Validate provided filters""" try: debug.lv5('Validating provided filters: %s', filter_dict) @@ -210,10 +223,10 @@ def do_filters(self): ]: self.filters.append({'filtertype': 'ilm', 'exclude': True}) try: - self.list_object.iterate_filters({'filters': self.filters}) + self.list_object.iterate_filters({"filters": self.filters}) self.list_object.empty_list_check() except (NoIndices, NoSnapshots) as exc: - otype = 'index' if isinstance(exc, NoIndices) else 'snapshot' + otype = "index" if isinstance(exc, NoIndices) else "snapshot" if self.ignore: logger.info('Singleton action not performed: empty %s list', otype) sys.exit(0) @@ -237,13 +250,13 @@ def get_list_object(self) -> t.Union[IndexList, SnapshotList]: def get_alias_obj(self): """Get the Alias object""" action_obj = Alias( - name=self.alias['name'], extra_settings=self.alias['extra_settings'] + name=self.alias["name"], extra_settings=self.alias["extra_settings"] ) - for k in ['remove', 'add']: + for k in ["remove", "add"]: if k in self.alias: msg = ( f"{'Add' if k == 'add' else 'Remov'}ing matching indices " - f"{'to' if k == 'add' else 'from'} alias \"{self.alias['name']}\"" + f'{"to" if k == "add" else "from"} alias "{self.alias["name"]}"' ) debug.lv4(msg) self.alias[k]['ilo'] = IndexList( @@ -255,17 +268,23 @@ def get_alias_obj(self): {'filters': self.alias[k]['filters']} ) fltr = getattr(action_obj, k) - fltr(self.alias[k]['ilo'], warn_if_no_indices=self.alias['wini']) + fltr(self.alias[k]["ilo"], warn_if_no_indices=self.alias["wini"]) return action_obj def do_singleton_action(self, dry_run=False): """Execute the (ostensibly) completely ready to run action""" debug.lv3('Doing the singleton "%s" action here.', self.action) try: - if self.action == 'alias': + if self.action == "alias": action_obj = self.get_alias_obj() - elif self.action in ['cluster_routing', 'create_index', 'rollover']: + elif self.action in ["cluster_routing", "create_index", "rollover"]: + action_obj = self.action_class(self.client, **self.options) + elif self.action in ["cleanup", "refreeze", "setup", "rotate", "status", "thaw"]: + logger.debug( + f"Declaring Deepfreeze action object with options: {self.options}" + ) action_obj = self.action_class(self.client, **self.options) + logger.debug("Deepfreeze action object declared") else: self.get_list_object() self.do_filters() diff --git a/curator/defaults/option_defaults.py b/curator/defaults/option_defaults.py index 83b96231..78c7f2aa 100644 --- a/curator/defaults/option_defaults.py +++ b/curator/defaults/option_defaults.py @@ -1,5 +1,7 @@ """Action Option Schema definitions""" +from datetime import datetime + from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required # pylint: disable=E1120 @@ -39,10 +41,10 @@ def conditions(): Coerce(int), Optional('max_size'): Any(str)}} """ return { - Optional('conditions'): { - Optional('max_age'): Any(str), - Optional('max_docs'): Coerce(int), - Optional('max_size'): Any(str), + Optional("conditions"): { + Optional("max_age"): Any(str), + Optional("max_docs"): Coerce(int), + Optional("max_size"): Any(str), } } @@ -64,7 +66,7 @@ def count(): """ :returns: {Required('count'): All(Coerce(int), Range(min=0, max=10))} """ - return {Required('count'): All(Coerce(int), Range(min=0, max=10))} + return {Required("count"): All(Coerce(int), Range(min=0, max=10))} def delay(): @@ -209,7 +211,7 @@ def include_global_state(action): Any(bool, All(Any(str), Boolean()))} """ default = False - if action == 'snapshot': + if action == "snapshot": default = True return { Optional('include_global_state', default=default): Any( # type: ignore @@ -268,7 +270,7 @@ def index_settings(): """ :returns: {Required('index_settings'): {'index': dict}} """ - return {Required('index_settings'): {'index': dict}} + return {Required("index_settings"): {"index": dict}} def indices(): @@ -282,7 +284,7 @@ def key(): """ :returns: {Required('key'): Any(str)} """ - return {Required('key'): Any(str)} + return {Required("key"): Any(str)} def max_num_segments(): @@ -291,7 +293,7 @@ def max_num_segments(): {Required('max_num_segments'): All(Coerce(int), Range(min=1, max=32768))} """ - return {Required('max_num_segments'): All(Coerce(int), Range(min=1, max=32768))} + return {Required("max_num_segments"): All(Coerce(int), Range(min=1, max=32768))} # pylint: disable=unused-argument @@ -463,7 +465,7 @@ def remote_filters(): # validate_actions() method in utils.py return { Optional( - 'remote_filters', + "remote_filters", default=[ { 'filtertype': 'pattern', @@ -480,21 +482,21 @@ def rename_pattern(): """ :returns: {Optional('rename_pattern'): Any(str)} """ - return {Optional('rename_pattern'): Any(str)} + return {Optional("rename_pattern"): Any(str)} def rename_replacement(): """ :returns: {Optional('rename_replacement'): Any(str)} """ - return {Optional('rename_replacement'): Any(str)} + return {Optional("rename_replacement"): Any(str)} def repository(): """ :returns: {Required('repository'): Any(str)} """ - return {Required('repository'): Any(str)} + return {Required("repository"): Any(str)} def request_body(): @@ -503,34 +505,34 @@ def request_body(): See code for more details. """ return { - Required('request_body'): { - Optional('conflicts'): Any('proceed', 'abort'), - Optional('max_docs'): Coerce(int), - Required('source'): { - Required('index'): Any(Any(str), list), - Optional('query'): dict, - Optional('remote'): { - Optional('host'): Any(str), - Optional('username'): Any(str), - Optional('password'): Any(str), - Optional('socket_timeout'): Any(str), - Optional('connect_timeout'): Any(str), - Optional('headers'): Any(str), + Required("request_body"): { + Optional("conflicts"): Any("proceed", "abort"), + Optional("max_docs"): Coerce(int), + Required("source"): { + Required("index"): Any(Any(str), list), + Optional("query"): dict, + Optional("remote"): { + Optional("host"): Any(str), + Optional("username"): Any(str), + Optional("password"): Any(str), + Optional("socket_timeout"): Any(str), + Optional("connect_timeout"): Any(str), + Optional("headers"): Any(str), }, Optional('size'): Coerce(int), Optional('_source'): Any(bool, Boolean()), # type: ignore }, - Required('dest'): { - Required('index'): Any(str), - Optional('version_type'): Any( - 'internal', 'external', 'external_gt', 'external_gte' + Required("dest"): { + Required("index"): Any(str), + Optional("version_type"): Any( + "internal", "external", "external_gt", "external_gte" ), - Optional('op_type'): Any(str), - Optional('pipeline'): Any(str), + Optional("op_type"): Any(str), + Optional("pipeline"): Any(str), }, - Optional('script'): { - Optional('source'): Any(str), - Optional('lang'): Any('painless', 'expression', 'mustache', 'java'), + Optional("script"): { + Optional("source"): Any(str), + Optional("lang"): Any("painless", "expression", "mustache", "java"), }, } } @@ -568,14 +570,14 @@ def routing_type(): """ :returns: {Required('routing_type'): Any('allocation', 'rebalance')} """ - return {Required('routing_type'): Any('allocation', 'rebalance')} + return {Required("routing_type"): Any("allocation", "rebalance")} def cluster_routing_setting(): """ :returns: {Required('setting'): Any('enable')} """ - return {Required('setting'): Any('enable')} + return {Required("setting"): Any("enable")} def cluster_routing_value(): @@ -585,7 +587,7 @@ def cluster_routing_value(): Any('all', 'primaries', 'none', 'new_primaries', 'replicas')} """ return { - Required('value'): Any('all', 'primaries', 'none', 'new_primaries', 'replicas') + Required("value"): Any("all", "primaries", "none", "new_primaries", "replicas") } @@ -600,7 +602,7 @@ def shrink_node(): """ :returns: {Required('shrink_node'): Any(str)} """ - return {Required('shrink_node'): Any(str)} + return {Required("shrink_node"): Any(str)} def shrink_prefix(): @@ -664,11 +666,11 @@ def timeout_override(action): ``delete_snapshots`` = ``300`` """ - if action in ['forcemerge', 'restore', 'snapshot']: + if action in ["forcemerge", "restore", "snapshot"]: defval = 21600 - elif action == 'close': + elif action == "close": defval = 180 - elif action == 'delete_snapshots': + elif action == "delete_snapshots": defval = 300 else: defval = None @@ -691,7 +693,7 @@ def wait_for_active_shards(action): ``shrink`` actions. """ defval = 0 - if action in ['reindex', 'shrink']: + if action in ["reindex", "shrink"]: defval = 1 return { Optional('wait_for_active_shards', default=defval): Any( # type: ignore @@ -710,7 +712,7 @@ def wait_for_completion(action): """ # if action in ['cold2frozen', 'reindex', 'restore', 'snapshot']: defval = True - if action in ['allocation', 'cluster_routing', 'replicas']: + if action in ["allocation", "cluster_routing", "replicas"]: defval = False return { Optional('wait_for_completion', default=defval): Any( # type: ignore @@ -746,7 +748,7 @@ def wait_interval(action): maxval = 30 # if action in ['allocation', 'cluster_routing', 'replicas']: defval = 3 - if action in ['restore', 'snapshot', 'reindex', 'shrink']: + if action in ["restore", "snapshot", "reindex", "shrink"]: defval = 9 return { Optional('wait_interval', default=defval): Any( # type: ignore @@ -766,3 +768,240 @@ def warn_if_no_indices(): bool, All(Any(str), Boolean()) # type: ignore ) } + + +def create_sample_ilm_policy(): + """ + Setting to allow creating a sample ILM policy + """ + return { + Optional("create_sample_ilm_policy", default=False): Any( + bool, All(Any(str), Boolean()) + ) + } + + +def ilm_policy_name(): + """ + Setting to allow setting a custom ILM policy name + """ + return {Optional("ilm_policy_name", default="deepfreeze-sample-policy"): Any(str)} + + +def year(): + """ + Year for deepfreeze operations + """ + return {Optional("year", default=datetime.today().year): Coerce(int)} + + +def month(): + """ + Month for deepfreeze operations + """ + return {Optional("month", default=datetime.today().month): All(Coerce(int), Range(min=1, max=12))} + + +def repo_name_prefix(): + """ + Repository name prefix for deepfreeze + """ + return {Optional("repo_name_prefix", default="deepfreeze"): Any(str)} + + +def bucket_name_prefix(): + """ + Bucket name prefix for deepfreeze + """ + return {Optional("bucket_name_prefix", default="deepfreeze"): Any(str)} + + +def base_path_prefix(): + """ + Base path prefix for deepfreeze snapshots + """ + return {Optional("base_path_prefix", default="snapshots"): Any(str)} + + +def canned_acl(): + """ + Canned ACL for S3 objects + """ + return { + Optional("canned_acl", default="private"): Any( + "private", + "public-read", + "public-read-write", + "authenticated-read", + "log-delivery-write", + "bucket-owner-read", + "bucket-owner-full-control", + ) + } + + +def storage_class(): + """ + Storage class for S3 objects + """ + return { + Optional("storage_class", default="intelligent_tiering"): Any( + "standard", + "reduced_redundancy", + "standard_ia", + "intelligent_tiering", + "onezone_ia", + "GLACIER", # Also support uppercase for backwards compatibility + ) + } + + +def provider(): + """ + Cloud provider for deepfreeze + """ + return {Optional("provider", default="aws"): Any("aws")} + + +def rotate_by(): + """ + Rotation strategy for deepfreeze + """ + return {Optional("rotate_by", default="path"): Any("path", "bucket")} + + +def style(): + """ + Naming style for deepfreeze repositories + """ + return {Optional("style", default="oneup"): Any("oneup", "date", "monthly", "weekly")} + + +def keep(): + """ + Number of repositories to keep mounted + """ + return {Optional("keep", default=6): All(Coerce(int), Range(min=1, max=100))} + + +def start_date(): + """ + Start date for thaw operation (ISO 8601 format) + """ + return {Optional("start_date", default=None): Any(None, str)} + + +def end_date(): + """ + End date for thaw operation (ISO 8601 format) + """ + return {Optional("end_date", default=None): Any(None, str)} + + +def sync(): + """ + Sync mode for thaw - wait for restore and mount (True) or return immediately (False) + """ + return {Optional("sync", default=False): Any(bool, All(Any(str), Boolean()))} + + +def duration(): + """ + Number of days to keep objects restored from Glacier + """ + return {Optional("duration", default=7): All(Coerce(int), Range(min=1, max=30))} + + +def retrieval_tier(): + """ + AWS Glacier retrieval tier for thaw operation + """ + return { + Optional("retrieval_tier", default="Standard"): Any( + "Standard", "Expedited", "Bulk" + ) + } + + +def check_status(): + """ + Thaw request ID to check status + """ + return {Optional("check_status", default=None): Any(None, str)} + + +def list_requests(): + """ + Flag to list all thaw requests + """ + return {Optional("list_requests", default=False): Any(bool, All(Any(str), Boolean()))} + + +def limit(): + """ + Number of most recent repositories to display in status + """ + return {Optional("limit", default=None): Any(None, All(Coerce(int), Range(min=1, max=10000)))} + + +def show_repos(): + """ + Show repositories section in status output + """ + return {Optional("show_repos", default=False): Any(bool, All(Any(str), Boolean()))} + + +def show_thawed(): + """ + Show thawed repositories section in status output + """ + return {Optional("show_thawed", default=False): Any(bool, All(Any(str), Boolean()))} + + +def show_buckets(): + """ + Show buckets section in status output + """ + return {Optional("show_buckets", default=False): Any(bool, All(Any(str), Boolean()))} + + +def show_ilm(): + """ + Show ILM policies section in status output + """ + return {Optional("show_ilm", default=False): Any(bool, All(Any(str), Boolean()))} + + +def show_config(): + """ + Show configuration section in status output + """ + return {Optional("show_config", default=False): Any(bool, All(Any(str), Boolean()))} + + +def porcelain(): + """ + Output plain text without formatting (suitable for scripting) + """ + return {Optional("porcelain", default=False): Any(bool, All(Any(str), Boolean()))} + + +def repo_id(): + """ + Repository name/ID to refreeze (if not provided, all thawed repos will be refrozen) + """ + return {Optional("repo_id", default=None): Any(None, str)} + + +def thaw_request_id(): + """ + Thaw request ID to refreeze (if not provided, all open thaw requests will be refrozen) + """ + return {Optional("thaw_request_id", default=None): Any(None, str)} + + +def include_completed(): + """ + Include completed requests when listing thaw requests (default: exclude completed) + """ + return {Optional("include_completed", default=False): Any(bool, All(Any(str), Boolean()))} diff --git a/curator/s3client.py b/curator/s3client.py new file mode 100644 index 00000000..808d856f --- /dev/null +++ b/curator/s3client.py @@ -0,0 +1,624 @@ +""" +s3client.py + +import boto3 + +Encapsulate the S3 client here so it can be used by all Curator classes, not just +deepfreeze. +""" + +import abc +import logging + +import boto3 +from botocore.exceptions import ClientError + +from curator.exceptions import ActionError + +# from botocore.exceptions import ClientError + + +class S3Client(metaclass=abc.ABCMeta): + """ + Superclass for S3 Clients. + + This class should *only* perform actions that are common to all S3 clients. It + should not handle record-keeping or anything unrelated to S3 actions. The calling + methods should handle that. + """ + + @abc.abstractmethod + def create_bucket(self, bucket_name: str) -> None: + """ + Create a bucket with the given name. + + Args: + bucket_name (str): The name of the bucket to create. + + Returns: + None + """ + return + + @abc.abstractmethod + def test_connection(self) -> bool: + """ + Test S3 connection and validate credentials. + + :return: True if credentials are valid and S3 is accessible + :rtype: bool + """ + return + + @abc.abstractmethod + def bucket_exists(self, bucket_name: str) -> bool: + """ + Test whether or not the named bucket exists + + :param bucket_name: Bucket name to check + :type bucket_name: str + :return: Existence state of named bucket + :rtype: bool + """ + return + + @abc.abstractmethod + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[str], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Return a bucket from deepfreeze. + + Args: + bucket_name (str): The name of the bucket to return. + path (str): The path to the bucket to return. + object_keys (list[str]): A list of object keys to return. + restore_days (int): The number of days to keep the object restored. + retrieval_tier (str): The retrieval tier to use. + + Returns: + None + """ + return + + @abc.abstractmethod + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Return a bucket to deepfreeze. + + Args: + bucket_name (str): The name of the bucket to return. + path (str): The path to the bucket to return. + storage_class (str): The storage class to send the data to. + + """ + return + + @abc.abstractmethod + def list_objects(self, bucket_name: str, prefix: str) -> list[str]: + """ + List objects in a bucket with a given prefix. + + Args: + bucket_name (str): The name of the bucket to list objects from. + prefix (str): The prefix to use when listing objects. + + Returns: + list[str]: A list of object keys. + """ + return + + @abc.abstractmethod + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete a bucket with the given name. + + Args: + bucket_name (str): The name of the bucket to delete. + force (bool): If True, empty the bucket before deleting it. + + Returns: + None + """ + return + + @abc.abstractmethod + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Put an object in a bucket at the given path. + + Args: + bucket_name (str): The name of the bucket to put the object in. + key (str): The key of the object to put. + body (str): The body of the object to put. + + Returns: + None + """ + return + + @abc.abstractmethod + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all buckets. + + Returns: + list[str]: A list of bucket names. + """ + return + + @abc.abstractmethod + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for an object without downloading it. + + Args: + bucket_name (str): The name of the bucket. + key (str): The object key. + + Returns: + dict: Object metadata including Restore status if applicable. + """ + return + + @abc.abstractmethod + def copy_object( + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str, + ) -> None: + """ + Copy an object from one bucket to another. + + Args: + source_bucket (str): The name of the source bucket. + source_key (str): The key of the object to copy. + dest_bucket (str): The name of the destination bucket. + dest_key (str): The key for the copied object. + + Returns: + None + """ + return + + +class AwsS3Client(S3Client): + """ + An S3 client object for use with AWS. + """ + + def __init__(self) -> None: + self.loggit = logging.getLogger("AWS S3 Client") + try: + self.client = boto3.client("s3") + # HIGH PRIORITY FIX: Validate credentials by attempting a simple operation + self.loggit.debug("Validating AWS credentials") + self.client.list_buckets() + self.loggit.info("AWS S3 Client initialized successfully") + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + self.loggit.error("Failed to initialize AWS S3 Client: %s - %s", error_code, e) + if error_code in ["InvalidAccessKeyId", "SignatureDoesNotMatch"]: + raise ActionError( + "AWS credentials are invalid or not configured. " + "Check AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY." + ) + elif error_code == "AccessDenied": + raise ActionError( + "AWS credentials do not have sufficient permissions. " + "Minimum required: s3:ListAllMyBuckets" + ) + raise ActionError(f"Failed to initialize AWS S3 Client: {e}") + except Exception as e: + self.loggit.error("Failed to initialize AWS S3 Client: %s", e, exc_info=True) + raise ActionError(f"Failed to initialize AWS S3 Client: {e}") + + def test_connection(self) -> bool: + """ + Test S3 connection and validate credentials. + + :return: True if credentials are valid and S3 is accessible + :rtype: bool + """ + try: + self.loggit.debug("Testing S3 connection") + self.client.list_buckets() + return True + except ClientError as e: + self.loggit.error("S3 connection test failed: %s", e) + return False + + def create_bucket(self, bucket_name: str) -> None: + self.loggit.info(f"Creating bucket: {bucket_name}") + if self.bucket_exists(bucket_name): + self.loggit.info(f"Bucket {bucket_name} already exists") + raise ActionError(f"Bucket {bucket_name} already exists") + try: + # HIGH PRIORITY FIX: Add region handling for bucket creation + # Get the region from the client configuration + region = self.client.meta.region_name + self.loggit.debug(f"Creating bucket in region: {region}") + + # AWS requires LocationConstraint for all regions except us-east-1 + if region and region != 'us-east-1': + self.client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={'LocationConstraint': region} + ) + self.loggit.info(f"Successfully created bucket {bucket_name} in region {region}") + else: + self.client.create_bucket(Bucket=bucket_name) + self.loggit.info(f"Successfully created bucket {bucket_name} in us-east-1") + except ClientError as e: + error_code = e.response.get("Error", {}).get("Code", "Unknown") + self.loggit.error(f"Error creating bucket {bucket_name}: {error_code} - {e}") + raise ActionError(f"Error creating bucket {bucket_name}: {e}") + + def bucket_exists(self, bucket_name: str) -> bool: + self.loggit.debug(f"Checking if bucket {bucket_name} exists") + try: + self.client.head_bucket(Bucket=bucket_name) + self.loggit.debug(f"Bucket {bucket_name} exists") + return True + except ClientError as e: + if e.response["Error"]["Code"] == "404": + self.loggit.debug(f"Bucket {bucket_name} does not exist") + return False + else: + self.loggit.error("Error checking bucket existence for %s: %s", bucket_name, e) + raise ActionError(e) + + def thaw( + self, + bucket_name: str, + base_path: str, + object_keys: list[str], + restore_days: int = 7, + retrieval_tier: str = "Standard", + ) -> None: + """ + Restores objects from Glacier storage class back to an instant access tier. + + Args: + bucket_name (str): The name of the bucket + base_path (str): The base path (prefix) of the objects to thaw + object_keys (list[str]): A list of object keys to thaw + restore_days (int): The number of days to keep the object restored + retrieval_tier (str): The retrieval tier to use + + Returns: + None + """ + # ENHANCED LOGGING: Add parameter details and progress tracking + self.loggit.info( + "Starting thaw operation - bucket: %s, base_path: %s, objects: %d, restore_days: %d, tier: %s", + bucket_name, + base_path, + len(object_keys), + restore_days, + retrieval_tier + ) + + restored_count = 0 + skipped_count = 0 + error_count = 0 + + for idx, key in enumerate(object_keys, 1): + if not key.startswith(base_path): + skipped_count += 1 + continue # Skip objects outside the base path + + # ? Do we need to keep track of what tier this came from instead of just assuming Glacier? + try: + response = self.client.head_object(Bucket=bucket_name, Key=key) + storage_class = response.get("StorageClass", "") + + if storage_class in ["GLACIER", "DEEP_ARCHIVE", "GLACIER_IR"]: + self.loggit.debug( + "Restoring object %d/%d: %s from %s", + idx, + len(object_keys), + key, + storage_class + ) + self.client.restore_object( + Bucket=bucket_name, + Key=key, + RestoreRequest={ + "Days": restore_days, + "GlacierJobParameters": {"Tier": retrieval_tier}, + }, + ) + restored_count += 1 + else: + self.loggit.debug( + "Skipping object %d/%d: %s (storage class: %s, not in Glacier)", + idx, + len(object_keys), + key, + storage_class + ) + skipped_count += 1 + + except Exception as e: + error_count += 1 + self.loggit.error( + "Error restoring object %d/%d (%s): %s (type: %s)", + idx, + len(object_keys), + key, + str(e), + type(e).__name__ + ) + + # Log summary + self.loggit.info( + "Thaw operation completed - restored: %d, skipped: %d, errors: %d (total: %d)", + restored_count, + skipped_count, + error_count, + len(object_keys) + ) + + def refreeze( + self, bucket_name: str, path: str, storage_class: str = "GLACIER" + ) -> None: + """ + Moves objects back to a Glacier-tier storage class. + + Args: + bucket_name (str): The name of the bucket + path (str): The path to the objects to refreeze + storage_class (str): The storage class to move the objects to + + Returns: + None + """ + # ENHANCED LOGGING: Add parameter details and progress tracking + self.loggit.info( + "Starting refreeze operation - bucket: %s, path: %s, target_storage_class: %s", + bucket_name, + path, + storage_class + ) + + refrozen_count = 0 + error_count = 0 + + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=path) + + for page_num, page in enumerate(pages, 1): + if "Contents" in page: + page_objects = len(page["Contents"]) + self.loggit.debug("Processing page %d with %d objects", page_num, page_objects) + + for obj_num, obj in enumerate(page["Contents"], 1): + key = obj["Key"] + current_storage = obj.get("StorageClass", "STANDARD") + + try: + # Copy the object with a new storage class + self.loggit.debug( + "Refreezing object %d/%d in page %d: %s (from %s to %s)", + obj_num, + page_objects, + page_num, + key, + current_storage, + storage_class + ) + self.client.copy_object( + Bucket=bucket_name, + CopySource={"Bucket": bucket_name, "Key": key}, + Key=key, + StorageClass=storage_class, + ) + refrozen_count += 1 + + except Exception as e: + error_count += 1 + self.loggit.error( + "Error refreezing object %s: %s (type: %s)", + key, + str(e), + type(e).__name__, + exc_info=True + ) + + # Log summary + self.loggit.info( + "Refreeze operation completed - refrozen: %d, errors: %d", + refrozen_count, + error_count + ) + + def list_objects(self, bucket_name: str, prefix: str) -> list[str]: + """ + List objects in a bucket with a given prefix. + + Args: + bucket_name (str): The name of the bucket to list objects from. + prefix (str): The prefix to use when listing objects. + + Returns: + list[str]: A list of object keys. + """ + self.loggit.info( + f"Listing objects in bucket: {bucket_name} with prefix: {prefix}" + ) + paginator = self.client.get_paginator("list_objects_v2") + pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + objects = [] + + for page in pages: + if "Contents" in page: + for obj in page["Contents"]: + objects.append(obj) + + return objects + + def delete_bucket(self, bucket_name: str, force: bool = False) -> None: + """ + Delete a bucket with the given name. + + Args: + bucket_name (str): The name of the bucket to delete. + force (bool): If True, empty the bucket before deleting it. + + Returns: + None + """ + self.loggit.info(f"Deleting bucket: {bucket_name}") + try: + # If force=True, empty the bucket first + if force: + self.loggit.info(f"Emptying bucket {bucket_name} before deletion") + try: + # List and delete all objects + paginator = self.client.get_paginator('list_objects_v2') + pages = paginator.paginate(Bucket=bucket_name) + + for page in pages: + if 'Contents' in page: + objects = [{'Key': obj['Key']} for obj in page['Contents']] + if objects: + self.client.delete_objects( + Bucket=bucket_name, + Delete={'Objects': objects} + ) + self.loggit.debug(f"Deleted {len(objects)} objects from {bucket_name}") + except ClientError as e: + if e.response['Error']['Code'] != 'NoSuchBucket': + self.loggit.warning(f"Error emptying bucket {bucket_name}: {e}") + + self.client.delete_bucket(Bucket=bucket_name) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) + + def put_object(self, bucket_name: str, key: str, body: str = "") -> None: + """ + Put an object in a bucket. + + Args: + bucket_name (str): The name of the bucket to put the object in. + key (str): The key of the object to put. + body (str): The body of the object to put. + + Returns: + None + """ + self.loggit.info(f"Putting object: {key} in bucket: {bucket_name}") + try: + self.client.put_object(Bucket=bucket_name, Key=key, Body=body) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) + + def list_buckets(self, prefix: str = None) -> list[str]: + """ + List all buckets. + + Returns: + list[str]: A list of bucket names. + """ + self.loggit.info("Listing buckets") + try: + response = self.client.list_buckets() + buckets = response.get("Buckets", []) + bucket_names = [bucket["Name"] for bucket in buckets] + if prefix: + bucket_names = [ + name for name in bucket_names if name.startswith(prefix) + ] + return bucket_names + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) + + def head_object(self, bucket_name: str, key: str) -> dict: + """ + Retrieve metadata for an object without downloading it. + + Args: + bucket_name (str): The name of the bucket. + key (str): The object key. + + Returns: + dict: Object metadata including Restore status if applicable. + """ + self.loggit.debug(f"Getting metadata for s3://{bucket_name}/{key}") + try: + response = self.client.head_object(Bucket=bucket_name, Key=key) + return response + except ClientError as e: + self.loggit.error(f"Error getting metadata for {key}: {e}") + raise ActionError(f"Error getting metadata for {key}: {e}") + + def copy_object( + self, + Bucket: str, + Key: str, + CopySource: dict[str, str], + StorageClass: str = "GLACIER", + ) -> None: + """ + Copy an object from one bucket to another. + + Args: + Bucket (str): The name of the destination bucket. + Key (str): The key for the copied object. + CopySource (dict[str, str]): The source bucket and key. + StorageClass (str): The storage class to use. + + Returns: + None + """ + self.loggit.info(f"Copying object {Key} to bucket {Bucket}") + try: + self.client.copy_object( + Bucket=Bucket, + CopySource=CopySource, + Key=Key, + StorageClass=StorageClass, + ) + except ClientError as e: + self.loggit.error(e) + raise ActionError(e) + + +def s3_client_factory(provider: str) -> S3Client: + """ + s3_client_factory method, returns an S3Client object implemented specific to + the value of the provider argument. + + Args: + provider (str): The provider to use for the S3Client object. Should + reference an implemented provider (aws, gcp, azure, etc) + + Raises: + NotImplementedError: raised if the provider is not implemented + ValueError: raised if the provider string is invalid. + + Returns: + S3Client: An S3Client object specific to the provider argument. + """ + if provider == "aws": + return AwsS3Client() + elif provider == "gcp": + # Placeholder for GCP S3Client implementation + raise NotImplementedError("GCP S3Client is not implemented yet") + elif provider == "azure": + # Placeholder for Azure S3Client implementation + raise NotImplementedError("Azure S3Client is not implemented yet") + else: + raise ValueError(f"Unsupported provider: {provider}") diff --git a/curator/singletons.py b/curator/singletons.py index eb0053e3..124cd215 100644 --- a/curator/singletons.py +++ b/curator/singletons.py @@ -1,7 +1,11 @@ """CLI module for curator_cli""" +import warnings import click from es_client.defaults import SHOW_EVERYTHING + +# Suppress urllib3 InsecureRequestWarning when verify_certs is disabled +warnings.filterwarnings('ignore', message='Unverified HTTPS request') from es_client.helpers.config import ( cli_opts, context_settings, @@ -23,6 +27,7 @@ alias, allocation, close, + deepfreeze, delete_indices, delete_snapshots, forcemerge, @@ -103,6 +108,7 @@ def curator_cli( curator_cli.add_command(close) curator_cli.add_command(delete_indices) curator_cli.add_command(delete_snapshots) +curator_cli.add_command(deepfreeze) curator_cli.add_command(forcemerge) curator_cli.add_command(open_indices) curator_cli.add_command(replicas) diff --git a/curator/validators/options.py b/curator/validators/options.py index 3f83ded7..5688c7b7 100644 --- a/curator/validators/options.py +++ b/curator/validators/options.py @@ -1,6 +1,7 @@ """Set up voluptuous Schema defaults for various actions""" from voluptuous import Schema + from curator.defaults import option_defaults @@ -18,12 +19,12 @@ def action_specific(action): :rtype: list """ options = { - 'alias': [ + "alias": [ option_defaults.name(action), option_defaults.warn_if_no_indices(), option_defaults.extra_settings(), ], - 'allocation': [ + "allocation": [ option_defaults.search_pattern(), option_defaults.key(), option_defaults.value(), @@ -32,12 +33,12 @@ def action_specific(action): option_defaults.wait_interval(action), option_defaults.max_wait(action), ], - 'close': [ + "close": [ option_defaults.search_pattern(), option_defaults.delete_aliases(), option_defaults.skip_flush(), ], - 'cluster_routing': [ + "cluster_routing": [ option_defaults.routing_type(), option_defaults.cluster_routing_setting(), option_defaults.cluster_routing_value(), @@ -45,40 +46,86 @@ def action_specific(action): option_defaults.wait_interval(action), option_defaults.max_wait(action), ], - 'cold2frozen': [ + "cold2frozen": [ option_defaults.search_pattern(), option_defaults.c2f_index_settings(), option_defaults.c2f_ignore_index_settings(), - option_defaults.wait_for_completion('cold2frozen'), + option_defaults.wait_for_completion("cold2frozen"), ], - 'create_index': [ + "create_index": [ option_defaults.name(action), option_defaults.ignore_existing(), option_defaults.extra_settings(), ], + 'setup': [ + option_defaults.year(), + option_defaults.month(), + option_defaults.repo_name_prefix(), + option_defaults.bucket_name_prefix(), + option_defaults.base_path_prefix(), + option_defaults.canned_acl(), + option_defaults.storage_class(), + option_defaults.provider(), + option_defaults.rotate_by(), + option_defaults.style(), + option_defaults.create_sample_ilm_policy(), + option_defaults.ilm_policy_name(), + option_defaults.porcelain(), + ], + 'rotate': [ + option_defaults.keep(), + option_defaults.year(), + option_defaults.month(), + ], + 'cleanup': [ + ], + 'status': [ + option_defaults.limit(), + option_defaults.show_repos(), + option_defaults.show_thawed(), + option_defaults.show_buckets(), + option_defaults.show_ilm(), + option_defaults.show_config(), + option_defaults.porcelain(), + ], + 'thaw': [ + option_defaults.start_date(), + option_defaults.end_date(), + option_defaults.sync(), + option_defaults.duration(), + option_defaults.retrieval_tier(), + option_defaults.check_status(), + option_defaults.list_requests(), + option_defaults.include_completed(), + option_defaults.porcelain(), + ], + 'refreeze': [ + option_defaults.thaw_request_id(), + option_defaults.porcelain(), + ], 'delete_indices': [ option_defaults.search_pattern(), ], - 'delete_snapshots': [ + "delete_snapshots": [ option_defaults.repository(), option_defaults.retry_interval(), option_defaults.retry_count(), ], - 'forcemerge': [ + "forcemerge": [ option_defaults.search_pattern(), option_defaults.delay(), option_defaults.max_num_segments(), ], - 'index_settings': [ + "index_settings": [ option_defaults.search_pattern(), option_defaults.index_settings(), option_defaults.ignore_unavailable(), option_defaults.preserve_existing(), ], - 'open': [ + "open": [ option_defaults.search_pattern(), ], - 'reindex': [ + "reindex": [ option_defaults.request_body(), option_defaults.refresh(), option_defaults.requests_per_second(), @@ -95,21 +142,21 @@ def action_specific(action): option_defaults.migration_prefix(), option_defaults.migration_suffix(), ], - 'replicas': [ + "replicas": [ option_defaults.search_pattern(), option_defaults.count(), option_defaults.wait_for_completion(action), option_defaults.wait_interval(action), option_defaults.max_wait(action), ], - 'rollover': [ + "rollover": [ option_defaults.name(action), option_defaults.new_index(), option_defaults.conditions(), option_defaults.extra_settings(), option_defaults.wait_for_active_shards(action), ], - 'restore': [ + "restore": [ option_defaults.repository(), option_defaults.name(action), option_defaults.indices(), @@ -125,7 +172,7 @@ def action_specific(action): option_defaults.max_wait(action), option_defaults.skip_repo_fs_check(), ], - 'snapshot': [ + "snapshot": [ option_defaults.search_pattern(), option_defaults.repository(), option_defaults.name(action), @@ -137,7 +184,7 @@ def action_specific(action): option_defaults.max_wait(action), option_defaults.skip_repo_fs_check(), ], - 'shrink': [ + "shrink": [ option_defaults.search_pattern(), option_defaults.shrink_node(), option_defaults.node_filters(), diff --git a/docker_test/scripts/add_s3_credentials.sh b/docker_test/scripts/add_s3_credentials.sh new file mode 100755 index 00000000..78bcc92d --- /dev/null +++ b/docker_test/scripts/add_s3_credentials.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Prompt for S3 credentials (silent input for security) +read -sp "Enter S3 Access Key: " ACCESS_KEY +echo +read -sp "Enter S3 Secret Key: " SECRET_KEY +echo +read -p "Enter Elasticsearch version: " VERSION +echo + +# Get a list of running Elasticsearch container IDs +CONTAINERS=$(docker ps --filter "ancestor=curator_estest:${VERSION}" --format "{{.ID}}") + +if [ -z "$CONTAINERS" ]; then + echo "No running Elasticsearch containers found." + exit 1 +fi + +# Loop through each container and set the credentials +for CONTAINER in $CONTAINERS; do + echo "Setting credentials in container $CONTAINER..." + echo "$ACCESS_KEY" | docker exec -i "$CONTAINER" bin/elasticsearch-keystore add s3.client.default.access_key --stdin + echo "$SECRET_KEY" | docker exec -i "$CONTAINER" bin/elasticsearch-keystore add s3.client.default.secret_key --stdin + docker restart "$CONTAINER" + echo "Restarted container $CONTAINER." +done + +echo "S3 credentials have been set in all Elasticsearch containers." + +echo "Adding enterprise license" +if [[ -f license.json ]]; then + curl -X PUT "http://localhost:9200/_license" \ + -H "Content-Type: application/json" \ + -d @license-release-stack-enterprise.json +else + curl -X POST "http://localhost:9200/_license/start_trial?acknowledge=true" +fi diff --git a/fix_repo_dates.py b/fix_repo_dates.py new file mode 100644 index 00000000..f6df991d --- /dev/null +++ b/fix_repo_dates.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Fix incorrect date ranges for specific repositories""" + +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +from elasticsearch8 import Elasticsearch + +# Connect to Elasticsearch (adjust if needed) +client = Elasticsearch( + ["https://192.168.10.81:9200"], + verify_certs=False +) + +STATUS_INDEX = "deepfreeze-status" + +# Repositories to fix (set start=None, end=None to clear bad dates) +repos_to_fix = { + "deepfreeze-000093": {"start": None, "end": None}, +} + +for repo_name, new_dates in repos_to_fix.items(): + print(f"\nFixing {repo_name}...") + + # Find the repo document + query = {"query": {"term": {"name.keyword": repo_name}}} + try: + response = client.search(index=STATUS_INDEX, body=query) + + if response["hits"]["total"]["value"] == 0: + print(f" Repository {repo_name} not found in status index") + continue + + doc_id = response["hits"]["hits"][0]["_id"] + current_doc = response["hits"]["hits"][0]["_source"] + + print(f" Current dates: {current_doc.get('start')} to {current_doc.get('end')}") + + # Update with new dates + update_body = {"doc": new_dates} + client.update(index=STATUS_INDEX, id=doc_id, body=update_body) + + print(f" Updated to: {new_dates['start']} to {new_dates['end']}") + + except Exception as e: + print(f" Error: {e}") + +print("\nDone!") diff --git a/pyproject.toml b/pyproject.toml index c2858162..c7d4acb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,9 @@ keywords = [ 'index-expiry' ] dependencies = [ - "es_client==8.19.5" + "boto3", + "es_client==8.19.5", + "rich" ] [project.optional-dependencies] diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..5b14ed6c --- /dev/null +++ b/ruff.toml @@ -0,0 +1,77 @@ +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 88 +indent-width = 4 + +# Assume Python 3.9 +target-version = "py39" + +[lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" \ No newline at end of file diff --git a/run_singleton.py b/run_singleton.py index d8e99de3..06397471 100755 --- a/run_singleton.py +++ b/run_singleton.py @@ -17,8 +17,13 @@ Be sure to substitute your unicode variant for en_US.utf8 """ +import warnings import sys import click + +# Suppress urllib3 InsecureRequestWarning when verify_certs is disabled +warnings.filterwarnings('ignore', message='Unverified HTTPS request') + from curator.singletons import curator_cli if __name__ == '__main__': diff --git a/seed_data_to_ds.py b/seed_data_to_ds.py new file mode 100755 index 00000000..37a75e23 --- /dev/null +++ b/seed_data_to_ds.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import time +from datetime import datetime + +from elasticsearch import Elasticsearch, NotFoundError + +# Configuration +ES_HOST = "https://es-test.bwortman.us" # Change if needed +DATASTREAM_NAME = "test_datastream" +ES_USERNAME = "bret" +ES_PASSWORD = "2xqT2IO1OQ%tfMHP" + +# Initialize Elasticsearch client with authentication +es = Elasticsearch(ES_HOST, basic_auth=(ES_USERNAME, ES_PASSWORD)) + + +def create_index_template(es, alias_name): + """Creates an index template with a rollover alias.""" + template_body = { + "index_patterns": [f"{alias_name}-*"], + "settings": {"number_of_shards": 1, "number_of_replicas": 1}, + "aliases": {alias_name: {"is_write_index": True}}, + } + es.indices.put_template(name=alias_name, body=template_body) + + +def create_initial_index(es, alias_name): + """Creates the initial index for rollover if it doesn't exist.""" + first_index = f"{alias_name}-000001" + try: + if not es.indices.exists(index=first_index): + es.indices.create( + index=first_index, + body={"aliases": {alias_name: {"is_write_index": True}}}, + ) + except NotFoundError: + print(f"Index {first_index} not found, creating a new one.") + es.indices.create( + index=first_index, body={"aliases": {alias_name: {"is_write_index": True}}} + ) + + +# Ensure the index template and initial index exist +create_index_template(es, DATASTREAM_NAME) +create_initial_index(es, DATASTREAM_NAME) + +while True: + document = { + "timestamp": datetime.utcnow().isoformat(), + "message": "Hello, Elasticsearch!", + } + + es.index(index=DATASTREAM_NAME, document=document) + # print(f"Indexed document: {document}") + + # Perform rollover if conditions are met + try: + es.indices.rollover( + alias=DATASTREAM_NAME, body={"conditions": {"max_docs": 1000}} + ) + except NotFoundError: + print("Rollover failed: Alias not found. Ensure the initial index is created.") + + time.sleep(1) diff --git a/tests/integration/DEEPFREEZE_HOW_IT_WORKS.md b/tests/integration/DEEPFREEZE_HOW_IT_WORKS.md new file mode 100644 index 00000000..737f6970 --- /dev/null +++ b/tests/integration/DEEPFREEZE_HOW_IT_WORKS.md @@ -0,0 +1,552 @@ +# How Deepfreeze Works: A Complete Guide + +## Overview + +Deepfreeze is a system for archiving Elasticsearch data to AWS S3 Glacier using Elasticsearch's native **searchable snapshots** feature integrated with **Index Lifecycle Management (ILM)**. + +## Core Concept + +**Deepfreeze does NOT manage snapshots directly.** Instead, it manages: +1. **Elasticsearch snapshot repositories** (S3-backed) +2. **ILM policies** that control when indices become searchable snapshots +3. **Repository rotation** to move old snapshots to Glacier Deep Archive + +The actual snapshot creation and mounting is handled by **Elasticsearch ILM**. + +--- + +## The Complete Workflow + +### Phase 1: Initial Setup (`deepfreeze setup`) + +**What happens:** +1. Creates an S3 bucket (e.g., `my-bucket`) +2. Creates an Elasticsearch snapshot repository pointing to that bucket (e.g., `deepfreeze-000001`) +3. Saves configuration to a status index (`.deepfreeze-status-idx`) + +**Result:** +- You now have a repository that ILM policies can reference for searchable snapshots +- NO snapshots exist yet +- NO indices are frozen yet + +**Key Point:** Setup is a one-time operation. It creates the **first repository**. + +--- + +### Phase 2: ILM Manages Data (`elasticsearch` handles this) + +**User creates ILM policies** that reference the deepfreeze repository: + +```json +{ + "policy": { + "phases": { + "frozen": { + "min_age": "30m", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "backups", + "force_merge_index": true + } + } + }, + "delete": { + "min_age": "60m", + "actions": { + "delete": { + "delete_searchable_snapshot": false + } + } + }, + "cold": { + "min_age": "7m", + "actions": { + "allocate": { + "number_of_replicas": 0, + "include": {}, + "exclude": {}, + "require": {} + }, + "searchable_snapshot": { + "snapshot_repository": "backups", + "force_merge_index": true + }, + "set_priority": { + "priority": 0 + } + } + }, + "hot": { + "min_age": "0ms", + "actions": { + "forcemerge": { + "max_num_segments": 1 + }, + "rollover": { + "max_age": "3m", + "max_primary_shard_size": "40gb" + }, + "set_priority": { + "priority": 100 + }, + "shrink": { + "number_of_shards": 1, + "allow_write_after_shrink": false + } + } + } + } + } +} +``` + +**What Elasticsearch does automatically:** +1. **Hot phase**: Index is writable, stored on local disk with fast SSD access +2. **Rollover**: When index hits max_age/max_size, new index is created +3. **Cold phase**: Index transitions to cold tier (still on disk, but can be on slower/cheaper storage) + - Index remains fully searchable + - Data is on disk but may be moved to less expensive nodes + - The index name changes: `my-index-000001` → `restored-my-index-000001` +4. **Frozen phase**: Elasticsearch: + - Creates a snapshot in `deepfreeze-000001` repository + - Deletes the local index + - Mounts the snapshot as a **searchable snapshot** (read-only, backed by S3) + - The index name changes: `restored-my-index-000001` → `partial-restored-my-index-000001` +5. **Delete phase**: Elasticsearch: + - Deletes the mounted searchable snapshot index + - KEEPS the snapshot in S3 (because `delete_searchable_snapshot: false`) + +**Key Point:** Deepfreeze does NOT trigger snapshots. ILM does this automatically based on index age. + +--- + +### Phase 3: Repository Rotation (`deepfreeze rotate`) + +**Rotation happens periodically** (e.g., monthly, or on-demand) to: +1. Create a **new repository** (e.g., `deepfreeze-000002`) +2. Create a new, versioned ILM policy which uses the **new repository** for future snapshots +3. Unmount old repositories and push them to Glacier Deep Archive +4. Clean up old ILM policy versions + +**Step-by-step what happens:** + +#### 3.1: Create New Repository +```python +# Creates: deepfreeze-000002 +# With either: +# - New S3 bucket: my-bucket-000002 (if rotate_by=bucket) +# - New S3 path: my-bucket/snapshots-000002 (if rotate_by=path) +``` + +#### 3.2: Version ILM Policies + +**CRITICAL**: Deepfreeze does NOT modify existing policies. It creates **versioned copies**: + +``` +Old policy: my-ilm-policy-000001 → references deepfreeze-000001 +New policy: my-ilm-policy-000002 → references deepfreeze-000002 +``` + +This ensures: +- Old indices keep their old policies and can still access old snapshots +- New indices use new policies with the new repository +- No disruption to existing data +- Index template updates to point to latest versioned ILM policy + +#### 3.3: Update Index Templates + +All index templates are updated to use the new versioned policies: + +```yaml +# Before rotation: +template: logs-* + settings: + index.lifecycle.name: my-ilm-policy-000001 + +# After rotation: +template: logs-* + settings: + index.lifecycle.name: my-ilm-policy-000002 +``` + +**Result**: New indices created from this template will use the new policy. + +#### 3.4: Update Repository Date Ranges + +For each **mounted** repository, deepfreeze scans the searchable snapshot indices to determine: +- `earliest`: Timestamp of oldest document across all mounted indices +- `latest`: Timestamp of newest document across all mounted indices + +These are stored in the status index for tracking. + +#### 3.5: Unmount Old Repositories + +Based on the `keep` parameter (default: 6), deepfreeze: +1. Sorts repositories by version (newest first) +2. Keeps the first N repositories mounted +3. Unmounts older repositories: + - Deletes all searchable snapshot indices from that repo (e.g., `partial-my-index-*`) + - Deletes the Elasticsearch repository definition + - Marks the repository as "unmounted" in the status index + - The underlying S3 bucket/path still contains the snapshots + +#### 3.6: Push to Glacier Deep Archive + +For each unmounted repository: +```python +# Changes S3 storage class from Intelligent-Tiering to Glacier Deep Archive +push_to_glacier(s3_client, repository) +``` + +This reduces storage costs dramatically (S3 → Glacier Deep Archive = ~95% cost reduction). + +#### 3.7: Cleanup Old ILM Policies + +For each unmounted repository, deepfreeze: +1. Finds all ILM policies with the same version suffix (e.g., `-000001`) +2. Checks if they're still in use by any: + - Indices + - Data streams + - Index templates +3. Deletes policies that are no longer in use + +**Example**: +- Repository `deepfreeze-000001` is unmounted +- Policy `my-ilm-policy-000001` exists +- No indices use this policy +- No templates reference this policy +- → Policy is deleted + +--- + +## Storage Lifecycle Summary + +``` +1. Hot Index (local disk - hot tier): + - Writable + - Fast queries (SSD) + - Stored on ES hot tier data nodes + - Cost: High (fast SSD storage) + +2. Cold Index (local disk - cold tier): + - Read-only + - Good query performance + - Stored on ES cold tier data nodes (cheaper disks) + - Cost: Medium (standard disk storage) + +3. Frozen Index (searchable snapshot, S3): + - Read-only + - Slower queries (S3 latency) + - Stored in S3 (Intelligent-Tiering) + - Repository is "mounted" + - Cost: Low (S3) + +4. Archived Snapshot (Glacier Deep Archive): + - Not queryable + - Repository is "unmounted" + - Stored in Glacier Deep Archive + - Cost: Very low (~$1/TB/month) + - Retrieval time: 12-48 hours (if needed) +``` + +--- + +## Key Data Structures + +### 1. Status Index (`.deepfreeze-status-idx`) + +Stores two types of documents: + +**Settings Document** (`_id: deepfreeze-settings`): +```json +{ + "repo_name_prefix": "deepfreeze", + "bucket_name_prefix": "my-bucket", + "base_path_prefix": "snapshots", + "storage_class": "intelligent_tiering", + "rotate_by": "path", + "last_suffix": "000003", + "provider": "aws", + "style": "oneup" +} +``` + +**Repository Documents** (`_id: {repo_name}`): +```json +{ + "name": "deepfreeze-000002", + "bucket": "my-bucket", + "base_path": "/snapshots-000002", + "earliest": 1704067200000, // Unix timestamp + "latest": 1735689600000, // Unix timestamp + "is_thawed": false, + "is_mounted": true, + "indices": [ + "partial-logs-2024.01.01-000001", + "partial-logs-2024.01.02-000001" + ] +} +``` + +### 2. Repository Naming + +**Format**: `{prefix}-{suffix}` + +**Two styles:** +- **oneup** (default): `deepfreeze-000001`, `deepfreeze-000002`, etc. +- **date**: `deepfreeze-2024.01`, `deepfreeze-2024.02`, etc. + +### 3. ILM Policy Versioning + +**Pattern**: `{base_name}-{suffix}` + +Example progression: +``` +Setup: my-policy (created by user) +Rotate 1: my-policy-000001 (created by deepfreeze) +Rotate 2: my-policy-000002 (created by deepfreeze) +Rotate 3: my-policy-000003 (created by deepfreeze) +``` + +The original `my-policy` can be deleted after first rotation. + +--- + +## Critical Configuration Points + +### 1. ILM Delete Action + +**MUST set** `delete_searchable_snapshot: false`: + +```json +{ + "delete": { + "actions": { + "delete": { + "delete_searchable_snapshot": false // ← CRITICAL! + } + } + } +} +``` + +Without this, Elasticsearch will delete snapshots when indices are deleted, defeating the entire purpose of deepfreeze. + +### 2. Rotation Frequency + +Rotation should happen **BEFORE** repositories get too large: + +**Recommended**: Rotate every 30-90 days depending on: +- Snapshot size +- Number of searchable snapshot indices +- S3 transfer costs for Glacier transitions +- Only push to Glacier after the value of the data has decreased to the point that it's unlikely to be queried any longer. + +**Why**: Once a repository is pushed to Glacier, you cannot query those snapshots without restoring them first (12-48 hour delay). + +### 3. Keep Parameter + +**Default**: `keep=6` + +Keeps the 6 most recent repositories mounted (queryable). Older repositories are unmounted and pushed to Glacier. + +**Tuning**: +- **Higher keep**: More data queryable, higher S3 costs +- **Lower keep**: Less data queryable, lower costs, more in Glacier + +--- + +## Testing Workflow + +### Manual Testing Steps: + +1. **Setup** (once): + ```bash + curator_cli deepfreeze setup \ + --bucket-name my-test-bucket \ + --repo-name deepfreeze + ``` + +2. **Create ILM Policy** (once): + ```bash + curl -X PUT "localhost:9200/_ilm/policy/logs-policy" \ + -H 'Content-Type: application/json' \ + -d '{ + "policy": { + "phases": { + "frozen": { + "min_age": "30m", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "backups", + "force_merge_index": true + } + } + }, + "delete": { + "min_age": "60m", + "actions": { + "delete": { + "delete_searchable_snapshot": false + } + } + }, + "cold": { + "min_age": "7m", + "actions": { + "allocate": { + "number_of_replicas": 0, + "include": {}, + "exclude": {}, + "require": {} + }, + "searchable_snapshot": { + "snapshot_repository": "backups", + "force_merge_index": true + }, + "set_priority": { + "priority": 0 + } + } + }, + "hot": { + "min_age": "0ms", + "actions": { + "forcemerge": { + "max_num_segments": 1 + }, + "rollover": { + "max_age": "3m", + "max_primary_shard_size": "40gb" + }, + "set_priority": { + "priority": 100 + }, + "shrink": { + "number_of_shards": 1, + "allow_write_after_shrink": false + } + } + } + } + } +}' + ``` + +3. **Create Index Template** (once): + ```bash + curl -X PUT "localhost:9200/_index_template/logs-template" \ + -H 'Content-Type: application/json' \ + -d '{ + "index_patterns": ["logs-*"], + "template": { + "settings": { + "index.lifecycle.name": "logs-policy", + "index.lifecycle.rollover_alias": "logs" + } + } + }' + ``` + +4. **Create Initial Index** (once): + ```bash + curl -X PUT "localhost:9200/logs-2024.01.01-000001" \ + -H 'Content-Type: application/json' \ + -d '{ + "aliases": { + "logs": {"is_write_index": true} + } + }' + ``` + +5. **Index Data** (ongoing): + ```bash + curl -X POST "localhost:9200/logs/_doc" \ + -H 'Content-Type: application/json' \ + -d '{"message": "test log", "timestamp": "2024-01-01T00:00:00Z"}' + ``` + +6. **Wait for ILM** (automatic): + - After 1 day: Index rolls over + - After 7 days from creation: Index moves to cold phase + - After 30 days from creation: Index becomes frozen (searchable snapshot) + - After 365 days from creation: Index is deleted (snapshot remains) + +7. **Rotate** (periodic): + ```bash + curator_cli deepfreeze rotate --keep 6 + ``` + +--- + +## Common Misconceptions + +### ❌ "Deepfreeze creates snapshots" +**NO.** Elasticsearch ILM creates snapshots when indices reach the frozen phase. + +### ❌ "Rotate command snapshots data" +**NO.** Rotate creates a new repository, updates policies, and unmounts old repos. ILM handles snapshots. + +### ❌ "I need to run rotate after every snapshot" +**NO.** Rotate is periodic (monthly/quarterly). ILM creates snapshots automatically whenever indices age into frozen phase. + +### ❌ "Unmounted repos are deleted" +**NO.** Unmounted repos have their snapshots preserved in S3, just moved to Glacier Deep Archive for cheaper storage. + +### ❌ "Old ILM policies are modified" +**NO.** Old policies are left unchanged. New versioned policies are created. + +--- + +## Integration Test Requirements + +Given the above, integration tests should verify: + +1. **Setup**: + - Creates repository + - Creates status index + - Saves settings + +2. **ILM Integration** (NOT deepfreeze responsibility): + - Indices transition to frozen phase + - Snapshots are created + - Searchable snapshots are mounted + +3. **Rotate**: + - Creates new repository + - Creates versioned ILM policies + - Updates templates + - Updates repository date ranges + - Unmounts old repositories + - Pushes to Glacier + - Cleans up old policies + +4. **Status**: + - Reports current repositories + - Shows mounted vs unmounted + - Shows date ranges + +5. **Cleanup**: + - Removes thawed repositories after expiration + +--- + +## Timing Considerations for Tests + +**Real-world timing:** +- Rollover: 7 days +- Move to Cold: 7 days after creation +- Move to Frozen: 30 days after creation +- Delete: 365 days after creation +- Rotate: Monthly (30 days) + +**Test timing options:** +1. **Mock ILM**: Don't wait for real ILM, manually create searchable snapshots +2. **Fast ILM**: Set phases to seconds (hot=7s, cold=7s, frozen=30s, delete=45s) +3. **Hybrid**: Use fast ILM for lifecycle tests, mocks for rotate tests + +**Recommended for testing:** +- Use environment variable to control interval scaling +- All timing expressed as multiples of a base interval +- Default interval=1s for CI/CD, interval=60s for validation + diff --git a/tests/integration/DEEPFREEZE_THAW_TESTS.md b/tests/integration/DEEPFREEZE_THAW_TESTS.md new file mode 100644 index 00000000..4a1a4f28 --- /dev/null +++ b/tests/integration/DEEPFREEZE_THAW_TESTS.md @@ -0,0 +1,383 @@ +# Deepfreeze Thaw Integration Tests + +This document describes the integration tests for deepfreeze thaw operations. + +## Overview + +The thaw integration tests (`test_deepfreeze_thaw.py`) verify the complete lifecycle of thawing repositories from Glacier storage, including: + +1. Creating thaw requests with specific date ranges +2. Monitoring restore progress using porcelain output +3. Verifying indices are mounted correctly after restoration +4. Verifying data can be searched in mounted indices +5. Running cleanup operations +6. Verifying repositories are unmounted after cleanup + +## Test Modes + +These tests support two modes of operation: + +### Fast Mode (Development/CI) + +Fast mode uses mocked operations to complete quickly, suitable for CI/CD pipelines. + +```bash +DEEPFREEZE_FAST_MODE=1 pytest tests/integration/test_deepfreeze_thaw.py -v +``` + +**Duration**: ~5-10 minutes per test +**Use case**: Local development, CI/CD, quick verification + +**What's mocked:** +- Glacier restore operations (instant completion) +- S3 object restoration progress +- Time-based expiration (accelerated) + +### Full Test Mode (Production Validation) + +Full test mode runs against real AWS Glacier, taking up to 6 hours for complete restoration. + +```bash +DEEPFREEZE_FULL_TEST=1 pytest tests/integration/test_deepfreeze_thaw.py -v +``` + +**Duration**: Up to 6 hours per test (depending on AWS Glacier restore tier) +**Use case**: Pre-release validation, production readiness testing + +**Requirements:** +- Valid AWS credentials configured +- S3 bucket access +- Glacier restore permissions +- Elasticsearch instance with snapshot repository support + +## Test Suite + +### Test Cases + +#### 1. `test_thaw_single_repository` + +Tests thawing a single repository containing data for a specific date range. + +**What it tests:** +- Creating test indices with timestamped data +- Snapshotting indices to a repository +- Pushing repository to Glacier +- Creating a thaw request for a specific date range +- Monitoring restore progress using porcelain output +- Verifying correct indices are mounted +- Verifying data is searchable +- Refreezing the repository + +**Date Range:** January 2024 (single month) +**Expected Result:** 1 repository thawed and mounted + +#### 2. `test_thaw_multiple_repositories` + +Tests thawing multiple repositories spanning a date range. + +**What it tests:** +- Creating multiple repositories via rotation +- Creating test data across multiple time periods +- Pushing all repositories to Glacier +- Creating a thaw request spanning multiple repositories +- Verifying all relevant repositories are restored +- Verifying repositories outside the date range are NOT thawed +- Searching data across multiple thawed repositories + +**Date Range:** January-February 2024 (two months) +**Expected Result:** 2 repositories thawed, 1 repository remains frozen + +#### 3. `test_thaw_with_porcelain_output_parsing` + +Tests the porcelain output format and parsing logic. + +**What it tests:** +- Porcelain output format from thaw commands +- Parsing REQUEST and REPO lines +- Checking restore completion status +- Monitoring repository mount status +- Progress tracking (0/100, Complete, etc.) + +**Output Format:** +``` +REQUEST {request_id} {status} {created_at} {start_date} {end_date} +REPO {name} {bucket} {path} {state} {mounted} {progress} +``` + +#### 4. `test_cleanup_removes_expired_repositories` + +Tests automatic cleanup of expired thaw requests. + +**What it tests:** +- Creating a thaw request with short duration +- Manually expiring the request +- Running cleanup operation +- Verifying repositories are unmounted +- Verifying thaw state is reset to frozen +- Verifying thaw request is marked as completed + +**Duration:** 1 day (manually expired for testing) + +## Running the Tests + +### Prerequisites + +1. **Curator Configuration File** + + The tests use the configuration from `~/.curator/curator.yml` by default. + + Create the configuration file if it doesn't exist: + ```bash + mkdir -p ~/.curator + cat > ~/.curator/curator.yml < ~/.curator/curator.yml < 0: - # ElasticsearchWarning: this request accesses system indices: [.tasks], - # but in a future major version, direct access to system indices will be - # prevented by default - warnings.filterwarnings("ignore", category=ElasticsearchWarning) - self.client.indices.delete(index=','.join(indices)) - for path_arg in ['location', 'configdir']: + self.client.indices.delete(index=",".join(indices)) + for path_arg in ["location", "configdir"]: if os.path.exists(self.args[path_arg]): shutil.rmtree(self.args[path_arg]) @@ -162,13 +171,13 @@ def parse_args(self): def create_indices(self, count, unit=None, ilm_policy=None): now = datetime.now(timezone.utc) - unit = unit if unit else self.args['time_unit'] + unit = unit if unit else self.args["time_unit"] fmt = DATEMAP[unit] - if not unit == 'months': + if not unit == "months": step = timedelta(**{unit: 1}) for _ in range(count): self.create_index( - self.args['prefix'] + now.strftime(fmt), + self.args["prefix"] + now.strftime(fmt), wait_for_yellow=False, ilm_policy=ilm_policy, ) @@ -177,7 +186,7 @@ def create_indices(self, count, unit=None, ilm_policy=None): now = date.today() d = date(now.year, now.month, 1) self.create_index( - self.args['prefix'] + now.strftime(fmt), + self.args["prefix"] + now.strftime(fmt), wait_for_yellow=False, ilm_policy=ilm_policy, ) @@ -188,16 +197,16 @@ def create_indices(self, count, unit=None, ilm_policy=None): else: d = date(d.year, d.month - 1, 1) self.create_index( - self.args['prefix'] + datetime(d.year, d.month, 1).strftime(fmt), + self.args["prefix"] + datetime(d.year, d.month, 1).strftime(fmt), wait_for_yellow=False, ilm_policy=ilm_policy, ) # pylint: disable=E1123 - self.client.cluster.health(wait_for_status='yellow') + self.client.cluster.health(wait_for_status="yellow") def wfy(self): # pylint: disable=E1123 - self.client.cluster.health(wait_for_status='yellow') + self.client.cluster.health(wait_for_status="yellow") def create_index( self, @@ -207,13 +216,9 @@ def create_index( ilm_policy=None, wait_for_active_shards=1, ): - request_body = {'index': {'number_of_shards': shards, 'number_of_replicas': 0}} + request_body = {"index": {"number_of_shards": shards, "number_of_replicas": 0}} if ilm_policy is not None: - request_body['index']['lifecycle'] = {'name': ilm_policy} - # ElasticsearchWarning: index name [.shouldbehidden] starts with a dot '.', - # in the next major version, index names starting with a dot are reserved - # for hidden indices and system indices - warnings.filterwarnings("ignore", category=ElasticsearchWarning) + request_body["index"]["lifecycle"] = {"name": ilm_policy} self.client.indices.create( index=name, settings=request_body, @@ -224,7 +229,7 @@ def create_index( def add_docs(self, idx): for i in ["1", "2", "3"]: - self.client.create(index=idx, id=i, document={"doc" + i: 'TEST DOCUMENT'}) + self.client.create(index=idx, id=i, document={"doc" + i: "TEST DOCUMENT"}) # This should force each doc to be in its own segment. # pylint: disable=E1123 self.client.indices.flush(index=idx, force=True) @@ -233,7 +238,7 @@ def add_docs(self, idx): def create_snapshot(self, name, csv_indices): self.create_repository() self.client.snapshot.create( - repository=self.args['repository'], + repository=self.args["repository"], snapshot=name, ignore_unavailable=False, include_global_state=True, @@ -243,53 +248,48 @@ def create_snapshot(self, name, csv_indices): ) def delete_snapshot(self, name): - try: - self.client.snapshot.delete( - repository=self.args['repository'], snapshot=name - ) - except NotFoundError: - pass + self.client.snapshot.delete(repository=self.args["repository"], snapshot=name) def create_repository(self): - request_body = {'type': 'fs', 'settings': {'location': self.args['location']}} + request_body = {"type": "fs", "settings": {"location": self.args["location"]}} self.client.snapshot.create_repository( - name=self.args['repository'], body=request_body + name=self.args["repository"], body=request_body ) + def create_named_repository(self, repo_name): + request_body = {"type": "fs", "settings": {"location": self.args["location"]}} + self.client.snapshot.create_repository(name=repo_name, body=request_body) + def delete_repositories(self): - result = [] - try: - result = self.client.snapshot.get_repository(name='*') - except NotFoundError: - pass + result = self.client.snapshot.get_repository(name="*") for repo in result: try: - cleanup = self.client.snapshot.get(repository=repo, snapshot='*') + cleanup = self.client.snapshot.get(repository=repo, snapshot="*") # pylint: disable=broad-except except Exception: - cleanup = {'snapshots': []} - for listitem in cleanup['snapshots']: - self.delete_snapshot(listitem['snapshot']) + cleanup = {"snapshots": []} + for listitem in cleanup["snapshots"]: + self.delete_snapshot(listitem["snapshot"]) self.client.snapshot.delete_repository(name=repo) def close_index(self, name): self.client.indices.close(index=name) def write_config(self, fname, data): - with open(fname, 'w', encoding='utf-8') as fhandle: + with open(fname, "w", encoding="utf-8") as fhandle: fhandle.write(data) def get_runner_args(self): - self.write_config(self.args['configfile'], testvars.client_config.format(HOST)) - runner = os.path.join(os.getcwd(), 'run_singleton.py') + self.write_config(self.args["configfile"], testvars.client_config.format(HOST)) + runner = os.path.join(os.getcwd(), "run_singleton.py") return [sys.executable, runner] - def run_subprocess(self, args, logname='subprocess'): + def run_subprocess(self, args, logname="subprocess"): local_logger = logging.getLogger(logname) p = Popen(args, stderr=PIPE, stdout=PIPE) stdout, stderr = p.communicate() - local_logger.debug('STDOUT = %s', stdout.decode('utf-8')) - local_logger.debug('STDERR = %s', stderr.decode('utf-8')) + local_logger.debug("STDOUT = %s", stdout.decode("utf-8")) + local_logger.debug("STDERR = %s", stderr.decode("utf-8")) return p.returncode def invoke_runner(self, dry_run=False): @@ -312,7 +312,118 @@ def invoke_runner_alt(self, **kwargs): myargs = [] if kwargs: for key, value in kwargs.items(): - myargs.append(f'--{key}') + myargs.append(f"--{key}") myargs.append(value) - myargs.append(self.args['actionfile']) + myargs.append(self.args["actionfile"]) self.result = self.runner.invoke(cli, myargs) + + +class DeepfreezeTestCase(CuratorTestCase): + # TODO: Augment setup, tearDown methods to remove buckets + # TODO: Add helper methods from deepfreeze_helpers so they're part of the test case + + def setUp(self): + self.bucket_name = "" + return super().setUp() + + def tearDown(self): + s3 = s3_client_factory(self.provider) + buckets = s3.list_buckets(testvars.df_bucket_name) + for bucket in buckets: + # if bucket['Name'].startswith(testvars.df_bucket_name): + s3.delete_bucket(bucket_name=bucket) + return super().tearDown() + + def do_setup( + self, do_action=True, rotate_by: str = None, create_ilm_policy: bool = False + ) -> Setup: + s3 = s3_client_factory(self.provider) + + # Clean up any existing settings + try: + self.client.indices.delete(index=STATUS_INDEX) + except Exception: + pass + try: + self.client.snapshot.delete_repository( + name=f"{testvars.df_repo_name}-000001" + ) + except Exception: + pass + try: + self.client.snapshot.delete_repository(name=f"{testvars.df_repo_name}*") + except Exception: + pass + try: + s3 = s3_client_factory(self.provider) + s3.delete_bucket(self.bucket_name) + except Exception: + pass + # Clean up any existing ILM policy + try: + self.client.ilm.delete_lifecycle(name=testvars.df_ilm_policy) + except Exception: + pass + + if rotate_by: + testvars.df_rotate_by = rotate_by + + setup = Setup( + client, + bucket_name_prefix=self.bucket_name, + repo_name_prefix=testvars.df_repo_name, + base_path_prefix=testvars.df_base_path, + storage_class=testvars.df_storage_class, + rotate_by=testvars.df_rotate_by, + style=testvars.df_style, + create_sample_ilm_policy=create_ilm_policy, + ilm_policy_name=testvars.df_ilm_policy, + ) + if do_action: + setup.do_action() + time.sleep(INTERVAL) + return setup + + def do_rotate( + self, iterations: int = 1, keep: int = None, populate_index=False + ) -> Rotate: + rotate = None + for _ in range(iterations): + if keep: + rotate = Rotate( + client=self.client, + keep=keep, + ) + else: + rotate = Rotate( + client=self.client, + ) + rotate.do_action() + if populate_index: + # Alter this so it creates an index which the ILM policy will rotate + idx = f"{testvars.df_test_index}-{random_suffix()}" + self._populate_index(index=idx) + self.client.indices.put_settings( + index=idx, + body={"index": {"lifecycle": {"name": testvars.df_ilm_policy}}}, + ) + time.sleep(INTERVAL) + return rotate + + def _populate_index(self, index: str, doc_count: int = 1000) -> None: + # Sleep for a seocond every 100 docs to spread out the timestamps a bit + for i in range(doc_count): + if i % 100 == 0 and i != 0: + time.sleep(1) + for _ in range(doc_count): + self.client.index(index=index, body={"foo": "bar"}) + + def delete_ilm_policy(self, name): + try: + self.client.ilm.delete_lifecycle(name=name) + finally: + pass + + def get_settings(self): + doc = self.client.get(index=STATUS_INDEX, id=SETTINGS_ID) + return Settings(**doc["_source"]) diff --git a/tests/integration/run_thaw_tests.sh b/tests/integration/run_thaw_tests.sh new file mode 100644 index 00000000..953dde3e --- /dev/null +++ b/tests/integration/run_thaw_tests.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# Script to run deepfreeze thaw integration tests +# Usage: ./run_thaw_tests.sh [fast|full] [test_name] + +set -e + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +# Get the script directory +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$SCRIPT_DIR/../.." + +# Default values +MODE="${1:-fast}" +TEST_NAME="${2:-}" + +# Print usage +usage() { + echo "Usage: $0 [fast|full] [test_name]" + echo "" + echo "Modes:" + echo " fast - Run tests with mocked operations (5-10 minutes)" + echo " full - Run tests against real AWS Glacier (up to 6 hours)" + echo "" + echo "Examples:" + echo " $0 fast # Run all tests in fast mode" + echo " $0 fast test_thaw_single_repository # Run specific test in fast mode" + echo " $0 full # Run all tests against real Glacier" + echo "" + exit 1 +} + +# Check if help is requested +if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then + usage +fi + +# Validate mode +if [ "$MODE" != "fast" ] && [ "$MODE" != "full" ]; then + echo -e "${RED}Error: Invalid mode '$MODE'. Must be 'fast' or 'full'${NC}" + usage +fi + +# Check for curator configuration file +CURATOR_CONFIG="${CURATOR_CONFIG:-$HOME/.curator/curator.yml}" +echo -e "${YELLOW}Checking for curator configuration...${NC}" +if [ ! -f "$CURATOR_CONFIG" ]; then + echo -e "${RED}Error: Configuration file not found: $CURATOR_CONFIG${NC}" + echo "Create ~/.curator/curator.yml or set CURATOR_CONFIG environment variable" + exit 1 +fi +echo -e "${GREEN}✓ Configuration file found: $CURATOR_CONFIG${NC}" + +# Extract Elasticsearch host from config and check connection +echo -e "${YELLOW}Checking Elasticsearch connection from config...${NC}" +# Try to extract the host from the YAML config (simple grep approach) +ES_HOST=$(grep -A 5 "^elasticsearch:" "$CURATOR_CONFIG" | grep "hosts:" | sed 's/.*hosts: *//;s/[][]//g;s/,.*//;s/ //g' | head -1) +if [ -z "$ES_HOST" ]; then + echo -e "${YELLOW}Warning: Could not extract Elasticsearch host from config${NC}" + ES_HOST="http://127.0.0.1:9200" +fi + +if ! curl -s "$ES_HOST" > /dev/null 2>&1; then + echo -e "${RED}Error: Cannot connect to Elasticsearch at $ES_HOST${NC}" + echo "Check your configuration file: $CURATOR_CONFIG" + exit 1 +fi +echo -e "${GREEN}✓ Elasticsearch is running at $ES_HOST${NC}" + +# Check AWS credentials for full mode +if [ "$MODE" = "full" ]; then + echo -e "${YELLOW}Checking AWS credentials...${NC}" + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then + echo -e "${RED}Error: AWS credentials not found${NC}" + echo "For full test mode, set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY" + exit 1 + fi + echo -e "${GREEN}✓ AWS credentials found${NC}" + + echo -e "${YELLOW}WARNING: Full test mode will take up to 6 hours to complete!${NC}" + echo -e "${YELLOW}Press Ctrl+C within 5 seconds to cancel...${NC}" + sleep 5 +fi + +# Set environment variables based on mode +if [ "$MODE" = "fast" ]; then + export DEEPFREEZE_FAST_MODE=1 + echo -e "${GREEN}Running in FAST mode (mocked operations)${NC}" +else + export DEEPFREEZE_FULL_TEST=1 + echo -e "${YELLOW}Running in FULL TEST mode (real AWS Glacier)${NC}" +fi + +# Build test command +TEST_FILE="$SCRIPT_DIR/test_deepfreeze_thaw.py" +if [ -n "$TEST_NAME" ]; then + TEST_PATH="$TEST_FILE::TestDeepfreezeThaw::$TEST_NAME" + echo -e "${GREEN}Running test: $TEST_NAME${NC}" +else + TEST_PATH="$TEST_FILE" + echo -e "${GREEN}Running all thaw tests${NC}" +fi + +# Run tests +echo -e "${YELLOW}Starting tests...${NC}" +cd "$PROJECT_ROOT" + +# Run pytest with verbose output +if pytest "$TEST_PATH" -v -s --tb=short; then + echo -e "${GREEN}✓ All tests passed!${NC}" + exit 0 +else + echo -e "${RED}✗ Some tests failed${NC}" + exit 1 +fi diff --git a/tests/integration/test_deepfreeze_rotate.py b/tests/integration/test_deepfreeze_rotate.py new file mode 100644 index 00000000..8760bb92 --- /dev/null +++ b/tests/integration/test_deepfreeze_rotate.py @@ -0,0 +1,253 @@ +""" +Test deepfreeze setup functionality +""" + +# pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long +import os +import random +import warnings + +from curator.actions.deepfreeze import PROVIDERS +from curator.actions.deepfreeze.constants import STATUS_INDEX +from curator.actions.deepfreeze.exceptions import MissingIndexError +from curator.actions.deepfreeze.rotate import Rotate +from curator.actions.deepfreeze.utilities import get_all_repos, get_repository +from curator.exceptions import ActionError +from curator.s3client import s3_client_factory +from tests.integration import testvars + +from . import DeepfreezeTestCase, random_suffix + +HOST = os.environ.get("TEST_ES_SERVER", "http://127.0.0.1:9200") +MET = "metadata" + + +class TestDeepfreezeRotate(DeepfreezeTestCase): + def test_rotate_happy_path(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + setup = self.do_setup(create_ilm_policy=True) + prefix = setup.settings.repo_name_prefix + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + + # Specific assertions + # Settings index should exist + assert csi[STATUS_INDEX] + + # Assert that there is only one document in the STATUS_INDEX + status_index_docs = self.client.search(index=STATUS_INDEX, size=0) + assert status_index_docs["hits"]["total"]["value"] == 2 + rotate = Rotate( + self.client, + ) + assert len(rotate.repo_list) == 1 + assert rotate.repo_list == [f"{prefix}-000001"] + # Perform the first rotation + rotate.do_action() + # There should now be one repositories. + + # Save off the current repo list + orig_list = rotate.repo_list + # Do another rotation with keep=1 + rotate = Rotate( + self.client, + keep=1, + ) + rotate.do_action() + # There should now be two (one kept and one new) + assert len(rotate.repo_list) == 2 + assert rotate.repo_list == [f"{prefix}-000002", f"{prefix}-000001"] + # They should not be the same two as before + assert rotate.repo_list != orig_list + + # Save off the current repo list + orig_list = rotate.repo_list + # Do another rotation with keep=1 + rotate = Rotate( + self.client, + keep=1, + ) + rotate.do_action() + # There should now be two (one kept and one new) + assert len(rotate.repo_list) == 2 + assert rotate.repo_list == [f"{prefix}-000003", f"{prefix}-000002"] + # They should not be the same two as before + assert rotate.repo_list != orig_list + # Query the settings index to get the unmountd repos + unmounted = get_all_repos(self.client) + assert len(unmounted) == 1 + assert unmounted[0].name == f"{prefix}-000001" + + def test_rotate_with_data(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + setup = self.do_setup(create_ilm_policy=True) + prefix = setup.settings.repo_name_prefix + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + + # Specific assertions + # Settings index should exist + assert csi[STATUS_INDEX] + + # Assert that there is only one document in the STATUS_INDEX + status_index_docs = self.client.search(index=STATUS_INDEX, size=0) + assert status_index_docs["hits"]["total"]["value"] == 2 + rotate = self.do_rotate(populate_index=True) + # There should now be one repositories. + assert len(rotate.repo_list) == 1 + + # Save off the current repo list + orig_list = rotate.repo_list + # Do another rotation with keep=1 + rotate = self.do_rotate(populate_index=True) + # There should now be two (one kept and one new) + assert len(rotate.repo_list) == 2 + assert rotate.repo_list == [f"{prefix}-000002", f"{prefix}-000001"] + # They should not be the same two as before + assert rotate.repo_list != orig_list + + # Save off the current repo list + orig_list = rotate.repo_list + # Do another rotation with keep=1 + rotate = self.do_rotate(populate_index=True, keep=1) + # There should now be two (one kept and one new) + assert len(rotate.repo_list) == 3 + assert rotate.repo_list == [ + f"{prefix}-000003", + f"{prefix}-000002", + f"{prefix}-000001", + ] + # Query the settings index to get the unmounted repos + unmounted = get_all_repos(self.client) + assert len(unmounted) == 2 + assert f"{prefix}-000001" in [x.name for x in unmounted] + assert f"{prefix}-000002" in [x.name for x in unmounted] + repos = [get_repository(self.client, name=r) for r in rotate.repo_list] + assert len(repos) == 3 + for repo in repos: + if repo: + assert repo.earliest is not None + assert repo.latest is not None + assert repo.earliest < repo.latest + assert len(repo.indices) > 1 + else: + print(f"{repo} is None") + + def test_missing_status_index(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + setup = self.do_setup(create_ilm_policy=True) + prefix = setup.settings.repo_name_prefix + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + + # Specific assertions + # Settings index should exist + assert csi[STATUS_INDEX] + + # Assert that there is only one document in the STATUS_INDEX + status_index_docs = self.client.search(index=STATUS_INDEX, size=0) + assert status_index_docs["hits"]["total"]["value"] == 2 + + # Now, delete the status index completely + self.client.indices.delete(index=STATUS_INDEX) + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + assert STATUS_INDEX not in csi + + with self.assertRaises(MissingIndexError): + rotate = self.do_rotate(populate_index=True) + + def test_missing_repo(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + setup = self.do_setup(create_ilm_policy=True) + prefix = setup.settings.repo_name_prefix + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + + # Specific assertions + # Settings index should exist + assert csi[STATUS_INDEX] + + # Assert that there is only one document in the STATUS_INDEX + status_index_docs = self.client.search(index=STATUS_INDEX, size=0) + assert status_index_docs["hits"]["total"]["value"] == 2 + + rotate = self.do_rotate(6) + # There should now be one repositories. + assert len(rotate.repo_list) == 6 + + # Delete a random repo + repo_to_delete = rotate.repo_list[random.randint(0, 5)] + self.client.snapshot.delete_repository( + name=repo_to_delete, + ) + + # Do another rotation with keep=1 + rotate = self.do_rotate(populate_index=True) + # There should now be two (one kept and one new) + assert len(rotate.repo_list) == 6 + assert repo_to_delete not in rotate.repo_list + + def test_missing_bucket(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + setup = self.do_setup(create_ilm_policy=True) + prefix = setup.settings.repo_name_prefix + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + + # Specific assertions + # Settings index should exist + assert csi[STATUS_INDEX] + + # Assert that there is only one document in the STATUS_INDEX + status_index_docs = self.client.search(index=STATUS_INDEX, size=0) + assert status_index_docs["hits"]["total"]["value"] == 2 + + rotate = self.do_rotate(6, populate_index=True) + # There should now be one repositories. + assert len(rotate.repo_list) == 6 + + # Delete the bucket + s3 = s3_client_factory(self.provider) + s3.delete_bucket(setup.settings.bucket_name_prefix) + + # Do another rotation with keep=1 + with self.assertRaises(ActionError): + rotate = self.do_rotate(populate_index=True) + + # This indicates a Bad Thing, but I'm not sure what the correct response + # should be from a DF standpoint. diff --git a/tests/integration/test_deepfreeze_setup.py b/tests/integration/test_deepfreeze_setup.py new file mode 100644 index 00000000..1e133824 --- /dev/null +++ b/tests/integration/test_deepfreeze_setup.py @@ -0,0 +1,156 @@ +""" +Test deepfreeze setup functionality +""" + +# pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long +import os +import time +import warnings + +from curator.actions.deepfreeze import PROVIDERS, SETTINGS_ID, STATUS_INDEX, Setup +from curator.exceptions import ActionError, RepositoryException +from curator.s3client import s3_client_factory + +from . import DeepfreezeTestCase, random_suffix, testvars + +HOST = os.environ.get("TEST_ES_SERVER", "http://127.0.0.1:9200") +MET = "metadata" +INTERVAL = 1 # Because we can't go too fast or cloud providers can't keep up. + + +class TestDeepfreezeSetup(DeepfreezeTestCase): + def test_setup(self): + for provider in PROVIDERS: + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + self.do_setup() + csi = self.client.cluster.state(metric=MET)[MET]["indices"] + + # Specific assertions + # Settings index should exist + assert csi[STATUS_INDEX] + # Settings doc should exist within index + assert self.client.get(index=STATUS_INDEX, id=SETTINGS_ID) + # Settings index should only have settings doc (count == 1) + assert 1 == self.client.count(index=STATUS_INDEX)["count"] + # Repo should exist + assert self.client.snapshot.get_repository( + name=f"{testvars.df_repo_name}-000001" + ) + # Bucket should exist + s3 = s3_client_factory(provider) + assert s3.bucket_exists(self.bucket_name) + # We can't test the base path on AWS because it won't be created until the + # first object is written, but we can test the settings to see if it's correct + # there. + s = self.get_settings() + assert s.base_path_prefix == testvars.df_base_path + assert s.last_suffix == "000001" + assert s.canned_acl == testvars.df_acl + assert s.storage_class == testvars.df_storage_class + assert s.provider == "aws" + assert s.rotate_by == testvars.df_rotate_by + assert s.style == testvars.df_style + assert s.repo_name_prefix == testvars.df_repo_name + assert s.bucket_name_prefix == self.bucket_name + + # Clean up + self.client.snapshot.delete_repository( + name=f"{testvars.df_repo_name}-000001" + ) + + def test_setup_with_ilm(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + + self.do_setup(create_ilm_policy=True) + # ILM policy should exist + assert self.client.ilm.get_lifecycle(name=testvars.df_ilm_policy) + # We can't test the base path on AWS because it won't be created until the + # first object is written, but we can test the settings to see if it's correct + # there. + s = self.get_settings() + assert s.base_path_prefix == testvars.df_base_path + assert s.last_suffix == "000001" + assert s.canned_acl == testvars.df_acl + assert s.storage_class == testvars.df_storage_class + assert s.provider == "aws" + assert s.rotate_by == testvars.df_rotate_by + assert s.style == testvars.df_style + assert s.repo_name_prefix == testvars.df_repo_name + assert s.bucket_name_prefix == self.bucket_name + + def test_setup_bucket_exists(self): + for provider in PROVIDERS: + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + s3 = s3_client_factory(provider) + print(f"Pre-creating {provider} with {self.bucket_name}") + s3.create_bucket(f"{self.bucket_name}-000001") + time.sleep(INTERVAL) + # This should raise an ActionError because the bucket already exists + setup = self.do_setup(do_action=False, rotate_by="bucket") + s = setup.settings + print(f"Settings: {s}") + with self.assertRaises(ActionError): + setup.do_action() + + def test_setup_repo_exists(self): + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + for provider in PROVIDERS: + self.provider = provider + if self.bucket_name == "": + self.bucket_name = f"{testvars.df_bucket_name}-{random_suffix()}" + s3 = s3_client_factory(provider) + self.bucket_name_2 = f"{testvars.df_bucket_name_2}-{random_suffix()}" + + # Pre-create the bucket and repo to simulate picking a repo that already \ + # exists. We use a different bucket name to avoid the bucket already exists + # error. + s3.create_bucket(self.bucket_name_2) + time.sleep(INTERVAL) + self.client.snapshot.create_repository( + name=f"{testvars.df_repo_name}-000001", + body={ + "type": "s3", + "settings": { + "bucket": self.bucket_name_2, + "base_path": testvars.df_base_path_2, + "storage_class": testvars.df_storage_class, + }, + }, + ) + + with self.assertRaises(RepositoryException): + setup = Setup( + self.client, + bucket_name_prefix=self.bucket_name, + repo_name_prefix=testvars.df_repo_name, + base_path_prefix=testvars.df_base_path, + storage_class=testvars.df_storage_class, + rotate_by=testvars.df_rotate_by, + style=testvars.df_style, + ) + setup.do_action() + + # Clean up + self.client.snapshot.delete_repository( + name=f"{testvars.df_repo_name}-000001" + ) diff --git a/tests/integration/test_deepfreeze_thaw.py b/tests/integration/test_deepfreeze_thaw.py new file mode 100644 index 00000000..176cc9b5 --- /dev/null +++ b/tests/integration/test_deepfreeze_thaw.py @@ -0,0 +1,861 @@ +""" +Test deepfreeze thaw functionality + +These are long-running integration tests that test the complete thaw lifecycle: +1. Creating thaw requests +2. Monitoring restore progress using porcelain output +3. Verifying indices are mounted correctly +4. Verifying data can be searched +5. Cleaning up and verifying repositories are unmounted + +IMPORTANT: Real thaw operations can take up to 6 hours due to AWS Glacier restore times. +Set DEEPFREEZE_FAST_MODE=1 to use mocked/accelerated tests for CI. +Set DEEPFREEZE_FULL_TEST=1 to run full integration tests against real AWS Glacier. + +Configuration is loaded from ~/.curator/curator.yml by default. +Set CURATOR_CONFIG environment variable to use a different config file. +""" + +# pylint: disable=missing-function-docstring, missing-class-docstring, line-too-long +import os +import time +import warnings +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Tuple + +import pytest +from es_client.builder import Builder +from es_client.helpers.config import get_config + +from curator.actions.deepfreeze import STATUS_INDEX, Cleanup, Refreeze, Thaw +from curator.actions.deepfreeze.utilities import ( + get_repositories_by_names, + get_settings, + get_thaw_request, + list_thaw_requests, +) +from curator.defaults.settings import VERSION_MAX, VERSION_MIN, default_config_file +from curator.s3client import s3_client_factory + +from . import DeepfreezeTestCase, random_suffix, testvars + +# Configuration file path +CONFIG_FILE = os.environ.get("CURATOR_CONFIG", default_config_file()) +INTERVAL = 1 # Base interval for sleep operations + +# Test mode configuration +FAST_MODE = os.environ.get("DEEPFREEZE_FAST_MODE", "0") == "1" +FULL_TEST = os.environ.get("DEEPFREEZE_FULL_TEST", "0") == "1" + +# Skip long-running tests unless explicitly enabled +pytestmark = pytest.mark.skipif( + not FULL_TEST and not FAST_MODE, + reason="Thaw tests are long-running. Set DEEPFREEZE_FULL_TEST=1 or DEEPFREEZE_FAST_MODE=1 to run.", +) + + +class ThawStatusParser: + """Helper class to parse porcelain output from thaw commands""" + + @staticmethod + def parse_status_output(output: str) -> Dict: + """ + Parse porcelain output from thaw --check-status command. + + Expected format: + REQUEST {request_id} {status} {created_at} {start_date} {end_date} + REPO {name} {bucket} {path} {state} {mounted} {progress} + + :param output: Raw porcelain output string + :type output: str + :return: Parsed status information + :rtype: Dict + """ + result = {"request": None, "repos": []} + + for line in output.strip().split("\n"): + if not line.strip(): + continue + + parts = line.split("\t") + record_type = parts[0] + + if record_type == "REQUEST": + result["request"] = { + "id": parts[1], + "status": parts[2], + "created_at": parts[3], + "start_date": parts[4], + "end_date": parts[5], + } + elif record_type == "REPO": + result["repos"].append( + { + "name": parts[1], + "bucket": parts[2], + "path": parts[3], + "state": parts[4], + "mounted": parts[5] == "yes", + "progress": parts[6], + } + ) + + return result + + @staticmethod + def parse_list_output(output: str) -> List[Dict]: + """ + Parse porcelain output from thaw --list command. + + Expected format: + THAW_REQUEST {request_id} {status} {created_at} {start_date} {end_date} {repo_count} + + :param output: Raw porcelain output string + :type output: str + :return: List of thaw request information + :rtype: List[Dict] + """ + requests = [] + + for line in output.strip().split("\n"): + if not line.strip(): + continue + + parts = line.split("\t") + if parts[0] == "THAW_REQUEST": + requests.append( + { + "id": parts[1], + "status": parts[2], + "created_at": parts[3], + "start_date": parts[4], + "end_date": parts[5], + "repo_count": int(parts[6]), + } + ) + + return requests + + @staticmethod + def is_restore_complete(status_data: Dict) -> bool: + """ + Check if restoration is complete for all repositories. + + :param status_data: Parsed status data from parse_status_output + :type status_data: Dict + :return: True if all repos show "Complete" progress + :rtype: bool + """ + if not status_data.get("repos"): + return False + + return all(repo["progress"] == "Complete" for repo in status_data["repos"]) + + @staticmethod + def all_repos_mounted(status_data: Dict) -> bool: + """ + Check if all repositories are mounted. + + :param status_data: Parsed status data from parse_status_output + :type status_data: Dict + :return: True if all repos are mounted + :rtype: bool + """ + if not status_data.get("repos"): + return False + + return all(repo["mounted"] for repo in status_data["repos"]) + + +class TestDeepfreezeThaw(DeepfreezeTestCase): + """Test suite for deepfreeze thaw operations""" + + def setUp(self): + """Set up test environment""" + # Load configuration from curator.yml + if not os.path.exists(CONFIG_FILE): + pytest.skip(f"Configuration file not found: {CONFIG_FILE}") + + # Get configuration dictionary + try: + config = get_config(CONFIG_FILE) + configdict = config['elasticsearch'] + except Exception as e: + pytest.skip(f"Failed to load configuration from {CONFIG_FILE}: {e}") + + # Build client using configuration + try: + builder = Builder( + configdict=configdict, + version_max=VERSION_MAX, + version_min=VERSION_MIN, + ) + builder.connect() + self.client = builder.client + except Exception as e: + pytest.skip(f"Failed to connect to Elasticsearch using config from {CONFIG_FILE}: {e}") + + # Initialize logger + import logging + self.logger = logging.getLogger("TestDeepfreezeThaw") + + # Set provider and suppress warnings + self.provider = "aws" + warnings.filterwarnings( + "ignore", category=DeprecationWarning, module="botocore.auth" + ) + + # Initialize bucket name for cleanup + self.bucket_name = "" + + def tearDown(self): + """Clean up test resources""" + # Clean up S3 buckets + if self.bucket_name: + try: + s3 = s3_client_factory(self.provider) + buckets = s3.list_buckets(testvars.df_bucket_name) + for bucket in buckets: + s3.delete_bucket(bucket_name=bucket) + except Exception as e: + self.logger.warning(f"Failed to clean up buckets: {e}") + + # Clean up Elasticsearch resources + try: + # Delete status index + if self.client.indices.exists(index=STATUS_INDEX): + self.client.indices.delete(index=STATUS_INDEX) + + # Delete all test repositories + repos = self.client.snapshot.get_repository(name="*") + for repo in repos: + if repo.startswith(testvars.df_repo_name): + try: + self.client.snapshot.delete_repository(name=repo) + except Exception: + pass + + # Delete all test indices + indices = list( + self.client.indices.get( + index="test-logs-*,df-*", + expand_wildcards="open,closed", + ignore_unavailable=True + ).keys() + ) + if indices: + self.client.indices.delete(index=",".join(indices), ignore_unavailable=True) + + except Exception as e: + self.logger.warning(f"Failed to clean up Elasticsearch resources: {e}") + + def _setup_test_environment(self) -> Tuple[str, str]: + """ + Set up the test environment with repositories and test data. + + :return: Tuple of (bucket_name, repo_name_prefix) + :rtype: Tuple[str, str] + """ + # Generate unique test identifiers + self.bucket_name = f"{testvars.df_bucket_name}-thaw-{random_suffix()}" + + # Run deepfreeze setup + self.do_setup() + + repo_name = f"{testvars.df_repo_name}-000001" + + return self.bucket_name, repo_name + + def _create_test_indices_with_dates( + self, repo_name: str, date_ranges: List[Tuple[datetime, datetime]], docs_per_index: int = 100 + ) -> List[str]: + """ + Create test indices with specific date ranges and snapshot them. + + :param repo_name: The repository to snapshot to + :type repo_name: str + :param date_ranges: List of (start_date, end_date) tuples for each index + :type date_ranges: List[Tuple[datetime, datetime]] + :param docs_per_index: Number of documents to create per index + :type docs_per_index: int + :return: List of created index names + :rtype: List[str] + """ + created_indices = [] + + for i, (start_date, end_date) in enumerate(date_ranges): + # Create index name based on date range + index_name = f"test-logs-{start_date.strftime('%Y%m%d')}-{i:03d}" + + # Create the index + self.create_index(index_name) + + # Add documents with timestamps in the date range + doc_count = docs_per_index + time_delta = (end_date - start_date) / doc_count + + for j in range(doc_count): + doc_time = start_date + (time_delta * j) + self.client.index( + index=index_name, + document={ + "@timestamp": doc_time.isoformat(), + "message": f"Test document {j} for index {index_name}", + "test_id": f"{index_name}-{j}", + }, + ) + + # Refresh the index + self.client.indices.refresh(index=index_name) + + # Create a snapshot of this index + snapshot_name = f"snap-{index_name}" + self.client.snapshot.create( + repository=repo_name, + snapshot=snapshot_name, + body={ + "indices": index_name, + "include_global_state": False, + "partial": False, + }, + wait_for_completion=True, + ) + + created_indices.append(index_name) + + # Small delay to ensure snapshots are distinct + time.sleep(INTERVAL) + + return created_indices + + def _push_repo_to_glacier(self, repo_name: str): + """ + Push a repository to Glacier storage (simulated in fast mode). + + :param repo_name: The repository name to push to Glacier + :type repo_name: str + """ + # Get repository object + repos = get_repositories_by_names(self.client, [repo_name]) + if not repos: + raise ValueError(f"Repository {repo_name} not found") + + repo = repos[0] + + if FAST_MODE: + # In fast mode, just mark as unmounted + repo.is_mounted = False + repo.persist(self.client) + self.client.snapshot.delete_repository(name=repo_name) + else: + # In full mode, actually push to Glacier + from curator.actions.deepfreeze.utilities import push_to_glacier + + s3 = s3_client_factory(self.provider) + push_to_glacier(s3, repo) + repo.is_mounted = False + repo.persist(self.client) + self.client.snapshot.delete_repository(name=repo_name) + + def _wait_for_restore_completion( + self, thaw_request_id: str, timeout_seconds: int = 300, poll_interval: int = 10 + ) -> bool: + """ + Wait for thaw restore operation to complete using porcelain output. + + :param thaw_request_id: The thaw request ID to monitor + :type thaw_request_id: str + :param timeout_seconds: Maximum time to wait in seconds + :type timeout_seconds: int + :param poll_interval: Seconds between status checks + :type poll_interval: int + :return: True if restore completed, False if timeout + :rtype: bool + """ + start_time = time.time() + parser = ThawStatusParser() + + while (time.time() - start_time) < timeout_seconds: + # Create Thaw action to check status + thaw = Thaw( + self.client, + check_status=thaw_request_id, + porcelain=True, + ) + + # In fast mode, we simulate completion + if FAST_MODE: + # After first poll, mark as complete + request = get_thaw_request(self.client, thaw_request_id) + repo_names = request.get("repos", []) + repos = get_repositories_by_names(self.client, repo_names) + + # Mount all repositories + for repo in repos: + if not repo.is_mounted: + repo.is_mounted = True + repo.thaw_state = "active" + repo.persist(self.client) + + # Re-register the repository with Elasticsearch + self.client.snapshot.create_repository( + name=repo.name, + body={ + "type": "s3", + "settings": { + "bucket": repo.bucket, + "base_path": repo.base_path, + }, + }, + ) + + return True + + # In full mode, actually poll for status + # This would use the real porcelain output + # For now, we'll use the action's internal check + try: + thaw_action = Thaw( + self.client, + check_status=thaw_request_id, + porcelain=False, + ) + + # Check if all repos are mounted + request = get_thaw_request(self.client, thaw_request_id) + repo_names = request.get("repos", []) + repos = get_repositories_by_names(self.client, repo_names) + + if all(repo.is_mounted for repo in repos): + return True + + except Exception as e: + self.logger.warning(f"Error checking thaw status: {e}") + + time.sleep(poll_interval) + + return False + + def test_thaw_single_repository(self): + """ + Test thawing a single repository with a specific date range. + + This test: + 1. Sets up a repository with test data spanning multiple dates + 2. Pushes the repository to Glacier + 3. Creates a thaw request for a specific date range + 4. Monitors restore progress using porcelain output + 5. Verifies indices are mounted correctly + 6. Verifies data can be searched + """ + # Set up environment + bucket_name, repo_name = self._setup_test_environment() + + # Create test indices with specific date ranges + # We'll create 3 indices spanning January, February, March 2024 + now = datetime.now(timezone.utc) + date_ranges = [ + ( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 1, 31, tzinfo=timezone.utc), + ), + ( + datetime(2024, 2, 1, tzinfo=timezone.utc), + datetime(2024, 2, 28, tzinfo=timezone.utc), + ), + ( + datetime(2024, 3, 1, tzinfo=timezone.utc), + datetime(2024, 3, 31, tzinfo=timezone.utc), + ), + ] + + created_indices = self._create_test_indices_with_dates(repo_name, date_ranges) + self.logger.info(f"Created indices: {created_indices}") + + # Push repository to Glacier + self.logger.info(f"Pushing repository {repo_name} to Glacier") + self._push_repo_to_glacier(repo_name) + + # Wait a moment for the unmount to complete + time.sleep(INTERVAL * 2) + + # Create a thaw request for January data only + start_date = datetime(2024, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2024, 1, 31, 23, 59, 59, tzinfo=timezone.utc) + + self.logger.info( + f"Creating thaw request for date range: {start_date} to {end_date}" + ) + + thaw = Thaw( + self.client, + start_date=start_date.isoformat(), + end_date=end_date.isoformat(), + sync=False, # Async mode + duration=7, + retrieval_tier="Standard", + porcelain=True, + ) + + # Capture the thaw request ID + # In a real scenario, we'd parse porcelain output + # For now, we'll get it from the status index + thaw.do_action() + + # Get the thaw request ID + requests = list_thaw_requests(self.client) + assert len(requests) > 0, "No thaw requests found after thaw action" + thaw_request_id = requests[-1]["id"] + + self.logger.info(f"Created thaw request: {thaw_request_id}") + + # Wait for restore to complete (with timeout) + timeout = 300 if FAST_MODE else 21600 # 5 min for fast, 6 hours for full + restore_completed = self._wait_for_restore_completion( + thaw_request_id, timeout_seconds=timeout + ) + + assert restore_completed, "Restore did not complete within timeout period" + + # Verify indices are mounted + self.logger.info("Verifying mounted indices") + request = get_thaw_request(self.client, thaw_request_id) + repo_names = request.get("repos", []) + repos = get_repositories_by_names(self.client, repo_names) + + # Should have exactly one repository (January data) + assert len(repos) == 1, f"Expected 1 repository, got {len(repos)}" + assert repos[0].is_mounted, "Repository should be mounted" + + # Verify we can search the data + self.logger.info("Verifying data can be searched") + january_index = created_indices[0] # The January index + + # Try to search the index + search_result = self.client.search( + index=january_index, + body={"query": {"match_all": {}}, "size": 1}, + ) + + assert search_result["hits"]["total"]["value"] > 0, "No documents found in index" + + # Verify the document has correct timestamp + doc = search_result["hits"]["hits"][0]["_source"] + assert "@timestamp" in doc, "Document missing @timestamp field" + + doc_time = datetime.fromisoformat(doc["@timestamp"].replace("Z", "+00:00")) + assert start_date <= doc_time <= end_date, "Document timestamp outside expected range" + + # Refreeze the repository + self.logger.info("Refreezing repository") + refreeze = Refreeze(self.client, thaw_request_id=thaw_request_id, porcelain=True) + refreeze.do_action() + + # Verify repository is unmounted + time.sleep(INTERVAL * 2) + repos = get_repositories_by_names(self.client, [repos[0].name]) + assert not repos[0].is_mounted, "Repository should be unmounted after refreeze" + + def test_thaw_multiple_repositories(self): + """ + Test thawing multiple repositories spanning a date range. + + This test: + 1. Sets up multiple repositories with different date ranges + 2. Pushes all repositories to Glacier + 3. Creates a thaw request spanning multiple repositories + 4. Verifies all relevant repositories are restored and mounted + 5. Verifies indices outside the date range are NOT mounted + """ + # Set up initial environment + bucket_name, first_repo = self._setup_test_environment() + + # Create multiple repositories by rotating + # We'll create 3 repositories for Jan, Feb, Mar 2024 + from curator.actions.deepfreeze.rotate import Rotate + + repos_created = [first_repo] + + # Create additional repositories + for _ in range(2): + rotate = Rotate(self.client, keep=10) # Keep all repos mounted + rotate.do_action() + time.sleep(INTERVAL) + + # Get the latest repository + settings = get_settings(self.client) + last_suffix = settings.last_suffix + latest_repo = f"{testvars.df_repo_name}-{last_suffix}" + repos_created.append(latest_repo) + + self.logger.info(f"Created repositories: {repos_created}") + + # Create test data in each repository + all_indices = [] + date_ranges_per_repo = [ + [ + ( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 1, 31, tzinfo=timezone.utc), + ) + ], + [ + ( + datetime(2024, 2, 1, tzinfo=timezone.utc), + datetime(2024, 2, 28, tzinfo=timezone.utc), + ) + ], + [ + ( + datetime(2024, 3, 1, tzinfo=timezone.utc), + datetime(2024, 3, 31, tzinfo=timezone.utc), + ) + ], + ] + + for repo_name, date_ranges in zip(repos_created, date_ranges_per_repo): + indices = self._create_test_indices_with_dates( + repo_name, date_ranges, docs_per_index=50 + ) + all_indices.extend(indices) + + self.logger.info(f"Created total indices: {all_indices}") + + # Push all repositories to Glacier + for repo_name in repos_created: + self.logger.info(f"Pushing repository {repo_name} to Glacier") + self._push_repo_to_glacier(repo_name) + time.sleep(INTERVAL) + + # Wait for unmounting to complete + time.sleep(INTERVAL * 2) + + # Create a thaw request spanning January and February (2 repos) + start_date = datetime(2024, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2024, 2, 28, 23, 59, 59, tzinfo=timezone.utc) + + self.logger.info( + f"Creating thaw request for date range: {start_date} to {end_date}" + ) + + thaw = Thaw( + self.client, + start_date=start_date.isoformat(), + end_date=end_date.isoformat(), + sync=False, + duration=7, + retrieval_tier="Standard", + porcelain=True, + ) + + thaw.do_action() + + # Get the thaw request ID + requests = list_thaw_requests(self.client) + thaw_request_id = requests[-1]["id"] + + self.logger.info(f"Created thaw request: {thaw_request_id}") + + # Wait for restore to complete + timeout = 300 if FAST_MODE else 21600 + restore_completed = self._wait_for_restore_completion( + thaw_request_id, timeout_seconds=timeout + ) + + assert restore_completed, "Restore did not complete within timeout period" + + # Verify exactly 2 repositories are mounted (Jan and Feb) + request = get_thaw_request(self.client, thaw_request_id) + repo_names = request.get("repos", []) + repos = get_repositories_by_names(self.client, repo_names) + + assert len(repos) == 2, f"Expected 2 repositories, got {len(repos)}" + assert all(repo.is_mounted for repo in repos), "All repos should be mounted" + + # Verify the March repository is NOT in the thaw request + march_repo = repos_created[2] + assert march_repo not in repo_names, "March repository should not be in thaw request" + + # Verify we can search data in both January and February indices + for index_name in [all_indices[0], all_indices[1]]: + search_result = self.client.search( + index=index_name, body={"query": {"match_all": {}}, "size": 1} + ) + assert search_result["hits"]["total"]["value"] > 0, f"No documents found in {index_name}" + + # Cleanup - run refreeze + self.logger.info("Running cleanup") + cleanup = Cleanup(self.client) + cleanup.do_action() + + # Verify repositories are unmounted after cleanup + time.sleep(INTERVAL * 2) + repos_after = get_repositories_by_names(self.client, repo_names) + # Note: After cleanup, repos should be unmounted if they've expired + # In this test, they won't have expired yet, so they'll still be mounted + # This is expected behavior + + def test_thaw_with_porcelain_output_parsing(self): + """ + Test parsing porcelain output from thaw operations. + + This test focuses on the porcelain output format and parsing logic. + """ + # Set up environment + bucket_name, repo_name = self._setup_test_environment() + + # Create simple test data + date_ranges = [ + ( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 1, 31, tzinfo=timezone.utc), + ) + ] + created_indices = self._create_test_indices_with_dates( + repo_name, date_ranges, docs_per_index=10 + ) + + # Push to Glacier + self._push_repo_to_glacier(repo_name) + time.sleep(INTERVAL * 2) + + # Create thaw request + start_date = datetime(2024, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2024, 1, 31, tzinfo=timezone.utc) + + thaw = Thaw( + self.client, + start_date=start_date.isoformat(), + end_date=end_date.isoformat(), + sync=False, + duration=7, + retrieval_tier="Standard", + porcelain=True, + ) + + thaw.do_action() + + # Get the thaw request + requests = list_thaw_requests(self.client) + thaw_request_id = requests[-1]["id"] + + # Test porcelain output parsing + parser = ThawStatusParser() + + # Simulate porcelain output (in real scenario, we'd capture stdout) + sample_output = f"""REQUEST\t{thaw_request_id}\tin_progress\t2024-01-01T00:00:00Z\t2024-01-01T00:00:00Z\t2024-01-31T23:59:59Z +REPO\t{repo_name}\t{bucket_name}\t/df-test-path-000001\tthawing\tno\t0/100""" + + parsed = parser.parse_status_output(sample_output) + + # Verify parsed structure + assert parsed["request"] is not None, "Request data not parsed" + assert parsed["request"]["id"] == thaw_request_id, "Request ID mismatch" + assert len(parsed["repos"]) == 1, "Expected 1 repository in parsed output" + + repo_data = parsed["repos"][0] + assert repo_data["name"] == repo_name, "Repository name mismatch" + assert not repo_data["mounted"], "Repository should not be mounted yet" + assert not parser.is_restore_complete(parsed), "Restore should not be complete" + assert not parser.all_repos_mounted(parsed), "Repos should not be mounted" + + # Simulate completed output + completed_output = f"""REQUEST\t{thaw_request_id}\tin_progress\t2024-01-01T00:00:00Z\t2024-01-01T00:00:00Z\t2024-01-31T23:59:59Z +REPO\t{repo_name}\t{bucket_name}\t/df-test-path-000001\tactive\tyes\tComplete""" + + parsed_complete = parser.parse_status_output(completed_output) + + assert parser.is_restore_complete(parsed_complete), "Restore should be complete" + assert parser.all_repos_mounted(parsed_complete), "All repos should be mounted" + + def test_cleanup_removes_expired_repositories(self): + """ + Test that cleanup properly removes expired thawed repositories. + + This test: + 1. Creates a thaw request + 2. Manually sets the expiration to past + 3. Runs cleanup + 4. Verifies repositories are unmounted and marked as frozen + """ + # Set up environment + bucket_name, repo_name = self._setup_test_environment() + + # Create test data + date_ranges = [ + ( + datetime(2024, 1, 1, tzinfo=timezone.utc), + datetime(2024, 1, 31, tzinfo=timezone.utc), + ) + ] + self._create_test_indices_with_dates(repo_name, date_ranges, docs_per_index=10) + + # Push to Glacier + self._push_repo_to_glacier(repo_name) + time.sleep(INTERVAL * 2) + + # Create thaw request with short duration + start_date = datetime(2024, 1, 1, tzinfo=timezone.utc) + end_date = datetime(2024, 1, 31, tzinfo=timezone.utc) + + thaw = Thaw( + self.client, + start_date=start_date.isoformat(), + end_date=end_date.isoformat(), + sync=False, + duration=1, # 1 day duration + retrieval_tier="Standard", + porcelain=False, + ) + + thaw.do_action() + + # Wait for restore in fast mode + if FAST_MODE: + requests = list_thaw_requests(self.client) + thaw_request_id = requests[-1]["id"] + self._wait_for_restore_completion(thaw_request_id, timeout_seconds=60) + + # Manually expire the thaw request by updating its timestamp + requests = list_thaw_requests(self.client) + thaw_request_id = requests[-1]["id"] + + # Update the request to have an expiration in the past + past_time = datetime.now(timezone.utc) - timedelta(days=2) + self.client.update( + index=STATUS_INDEX, + id=thaw_request_id, + body={ + "doc": { + "created_at": past_time.isoformat(), + "expires_at": (past_time + timedelta(days=1)).isoformat(), + } + }, + ) + self.client.indices.refresh(index=STATUS_INDEX) + + # Get repository state before cleanup + request = get_thaw_request(self.client, thaw_request_id) + repo_names = request.get("repos", []) + + # Run cleanup + self.logger.info("Running cleanup on expired thaw request") + cleanup = Cleanup(self.client) + cleanup.do_action() + + time.sleep(INTERVAL * 2) + + # Verify repositories are unmounted + repos_after = get_repositories_by_names(self.client, repo_names) + for repo in repos_after: + assert not repo.is_mounted, f"Repository {repo.name} should be unmounted after cleanup" + assert repo.thaw_state == "frozen", f"Repository {repo.name} should be frozen after cleanup" + + # Verify the thaw request is marked as completed + request_after = get_thaw_request(self.client, thaw_request_id) + assert request_after["status"] == "completed", "Thaw request should be marked as completed" + + +if __name__ == "__main__": + # Allow running individual tests + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/integration/testvars.py b/tests/integration/testvars.py index 4359da58..200b05e0 100644 --- a/tests/integration/testvars.py +++ b/tests/integration/testvars.py @@ -1,7 +1,3 @@ -"""Test variables""" - -# pylint: disable=C0103, C0302 - client_config = ( '---\n' 'elasticsearch:\n' @@ -571,21 +567,6 @@ ' exclude: {1}\n' ) -filter_closed = ( - '---\n' - 'actions:\n' - ' 1:\n' - ' description: "Delete indices as filtered"\n' - ' action: delete_indices\n' - ' options:\n' - ' ignore_empty_list: True\n' - ' continue_if_exception: False\n' - ' disable_action: False\n' - ' filters:\n' - ' - filtertype: closed\n' - ' exclude: {0}\n' -) - bad_option_proto_test = ( '---\n' 'actions:\n' @@ -632,8 +613,7 @@ '---\n' 'actions:\n' ' 1:\n' - ' description: >-\n' - ' forceMerge segment count per shard to provided value with optional delay\n' + ' description: "forceMerge segment count per shard to provided value with optional delay"\n' ' action: forcemerge\n' ' options:\n' ' max_num_segments: {0}\n' @@ -1053,3 +1033,37 @@ ' stats_result: {7}\n' ' epoch: {8}\n' ) +df_ilm_policy = "df-test-ilm-policy" +df_ilm_body = { + "policy": { + "phases": { + "hot": { + "min_age": "0s", + "actions": {"rollover": {"max_size": "45gb", "max_age": "7s"}}, + }, + "frozen": { + "min_age": "7s", + "actions": { + "searchable_snapshot": {"snapshot_repository": "SNAPSHOT_REPO"} + }, + }, + "delete": { + "min_age": "30s", + "actions": {"delete": {"delete_searchable_snapshot": False}}, + }, + } + } +} +df_bucket_name = "df" +df_bucket_name_2 = "df-test" +df_repo_name = "df-test-repo" +df_providers = ["aws", "gcp", "azure"] +df_base_path = "/df-test-path" +df_base_path_2 = "/df-another-test-path" +df_acl = "private" +df_storage_class = "Standard" +df_rotate_by = "path" +df_style = "oneup" +df_month = "05" +df_year = "2024" +df_test_index = "df-test-idx" diff --git a/tests/unit/test_action_deepfreeze_helpers.py b/tests/unit/test_action_deepfreeze_helpers.py new file mode 100644 index 00000000..94c2d34f --- /dev/null +++ b/tests/unit/test_action_deepfreeze_helpers.py @@ -0,0 +1,335 @@ +"""Test deepfreeze helpers module""" +# pylint: disable=attribute-defined-outside-init +from unittest import TestCase +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime +import json +import pytest + +from curator.actions.deepfreeze.helpers import Deepfreeze, Repository, Settings +from curator.actions.deepfreeze.constants import STATUS_INDEX, SETTINGS_ID + + +class TestDeepfreeze(TestCase): + """Test Deepfreeze class""" + + def test_deepfreeze_init(self): + """Test Deepfreeze class initialization""" + df = Deepfreeze() + assert isinstance(df, Deepfreeze) + + +class TestRepository(TestCase): + """Test Repository dataclass""" + + def test_repository_init_with_all_params(self): + """Test Repository initialization with all parameters""" + start = datetime(2024, 1, 1) + end = datetime(2024, 12, 31) + + repo = Repository( + name="test-repo", + bucket="test-bucket", + base_path="/path/to/repo", + start=start, + end=end, + is_thawed=True, + is_mounted=False, + doctype="repository", + docid="repo-id-123" + ) + + assert repo.name == "test-repo" + assert repo.bucket == "test-bucket" + assert repo.base_path == "/path/to/repo" + assert repo.start == start + assert repo.end == end + assert repo.is_thawed is True + assert repo.is_mounted is False + assert repo.doctype == "repository" + assert repo.docid == "repo-id-123" + + def test_repository_init_with_defaults(self): + """Test Repository initialization with default values""" + repo = Repository(name="test-repo") + + assert repo.name == "test-repo" + assert repo.bucket is None + assert repo.base_path is None + assert repo.start is None + assert repo.end is None + assert repo.is_thawed is False + assert repo.is_mounted is True + assert repo.doctype == "repository" + assert repo.docid is None + + def test_repository_from_elasticsearch_success(self): + """Test Repository.from_elasticsearch successful retrieval""" + mock_client = Mock() + mock_response = { + 'hits': { + 'hits': [{ + '_id': 'repo-id-123', + '_source': { + 'name': 'test-repo', + 'bucket': 'test-bucket', + 'base_path': '/path/to/repo', + 'start': '2024-01-01T00:00:00', + 'end': '2024-12-31T23:59:59', + 'is_thawed': True, + 'is_mounted': False, + 'doctype': 'repository' + } + }] + } + } + mock_client.search.return_value = mock_response + + with patch('curator.actions.deepfreeze.helpers.logging'): + repo = Repository.from_elasticsearch(mock_client, 'test-repo') + + assert repo is not None + assert repo.name == 'test-repo' + assert repo.bucket == 'test-bucket' + assert repo.base_path == '/path/to/repo' + assert repo.docid == 'repo-id-123' + + mock_client.search.assert_called_once_with( + index=STATUS_INDEX, + query={"match": {"name.keyword": "test-repo"}}, + size=1 + ) + + def test_repository_from_elasticsearch_not_found(self): + """Test Repository.from_elasticsearch when repository not found""" + mock_client = Mock() + mock_response = { + 'hits': { + 'hits': [] + } + } + mock_client.search.return_value = mock_response + + with patch('curator.actions.deepfreeze.helpers.logging'): + repo = Repository.from_elasticsearch(mock_client, 'nonexistent-repo') + + assert repo is None + + def test_repository_from_elasticsearch_with_custom_index(self): + """Test Repository.from_elasticsearch with custom index""" + mock_client = Mock() + mock_response = { + 'hits': { + 'hits': [{ + '_id': 'repo-id', + '_source': { + 'name': 'test-repo', + 'doctype': 'repository' + } + }] + } + } + mock_client.search.return_value = mock_response + + with patch('curator.actions.deepfreeze.helpers.logging'): + repo = Repository.from_elasticsearch( + mock_client, + 'test-repo', + index='custom-index' + ) + + mock_client.search.assert_called_once_with( + index='custom-index', + query={"match": {"name.keyword": "test-repo"}}, + size=1 + ) + + def test_repository_to_dict(self): + """Test Repository.to_dict method""" + repo = Repository( + name="test-repo", + bucket="test-bucket", + base_path="/path/to/repo", + start="2024-01-01", + end="2024-12-31", + is_thawed=True, + is_mounted=False, + doctype="repository" + ) + + result = repo.to_dict() + + assert isinstance(result, dict) + assert result['name'] == "test-repo" + assert result['bucket'] == "test-bucket" + assert result['base_path'] == "/path/to/repo" + assert result['is_thawed'] is True + assert result['is_mounted'] is False + assert result['doctype'] == "repository" + assert result['start'] == "2024-01-01" + assert result['end'] == "2024-12-31" + + def test_repository_to_dict_with_none_dates(self): + """Test Repository.to_dict with None dates""" + repo = Repository( + name="test-repo", + start=None, + end=None + ) + + result = repo.to_dict() + + assert result['start'] is None + assert result['end'] is None + + def test_repository_to_json(self): + """Test Repository.to_json method""" + repo = Repository( + name="test-repo", + bucket="test-bucket", + base_path="/path/to/repo", + is_thawed=False, + is_mounted=True + ) + + result = repo.to_json() + + assert isinstance(result, str) + data = json.loads(result) + assert data['name'] == "test-repo" + assert data['bucket'] == "test-bucket" + assert data['base_path'] == "/path/to/repo" + assert data['is_thawed'] is False + assert data['is_mounted'] is True + + def test_repository_lt_comparison(self): + """Test Repository __lt__ comparison method""" + repo1 = Repository(name="repo-001") + repo2 = Repository(name="repo-002") + repo3 = Repository(name="repo-010") + + assert repo1 < repo2 + assert repo2 < repo3 + assert not repo2 < repo1 + assert not repo3 < repo2 + + def test_repository_persist(self): + """Test Repository.persist method""" + mock_client = Mock() + mock_client.update.return_value = {'_id': 'updated-id-123'} + + repo = Repository( + name="test-repo", + bucket="test-bucket", + base_path="/path/to/repo", + docid="existing-id-123" + ) + + with patch('curator.actions.deepfreeze.helpers.logging'): + repo.persist(mock_client) + + # Should call update with existing ID + mock_client.update.assert_called_once() + call_args = mock_client.update.call_args + assert call_args[1]['index'] == STATUS_INDEX + assert call_args[1]['id'] == 'existing-id-123' + assert call_args[1]['doc']['name'] == 'test-repo' + + def test_repository_unmount(self): + """Test Repository.unmount method""" + repo = Repository( + name="test-repo", + is_mounted=True + ) + + repo.unmount() + + # Should update is_mounted + assert repo.is_mounted is False + + +class TestSettings(TestCase): + """Test Settings dataclass""" + + def test_settings_init_with_all_params(self): + """Test Settings initialization with all parameters""" + settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + canned_acl="private", + storage_class="GLACIER", + provider="aws", + rotate_by="path", + style="oneup", + last_suffix="000001" + ) + + assert settings.repo_name_prefix == "deepfreeze" + assert settings.bucket_name_prefix == "deepfreeze" + assert settings.base_path_prefix == "snapshots" + assert settings.canned_acl == "private" + assert settings.storage_class == "GLACIER" + assert settings.provider == "aws" + assert settings.rotate_by == "path" + assert settings.style == "oneup" + assert settings.last_suffix == "000001" + + def test_settings_init_with_defaults(self): + """Test Settings initialization with default values""" + settings = Settings() + + assert settings.repo_name_prefix == "deepfreeze" + assert settings.bucket_name_prefix == "deepfreeze" + assert settings.base_path_prefix == "snapshots" + assert settings.canned_acl == "private" + assert settings.storage_class == "intelligent_tiering" + assert settings.provider == "aws" + assert settings.rotate_by == "path" + assert settings.style == "oneup" + assert settings.last_suffix is None + + def test_settings_init_with_hash(self): + """Test Settings initialization with settings hash""" + settings_hash = { + 'repo_name_prefix': 'custom-prefix', + 'storage_class': 'STANDARD_IA', + 'rotate_by': 'bucket' + } + + settings = Settings(settings_hash=settings_hash) + + # Settings constructor overrides hash values with defaults if they're passed as parameters + # Since we're not passing explicit parameters, the hash should be applied first, + # then defaults override them + assert settings.repo_name_prefix == "deepfreeze" # Default overrides hash + assert settings.storage_class == "intelligent_tiering" # Default overrides hash + assert settings.rotate_by == "path" # Default overrides hash + # But the hash values should be set via setattr + # Let's test with no default parameters + settings2 = Settings(settings_hash=settings_hash, repo_name_prefix=None, storage_class=None, rotate_by=None) + assert settings2.repo_name_prefix == "custom-prefix" + assert settings2.storage_class == "STANDARD_IA" + assert settings2.rotate_by == "bucket" + + def test_settings_dataclass_behavior(self): + """Test Settings dataclass behavior""" + settings = Settings( + repo_name_prefix="test-prefix", + bucket_name_prefix="test-bucket", + provider="gcp" + ) + + # Settings is a dataclass, so we can access attributes directly + assert settings.repo_name_prefix == "test-prefix" + assert settings.bucket_name_prefix == "test-bucket" + assert settings.provider == "gcp" + assert settings.doctype == "settings" + + # Test that we can convert to dict using dataclasses + import dataclasses + result = dataclasses.asdict(settings) + assert isinstance(result, dict) + assert result['repo_name_prefix'] == "test-prefix" + assert result['bucket_name_prefix'] == "test-bucket" + assert result['provider'] == "gcp" \ No newline at end of file diff --git a/tests/unit/test_action_deepfreeze_rotate.py b/tests/unit/test_action_deepfreeze_rotate.py new file mode 100644 index 00000000..44b56b84 --- /dev/null +++ b/tests/unit/test_action_deepfreeze_rotate.py @@ -0,0 +1,336 @@ +"""Test deepfreeze Rotate action""" +# pylint: disable=attribute-defined-outside-init +from unittest import TestCase +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime +import pytest + +from curator.actions.deepfreeze.rotate import Rotate +from curator.actions.deepfreeze.helpers import Settings, Repository +from curator.actions.deepfreeze.constants import STATUS_INDEX +from curator.actions.deepfreeze.exceptions import MissingIndexError, PreconditionError, ActionException + + +class TestDeepfreezeRotate(TestCase): + """Test Deepfreeze Rotate action""" + + def setUp(self): + """Set up test fixtures""" + self.client = Mock() + self.mock_settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + rotate_by="path", + style="oneup", + last_suffix="000001" + ) + self.mock_latest_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + is_mounted=True, + is_thawed=False + ) + + def test_init_defaults(self): + """Test Rotate initialization with default values""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory') as mock_factory: + mock_s3 = Mock() + mock_factory.return_value = mock_s3 + self.client.indices.exists.return_value = True + + rotate = Rotate(self.client) + + assert rotate.client == self.client + assert rotate.s3 == mock_s3 + assert rotate.settings == self.mock_settings + assert rotate.latest_repo == "deepfreeze-000001" + assert rotate.keep == 6 # default value + + def test_calculate_new_names_rotate_by_path_oneup(self): + """Test name calculation for path rotation with oneup style""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + self.client.indices.exists.return_value = True + rotate = Rotate(self.client) + + assert rotate.new_repo_name == "deepfreeze-000002" + assert rotate.new_bucket_name == "deepfreeze" + assert rotate.base_path == "snapshots-000002" + + def test_calculate_new_names_rotate_by_bucket(self): + """Test name calculation for bucket rotation""" + settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + rotate_by="bucket", + style="oneup", + last_suffix="000003" + ) + + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000003"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000004"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + self.client.indices.exists.return_value = True + rotate = Rotate(self.client) + + assert rotate.new_repo_name == "deepfreeze-000004" + assert rotate.new_bucket_name == "deepfreeze-000004" + assert rotate.base_path == "snapshots" + + def test_calculate_new_names_monthly_style(self): + """Test name calculation with monthly style""" + settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + rotate_by="path", + style="monthly", + last_suffix="2024.02" + ) + + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-2024.02"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="2024.03"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + self.client.indices.exists.return_value = True + rotate = Rotate(self.client) + + assert rotate.new_repo_name == "deepfreeze-2024.03" + assert rotate.base_path == "snapshots-2024.03" + + def test_check_preconditions_missing_index(self): + """Test preconditions check when status index is missing""" + from elasticsearch8 import NotFoundError + + with patch('curator.actions.deepfreeze.rotate.get_settings') as mock_get_settings: + mock_get_settings.side_effect = MissingIndexError("Status index missing") + + with pytest.raises(MissingIndexError): + Rotate(self.client) + + def test_check_preconditions_new_repo_exists(self): + """Test preconditions check when new repository already exists""" + # Return repo list that includes the new repo name that will be calculated + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001", "deepfreeze-000002"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + self.client.indices.exists.return_value = True + from curator.exceptions import RepositoryException + with pytest.raises(RepositoryException, match="already exists"): + Rotate(self.client) + + def test_check_preconditions_success(self): + """Test successful preconditions check""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory') as mock_factory: + mock_s3 = Mock() + mock_factory.return_value = mock_s3 + self.client.indices.exists.return_value = True + + # Should not raise any exceptions + rotate = Rotate(self.client) + assert rotate is not None + + def test_update_ilm_policies_creates_versioned_policies(self): + """Test that update_ilm_policies creates versioned policies instead of modifying existing ones""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.get_policies_for_repo') as mock_get_policies: + with patch('curator.actions.deepfreeze.rotate.create_versioned_ilm_policy') as mock_create: + with patch('curator.actions.deepfreeze.rotate.get_composable_templates') as mock_get_composable: + with patch('curator.actions.deepfreeze.rotate.get_index_templates') as mock_get_templates: + with patch('curator.actions.deepfreeze.rotate.update_template_ilm_policy') as mock_update_template: + self.client.indices.exists.return_value = True + + # Mock policy that references the old repo + mock_get_policies.return_value = { + "my-policy": { + "policy": { + "phases": { + "cold": { + "actions": { + "searchable_snapshot": { + "snapshot_repository": "deepfreeze-000001" + } + } + } + } + } + } + } + + mock_create.return_value = "my-policy-000002" + mock_get_composable.return_value = {"index_templates": []} + mock_get_templates.return_value = {} + + rotate = Rotate(self.client) + rotate.update_ilm_policies(dry_run=False) + + # Verify versioned policy was created + mock_create.assert_called_once() + call_args = mock_create.call_args + assert call_args[0][1] == "my-policy" # base policy name + assert call_args[0][3] == "deepfreeze-000002" # new repo name + assert call_args[0][4] == "000002" # suffix + + def test_update_ilm_policies_updates_templates(self): + """Test that update_ilm_policies updates index templates to use new versioned policies""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.get_policies_for_repo') as mock_get_policies: + with patch('curator.actions.deepfreeze.rotate.create_versioned_ilm_policy') as mock_create: + with patch('curator.actions.deepfreeze.rotate.get_composable_templates') as mock_get_composable: + with patch('curator.actions.deepfreeze.rotate.get_index_templates') as mock_get_templates: + with patch('curator.actions.deepfreeze.rotate.update_template_ilm_policy') as mock_update_template: + self.client.indices.exists.return_value = True + + mock_get_policies.return_value = { + "my-policy": {"policy": {"phases": {}}} + } + mock_create.return_value = "my-policy-000002" + + # Mock templates + mock_get_composable.return_value = { + "index_templates": [{"name": "logs-template"}] + } + mock_get_templates.return_value = {"metrics-template": {}} + mock_update_template.return_value = True + + rotate = Rotate(self.client) + rotate.update_ilm_policies(dry_run=False) + + # Verify templates were updated (both composable and legacy) + assert mock_update_template.call_count >= 2 + + def test_update_ilm_policies_dry_run(self): + """Test that update_ilm_policies dry-run mode doesn't create policies""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.get_policies_for_repo') as mock_get_policies: + with patch('curator.actions.deepfreeze.rotate.create_versioned_ilm_policy') as mock_create: + with patch('curator.actions.deepfreeze.rotate.get_composable_templates') as mock_get_composable: + with patch('curator.actions.deepfreeze.rotate.get_index_templates') as mock_get_templates: + self.client.indices.exists.return_value = True + + mock_get_policies.return_value = { + "my-policy": {"policy": {"phases": {}}} + } + mock_get_composable.return_value = {"index_templates": []} + mock_get_templates.return_value = {} + + rotate = Rotate(self.client) + rotate.update_ilm_policies(dry_run=True) + + # Verify no policies were created in dry-run + mock_create.assert_not_called() + + def test_cleanup_policies_for_repo(self): + """Test cleanup_policies_for_repo deletes policies with matching suffix""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.get_policies_by_suffix') as mock_get_by_suffix: + with patch('curator.actions.deepfreeze.rotate.is_policy_safe_to_delete') as mock_is_safe: + self.client.indices.exists.return_value = True + + # Mock policies with suffix 000001 + mock_get_by_suffix.return_value = { + "my-policy-000001": {"policy": {}}, + "other-policy-000001": {"policy": {}} + } + mock_is_safe.return_value = True + + rotate = Rotate(self.client) + rotate.cleanup_policies_for_repo("deepfreeze-000001", dry_run=False) + + # Verify policies were deleted + assert self.client.ilm.delete_lifecycle.call_count == 2 + self.client.ilm.delete_lifecycle.assert_any_call(name="my-policy-000001") + self.client.ilm.delete_lifecycle.assert_any_call(name="other-policy-000001") + + def test_cleanup_policies_for_repo_skips_in_use(self): + """Test cleanup_policies_for_repo skips policies still in use""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.get_policies_by_suffix') as mock_get_by_suffix: + with patch('curator.actions.deepfreeze.rotate.is_policy_safe_to_delete') as mock_is_safe: + self.client.indices.exists.return_value = True + + mock_get_by_suffix.return_value = { + "my-policy-000001": {"policy": {}} + } + # Policy is still in use + mock_is_safe.return_value = False + + rotate = Rotate(self.client) + rotate.cleanup_policies_for_repo("deepfreeze-000001", dry_run=False) + + # Verify policy was NOT deleted + self.client.ilm.delete_lifecycle.assert_not_called() + + def test_cleanup_policies_for_repo_dry_run(self): + """Test cleanup_policies_for_repo dry-run mode doesn't delete policies""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000002"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.get_policies_by_suffix') as mock_get_by_suffix: + with patch('curator.actions.deepfreeze.rotate.is_policy_safe_to_delete') as mock_is_safe: + self.client.indices.exists.return_value = True + + mock_get_by_suffix.return_value = { + "my-policy-000001": {"policy": {}} + } + mock_is_safe.return_value = True + + rotate = Rotate(self.client) + rotate.cleanup_policies_for_repo("deepfreeze-000001", dry_run=True) + + # Verify no policies were deleted in dry-run + self.client.ilm.delete_lifecycle.assert_not_called() + + def test_unmount_oldest_repos_calls_cleanup(self): + """Test that unmount_oldest_repos calls cleanup_policies_for_repo""" + with patch('curator.actions.deepfreeze.rotate.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.rotate.get_matching_repo_names', return_value=["deepfreeze-000002", "deepfreeze-000001"]): + with patch('curator.actions.deepfreeze.rotate.get_next_suffix', return_value="000003"): + with patch('curator.actions.deepfreeze.rotate.s3_client_factory'): + with patch('curator.actions.deepfreeze.rotate.unmount_repo') as mock_unmount: + with patch('curator.actions.deepfreeze.rotate.push_to_glacier'): + with patch('curator.actions.deepfreeze.rotate.Repository') as mock_repo_class: + self.client.indices.exists.return_value = True + + mock_repo = Mock() + mock_repo.name = "deepfreeze-000001" + mock_repo_class.from_elasticsearch.return_value = mock_repo + + rotate = Rotate(self.client, keep="1") + + with patch.object(rotate, 'cleanup_policies_for_repo') as mock_cleanup: + rotate.unmount_oldest_repos(dry_run=False) + + # Verify cleanup was called for the unmounted repo + mock_cleanup.assert_called_once_with("deepfreeze-000001", dry_run=False) + + diff --git a/tests/unit/test_action_deepfreeze_setup.py b/tests/unit/test_action_deepfreeze_setup.py new file mode 100644 index 00000000..d6a8a0ff --- /dev/null +++ b/tests/unit/test_action_deepfreeze_setup.py @@ -0,0 +1,301 @@ +"""Test deepfreeze Setup action""" +# pylint: disable=attribute-defined-outside-init +from unittest import TestCase +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime +import pytest + +from curator.actions.deepfreeze.setup import Setup +from curator.actions.deepfreeze.helpers import Settings, Repository +from curator.actions.deepfreeze.constants import STATUS_INDEX, SETTINGS_ID +from curator.actions.deepfreeze.exceptions import PreconditionError, ActionException +from curator.s3client import AwsS3Client + + +class TestDeepfreezeSetup(TestCase): + """Test Deepfreeze Setup action""" + + def setUp(self): + """Set up test fixtures""" + self.client = Mock() + self.client.indices.exists.return_value = False + self.client.snapshot.get_repository.return_value = {} + self.client.ilm.get_lifecycle.return_value = {} + + def test_init_defaults(self): + """Test Setup initialization with default values""" + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + setup = Setup(self.client) + + assert setup.client == self.client + assert setup.s3 == mock_s3 + assert setup.settings.repo_name_prefix == "deepfreeze" + assert setup.settings.bucket_name_prefix == "deepfreeze" + assert setup.settings.base_path_prefix == "snapshots" + assert setup.settings.canned_acl == "private" + assert setup.settings.storage_class == "intelligent_tiering" + assert setup.settings.provider == "aws" + assert setup.settings.rotate_by == "path" + assert setup.settings.style == "oneup" + assert setup.ilm_policy_name == "deepfreeze-sample-policy" + assert setup.create_sample_ilm_policy is False + + def test_init_custom_values(self): + """Test Setup initialization with custom values""" + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + setup = Setup( + self.client, + year=2024, + month=3, + repo_name_prefix="custom-repo", + bucket_name_prefix="custom-bucket", + base_path_prefix="custom-path", + canned_acl="public-read", + storage_class="GLACIER", + provider="gcp", + rotate_by="bucket", + style="monthly", + ilm_policy_name="custom-policy", + create_sample_ilm_policy=True + ) + + assert setup.settings.repo_name_prefix == "custom-repo" + assert setup.settings.bucket_name_prefix == "custom-bucket" + assert setup.settings.base_path_prefix == "custom-path" + assert setup.settings.canned_acl == "public-read" + assert setup.settings.storage_class == "GLACIER" + assert setup.settings.provider == "gcp" + assert setup.settings.rotate_by == "bucket" + assert setup.settings.style == "monthly" + assert setup.ilm_policy_name == "custom-policy" + assert setup.create_sample_ilm_policy is True + + def test_check_preconditions_status_index_exists(self): + """Test preconditions check when status index exists""" + self.client.indices.exists.return_value = True + + with patch('curator.actions.deepfreeze.setup.s3_client_factory'): + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_get_repos.return_value = [] + setup = Setup(self.client) + + with pytest.raises(PreconditionError, match="already exists"): + setup._check_preconditions() + + def test_check_preconditions_repository_exists(self): + """Test preconditions check when repository already exists""" + self.client.indices.exists.return_value = False + self.client.snapshot.get_repository.return_value = { + 'deepfreeze-000001': {} + } + + with patch('curator.actions.deepfreeze.setup.s3_client_factory'): + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_get_repos.return_value = [] + setup = Setup(self.client) + + with pytest.raises(PreconditionError, match="Repository.*already exists"): + setup._check_preconditions() + + def test_check_preconditions_bucket_exists(self): + """Test preconditions check when bucket already exists""" + self.client.indices.exists.return_value = False + self.client.snapshot.get_repository.return_value = {} + + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = True + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + setup = Setup(self.client, rotate_by="bucket") + + with pytest.raises(PreconditionError, match="Bucket.*already exists"): + setup._check_preconditions() + + def test_check_preconditions_success(self): + """Test successful preconditions check""" + self.client.indices.exists.return_value = False + self.client.snapshot.get_repository.return_value = {} + + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = False + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + setup = Setup(self.client) + + # Should not raise any exceptions + setup._check_preconditions() + + def test_do_dry_run(self): + """Test dry run mode""" + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + with patch('curator.actions.deepfreeze.setup.create_repo') as mock_create_repo: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = False + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + setup = Setup(self.client) + setup.do_dry_run() + + # Should call create_repo with dry_run=True + mock_create_repo.assert_called_once() + call_args = mock_create_repo.call_args + assert call_args.kwargs.get('dry_run') is True + + def test_do_action_success_rotate_by_path(self): + """Test successful setup action with rotate_by='path'""" + self.client.indices.exists.return_value = False + + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = False + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + with patch('curator.actions.deepfreeze.setup.ensure_settings_index'): + with patch('curator.actions.deepfreeze.setup.save_settings'): + with patch('curator.actions.deepfreeze.setup.create_repo'): + setup = Setup(self.client, rotate_by="path") + + setup.do_action() + + # Should create bucket (only one for path rotation) + mock_s3.create_bucket.assert_called_once_with("deepfreeze") + + def test_do_action_success_rotate_by_bucket(self): + """Test successful setup action with rotate_by='bucket'""" + self.client.indices.exists.return_value = False + + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = False + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + with patch('curator.actions.deepfreeze.setup.ensure_settings_index'): + with patch('curator.actions.deepfreeze.setup.save_settings'): + with patch('curator.actions.deepfreeze.setup.create_repo'): + setup = Setup(self.client, rotate_by="bucket") + + setup.do_action() + + # Should create bucket with suffix for bucket rotation + mock_s3.create_bucket.assert_called_once_with("deepfreeze-000001") + + def test_do_action_with_ilm_policy(self): + """Test setup action creates ILM policy""" + self.client.indices.exists.return_value = False + + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = False + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + with patch('curator.actions.deepfreeze.setup.ensure_settings_index'): + with patch('curator.actions.deepfreeze.setup.save_settings'): + with patch('curator.actions.deepfreeze.setup.create_repo'): + with patch('curator.actions.deepfreeze.setup.create_ilm_policy') as mock_create_ilm: + setup = Setup( + self.client, + create_sample_ilm_policy=True, + ilm_policy_name="test-policy" + ) + + setup.do_action() + + # Should create ILM policy + mock_create_ilm.assert_called_once() + + def test_calculate_names_rotate_by_path(self): + """Test name calculation for path rotation""" + with patch('curator.actions.deepfreeze.setup.s3_client_factory'): + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_get_repos.return_value = [] + setup = Setup(self.client, rotate_by="path") + + # Should calculate names correctly + assert setup.new_repo_name == "deepfreeze-000001" + assert setup.new_bucket_name == "deepfreeze" + assert setup.base_path == "snapshots-000001" + + def test_calculate_names_rotate_by_bucket(self): + """Test name calculation for bucket rotation""" + with patch('curator.actions.deepfreeze.setup.s3_client_factory'): + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_get_repos.return_value = [] + setup = Setup(self.client, rotate_by="bucket") + + # Should calculate names correctly + assert setup.new_repo_name == "deepfreeze-000001" + assert setup.new_bucket_name == "deepfreeze-000001" + assert setup.base_path == "snapshots" + + def test_calculate_names_monthly_style(self): + """Test name calculation with monthly style""" + with patch('curator.actions.deepfreeze.setup.s3_client_factory'): + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_get_repos.return_value = [] + setup = Setup( + self.client, + year=2024, + month=3, + style="monthly", + rotate_by="path" + ) + + assert setup.new_repo_name == "deepfreeze-2024.03" + assert setup.base_path == "snapshots-2024.03" + + def test_action_with_existing_repo_name_fails(self): + """Test that setup fails if repository name already exists""" + self.client.indices.exists.return_value = False + self.client.snapshot.get_repository.return_value = { + 'deepfreeze-000001': {} # Repository already exists + } + + with patch('curator.actions.deepfreeze.setup.s3_client_factory'): + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_get_repos.return_value = [] + setup = Setup(self.client) + + with pytest.raises(PreconditionError, match="already exists"): + setup._check_preconditions() + + def test_action_with_existing_bucket_fails(self): + """Test that setup fails if bucket already exists for bucket rotation""" + self.client.indices.exists.return_value = False + self.client.snapshot.get_repository.return_value = {} + + with patch('curator.actions.deepfreeze.setup.s3_client_factory') as mock_factory: + with patch('curator.actions.deepfreeze.setup.get_matching_repo_names') as mock_get_repos: + mock_s3 = Mock() + mock_s3.bucket_exists.return_value = True # Bucket exists + mock_factory.return_value = mock_s3 + mock_get_repos.return_value = [] + + setup = Setup(self.client, rotate_by="bucket") + + with pytest.raises(PreconditionError, match="already exists"): + setup._check_preconditions() + diff --git a/tests/unit/test_action_deepfreeze_status.py b/tests/unit/test_action_deepfreeze_status.py new file mode 100644 index 00000000..cb9d1d69 --- /dev/null +++ b/tests/unit/test_action_deepfreeze_status.py @@ -0,0 +1,346 @@ +"""Test deepfreeze Status action""" +# pylint: disable=attribute-defined-outside-init +from unittest import TestCase +from unittest.mock import Mock, patch, MagicMock +import pytest + +from curator.actions.deepfreeze.status import Status +from curator.actions.deepfreeze.helpers import Settings, Repository + + +class TestDeepfreezeStatus(TestCase): + """Test Deepfreeze Status action""" + + def setUp(self): + """Set up test fixtures""" + self.client = Mock() + self.mock_settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + canned_acl="private", + storage_class="GLACIER", + provider="aws", + rotate_by="path", + style="oneup", + last_suffix="000003" + ) + + def test_init(self): + """Test Status initialization""" + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.Console') as mock_console: + status = Status(self.client) + + assert status.client == self.client + assert status.settings == self.mock_settings + mock_console.assert_called_once() + mock_console.return_value.clear.assert_called_once() + + def test_get_cluster_name_success(self): + """Test successful cluster name retrieval""" + self.client.cluster.health.return_value = { + 'cluster_name': 'test-cluster', + 'status': 'green' + } + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + status = Status(self.client) + cluster_name = status.get_cluster_name() + + assert cluster_name == 'test-cluster' + + def test_get_cluster_name_error(self): + """Test cluster name retrieval with error""" + self.client.cluster.health.side_effect = Exception("Connection failed") + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + status = Status(self.client) + cluster_name = status.get_cluster_name() + + assert cluster_name.startswith("Error:") + assert "Connection failed" in cluster_name + + def test_do_config(self): + """Test configuration display""" + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.Table') as mock_table_class: + with patch('curator.actions.deepfreeze.status.Console'): + mock_table = Mock() + mock_table_class.return_value = mock_table + + status = Status(self.client) + status.get_cluster_name = Mock(return_value="test-cluster") + + status.do_config() + + # Should create table with title "Configuration" + mock_table_class.assert_called_with(title="Configuration") + + # Should add columns + mock_table.add_column.assert_any_call("Setting", style="cyan") + mock_table.add_column.assert_any_call("Value", style="magenta") + + # Should add rows for all settings + expected_calls = [ + ("Repo Prefix", "deepfreeze"), + ("Bucket Prefix", "deepfreeze"), + ("Base Path Prefix", "snapshots"), + ("Canned ACL", "private"), + ("Storage Class", "GLACIER"), + ("Provider", "aws"), + ("Rotate By", "path"), + ("Style", "oneup"), + ("Last Suffix", "000003"), + ("Cluster Name", "test-cluster") + ] + + for expected_call in expected_calls: + mock_table.add_row.assert_any_call(*expected_call) + + def test_do_ilm_policies(self): + """Test ILM policies display""" + self.client.ilm.get_lifecycle.return_value = { + 'policy1': { + 'policy': { + 'phases': { + 'frozen': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'deepfreeze-000003' + } + } + } + } + }, + 'in_use_by': { + 'indices': ['index1', 'index2'], + 'data_streams': ['stream1'] + } + }, + 'policy2': { + 'policy': { + 'phases': { + 'cold': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'deepfreeze-000003' + } + } + } + } + }, + 'in_use_by': { + 'indices': ['index3'], + 'data_streams': [] + } + }, + 'policy3': { + 'policy': { + 'phases': { + 'hot': { + 'actions': {} + } + } + }, + 'in_use_by': { + 'indices': [], + 'data_streams': [] + } + } + } + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.Table') as mock_table_class: + with patch('curator.actions.deepfreeze.status.Console'): + mock_table = Mock() + mock_table_class.return_value = mock_table + + status = Status(self.client) + + status.do_ilm_policies() + + # Should create table with title "ILM Policies" + mock_table_class.assert_called_with(title="ILM Policies") + + # Should add columns + mock_table.add_column.assert_any_call("Policy", style="cyan") + mock_table.add_column.assert_any_call("Repository", style="magenta") + mock_table.add_column.assert_any_call("Indices", style="magenta") + mock_table.add_column.assert_any_call("Datastreams", style="magenta") + + # Should add rows for matching policies (policy1 and policy2) + mock_table.add_row.assert_any_call("policy1", "deepfreeze-000003*", "2", "1") + mock_table.add_row.assert_any_call("policy2", "deepfreeze-000003*", "1", "0") + + def test_do_buckets_path_rotation(self): + """Test buckets display for path rotation""" + mock_repos = [ + Repository( + name="deepfreeze-000003", + bucket="deepfreeze", + base_path="snapshots-000003" + ) + ] + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.get_all_repos', return_value=mock_repos): + with patch('curator.actions.deepfreeze.status.Table') as mock_table_class: + with patch('curator.actions.deepfreeze.status.Console'): + mock_table = Mock() + mock_table_class.return_value = mock_table + + status = Status(self.client) + + status.do_buckets() + + # Should create table with title "Buckets" + mock_table_class.assert_called_with(title="Buckets") + + # Should add columns + mock_table.add_column.assert_any_call("Provider", style="cyan") + mock_table.add_column.assert_any_call("Bucket", style="magenta") + mock_table.add_column.assert_any_call("Base_path", style="magenta") + + # For path rotation, should show single bucket with suffixed path + # Bucket gets marked with asterisk since it matches current bucket/base_path + mock_table.add_row.assert_called_with( + "aws", + "deepfreeze*", + "snapshots-000003" + ) + + def test_do_buckets_bucket_rotation(self): + """Test buckets display for bucket rotation""" + bucket_rotation_settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + rotate_by="bucket", + style="oneup", + last_suffix="000003", + provider="aws" + ) + + mock_repos = [ + Repository( + name="deepfreeze-000003", + bucket="deepfreeze-000003", + base_path="snapshots" + ) + ] + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=bucket_rotation_settings): + with patch('curator.actions.deepfreeze.status.get_all_repos', return_value=mock_repos): + with patch('curator.actions.deepfreeze.status.Table') as mock_table_class: + with patch('curator.actions.deepfreeze.status.Console'): + mock_table = Mock() + mock_table_class.return_value = mock_table + + status = Status(self.client) + + status.do_buckets() + + # For bucket rotation, should show suffixed bucket with static path + mock_table.add_row.assert_called_with( + "aws", + "deepfreeze-000003*", + "snapshots" + ) + + + def test_do_action(self): + """Test main action execution""" + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.Console'): + status = Status(self.client) + + # Mock all sub-methods + status.do_repositories = Mock() + status.do_buckets = Mock() + status.do_ilm_policies = Mock() + status.do_config = Mock() + + with patch('curator.actions.deepfreeze.status.print') as mock_print: + status.do_action() + + # Should call all display methods in order + status.do_repositories.assert_called_once() + status.do_buckets.assert_called_once() + status.do_ilm_policies.assert_called_once() + status.do_config.assert_called_once() + + # Should print empty line + mock_print.assert_called_once() + + def test_do_singleton_action(self): + """Test singleton action execution""" + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.Console'): + status = Status(self.client) + + with patch.object(status, 'do_action') as mock_do_action: + status.do_singleton_action() + + mock_do_action.assert_called_once() + + + def test_repository_status_with_snapshots(self): + """Test repository status display with snapshot counts""" + mock_repos = [ + Repository( + name="deepfreeze-000001", + is_mounted=True, + is_thawed=False + ) + ] + + # Mock successful snapshot retrieval + self.client.snapshot.get.return_value = { + 'snapshots': [ + {'name': 'snap1'}, + {'name': 'snap2'}, + {'name': 'snap3'} + ] + } + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.get_all_repos', return_value=mock_repos): + with patch('curator.actions.deepfreeze.status.Table') as mock_table_class: + with patch('curator.actions.deepfreeze.status.Console'): + mock_table = Mock() + mock_table_class.return_value = mock_table + + status = Status(self.client) + + status.do_repositories() + + # Should show snapshot count + mock_table.add_row.assert_called_with( + "deepfreeze-000001", "M", "3", "N/A", "N/A" + ) + + def test_repository_unmount_on_error(self): + """Test repository gets unmounted when snapshot check fails""" + mock_repo = Repository( + name="deepfreeze-000001", + is_mounted=True, + is_thawed=False + ) + + # Mock snapshot retrieval error + self.client.snapshot.get.side_effect = Exception("Repository not accessible") + + with patch('curator.actions.deepfreeze.status.get_settings', return_value=self.mock_settings): + with patch('curator.actions.deepfreeze.status.get_all_repos', return_value=[mock_repo]): + with patch('curator.actions.deepfreeze.status.Table') as mock_table_class: + with patch('curator.actions.deepfreeze.status.Console'): + mock_table = Mock() + mock_table_class.return_value = mock_table + + status = Status(self.client) + + status.do_repositories() + + # Repository should be unmounted after error + assert mock_repo.is_mounted is False \ No newline at end of file diff --git a/tests/unit/test_action_deepfreeze_thaw.py b/tests/unit/test_action_deepfreeze_thaw.py new file mode 100644 index 00000000..adb90e44 --- /dev/null +++ b/tests/unit/test_action_deepfreeze_thaw.py @@ -0,0 +1,645 @@ +"""Test deepfreeze Thaw action""" +# pylint: disable=attribute-defined-outside-init +from datetime import datetime, timezone +from unittest import TestCase +from unittest.mock import Mock, patch, call + +from curator.actions.deepfreeze.thaw import Thaw +from curator.actions.deepfreeze.helpers import Settings, Repository + + +class TestDeepfreezeThaw(TestCase): + """Test Deepfreeze Thaw action""" + + def setUp(self): + """Set up test fixtures""" + self.client = Mock() + self.mock_settings = Settings( + repo_name_prefix="deepfreeze", + bucket_name_prefix="deepfreeze", + base_path_prefix="snapshots", + canned_acl="private", + storage_class="GLACIER", + provider="aws", + rotate_by="path", + style="oneup", + last_suffix="000003", + ) + + self.start_date = "2025-01-01T00:00:00Z" + self.end_date = "2025-01-31T23:59:59Z" + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_init_success(self, mock_get_settings, mock_s3_factory): + """Test Thaw initialization with valid dates""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + assert thaw.client == self.client + assert thaw.sync is False + assert thaw.duration == 7 + assert thaw.retrieval_tier == "Standard" + assert thaw.start_date.year == 2025 + assert thaw.start_date.month == 1 + assert thaw.end_date.month == 1 + mock_get_settings.assert_called_once_with(self.client) + mock_s3_factory.assert_called_once_with("aws") + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_init_with_custom_params(self, mock_get_settings, mock_s3_factory): + """Test Thaw initialization with custom parameters""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + sync=True, + duration=14, + retrieval_tier="Expedited", + ) + + assert thaw.sync is True + assert thaw.duration == 14 + assert thaw.retrieval_tier == "Expedited" + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_init_invalid_date_format(self, mock_get_settings, mock_s3_factory): + """Test Thaw initialization with invalid date format""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + with self.assertRaises(ValueError) as context: + Thaw( + self.client, + start_date="not-a-date", + end_date=self.end_date, + ) + + assert "Invalid start_date" in str(context.exception) + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_init_start_after_end(self, mock_get_settings, mock_s3_factory): + """Test Thaw initialization with start_date after end_date""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + with self.assertRaises(ValueError) as context: + Thaw( + self.client, + start_date=self.end_date, + end_date=self.start_date, + ) + + assert "start_date must be before or equal to end_date" in str( + context.exception + ) + + @patch("curator.actions.deepfreeze.thaw.find_repos_by_date_range") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_dry_run_no_repos( + self, mock_get_settings, mock_s3_factory, mock_find_repos + ): + """Test dry run with no matching repositories""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + mock_find_repos.return_value = [] + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + thaw.do_dry_run() + + mock_find_repos.assert_called_once() + + @patch("curator.actions.deepfreeze.thaw.find_repos_by_date_range") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_dry_run_with_repos( + self, mock_get_settings, mock_s3_factory, mock_find_repos + ): + """Test dry run with matching repositories""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + mock_repos = [ + Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + start="2025-01-01T00:00:00Z", + end="2025-01-15T23:59:59Z", + is_mounted=False, + is_thawed=False, + ), + Repository( + name="deepfreeze-000002", + bucket="deepfreeze", + base_path="snapshots-000002", + start="2025-01-16T00:00:00Z", + end="2025-01-31T23:59:59Z", + is_mounted=False, + is_thawed=False, + ), + ] + mock_find_repos.return_value = mock_repos + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + thaw.do_dry_run() + + mock_find_repos.assert_called_once() + + @patch("curator.actions.deepfreeze.thaw.save_thaw_request") + @patch("curator.actions.deepfreeze.thaw.find_repos_by_date_range") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_action_async_mode( + self, + mock_get_settings, + mock_s3_factory, + mock_find_repos, + mock_save_request, + ): + """Test thaw action in async mode""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + start="2025-01-01T00:00:00Z", + end="2025-01-15T23:59:59Z", + is_mounted=False, + is_thawed=False, + ) + mock_find_repos.return_value = [mock_repo] + + # Mock list_objects to return some objects + mock_s3.list_objects.return_value = [ + {"Key": "snapshots-000001/index1/data.dat"}, + {"Key": "snapshots-000001/index2/data.dat"}, + ] + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + sync=False, + ) + + thaw.do_action() + + # Should list objects and call thaw + mock_s3.list_objects.assert_called_once_with( + "deepfreeze", "snapshots-000001" + ) + mock_s3.thaw.assert_called_once() + + # Should save thaw request in async mode + mock_save_request.assert_called_once() + args = mock_save_request.call_args[0] + assert args[0] == self.client + assert args[2] == [mock_repo] # repos list + assert args[3] == "in_progress" # status + + @patch("curator.actions.deepfreeze.thaw.mount_repo") + @patch("curator.actions.deepfreeze.thaw.check_restore_status") + @patch("curator.actions.deepfreeze.thaw.find_repos_by_date_range") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_action_sync_mode( + self, + mock_get_settings, + mock_s3_factory, + mock_find_repos, + mock_check_status, + mock_mount_repo, + ): + """Test thaw action in sync mode""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + start="2025-01-01T00:00:00Z", + end="2025-01-15T23:59:59Z", + is_mounted=False, + is_thawed=False, + ) + mock_find_repos.return_value = [mock_repo] + + # Mock list_objects to return some objects + mock_s3.list_objects.return_value = [ + {"Key": "snapshots-000001/index1/data.dat"}, + ] + + # Mock restore status to indicate completion + mock_check_status.return_value = { + "total": 1, + "restored": 1, + "in_progress": 0, + "not_restored": 0, + "complete": True, + } + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + sync=True, + ) + + thaw.do_action() + + # Should list objects and call thaw + mock_s3.list_objects.assert_called_once() + mock_s3.thaw.assert_called_once() + + # Should check restore status and mount in sync mode + mock_check_status.assert_called() + mock_mount_repo.assert_called_once_with(self.client, mock_repo) + + @patch("curator.actions.deepfreeze.thaw.find_repos_by_date_range") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_action_no_repos( + self, mock_get_settings, mock_s3_factory, mock_find_repos + ): + """Test thaw action with no matching repositories""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + mock_find_repos.return_value = [] + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + thaw.do_action() + + mock_find_repos.assert_called_once() + + @patch("curator.actions.deepfreeze.thaw.find_repos_by_date_range") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_thaw_repository_already_thawed( + self, mock_get_settings, mock_s3_factory, mock_find_repos + ): + """Test thawing a repository that is already thawed""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + is_mounted=True, + is_thawed=True, + ) + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + result = thaw._thaw_repository(mock_repo) + + assert result is True + # Should not call S3 operations for already thawed repo + mock_s3.list_objects.assert_not_called() + mock_s3.thaw.assert_not_called() + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_thaw_repository_s3_error(self, mock_get_settings, mock_s3_factory): + """Test thawing a repository when S3 operations fail""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + is_mounted=False, + is_thawed=False, + ) + + # Mock list_objects to return objects + mock_s3.list_objects.return_value = [ + {"Key": "snapshots-000001/index1/data.dat"}, + ] + + # Mock thaw to raise an exception + mock_s3.thaw.side_effect = Exception("S3 error") + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + result = thaw._thaw_repository(mock_repo) + + assert result is False + + @patch("curator.actions.deepfreeze.thaw.check_restore_status") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + @patch("curator.actions.deepfreeze.thaw.time.sleep") + def test_wait_for_restore_success( + self, mock_sleep, mock_get_settings, mock_s3_factory, mock_check_status + ): + """Test waiting for restore to complete""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + ) + + # Three calls: initial, in-progress, then complete + # (the initial call is made to get total objects count) + mock_check_status.side_effect = [ + { + "total": 2, + "restored": 0, + "in_progress": 2, + "not_restored": 0, + "complete": False, + }, + { + "total": 2, + "restored": 1, + "in_progress": 1, + "not_restored": 0, + "complete": False, + }, + { + "total": 2, + "restored": 2, + "in_progress": 0, + "not_restored": 0, + "complete": True, + }, + ] + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + result = thaw._wait_for_restore(mock_repo, poll_interval=1, show_progress=False) + + assert result is True + assert mock_check_status.call_count == 3 + # Should sleep once between the second and third check + mock_sleep.assert_called_once_with(1) + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_singleton_action(self, mock_get_settings, mock_s3_factory): + """Test singleton action execution""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + thaw = Thaw( + self.client, + start_date=self.start_date, + end_date=self.end_date, + ) + + with patch.object(thaw, "do_action") as mock_do_action: + thaw.do_singleton_action() + + mock_do_action.assert_called_once() + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + @patch("curator.actions.deepfreeze.thaw.get_repositories_by_names") + @patch("curator.actions.deepfreeze.thaw.get_thaw_request") + def test_check_status_mode_initialization( + self, mock_get_request, mock_get_repos, mock_get_settings, mock_s3_factory + ): + """Test initialization in check_status mode""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + thaw = Thaw( + self.client, + check_status="test-request-id", + ) + + assert thaw.mode == "check_status" + assert thaw.check_status == "test-request-id" + + def test_list_mode_initialization(self): + """Test initialization in list mode""" + thaw = Thaw( + self.client, + list_requests=True, + ) + + assert thaw.mode == "list" + assert thaw.list_requests is True + + def test_create_mode_missing_dates_error(self): + """Test error when creating thaw without dates""" + with self.assertRaises(ValueError) as context: + Thaw(self.client) + + assert "start_date and end_date are required" in str(context.exception) + + @patch("curator.actions.deepfreeze.thaw.update_thaw_request") + @patch("curator.actions.deepfreeze.thaw.mount_repo") + @patch("curator.actions.deepfreeze.thaw.check_restore_status") + @patch("curator.actions.deepfreeze.thaw.get_repositories_by_names") + @patch("curator.actions.deepfreeze.thaw.get_thaw_request") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_check_status_restoration_complete( + self, + mock_get_settings, + mock_s3_factory, + mock_get_request, + mock_get_repos, + mock_check_status, + mock_mount_repo, + mock_update_request, + ): + """Test check_status when restoration is complete""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + # Mock thaw request + mock_get_request.return_value = { + "request_id": "test-id", + "repos": ["deepfreeze-000001"], + "status": "in_progress", + "created_at": "2025-01-15T10:00:00Z", + } + + # Mock repository + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + is_mounted=False, + is_thawed=False, + ) + mock_get_repos.return_value = [mock_repo] + + # Mock complete restoration status + mock_check_status.return_value = { + "total": 10, + "restored": 10, + "in_progress": 0, + "not_restored": 0, + "complete": True, + } + + thaw = Thaw(self.client, check_status="test-id") + thaw.do_check_status() + + # Should mount the repository + mock_mount_repo.assert_called_once_with(self.client, mock_repo) + # Should update request status to completed + mock_update_request.assert_called_once_with( + self.client, "test-id", status="completed" + ) + + @patch("curator.actions.deepfreeze.thaw.check_restore_status") + @patch("curator.actions.deepfreeze.thaw.get_repositories_by_names") + @patch("curator.actions.deepfreeze.thaw.get_thaw_request") + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_do_check_status_restoration_in_progress( + self, + mock_get_settings, + mock_s3_factory, + mock_get_request, + mock_get_repos, + mock_check_status, + ): + """Test check_status when restoration is still in progress""" + mock_get_settings.return_value = self.mock_settings + mock_s3 = Mock() + mock_s3_factory.return_value = mock_s3 + + mock_get_request.return_value = { + "request_id": "test-id", + "repos": ["deepfreeze-000001"], + "status": "in_progress", + "created_at": "2025-01-15T10:00:00Z", + } + + mock_repo = Repository( + name="deepfreeze-000001", + bucket="deepfreeze", + base_path="snapshots-000001", + is_mounted=False, + is_thawed=False, + ) + mock_get_repos.return_value = [mock_repo] + + # Mock in-progress restoration status + mock_check_status.return_value = { + "total": 10, + "restored": 5, + "in_progress": 5, + "not_restored": 0, + "complete": False, + } + + thaw = Thaw(self.client, check_status="test-id") + thaw.do_check_status() + + # Should check status but not mount + mock_check_status.assert_called_once() + + @patch("curator.actions.deepfreeze.thaw.list_thaw_requests") + def test_do_list_requests_empty(self, mock_list_requests): + """Test listing thaw requests when none exist""" + mock_list_requests.return_value = [] + + thaw = Thaw(self.client, list_requests=True) + thaw.do_list_requests() + + mock_list_requests.assert_called_once_with(self.client) + + @patch("curator.actions.deepfreeze.thaw.list_thaw_requests") + def test_do_list_requests_with_data(self, mock_list_requests): + """Test listing thaw requests with data""" + mock_list_requests.return_value = [ + { + "id": "request-1", + "request_id": "request-1", + "repos": ["deepfreeze-000001", "deepfreeze-000002"], + "status": "in_progress", + "created_at": "2025-01-15T10:00:00Z", + }, + { + "id": "request-2", + "request_id": "request-2", + "repos": ["deepfreeze-000003"], + "status": "completed", + "created_at": "2025-01-14T14:00:00Z", + }, + ] + + thaw = Thaw(self.client, list_requests=True) + thaw.do_list_requests() + + mock_list_requests.assert_called_once_with(self.client) + + @patch("curator.actions.deepfreeze.thaw.s3_client_factory") + @patch("curator.actions.deepfreeze.thaw.get_settings") + def test_mode_routing_in_do_action(self, mock_get_settings, mock_s3_factory): + """Test that do_action routes to correct handler based on mode""" + mock_get_settings.return_value = self.mock_settings + mock_s3_factory.return_value = Mock() + + # Test list mode + thaw_list = Thaw(self.client, list_requests=True) + with patch.object(thaw_list, "do_list_requests") as mock_list: + thaw_list.do_action() + mock_list.assert_called_once() + + # Test check_status mode + thaw_check = Thaw(self.client, check_status="test-id") + with patch.object(thaw_check, "do_check_status") as mock_check: + thaw_check.do_action() + mock_check.assert_called_once() diff --git a/tests/unit/test_action_deepfreeze_utilities.py b/tests/unit/test_action_deepfreeze_utilities.py new file mode 100644 index 00000000..3d2dcd53 --- /dev/null +++ b/tests/unit/test_action_deepfreeze_utilities.py @@ -0,0 +1,1350 @@ +"""Test deepfreeze utilities module""" +# pylint: disable=attribute-defined-outside-init +from unittest import TestCase +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime, timezone +import pytest +import botocore.exceptions + +from curator.actions.deepfreeze.utilities import ( + push_to_glacier, + get_all_indices_in_repo, + get_timestamp_range, + get_repository, + get_all_repos, + get_settings, + save_settings, + get_next_suffix, + get_matching_repo_names, + get_matching_repos, + unmount_repo, + decode_date, + create_ilm_policy, + update_repository_date_range, + get_index_templates, + get_composable_templates, + update_template_ilm_policy, + create_versioned_ilm_policy, + get_policies_for_repo, + get_policies_by_suffix, + is_policy_safe_to_delete, + get_index_datastream_name, + add_index_to_datastream, +) +from curator.actions.deepfreeze.helpers import Repository, Settings +from curator.actions.deepfreeze.constants import STATUS_INDEX, SETTINGS_ID +from curator.actions.deepfreeze.exceptions import MissingIndexError +from curator.exceptions import ActionError + + +class TestPushToGlacier(TestCase): + """Test push_to_glacier function""" + + def test_push_to_glacier_success(self): + """Test successful push to Glacier""" + mock_s3 = Mock() + mock_s3.list_objects.return_value = [ + {'Key': 'snapshots/file1', 'StorageClass': 'STANDARD'}, + {'Key': 'snapshots/file2', 'StorageClass': 'STANDARD'} + ] + mock_s3.copy_object.return_value = None + + repo = Repository( + name='test-repo', + bucket='test-bucket', + base_path='snapshots' + ) + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = push_to_glacier(mock_s3, repo) + + assert result is True + assert mock_s3.copy_object.call_count == 2 + mock_s3.copy_object.assert_any_call( + Bucket='test-bucket', + Key='snapshots/file1', + CopySource={'Bucket': 'test-bucket', 'Key': 'snapshots/file1'}, + StorageClass='GLACIER' + ) + + def test_push_to_glacier_with_trailing_slash(self): + """Test push to Glacier with trailing slash in base_path""" + mock_s3 = Mock() + mock_s3.list_objects.return_value = [ + {'Key': 'snapshots/file1', 'StorageClass': 'STANDARD'} + ] + + repo = Repository( + name='test-repo', + bucket='test-bucket', + base_path='snapshots/' # With trailing slash + ) + + with patch('curator.actions.deepfreeze.utilities.logging'): + push_to_glacier(mock_s3, repo) + + # Should normalize the path + mock_s3.list_objects.assert_called_once_with('test-bucket', 'snapshots/') + + def test_push_to_glacier_partial_failure(self): + """Test push to Glacier with partial failure""" + mock_s3 = Mock() + mock_s3.list_objects.return_value = [ + {'Key': 'snapshots/file1', 'StorageClass': 'STANDARD'}, + {'Key': 'snapshots/file2', 'StorageClass': 'STANDARD'} + ] + + # First call succeeds, second fails + mock_s3.copy_object.side_effect = [ + None, + botocore.exceptions.ClientError({'Error': {'Code': 'AccessDenied'}}, 'copy_object') + ] + + repo = Repository( + name='test-repo', + bucket='test-bucket', + base_path='snapshots' + ) + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = push_to_glacier(mock_s3, repo) + + assert result is False # Should return False due to partial failure + assert mock_s3.copy_object.call_count == 2 + + def test_push_to_glacier_list_error(self): + """Test push to Glacier with list objects error""" + mock_s3 = Mock() + mock_s3.list_objects.side_effect = botocore.exceptions.ClientError( + {'Error': {'Code': 'NoSuchBucket'}}, 'list_objects' + ) + + repo = Repository( + name='test-repo', + bucket='test-bucket', + base_path='snapshots' + ) + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = push_to_glacier(mock_s3, repo) + + assert result is False + + +class TestGetAllIndicesInRepo(TestCase): + """Test get_all_indices_in_repo function""" + + def test_get_all_indices_success(self): + """Test successful retrieval of all indices""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [ + {'indices': ['index1', 'index2']}, + {'indices': ['index2', 'index3']}, + {'indices': ['index4']} + ] + } + + result = get_all_indices_in_repo(mock_client, 'test-repo') + + assert sorted(result) == ['index1', 'index2', 'index3', 'index4'] + mock_client.snapshot.get.assert_called_once_with( + repository='test-repo', + snapshot='_all' + ) + + def test_get_all_indices_empty_repo(self): + """Test get_all_indices with empty repository""" + mock_client = Mock() + mock_client.snapshot.get.return_value = {'snapshots': []} + + result = get_all_indices_in_repo(mock_client, 'test-repo') + + assert result == [] + + def test_get_all_indices_no_indices(self): + """Test get_all_indices with snapshots but no indices""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [ + {'indices': []}, + {'indices': []} + ] + } + + result = get_all_indices_in_repo(mock_client, 'test-repo') + + assert result == [] + + +class TestGetTimestampRange(TestCase): + """Test get_timestamp_range function""" + + def test_get_timestamp_range_success(self): + """Test successful timestamp range retrieval""" + mock_client = Mock() + mock_client.indices.exists.return_value = True + mock_client.search.return_value = { + 'aggregations': { + 'earliest': {'value_as_string': '2021-01-01T00:00:00.000Z'}, + 'latest': {'value_as_string': '2022-01-01T00:00:00.000Z'} + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + earliest, latest = get_timestamp_range(mock_client, ['index1', 'index2']) + + assert earliest == datetime(2021, 1, 1, 0, 0, tzinfo=timezone.utc) + assert latest == datetime(2022, 1, 1, 0, 0, tzinfo=timezone.utc) + + def test_get_timestamp_range_empty_indices(self): + """Test timestamp range with empty indices list""" + mock_client = Mock() + + with patch('curator.actions.deepfreeze.utilities.logging'): + earliest, latest = get_timestamp_range(mock_client, []) + + assert earliest is None + assert latest is None + + def test_get_timestamp_range_nonexistent_indices(self): + """Test timestamp range with non-existent indices""" + mock_client = Mock() + mock_client.indices.exists.return_value = False + # Mock search to raise exception when called with empty index + mock_client.search.side_effect = Exception("No indices to search") + + with patch('curator.actions.deepfreeze.utilities.logging'): + earliest, latest = get_timestamp_range(mock_client, ['index1', 'index2']) + + # Should return None, None when no valid indices after filtering (exception caught) + assert earliest is None + assert latest is None + + def test_get_timestamp_range_mixed_indices(self): + """Test timestamp range with mix of existing and non-existing indices""" + mock_client = Mock() + mock_client.indices.exists.side_effect = [True, False, True] # index1 exists, index2 doesn't, index3 exists + mock_client.search.return_value = { + 'aggregations': { + 'earliest': {'value_as_string': '2021-01-01T00:00:00.000Z'}, + 'latest': {'value_as_string': '2022-01-01T00:00:00.000Z'} + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + earliest, latest = get_timestamp_range( + mock_client, + ['index1', 'index2', 'index3'] + ) + + # Should only search on existing indices + mock_client.search.assert_called_once() + call_args = mock_client.search.call_args + assert call_args[1]['index'] == 'index1,index3' + + +class TestGetRepository(TestCase): + """Test get_repository function""" + + def test_get_repository_found(self): + """Test get_repository when repository exists""" + mock_client = Mock() + mock_response = { + 'hits': { + 'total': {'value': 1}, + 'hits': [{ + '_id': 'repo-id', + '_source': { + 'name': 'test-repo', + 'bucket': 'test-bucket' + } + }] + } + } + mock_client.search.return_value = mock_response + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_repository(mock_client, 'test-repo') + + assert result.name == 'test-repo' + assert result.bucket == 'test-bucket' + assert result.docid == 'repo-id' + + def test_get_repository_not_found(self): + """Test get_repository when repository doesn't exist""" + mock_client = Mock() + mock_response = { + 'hits': { + 'total': {'value': 0}, + 'hits': [] + } + } + mock_client.search.return_value = mock_response + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_repository(mock_client, 'test-repo') + + assert result.name == 'test-repo' + assert result.bucket is None + + +class TestGetAllRepos(TestCase): + """Test get_all_repos function""" + + def test_get_all_repos_success(self): + """Test successful retrieval of all repositories""" + mock_client = Mock() + mock_client.search.return_value = { + 'hits': { + 'hits': [ + { + '_id': 'id1', + '_source': { + 'name': 'repo1', + 'bucket': 'bucket1', + 'doctype': 'repository' + } + }, + { + '_id': 'id2', + '_source': { + 'name': 'repo2', + 'bucket': 'bucket2', + 'doctype': 'repository' + } + } + ] + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_all_repos(mock_client) + + assert len(result) == 2 + assert all(isinstance(repo, Repository) for repo in result) + assert result[0].name == 'repo1' + assert result[1].name == 'repo2' + + def test_get_all_repos_empty(self): + """Test get_all_repos when no repositories exist""" + mock_client = Mock() + mock_client.search.return_value = {'hits': {'hits': []}} + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_all_repos(mock_client) + + assert result == [] + + +class TestGetSettings(TestCase): + """Test get_settings function""" + + def test_get_settings_success(self): + """Test successful retrieval of settings""" + mock_client = Mock() + mock_client.indices.exists.return_value = True + mock_client.get.return_value = { + '_source': { + 'repo_name_prefix': 'deepfreeze', + 'bucket_name_prefix': 'deepfreeze', + 'storage_class': 'GLACIER', + 'provider': 'aws', + 'doctype': 'settings' # Include doctype to test filtering + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_settings(mock_client) + + assert isinstance(result, Settings) + assert result.repo_name_prefix == 'deepfreeze' + assert result.storage_class == 'GLACIER' + + def test_get_settings_index_missing(self): + """Test get_settings when status index doesn't exist""" + mock_client = Mock() + mock_client.indices.exists.return_value = False + + with patch('curator.actions.deepfreeze.utilities.logging'): + with pytest.raises(MissingIndexError): + get_settings(mock_client) + + def test_get_settings_not_found(self): + """Test get_settings when settings don't exist""" + mock_client = Mock() + mock_client.indices.exists.return_value = True + from elasticsearch8 import NotFoundError + mock_client.get.side_effect = NotFoundError(404, 'not_found', {}) + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_settings(mock_client) + + assert result is None + + +class TestSaveSettings(TestCase): + """Test save_settings function""" + + def test_save_settings_new(self): + """Test saving new settings""" + mock_client = Mock() + from elasticsearch8 import NotFoundError + mock_client.get.side_effect = NotFoundError(404, 'not_found', {}) + + settings = Settings( + repo_name_prefix='test', + storage_class='GLACIER' + ) + + with patch('curator.actions.deepfreeze.utilities.logging'): + save_settings(mock_client, settings) + + mock_client.create.assert_called_once() + call_args = mock_client.create.call_args + assert call_args[1]['index'] == STATUS_INDEX + assert call_args[1]['id'] == SETTINGS_ID + + def test_save_settings_update(self): + """Test updating existing settings""" + mock_client = Mock() + mock_client.get.return_value = {'_source': {}} + + settings = Settings( + repo_name_prefix='test', + storage_class='GLACIER' + ) + + with patch('curator.actions.deepfreeze.utilities.logging'): + save_settings(mock_client, settings) + + mock_client.update.assert_called_once() + call_args = mock_client.update.call_args + assert call_args[1]['index'] == STATUS_INDEX + assert call_args[1]['id'] == SETTINGS_ID + + +class TestGetNextSuffix(TestCase): + """Test get_next_suffix function""" + + def test_get_next_suffix_oneup(self): + """Test get_next_suffix with oneup style""" + assert get_next_suffix('oneup', '000001', None, None) == '000002' + assert get_next_suffix('oneup', '000009', None, None) == '000010' + assert get_next_suffix('oneup', '000099', None, None) == '000100' + assert get_next_suffix('oneup', '999999', None, None) == '1000000' + + def test_get_next_suffix_date(self): + """Test get_next_suffix with date style""" + assert get_next_suffix('date', '2024.01', 2024, 3) == '2024.03' + + def test_get_next_suffix_date_current(self): + """Test get_next_suffix with date style using current date""" + with patch('curator.actions.deepfreeze.utilities.datetime') as mock_dt: + mock_dt.now.return_value = datetime(2024, 3, 15) + assert get_next_suffix('date', '2024.02', None, None) == '2024.03' + + def test_get_next_suffix_invalid_style(self): + """Test get_next_suffix with invalid style""" + with pytest.raises(ValueError, match="Invalid style"): + get_next_suffix('invalid', '000001', None, None) + + +class TestGetMatchingRepoNames(TestCase): + """Test get_matching_repo_names function""" + + def test_get_matching_repo_names_success(self): + """Test successful retrieval of matching repository names""" + mock_client = Mock() + mock_client.snapshot.get_repository.return_value = { + 'deepfreeze-001': {}, + 'deepfreeze-002': {}, + 'other-repo': {}, + 'deepfreeze-003': {} + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_matching_repo_names(mock_client, 'deepfreeze-') + + assert sorted(result) == ['deepfreeze-001', 'deepfreeze-002', 'deepfreeze-003'] + + def test_get_matching_repo_names_no_matches(self): + """Test get_matching_repo_names with no matches""" + mock_client = Mock() + mock_client.snapshot.get_repository.return_value = { + 'other-repo-1': {}, + 'other-repo-2': {} + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_matching_repo_names(mock_client, 'deepfreeze-') + + assert result == [] + + +class TestGetMatchingRepos(TestCase): + """Test get_matching_repos function""" + + def test_get_matching_repos_success(self): + """Test successful retrieval of matching repositories""" + mock_client = Mock() + mock_client.search.return_value = { + 'hits': { + 'hits': [ + { + '_id': 'id1', + '_source': { + 'name': 'deepfreeze-001', + 'bucket': 'bucket1', + 'is_mounted': True + } + }, + { + '_id': 'id2', + '_source': { + 'name': 'other-repo', + 'bucket': 'bucket2', + 'is_mounted': False + } + }, + { + '_id': 'id3', + '_source': { + 'name': 'deepfreeze-002', + 'bucket': 'bucket3', + 'is_mounted': False + } + } + ] + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_matching_repos(mock_client, 'deepfreeze-') + + # Should return only deepfreeze repos + assert len(result) == 2 + repo_names = [repo.name for repo in result] + assert 'deepfreeze-001' in repo_names + assert 'deepfreeze-002' in repo_names + + def test_get_matching_repos_mounted_only(self): + """Test get_matching_repos with mounted filter""" + mock_client = Mock() + mock_client.search.return_value = { + 'hits': { + 'hits': [ + { + '_id': 'id1', + '_source': { + 'name': 'deepfreeze-001', + 'bucket': 'bucket1', + 'is_mounted': True + } + }, + { + '_id': 'id2', + '_source': { + 'name': 'deepfreeze-002', + 'bucket': 'bucket2', + 'is_mounted': False + } + } + ] + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_matching_repos(mock_client, 'deepfreeze-', mounted=True) + + # Should return only mounted repos + assert len(result) == 1 + assert result[0].name == 'deepfreeze-001' + + +class TestUnmountRepo(TestCase): + """Test unmount_repo function""" + + def test_unmount_repo_success(self): + """Test successful repository unmounting""" + mock_client = Mock() + mock_client.snapshot.get_repository.return_value = { + 'test-repo': { + 'settings': { + 'bucket': 'test-bucket', + 'base_path': 'test-path' + } + } + } + mock_client.search.return_value = { + 'hits': { + 'total': {'value': 1}, + 'hits': [{ + '_id': 'repo-id', + '_source': { + 'name': 'test-repo', + 'bucket': 'test-bucket' + } + }] + } + } + + with patch('curator.actions.deepfreeze.utilities.get_all_indices_in_repo', return_value=['index1']): + with patch('curator.actions.deepfreeze.utilities.get_timestamp_range', return_value=(None, None)): + with patch('curator.actions.deepfreeze.utilities.decode_date', return_value=datetime.now()): + with patch('curator.actions.deepfreeze.utilities.logging'): + result = unmount_repo(mock_client, 'test-repo') + + mock_client.snapshot.delete_repository.assert_called_once_with(name='test-repo') + mock_client.update.assert_called_once() + assert result.name == 'test-repo' + assert result.is_mounted is False + + +class TestDecodeDate(TestCase): + """Test decode_date function""" + + def test_decode_date_datetime_utc(self): + """Test decode_date with datetime object in UTC""" + dt = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + result = decode_date(dt) + assert result == dt + + def test_decode_date_datetime_naive(self): + """Test decode_date with naive datetime object""" + dt = datetime(2024, 1, 1, 12, 0, 0) + result = decode_date(dt) + assert result == dt.replace(tzinfo=timezone.utc) + + def test_decode_date_string(self): + """Test decode_date with ISO string""" + date_str = "2024-01-01T12:00:00" + with patch('curator.actions.deepfreeze.utilities.logging'): + result = decode_date(date_str) + + expected = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + assert result == expected + + def test_decode_date_invalid(self): + """Test decode_date with invalid input""" + with pytest.raises(ValueError): + decode_date(12345) + + +class TestCreateIlmPolicy(TestCase): + """Test create_ilm_policy function""" + + def test_create_ilm_policy_success(self): + """Test successful ILM policy creation""" + mock_client = Mock() + policy_body = {'phases': {'hot': {}}} + + with patch('curator.actions.deepfreeze.utilities.logging'): + create_ilm_policy(mock_client, 'test-policy', policy_body) + + mock_client.ilm.put_lifecycle.assert_called_once_with( + name='test-policy', + body=policy_body + ) + + def test_create_ilm_policy_error(self): + """Test ILM policy creation error""" + mock_client = Mock() + mock_client.ilm.put_lifecycle.side_effect = Exception('Policy creation failed') + policy_body = {'phases': {'hot': {}}} + + with patch('curator.actions.deepfreeze.utilities.logging'): + with pytest.raises(ActionError): + create_ilm_policy(mock_client, 'test-policy', policy_body) + +class TestUpdateRepositoryDateRange(TestCase): + """Test update_repository_date_range function""" + + def test_update_date_range_success(self): + """Test successful date range update""" + mock_client = Mock() + # Mock get_all_indices_in_repo + mock_client.snapshot.get.return_value = { + 'snapshots': [{'indices': ['index1', 'index2']}] + } + # Mock index existence checks - simulating partial- prefix + mock_client.indices.exists.side_effect = [False, True, False, True] + # Mock status index search for update + mock_client.search.return_value = { + 'hits': {'total': {'value': 1}, 'hits': [{'_id': 'repo-doc-id'}]} + } + + repo = Repository(name='test-repo') + + # Mock the get_timestamp_range function directly + earliest = datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc) + latest = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + + with patch('curator.actions.deepfreeze.utilities.get_timestamp_range', return_value=(earliest, latest)): + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is True + assert repo.start is not None + assert repo.end is not None + mock_client.update.assert_called_once() + + def test_update_date_range_no_mounted_indices(self): + """Test update with no mounted indices""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [{'indices': ['index1']}] + } + # All index existence checks return False + mock_client.indices.exists.return_value = False + + repo = Repository(name='test-repo') + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is False + mock_client.update.assert_not_called() + + def test_update_date_range_handles_original_names(self): + """Test update with indices mounted using original names""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [{'indices': ['index1']}] + } + # Original name exists + mock_client.indices.exists.side_effect = [True] + # Mock status index search for update + mock_client.search.return_value = { + 'hits': {'total': {'value': 1}, 'hits': [{'_id': 'repo-doc-id'}]} + } + + repo = Repository(name='test-repo') + + # Mock the get_timestamp_range function directly + earliest = datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc) + latest = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + + with patch('curator.actions.deepfreeze.utilities.get_timestamp_range', return_value=(earliest, latest)): + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is True + + def test_update_date_range_handles_restored_prefix(self): + """Test update with indices using restored- prefix""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [{'indices': ['index1']}] + } + # Original and partial- don't exist, restored- does + mock_client.indices.exists.side_effect = [False, False, True] + # Mock status index search for update + mock_client.search.return_value = { + 'hits': {'total': {'value': 1}, 'hits': [{'_id': 'repo-doc-id'}]} + } + + repo = Repository(name='test-repo') + + # Mock the get_timestamp_range function directly + earliest = datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc) + latest = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + + with patch('curator.actions.deepfreeze.utilities.get_timestamp_range', return_value=(earliest, latest)): + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is True + + def test_update_date_range_no_timestamp_data(self): + """Test update when timestamp query returns None""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [{'indices': ['index1']}] + } + mock_client.indices.exists.return_value = True + + repo = Repository(name='test-repo') + + with patch('curator.actions.deepfreeze.utilities.get_timestamp_range', return_value=(None, None)): + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is False + mock_client.update.assert_not_called() + + def test_update_date_range_exception_handling(self): + """Test update handles exceptions gracefully""" + mock_client = Mock() + mock_client.snapshot.get.side_effect = Exception("Repository error") + + repo = Repository(name='test-repo') + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is False + + def test_update_date_range_creates_new_document(self): + """Test update creates document if it doesn't exist""" + mock_client = Mock() + mock_client.snapshot.get.return_value = { + 'snapshots': [{'indices': ['index1']}] + } + mock_client.indices.exists.return_value = True + mock_client.search.side_effect = [ + # First search for timestamp data + { + 'aggregations': { + 'earliest': {'value_as_string': '2024-01-01T00:00:00.000Z'}, + 'latest': {'value_as_string': '2024-12-31T23:59:59.000Z'} + } + }, + # Second search for existing document - returns nothing + {'hits': {'total': {'value': 0}, 'hits': []}} + ] + + repo = Repository(name='test-repo') + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_repository_date_range(mock_client, repo) + + assert result is True + mock_client.index.assert_called_once() + + +class TestGetIndexTemplates(TestCase): + """Test get_index_templates function""" + + def test_get_index_templates_success(self): + """Test successful retrieval of legacy templates""" + mock_client = Mock() + mock_client.indices.get_template.return_value = { + 'template1': {'settings': {}}, + 'template2': {'settings': {}} + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_index_templates(mock_client) + + assert len(result) == 2 + assert 'template1' in result + assert 'template2' in result + + def test_get_index_templates_error(self): + """Test get_index_templates error handling""" + mock_client = Mock() + mock_client.indices.get_template.side_effect = Exception("API error") + + with patch('curator.actions.deepfreeze.utilities.logging'): + with pytest.raises(ActionError): + get_index_templates(mock_client) + + +class TestGetComposableTemplates(TestCase): + """Test get_composable_templates function""" + + def test_get_composable_templates_success(self): + """Test successful retrieval of composable templates""" + mock_client = Mock() + mock_client.indices.get_index_template.return_value = { + 'index_templates': [ + {'name': 'template1'}, + {'name': 'template2'} + ] + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_composable_templates(mock_client) + + assert 'index_templates' in result + assert len(result['index_templates']) == 2 + + def test_get_composable_templates_error(self): + """Test get_composable_templates error handling""" + mock_client = Mock() + mock_client.indices.get_index_template.side_effect = Exception("API error") + + with patch('curator.actions.deepfreeze.utilities.logging'): + with pytest.raises(ActionError): + get_composable_templates(mock_client) + + +class TestUpdateTemplateIlmPolicy(TestCase): + """Test update_template_ilm_policy function""" + + def test_update_composable_template_success(self): + """Test successful update of composable template""" + mock_client = Mock() + mock_client.indices.get_index_template.return_value = { + 'index_templates': [{ + 'name': 'test-template', + 'index_template': { + 'template': { + 'settings': { + 'index': { + 'lifecycle': {'name': 'old-policy'} + } + } + } + } + }] + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_template_ilm_policy( + mock_client, 'test-template', 'old-policy', 'new-policy', is_composable=True + ) + + assert result is True + mock_client.indices.put_index_template.assert_called_once() + + def test_update_legacy_template_success(self): + """Test successful update of legacy template""" + mock_client = Mock() + mock_client.indices.get_template.return_value = { + 'test-template': { + 'settings': { + 'index': { + 'lifecycle': {'name': 'old-policy'} + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_template_ilm_policy( + mock_client, 'test-template', 'old-policy', 'new-policy', is_composable=False + ) + + assert result is True + mock_client.indices.put_template.assert_called_once() + + def test_update_template_no_match(self): + """Test template update when policy doesn't match""" + mock_client = Mock() + mock_client.indices.get_index_template.return_value = { + 'index_templates': [{ + 'name': 'test-template', + 'index_template': { + 'template': { + 'settings': { + 'index': { + 'lifecycle': {'name': 'different-policy'} + } + } + } + } + }] + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = update_template_ilm_policy( + mock_client, 'test-template', 'old-policy', 'new-policy', is_composable=True + ) + + assert result is False + mock_client.indices.put_index_template.assert_not_called() + + +class TestCreateVersionedIlmPolicy(TestCase): + """Test create_versioned_ilm_policy function""" + + def test_create_versioned_policy_success(self): + """Test successful creation of versioned policy""" + mock_client = Mock() + policy_body = { + 'phases': { + 'cold': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'old-repo' + } + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = create_versioned_ilm_policy( + mock_client, 'my-policy', policy_body, 'new-repo', '000005' + ) + + assert result == 'my-policy-000005' + mock_client.ilm.put_lifecycle.assert_called_once() + call_args = mock_client.ilm.put_lifecycle.call_args + assert call_args[1]['name'] == 'my-policy-000005' + # Verify repo was updated in policy + policy_arg = call_args[1]['policy'] + assert policy_arg['phases']['cold']['actions']['searchable_snapshot']['snapshot_repository'] == 'new-repo' + + def test_create_versioned_policy_multiple_phases(self): + """Test versioned policy with multiple phases""" + mock_client = Mock() + policy_body = { + 'phases': { + 'cold': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'old-repo' + } + } + }, + 'frozen': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'old-repo' + } + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = create_versioned_ilm_policy( + mock_client, 'my-policy', policy_body, 'new-repo', '000005' + ) + + # Verify all phases were updated + call_args = mock_client.ilm.put_lifecycle.call_args + policy_arg = call_args[1]['policy'] + assert policy_arg['phases']['cold']['actions']['searchable_snapshot']['snapshot_repository'] == 'new-repo' + assert policy_arg['phases']['frozen']['actions']['searchable_snapshot']['snapshot_repository'] == 'new-repo' + + def test_create_versioned_policy_error(self): + """Test versioned policy creation error""" + mock_client = Mock() + mock_client.ilm.put_lifecycle.side_effect = Exception("Policy creation failed") + policy_body = {'phases': {}} + + with patch('curator.actions.deepfreeze.utilities.logging'): + with pytest.raises(ActionError): + create_versioned_ilm_policy( + mock_client, 'my-policy', policy_body, 'new-repo', '000005' + ) + + +class TestGetPoliciesForRepo(TestCase): + """Test get_policies_for_repo function""" + + def test_get_policies_for_repo_success(self): + """Test successful retrieval of policies for repository""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'policy1': { + 'policy': { + 'phases': { + 'cold': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'target-repo' + } + } + } + } + } + }, + 'policy2': { + 'policy': { + 'phases': { + 'frozen': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'other-repo' + } + } + } + } + } + }, + 'policy3': { + 'policy': { + 'phases': { + 'cold': { + 'actions': { + 'searchable_snapshot': { + 'snapshot_repository': 'target-repo' + } + } + } + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_policies_for_repo(mock_client, 'target-repo') + + assert len(result) == 2 + assert 'policy1' in result + assert 'policy3' in result + assert 'policy2' not in result + + def test_get_policies_for_repo_no_matches(self): + """Test get_policies_for_repo with no matches""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'policy1': { + 'policy': { + 'phases': { + 'cold': { + 'actions': {} + } + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_policies_for_repo(mock_client, 'target-repo') + + assert len(result) == 0 + + +class TestGetPoliciesBySuffix(TestCase): + """Test get_policies_by_suffix function""" + + def test_get_policies_by_suffix_success(self): + """Test successful retrieval of policies by suffix""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'my-policy-000003': {'policy': {}}, + 'other-policy-000003': {'policy': {}}, + 'different-policy-000004': {'policy': {}}, + 'my-policy': {'policy': {}} + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_policies_by_suffix(mock_client, '000003') + + assert len(result) == 2 + assert 'my-policy-000003' in result + assert 'other-policy-000003' in result + assert 'different-policy-000004' not in result + assert 'my-policy' not in result + + def test_get_policies_by_suffix_no_matches(self): + """Test get_policies_by_suffix with no matches""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'policy1': {'policy': {}}, + 'policy2': {'policy': {}} + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_policies_by_suffix(mock_client, '000003') + + assert len(result) == 0 + + +class TestIsPolicySafeToDelete(TestCase): + """Test is_policy_safe_to_delete function""" + + def test_policy_safe_to_delete(self): + """Test policy that is safe to delete""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'test-policy': { + 'policy': {}, + 'in_use_by': { + 'indices': [], + 'data_streams': [], + 'composable_templates': [] + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = is_policy_safe_to_delete(mock_client, 'test-policy') + + assert result is True + + def test_policy_in_use_by_indices(self): + """Test policy that is in use by indices""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'test-policy': { + 'policy': {}, + 'in_use_by': { + 'indices': ['index1', 'index2'], + 'data_streams': [], + 'composable_templates': [] + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = is_policy_safe_to_delete(mock_client, 'test-policy') + + assert result is False + + def test_policy_in_use_by_data_streams(self): + """Test policy that is in use by data streams""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'test-policy': { + 'policy': {}, + 'in_use_by': { + 'indices': [], + 'data_streams': ['logs-stream'], + 'composable_templates': [] + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = is_policy_safe_to_delete(mock_client, 'test-policy') + + assert result is False + + def test_policy_in_use_by_templates(self): + """Test policy that is in use by templates""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = { + 'test-policy': { + 'policy': {}, + 'in_use_by': { + 'indices': [], + 'data_streams': [], + 'composable_templates': ['template1'] + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = is_policy_safe_to_delete(mock_client, 'test-policy') + + assert result is False + + def test_policy_not_found(self): + """Test policy that doesn't exist""" + mock_client = Mock() + mock_client.ilm.get_lifecycle.return_value = {} + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = is_policy_safe_to_delete(mock_client, 'test-policy') + + assert result is False + + def test_policy_not_found_exception(self): + """Test policy check with NotFoundError""" + mock_client = Mock() + from elasticsearch8 import NotFoundError + mock_client.ilm.get_lifecycle.side_effect = NotFoundError(404, 'not_found', {}) + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = is_policy_safe_to_delete(mock_client, 'test-policy') + + assert result is False + + +class TestGetIndexDatastreamName(TestCase): + """Test get_index_datastream_name function""" + + def test_datastream_from_metadata(self): + """Test extracting data stream name from index metadata""" + mock_client = Mock() + mock_client.indices.get_settings.return_value = { + '.ds-logs-2024.01.01-000001': { + 'settings': { + 'index': { + 'provided_name': '.ds-logs-2024.01.01-000001' + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_index_datastream_name(mock_client, '.ds-logs-2024.01.01-000001') + + assert result == 'logs' + + def test_datastream_from_index_name_fallback(self): + """Test extracting data stream name from index name when metadata is missing""" + mock_client = Mock() + mock_client.indices.get_settings.return_value = { + '.ds-metrics-cpu-2024.01.01-000002': { + 'settings': { + 'index': { + # No provided_name - testing fallback to index name + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_index_datastream_name(mock_client, '.ds-metrics-cpu-2024.01.01-000002') + + assert result == 'metrics-cpu' + + def test_non_datastream_index(self): + """Test that non-datastream indices return None""" + mock_client = Mock() + mock_client.indices.get_settings.return_value = { + 'regular-index-2024.01.01': { + 'settings': { + 'index': { + 'provided_name': 'regular-index-2024.01.01' + } + } + } + } + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_index_datastream_name(mock_client, 'regular-index-2024.01.01') + + assert result is None + + def test_exception_handling(self): + """Test error handling when getting index settings fails""" + mock_client = Mock() + mock_client.indices.get_settings.side_effect = Exception('Connection error') + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = get_index_datastream_name(mock_client, '.ds-logs-2024.01.01-000001') + + assert result is None + + +class TestAddIndexToDatastream(TestCase): + """Test add_index_to_datastream function""" + + def test_add_index_successfully(self): + """Test successfully adding an index to a data stream""" + mock_client = Mock() + mock_client.indices.get_data_stream.return_value = {'data_streams': [{'name': 'logs'}]} + mock_client.indices.modify_data_stream.return_value = {'acknowledged': True} + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = add_index_to_datastream(mock_client, 'logs', '.ds-logs-2024.01.01-000001') + + assert result is True + mock_client.indices.modify_data_stream.assert_called_once_with( + body={ + 'actions': [ + {'add_backing_index': {'data_stream': 'logs', 'index': '.ds-logs-2024.01.01-000001'}} + ] + } + ) + + def test_datastream_not_found(self): + """Test adding index when data stream doesn't exist""" + mock_client = Mock() + from elasticsearch8 import NotFoundError + mock_client.indices.get_data_stream.side_effect = NotFoundError(404, 'not_found', {}) + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = add_index_to_datastream(mock_client, 'logs', '.ds-logs-2024.01.01-000001') + + assert result is False + + def test_add_index_fails(self): + """Test when adding index to data stream fails""" + mock_client = Mock() + mock_client.indices.get_data_stream.return_value = {'data_streams': [{'name': 'logs'}]} + mock_client.indices.modify_data_stream.side_effect = Exception('Failed to modify') + + with patch('curator.actions.deepfreeze.utilities.logging'): + result = add_index_to_datastream(mock_client, 'logs', '.ds-logs-2024.01.01-000001') + + assert result is False diff --git a/tests/unit/test_class_s3client.py b/tests/unit/test_class_s3client.py new file mode 100644 index 00000000..9ac790fc --- /dev/null +++ b/tests/unit/test_class_s3client.py @@ -0,0 +1,484 @@ +"""Test S3Client classes""" +from unittest.mock import MagicMock, patch, call +import pytest +from botocore.exceptions import ClientError +from curator.exceptions import ActionError +from curator.s3client import AwsS3Client, S3Client, s3_client_factory + + +class TestS3ClientAbstract: + """Test abstract S3Client class""" + + def test_abstract_methods_not_implemented(self): + """Test that abstract methods raise NotImplementedError""" + # S3Client is abstract, cannot instantiate directly + with pytest.raises(TypeError): + S3Client() + + +class TestAwsS3Client: + """Test AwsS3Client class""" + + def setup_method(self): + """Setup for each test""" + with patch('boto3.client'): + self.s3 = AwsS3Client() + self.s3.client = MagicMock() + + def test_init(self): + """Test AwsS3Client initialization""" + with patch('boto3.client') as mock_boto: + s3 = AwsS3Client() + mock_boto.assert_called_with("s3") + assert s3.loggit is not None + + def test_create_bucket_success(self): + """Test successful bucket creation""" + self.s3.bucket_exists = MagicMock(return_value=False) + self.s3.create_bucket("test-bucket") + self.s3.client.create_bucket.assert_called_with(Bucket="test-bucket") + + def test_create_bucket_already_exists(self): + """Test bucket creation when bucket already exists""" + self.s3.bucket_exists = MagicMock(return_value=True) + with pytest.raises(ActionError, match="already exists"): + self.s3.create_bucket("test-bucket") + + def test_create_bucket_client_error(self): + """Test bucket creation with ClientError""" + self.s3.bucket_exists = MagicMock(return_value=False) + self.s3.client.create_bucket.side_effect = ClientError( + {"Error": {"Code": "BucketAlreadyExists"}}, "create_bucket" + ) + with pytest.raises(ActionError): + self.s3.create_bucket("test-bucket") + + def test_bucket_exists_true(self): + """Test bucket_exists returns True when bucket exists""" + self.s3.client.head_bucket.return_value = {} + assert self.s3.bucket_exists("test-bucket") is True + self.s3.client.head_bucket.assert_called_with(Bucket="test-bucket") + + def test_bucket_exists_false(self): + """Test bucket_exists returns False when bucket doesn't exist""" + self.s3.client.head_bucket.side_effect = ClientError( + {"Error": {"Code": "404"}}, "head_bucket" + ) + assert self.s3.bucket_exists("test-bucket") is False + + def test_bucket_exists_other_error(self): + """Test bucket_exists raises ActionError for non-404 errors""" + self.s3.client.head_bucket.side_effect = ClientError( + {"Error": {"Code": "403"}}, "head_bucket" + ) + with pytest.raises(ActionError): + self.s3.bucket_exists("test-bucket") + + def test_thaw_glacier_objects(self): + """Test thawing objects from Glacier""" + self.s3.client.head_object.return_value = {"StorageClass": "GLACIER"} + + self.s3.thaw( + "test-bucket", + "base_path", + ["base_path/file1", "base_path/file2"], + 7, + "Standard" + ) + + assert self.s3.client.restore_object.call_count == 2 + self.s3.client.restore_object.assert_any_call( + Bucket="test-bucket", + Key="base_path/file1", + RestoreRequest={ + "Days": 7, + "GlacierJobParameters": {"Tier": "Standard"} + } + ) + + def test_thaw_deep_archive_objects(self): + """Test thawing objects from Deep Archive""" + self.s3.client.head_object.return_value = {"StorageClass": "DEEP_ARCHIVE"} + + self.s3.thaw( + "test-bucket", + "base_path", + ["base_path/file1"], + 7, + "Expedited" + ) + + self.s3.client.restore_object.assert_called_once_with( + Bucket="test-bucket", + Key="base_path/file1", + RestoreRequest={ + "Days": 7, + "GlacierJobParameters": {"Tier": "Expedited"} + } + ) + + def test_thaw_skip_non_glacier(self): + """Test thaw skips non-Glacier storage classes""" + self.s3.client.head_object.return_value = {"StorageClass": "STANDARD"} + + self.s3.thaw("test-bucket", "base_path", ["base_path/file1"], 7, "Standard") + self.s3.client.restore_object.assert_not_called() + + def test_thaw_skip_wrong_path(self): + """Test thaw skips objects outside base_path""" + self.s3.client.head_object.return_value = {"StorageClass": "GLACIER"} + + self.s3.thaw("test-bucket", "base_path", ["wrong_path/file1"], 7, "Standard") + self.s3.client.restore_object.assert_not_called() + + def test_thaw_exception_handling(self): + """Test thaw handles exceptions gracefully""" + self.s3.client.head_object.side_effect = Exception("Test error") + + # Should not raise, just log the error + self.s3.thaw("test-bucket", "base_path", ["base_path/file1"], 7, "Standard") + self.s3.client.restore_object.assert_not_called() + + def test_refreeze_success(self): + """Test successful refreezing of objects""" + self.s3.client.get_paginator.return_value.paginate.return_value = [ + {"Contents": [ + {"Key": "base_path/file1"}, + {"Key": "base_path/file2"} + ]} + ] + + self.s3.refreeze("test-bucket", "base_path", "GLACIER") + + assert self.s3.client.copy_object.call_count == 2 + self.s3.client.copy_object.assert_any_call( + Bucket="test-bucket", + CopySource={"Bucket": "test-bucket", "Key": "base_path/file1"}, + Key="base_path/file1", + StorageClass="GLACIER" + ) + + def test_refreeze_deep_archive(self): + """Test refreezing to Deep Archive""" + self.s3.client.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": "base_path/file1"}]} + ] + + self.s3.refreeze("test-bucket", "base_path", "DEEP_ARCHIVE") + + self.s3.client.copy_object.assert_called_with( + Bucket="test-bucket", + CopySource={"Bucket": "test-bucket", "Key": "base_path/file1"}, + Key="base_path/file1", + StorageClass="DEEP_ARCHIVE" + ) + + def test_refreeze_no_contents(self): + """Test refreeze when no contents returned""" + self.s3.client.get_paginator.return_value.paginate.return_value = [{}] + + self.s3.refreeze("test-bucket", "base_path", "GLACIER") + self.s3.client.copy_object.assert_not_called() + + def test_refreeze_exception_handling(self): + """Test refreeze handles exceptions gracefully""" + self.s3.client.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": "base_path/file1"}]} + ] + self.s3.client.copy_object.side_effect = Exception("Test error") + + # Should not raise, just log the error + self.s3.refreeze("test-bucket", "base_path", "GLACIER") + + def test_list_objects_success(self): + """Test successful listing of objects""" + mock_objects = [ + {"Key": "file1", "Size": 100}, + {"Key": "file2", "Size": 200} + ] + self.s3.client.get_paginator.return_value.paginate.return_value = [ + {"Contents": mock_objects} + ] + + result = self.s3.list_objects("test-bucket", "prefix") + + assert result == mock_objects + self.s3.client.get_paginator.assert_called_with("list_objects_v2") + + def test_list_objects_multiple_pages(self): + """Test listing objects across multiple pages""" + self.s3.client.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": "file1"}]}, + {"Contents": [{"Key": "file2"}]} + ] + + result = self.s3.list_objects("test-bucket", "prefix") + + assert len(result) == 2 + assert result[0]["Key"] == "file1" + assert result[1]["Key"] == "file2" + + def test_list_objects_no_contents(self): + """Test listing objects when no contents""" + self.s3.client.get_paginator.return_value.paginate.return_value = [{}] + + result = self.s3.list_objects("test-bucket", "prefix") + assert result == [] + + def test_delete_bucket_success(self): + """Test successful bucket deletion""" + self.s3.delete_bucket("test-bucket") + self.s3.client.delete_bucket.assert_called_with(Bucket="test-bucket") + + def test_delete_bucket_error(self): + """Test bucket deletion error""" + self.s3.client.delete_bucket.side_effect = ClientError( + {"Error": {"Code": "BucketNotEmpty"}}, "delete_bucket" + ) + + with pytest.raises(ActionError): + self.s3.delete_bucket("test-bucket") + + def test_put_object_success(self): + """Test successful object put""" + self.s3.put_object("test-bucket", "key", "body content") + self.s3.client.put_object.assert_called_with( + Bucket="test-bucket", + Key="key", + Body="body content" + ) + + def test_put_object_empty_body(self): + """Test putting object with empty body""" + self.s3.put_object("test-bucket", "key") + self.s3.client.put_object.assert_called_with( + Bucket="test-bucket", + Key="key", + Body="" + ) + + def test_put_object_error(self): + """Test put object error""" + self.s3.client.put_object.side_effect = ClientError( + {"Error": {"Code": "AccessDenied"}}, "put_object" + ) + + with pytest.raises(ActionError): + self.s3.put_object("test-bucket", "key", "body") + + def test_list_buckets_success(self): + """Test successful bucket listing""" + self.s3.client.list_buckets.return_value = { + "Buckets": [ + {"Name": "bucket1"}, + {"Name": "bucket2"}, + {"Name": "test-bucket3"} + ] + } + + result = self.s3.list_buckets() + assert result == ["bucket1", "bucket2", "test-bucket3"] + + def test_list_buckets_with_prefix(self): + """Test bucket listing with prefix filter""" + self.s3.client.list_buckets.return_value = { + "Buckets": [ + {"Name": "bucket1"}, + {"Name": "test-bucket2"}, + {"Name": "test-bucket3"} + ] + } + + result = self.s3.list_buckets(prefix="test-") + assert result == ["test-bucket2", "test-bucket3"] + + def test_list_buckets_empty(self): + """Test listing buckets when none exist""" + self.s3.client.list_buckets.return_value = {"Buckets": []} + + result = self.s3.list_buckets() + assert result == [] + + def test_list_buckets_error(self): + """Test bucket listing error""" + self.s3.client.list_buckets.side_effect = ClientError( + {"Error": {"Code": "AccessDenied"}}, "list_buckets" + ) + + with pytest.raises(ActionError): + self.s3.list_buckets() + + def test_copy_object_success(self): + """Test successful object copy""" + self.s3.copy_object( + Bucket="dest-bucket", + Key="dest-key", + CopySource={"Bucket": "src-bucket", "Key": "src-key"}, + StorageClass="STANDARD_IA" + ) + + self.s3.client.copy_object.assert_called_with( + Bucket="dest-bucket", + CopySource={"Bucket": "src-bucket", "Key": "src-key"}, + Key="dest-key", + StorageClass="STANDARD_IA" + ) + + def test_copy_object_default_storage_class(self): + """Test object copy with default storage class""" + self.s3.copy_object( + Bucket="dest-bucket", + Key="dest-key", + CopySource={"Bucket": "src-bucket", "Key": "src-key"} + ) + + self.s3.client.copy_object.assert_called_with( + Bucket="dest-bucket", + CopySource={"Bucket": "src-bucket", "Key": "src-key"}, + Key="dest-key", + StorageClass="GLACIER" + ) + + def test_copy_object_error(self): + """Test object copy error""" + self.s3.client.copy_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey"}}, "copy_object" + ) + + with pytest.raises(ActionError): + self.s3.copy_object( + Bucket="dest-bucket", + Key="dest-key", + CopySource={"Bucket": "src-bucket", "Key": "src-key"} + ) + + +class TestS3ClientFactory: + """Test s3_client_factory function""" + + def test_factory_aws(self): + """Test factory returns AwsS3Client for aws provider""" + with patch('boto3.client'): + client = s3_client_factory("aws") + assert isinstance(client, AwsS3Client) + + def test_factory_gcp_not_implemented(self): + """Test factory raises NotImplementedError for gcp provider""" + with pytest.raises(NotImplementedError, match="GCP S3Client is not implemented"): + s3_client_factory("gcp") + + def test_factory_azure_not_implemented(self): + """Test factory raises NotImplementedError for azure provider""" + with pytest.raises(NotImplementedError, match="Azure S3Client is not implemented"): + s3_client_factory("azure") + + def test_factory_unknown_provider(self): + """Test factory raises ValueError for unknown provider""" + with pytest.raises(ValueError, match="Unsupported provider"): + s3_client_factory("unknown") + + +# Legacy tests for backward compatibility +def test_create_bucket(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.bucket_exists = MagicMock(return_value=False) # Mock the method directly + + assert s3.bucket_exists("test-bucket") is False + + # FIXME: This test is not working as expected. Something in the way it's mocked up + # FIXME: means that the call to create_bucket gets a different result when + # bucket_exists() is called. + s3.create_bucket("test-bucket") + s3.client.create_bucket.assert_called_with(Bucket="test-bucket") + + +def test_create_bucket_error(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.client.create_bucket.side_effect = ClientError( + {"Error": {"Code": "Error"}}, "create_bucket" + ) + + with pytest.raises(Exception): + s3.create_bucket("test-bucket") + + +def test_thaw(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.client.head_object.return_value = {"StorageClass": "GLACIER"} + + s3.thaw( + "test-bucket", + "base_path", + ["base_path/file1", "base_path/file2"], + 7, + "Standard", + ) + assert s3.client.restore_object.call_count == 2 + + +def test_thaw_skip_non_glacier(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.client.head_object.return_value = {"StorageClass": "STANDARD"} + + s3.thaw("test-bucket", "base_path", ["base_path/file1"], 7, "Standard") + s3.client.restore_object.assert_not_called() + + +def test_refreeze(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.client.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": "base_path/file1"}]} + ] + + s3.refreeze("test-bucket", "base_path", "GLACIER") + s3.client.copy_object.assert_called_with( + Bucket="test-bucket", + CopySource={"Bucket": "test-bucket", "Key": "base_path/file1"}, + Key="base_path/file1", + StorageClass="GLACIER", + ) + + +def test_s3_client_factory(): + assert isinstance(s3_client_factory("aws"), AwsS3Client) + with pytest.raises(NotImplementedError): + s3_client_factory("gcp") + with pytest.raises(NotImplementedError): + s3_client_factory("azure") + with pytest.raises(ValueError): + s3_client_factory("unknown") + + +def test_s3_client_init(): + with patch("boto3.client") as mock_boto: + s3 = AwsS3Client() + mock_boto.assert_called_with("s3") + + +def test_thaw_invalid_key(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.client.head_object.return_value = {"StorageClass": "GLACIER"} + + s3.thaw("test-bucket", "base_path", ["wrong_path/file1"], 7, "Standard") + s3.client.restore_object.assert_not_called() + + +def test_refreeze_no_contents(): + s3 = AwsS3Client() + s3.client = MagicMock() + s3.client.get_paginator.return_value.paginate.return_value = [{}] + + s3.refreeze("test-bucket", "base_path", "GLACIER") + s3.client.copy_object.assert_not_called() + + +def test_uniimplemented(): + # S3Client is abstract and cannot be instantiated + with pytest.raises(TypeError): + S3Client() \ No newline at end of file diff --git a/tests/unit/testvars.py b/tests/unit/testvars.py index 63d29ffd..8184d8dd 100644 --- a/tests/unit/testvars.py +++ b/tests/unit/testvars.py @@ -1,450 +1,611 @@ from elasticsearch8 import ConflictError, NotFoundError, TransportError -fake_fail = Exception('Simulated Failure') -four_oh_one = TransportError(401, "simulated error") -four_oh_four = TransportError(404, "simulated error") -get_alias_fail = NotFoundError(404, 'simulated error', 'simulated error') -named_index = 'index_name' -named_indices = [ "index-2015.01.01", "index-2015.02.01" ] -open_index = {'metadata': {'indices' : { named_index : {'state' : 'open'}}}} -closed_index = {'metadata': {'indices' : { named_index : {'state' : 'close'}}}} -cat_open_index = [{'status': 'open'}] -cat_closed_index = [{'status': 'close'}] -open_indices = { 'metadata': { 'indices' : { 'index1' : { 'state' : 'open' }, - 'index2' : { 'state' : 'open' }}}} -closed_indices = { 'metadata': { 'indices' : { 'index1' : { 'state' : 'close' }, - 'index2' : { 'state' : 'close' }}}} -named_alias = 'alias_name' -alias_retval = { "pre_aliased_index": { "aliases" : { named_alias : { }}}} -rollable_alias = { "index-000001": { "aliases" : { named_alias : { }}}} -rollover_conditions = { 'conditions': { 'max_age': '1s' } } +fake_fail = Exception("Simulated Failure") +four_oh_one = TransportError(401, "simulated error") +four_oh_four = TransportError(404, "simulated error") +get_alias_fail = NotFoundError(404, "simulated error", "simulated error") +named_index = "index_name" +named_indices = ["index-2015.01.01", "index-2015.02.01"] +open_index = {"metadata": {"indices": {named_index: {"state": "open"}}}} +closed_index = {"metadata": {"indices": {named_index: {"state": "close"}}}} +cat_open_index = [{"status": "open"}] +cat_closed_index = [{"status": "close"}] +open_indices = { + "metadata": {"indices": {"index1": {"state": "open"}, "index2": {"state": "open"}}} +} +closed_indices = { + "metadata": { + "indices": {"index1": {"state": "close"}, "index2": {"state": "close"}} + } +} +named_alias = "alias_name" +alias_retval = {"pre_aliased_index": {"aliases": {named_alias: {}}}} +rollable_alias = {"index-000001": {"aliases": {named_alias: {}}}} +rollover_conditions = {"conditions": {"max_age": "1s"}} dry_run_rollover = { - "acknowledged": True, - "shards_acknowledged": True, - "old_index": "index-000001", - "new_index": "index-000002", - "rolled_over": False, - "dry_run": True, - "conditions": { - "max_age" : "1s" - } + "acknowledged": True, + "shards_acknowledged": True, + "old_index": "index-000001", + "new_index": "index-000002", + "rolled_over": False, + "dry_run": True, + "conditions": {"max_age": "1s"}, } aliases_retval = { - "index1": { "aliases" : { named_alias : { } } }, - "index2": { "aliases" : { named_alias : { } } }, + "index1": {"aliases": {named_alias: {}}}, + "index2": {"aliases": {named_alias: {}}}, +} +alias_one_add = [{"add": {"alias": "alias", "index": "index_name"}}] +alias_one_add_with_extras = [ + { + "add": { + "alias": "alias", + "index": "index_name", + "filter": {"term": {"user": "kimchy"}}, + } } -alias_one_add = [{'add': {'alias': 'alias', 'index': 'index_name'}}] -alias_one_add_with_extras = [ - { 'add': { - 'alias': 'alias', 'index': 'index_name', - 'filter' : { 'term' : { 'user' : 'kimchy' }} - } - }] -alias_one_rm = [{'remove': {'alias': 'my_alias', 'index': named_index}}] -alias_one_body = { "actions" : [ - {'remove': {'alias': 'alias', 'index': 'index_name'}}, - {'add': {'alias': 'alias', 'index': 'index_name'}} - ]} -alias_two_add = [ - {'add': {'alias': 'alias', 'index': 'index-2016.03.03'}}, - {'add': {'alias': 'alias', 'index': 'index-2016.03.04'}}, - ] -alias_two_rm = [ - {'remove': {'alias': 'my_alias', 'index': 'index-2016.03.03'}}, - {'remove': {'alias': 'my_alias', 'index': 'index-2016.03.04'}}, - ] -alias_success = { "acknowledged": True } -allocation_in = {named_index: {'settings': {'index': {'routing': {'allocation': {'require': {'foo': 'bar'}}}}}}} -allocation_out = {named_index: {'settings': {'index': {'routing': {'allocation': {'require': {'not': 'foo'}}}}}}} -indices_space = { 'indices' : { - 'index1' : { 'index' : { 'primary_size_in_bytes': 1083741824 }}, - 'index2' : { 'index' : { 'primary_size_in_bytes': 1083741824 }}}} -snap_name = 'snap_name' -repo_name = 'repo_name' -test_repo = {repo_name: {'type': 'fs', 'settings': {'compress': 'true', 'location': '/tmp/repos/repo_name'}}} -test_repos = {'TESTING': {'type': 'fs', 'settings': {'compress': 'true', 'location': '/tmp/repos/TESTING'}}, - repo_name: {'type': 'fs', 'settings': {'compress': 'true', 'location': '/rmp/repos/repo_name'}}} -snap_running = { 'snapshots': ['running'] } -nosnap_running = { 'snapshots': [] } -snapshot = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'SUCCESS', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }]} -oneinprogress = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-03-01T00:00:02.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'IN_PROGRESS', - 'snapshot': snap_name, 'end_time': '2015-03-01T00:00:03.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1425168002 - }]} -partial = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'PARTIAL', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }]} -failed = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'FAILED', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }]} -othersnap = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'SOMETHINGELSE', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }]} -snapshots = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'SUCCESS', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }, - { - 'duration_in_millis': 60000, 'start_time': '2015-03-01T00:00:02.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'SUCCESS', - 'snapshot': 'snapshot-2015.03.01', 'end_time': '2015-03-01T00:00:03.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1425168002 - }]} -inprogress = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'SUCCESS', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }, - { - 'duration_in_millis': 60000, 'start_time': '2015-03-01T00:00:02.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'IN_PROGRESS', - 'snapshot': 'snapshot-2015.03.01', 'end_time': '2015-03-01T00:00:03.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1425168002 - }]} -highly_unlikely = { 'snapshots': [ - { - 'duration_in_millis': 60000, 'start_time': '2015-02-01T00:00:00.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'IN_PROGRESS', - 'snapshot': snap_name, 'end_time': '2015-02-01T00:00:01.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1422748800 - }, - { - 'duration_in_millis': 60000, 'start_time': '2015-03-01T00:00:02.000Z', - 'shards': {'successful': 4, 'failed': 0, 'total': 4}, - 'end_time_in_millis': 0, 'state': 'IN_PROGRESS', - 'snapshot': 'snapshot-2015.03.01', 'end_time': '2015-03-01T00:00:03.000Z', - 'indices': named_indices, - 'failures': [], 'start_time_in_millis': 1425168002 - }]} -snap_body_all = { - "ignore_unavailable": False, - "include_global_state": True, - "partial": False, - "indices" : "_all" - } -snap_body = { - "ignore_unavailable": False, - "include_global_state": True, - "partial": False, - "indices" : "index-2015.01.01,index-2015.02.01" - } -verified_nodes = {'nodes': {'nodeid1': {'name': 'node1'}, 'nodeid2': {'name': 'node2'}}} -synced_pass = { - "_shards":{"total":1,"successful":1,"failed":0}, - "index_name":{ - "total":1,"successful":1,"failed":0, - "failures":[], - } - } -synced_fail = { - "_shards":{"total":1,"successful":0,"failed":1}, - "index_name":{ - "total":1,"successful":0,"failed":1, - "failures":[ - {"shard":0,"reason":"pending operations","routing":{"state":"STARTED","primary":True,"node":"nodeid1","relocating_node":None,"shard":0,"index":"index_name"}}, - ] - } - } -sync_conflict = ConflictError(409, '{"_shards":{"total":1,"successful":0,"failed":1},"index_name":{"total":1,"successful":0,"failed":1,"failures":[{"shard":0,"reason":"pending operations","routing":{"state":"STARTED","primary":true,"node":"nodeid1","relocating_node":null,"shard":0,"index":"index_name"}}]}})', synced_fail) -synced_fails = { - "_shards":{"total":2,"successful":1,"failed":1}, - "index1":{ - "total":1,"successful":0,"failed":1, - "failures":[ - {"shard":0,"reason":"pending operations","routing":{"state":"STARTED","primary":True,"node":"nodeid1","relocating_node":None,"shard":0,"index":"index_name"}}, - ] - }, - "index2":{ - "total":1,"successful":1,"failed":0, - "failures":[] - }, - } +] +alias_one_rm = [{"remove": {"alias": "my_alias", "index": named_index}}] +alias_one_body = { + "actions": [ + {"remove": {"alias": "alias", "index": "index_name"}}, + {"add": {"alias": "alias", "index": "index_name"}}, + ] +} +alias_two_add = [ + {"add": {"alias": "alias", "index": "index-2016.03.03"}}, + {"add": {"alias": "alias", "index": "index-2016.03.04"}}, +] +alias_two_rm = [ + {"remove": {"alias": "my_alias", "index": "index-2016.03.03"}}, + {"remove": {"alias": "my_alias", "index": "index-2016.03.04"}}, +] +alias_success = {"acknowledged": True} +allocation_in = { + named_index: { + "settings": {"index": {"routing": {"allocation": {"require": {"foo": "bar"}}}}} + } +} +allocation_out = { + named_index: { + "settings": {"index": {"routing": {"allocation": {"require": {"not": "foo"}}}}} + } +} +indices_space = { + "indices": { + "index1": {"index": {"primary_size_in_bytes": 1083741824}}, + "index2": {"index": {"primary_size_in_bytes": 1083741824}}, + } +} +snap_name = "snap_name" +repo_name = "repo_name" +test_repo = { + repo_name: { + "type": "fs", + "settings": {"compress": "true", "location": "/tmp/repos/repo_name"}, + } +} +test_repos = { + "TESTING": { + "type": "fs", + "settings": {"compress": "true", "location": "/tmp/repos/TESTING"}, + }, + repo_name: { + "type": "fs", + "settings": {"compress": "true", "location": "/rmp/repos/repo_name"}, + }, +} +snap_running = {"snapshots": ["running"]} +nosnap_running = {"snapshots": []} +snapshot = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "SUCCESS", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + } + ] +} +oneinprogress = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-03-01T00:00:02.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "IN_PROGRESS", + "snapshot": snap_name, + "end_time": "2015-03-01T00:00:03.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1425168002, + } + ] +} +partial = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "PARTIAL", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + } + ] +} +failed = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "FAILED", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + } + ] +} +othersnap = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "SOMETHINGELSE", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + } + ] +} +snapshots = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "SUCCESS", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + }, + { + "duration_in_millis": 60000, + "start_time": "2015-03-01T00:00:02.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "SUCCESS", + "snapshot": "snapshot-2015.03.01", + "end_time": "2015-03-01T00:00:03.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1425168002, + }, + ] +} +inprogress = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "SUCCESS", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + }, + { + "duration_in_millis": 60000, + "start_time": "2015-03-01T00:00:02.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "IN_PROGRESS", + "snapshot": "snapshot-2015.03.01", + "end_time": "2015-03-01T00:00:03.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1425168002, + }, + ] +} +highly_unlikely = { + "snapshots": [ + { + "duration_in_millis": 60000, + "start_time": "2015-02-01T00:00:00.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "IN_PROGRESS", + "snapshot": snap_name, + "end_time": "2015-02-01T00:00:01.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1422748800, + }, + { + "duration_in_millis": 60000, + "start_time": "2015-03-01T00:00:02.000Z", + "shards": {"successful": 4, "failed": 0, "total": 4}, + "end_time_in_millis": 0, + "state": "IN_PROGRESS", + "snapshot": "snapshot-2015.03.01", + "end_time": "2015-03-01T00:00:03.000Z", + "indices": named_indices, + "failures": [], + "start_time_in_millis": 1425168002, + }, + ] +} +snap_body_all = { + "ignore_unavailable": False, + "include_global_state": True, + "partial": False, + "indices": "_all", +} +snap_body = { + "ignore_unavailable": False, + "include_global_state": True, + "partial": False, + "indices": "index-2015.01.01,index-2015.02.01", +} +verified_nodes = {"nodes": {"nodeid1": {"name": "node1"}, "nodeid2": {"name": "node2"}}} +synced_pass = { + "_shards": {"total": 1, "successful": 1, "failed": 0}, + "index_name": { + "total": 1, + "successful": 1, + "failed": 0, + "failures": [], + }, +} +synced_fail = { + "_shards": {"total": 1, "successful": 0, "failed": 1}, + "index_name": { + "total": 1, + "successful": 0, + "failed": 1, + "failures": [ + { + "shard": 0, + "reason": "pending operations", + "routing": { + "state": "STARTED", + "primary": True, + "node": "nodeid1", + "relocating_node": None, + "shard": 0, + "index": "index_name", + }, + }, + ], + }, +} +sync_conflict = ConflictError( + 409, + '{"_shards":{"total":1,"successful":0,"failed":1},"index_name":{"total":1,"successful":0,"failed":1,"failures":[{"shard":0,"reason":"pending operations","routing":{"state":"STARTED","primary":true,"node":"nodeid1","relocating_node":null,"shard":0,"index":"index_name"}}]}})', + synced_fail, +) +synced_fails = { + "_shards": {"total": 2, "successful": 1, "failed": 1}, + "index1": { + "total": 1, + "successful": 0, + "failed": 1, + "failures": [ + { + "shard": 0, + "reason": "pending operations", + "routing": { + "state": "STARTED", + "primary": True, + "node": "nodeid1", + "relocating_node": None, + "shard": 0, + "index": "index_name", + }, + }, + ], + }, + "index2": {"total": 1, "successful": 1, "failed": 0, "failures": []}, +} -state_one = [{'index': named_index, 'status': 'open'}] +state_one = [{"index": named_index, "status": "open"}] -settings_one = { +settings_one = { named_index: { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'random_uuid_string_here', - 'number_of_shards': '2', 'creation_date': '1456963200172', - 'routing': {'allocation': {'include': {'tag': 'foo'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "random_uuid_string_here", + "number_of_shards": "2", + "creation_date": "1456963200172", + "routing": {"allocation": {"include": {"tag": "foo"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, } } -settings_1_get_aliases = { named_index: { "aliases" : { 'my_alias' : { } } } } +settings_1_get_aliases = {named_index: {"aliases": {"my_alias": {}}}} state_two = [ - {'index': 'index-2016.03.03', 'status': 'open'}, - {'index': 'index-2016.03.04', 'status': 'open'} + {"index": "index-2016.03.03", "status": "open"}, + {"index": "index-2016.03.04", "status": "open"}, ] -settings_two = { - 'index-2016.03.03': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'random_uuid_string_here', - 'number_of_shards': '5', 'creation_date': '1456963200172', - 'routing': {'allocation': {'include': {'tag': 'foo'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' +settings_two = { + "index-2016.03.03": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "random_uuid_string_here", + "number_of_shards": "5", + "creation_date": "1456963200172", + "routing": {"allocation": {"include": {"tag": "foo"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, }, - 'index-2016.03.04': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'another_random_uuid_string', - 'number_of_shards': '5', 'creation_date': '1457049600812', - 'routing': {'allocation': {'include': {'tag': 'bar'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "index-2016.03.04": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "another_random_uuid_string", + "number_of_shards": "5", + "creation_date": "1457049600812", + "routing": {"allocation": {"include": {"tag": "bar"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } - } + }, + }, } settings_2_get_aliases = { - "index-2016.03.03": { "aliases" : { 'my_alias' : { } } }, - "index-2016.03.04": { "aliases" : { 'my_alias' : { } } }, + "index-2016.03.03": {"aliases": {"my_alias": {}}}, + "index-2016.03.04": {"aliases": {"my_alias": {}}}, } -state_2_closed = [ - {'index': 'index-2016.03.03', 'status': 'close'}, - {'index': 'index-2016.03.04', 'status': 'open'}, +state_2_closed = [ + {"index": "index-2016.03.03", "status": "close"}, + {"index": "index-2016.03.04", "status": "open"}, ] settings_2_closed = { - 'index-2016.03.03': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'random_uuid_string_here', - 'number_of_shards': '5', 'creation_date': '1456963200172', - 'routing': {'allocation': {'include': {'tag': 'foo'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "index-2016.03.03": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "random_uuid_string_here", + "number_of_shards": "5", + "creation_date": "1456963200172", + "routing": {"allocation": {"include": {"tag": "foo"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, }, - 'index-2016.03.04': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'another_random_uuid_string', - 'number_of_shards': '5', 'creation_date': '1457049600812', - 'routing': {'allocation': {'include': {'tag': 'bar'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "index-2016.03.04": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "another_random_uuid_string", + "number_of_shards": "5", + "creation_date": "1457049600812", + "routing": {"allocation": {"include": {"tag": "bar"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } - } + }, + }, } -state_four = [ - {'index': 'a-2016.03.03', 'status': 'open'}, - {'index': 'b-2016.03.04', 'status': 'open'}, - {'index': 'c-2016.03.05', 'status': 'close'}, - {'index': 'd-2016.03.06', 'status': 'open'}, +state_four = [ + {"index": "a-2016.03.03", "status": "open"}, + {"index": "b-2016.03.04", "status": "open"}, + {"index": "c-2016.03.05", "status": "close"}, + {"index": "d-2016.03.06", "status": "open"}, ] -settings_four = { - 'a-2016.03.03': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'random_uuid_string_here', - 'number_of_shards': '5', 'creation_date': '1456963200172', - 'routing': {'allocation': {'include': {'tag': 'foo'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' +settings_four = { + "a-2016.03.03": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "random_uuid_string_here", + "number_of_shards": "5", + "creation_date": "1456963200172", + "routing": {"allocation": {"include": {"tag": "foo"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, }, - 'b-2016.03.04': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'another_random_uuid_string', - 'number_of_shards': '5', 'creation_date': '1457049600812', - 'routing': {'allocation': {'include': {'tag': 'bar'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "b-2016.03.04": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "another_random_uuid_string", + "number_of_shards": "5", + "creation_date": "1457049600812", + "routing": {"allocation": {"include": {"tag": "bar"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, }, - 'c-2016.03.05': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'random_uuid_string_here', - 'number_of_shards': '5', 'creation_date': '1457136000933', - 'routing': {'allocation': {'include': {'tag': 'foo'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "c-2016.03.05": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "random_uuid_string_here", + "number_of_shards": "5", + "creation_date": "1457136000933", + "routing": {"allocation": {"include": {"tag": "foo"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, }, - 'd-2016.03.06': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'another_random_uuid_string', - 'number_of_shards': '5', 'creation_date': '1457222400527', - 'routing': {'allocation': {'include': {'tag': 'bar'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "d-2016.03.06": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "another_random_uuid_string", + "number_of_shards": "5", + "creation_date": "1457222400527", + "routing": {"allocation": {"include": {"tag": "bar"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } - } + }, + }, } -state_named = [ - {'index': 'index-2015.01.01', 'status': 'open'}, - {'index': 'index-2015.02.01', 'status': 'open'}, +state_named = [ + {"index": "index-2015.01.01", "status": "open"}, + {"index": "index-2015.02.01", "status": "open"}, ] settings_named = { - 'index-2015.01.01': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'random_uuid_string_here', - 'number_of_shards': '5', 'creation_date': '1456963200172', - 'routing': {'allocation': {'include': {'tag': 'foo'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "index-2015.01.01": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "random_uuid_string_here", + "number_of_shards": "5", + "creation_date": "1456963200172", + "routing": {"allocation": {"include": {"tag": "foo"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } + }, }, - 'index-2015.02.01': { - 'aliases': ['my_alias'], - 'mappings': {}, - 'settings': { - 'index': { - 'number_of_replicas': '1', 'uuid': 'another_random_uuid_string', - 'number_of_shards': '5', 'creation_date': '1457049600812', - 'routing': {'allocation': {'include': {'tag': 'bar'}}}, - 'version': {'created': '2020099'}, 'refresh_interval': '5s' + "index-2015.02.01": { + "aliases": ["my_alias"], + "mappings": {}, + "settings": { + "index": { + "number_of_replicas": "1", + "uuid": "another_random_uuid_string", + "number_of_shards": "5", + "creation_date": "1457049600812", + "routing": {"allocation": {"include": {"tag": "bar"}}}, + "version": {"created": "2020099"}, + "refresh_interval": "5s", } - } - } + }, + }, } -stats_one = { - 'indices': { - named_index : { - 'total': { - 'docs': {'count': 6374962, 'deleted': 0}, - 'store': {'size_in_bytes': 1115219663, 'throttle_time_in_millis': 0} +stats_one = { + "indices": { + named_index: { + "total": { + "docs": {"count": 6374962, "deleted": 0}, + "store": {"size_in_bytes": 1115219663, "throttle_time_in_millis": 0}, + }, + "primaries": { + "docs": {"count": 3187481, "deleted": 0}, + "store": {"size_in_bytes": 557951789, "throttle_time_in_millis": 0}, }, - 'primaries': { - 'docs': {'count': 3187481, 'deleted': 0}, - 'store': {'size_in_bytes': 557951789, 'throttle_time_in_millis': 0} - } } } } -stats_two = { - 'indices': { - 'index-2016.03.03': { - 'total': { - 'docs': {'count': 6374962, 'deleted': 0}, - 'store': {'size_in_bytes': 1115219663, 'throttle_time_in_millis': 0} +stats_two = { + "indices": { + "index-2016.03.03": { + "total": { + "docs": {"count": 6374962, "deleted": 0}, + "store": {"size_in_bytes": 1115219663, "throttle_time_in_millis": 0}, + }, + "primaries": { + "docs": {"count": 3187481, "deleted": 0}, + "store": {"size_in_bytes": 557951789, "throttle_time_in_millis": 0}, }, - 'primaries': { - 'docs': {'count': 3187481, 'deleted': 0}, - 'store': {'size_in_bytes': 557951789, 'throttle_time_in_millis': 0} - } }, - 'index-2016.03.04': { - 'total': { - 'docs': {'count': 6377544, 'deleted': 0}, - 'store': {'size_in_bytes': 1120891046, 'throttle_time_in_millis': 0} + "index-2016.03.04": { + "total": { + "docs": {"count": 6377544, "deleted": 0}, + "store": {"size_in_bytes": 1120891046, "throttle_time_in_millis": 0}, }, - 'primaries': { - 'docs': {'count': 3188772, 'deleted': 0}, - 'store': {'size_in_bytes': 560677114, 'throttle_time_in_millis': 0} - } - } + "primaries": { + "docs": {"count": 3188772, "deleted": 0}, + "store": {"size_in_bytes": 560677114, "throttle_time_in_millis": 0}, + }, + }, } } -stats_four = { - 'indices': { - 'a-2016.03.03': { - 'total': { - 'docs': {'count': 6374962, 'deleted': 0}, - 'store': {'size_in_bytes': 1115219663, 'throttle_time_in_millis': 0} +stats_four = { + "indices": { + "a-2016.03.03": { + "total": { + "docs": {"count": 6374962, "deleted": 0}, + "store": {"size_in_bytes": 1115219663, "throttle_time_in_millis": 0}, + }, + "primaries": { + "docs": {"count": 3187481, "deleted": 0}, + "store": {"size_in_bytes": 557951789, "throttle_time_in_millis": 0}, }, - 'primaries': { - 'docs': {'count': 3187481, 'deleted': 0}, - 'store': {'size_in_bytes': 557951789, 'throttle_time_in_millis': 0} - } }, - 'b-2016.03.04': { - 'total': { - 'docs': {'count': 6377544, 'deleted': 0}, - 'store': {'size_in_bytes': 1120891046, 'throttle_time_in_millis': 0} + "b-2016.03.04": { + "total": { + "docs": {"count": 6377544, "deleted": 0}, + "store": {"size_in_bytes": 1120891046, "throttle_time_in_millis": 0}, + }, + "primaries": { + "docs": {"count": 3188772, "deleted": 0}, + "store": {"size_in_bytes": 560677114, "throttle_time_in_millis": 0}, }, - 'primaries': { - 'docs': {'count': 3188772, 'deleted': 0}, - 'store': {'size_in_bytes': 560677114, 'throttle_time_in_millis': 0} - } }, # CLOSED, ergo, not present # 'c-2016.03.05': { @@ -457,116 +618,165 @@ # 'store': {'size_in_bytes': 560441083, 'throttle_time_in_millis': 0} # } # }, - 'd-2016.03.06': { - 'total': { - 'docs': {'count': 6266436, 'deleted': 0}, - 'store': {'size_in_bytes': 1120882168, 'throttle_time_in_millis': 0} + "d-2016.03.06": { + "total": { + "docs": {"count": 6266436, "deleted": 0}, + "store": {"size_in_bytes": 1120882168, "throttle_time_in_millis": 0}, }, - 'primaries': { - 'docs': {'count': 3133218, 'deleted': 0}, - 'store': {'size_in_bytes': 560441084, 'throttle_time_in_millis': 0} - } - } - + "primaries": { + "docs": {"count": 3133218, "deleted": 0}, + "store": {"size_in_bytes": 560441084, "throttle_time_in_millis": 0}, + }, + }, } } fieldstats_one = { - 'indices': { - named_index : { - 'fields': { - 'timestamp': { - 'density': 100, - 'min_value_as_string': '2016-03-03T00:00:06.189Z', - 'max_value': 1457049599152, 'max_doc': 415651, - 'min_value': 1456963206189, 'doc_count': 415651, - 'max_value_as_string': '2016-03-03T23:59:59.152Z', - 'sum_total_term_freq': -1, 'sum_doc_freq': 1662604}}}} + "indices": { + named_index: { + "fields": { + "timestamp": { + "density": 100, + "min_value_as_string": "2016-03-03T00:00:06.189Z", + "max_value": 1457049599152, + "max_doc": 415651, + "min_value": 1456963206189, + "doc_count": 415651, + "max_value_as_string": "2016-03-03T23:59:59.152Z", + "sum_total_term_freq": -1, + "sum_doc_freq": 1662604, + } + } + } } +} fieldstats_two = { - 'indices': { - 'index-2016.03.03': { - 'fields': { - 'timestamp': { - 'density': 100, - 'min_value_as_string': '2016-03-03T00:00:06.189Z', - 'max_value': 1457049599152, 'max_doc': 415651, - 'min_value': 1456963206189, 'doc_count': 415651, - 'max_value_as_string': '2016-03-03T23:59:59.152Z', - 'sum_total_term_freq': -1, 'sum_doc_freq': 1662604}}}, - 'index-2016.03.04': { - 'fields': { - 'timestamp': { - 'density': 100, - 'min_value_as_string': '2016-03-04T00:00:00.812Z', - 'max_value': 1457135999223, 'max_doc': 426762, - 'min_value': 1457049600812, 'doc_count': 426762, - 'max_value_as_string': '2016-03-04T23:59:59.223Z', - 'sum_total_term_freq': -1, 'sum_doc_freq': 1673715}}}, + "indices": { + "index-2016.03.03": { + "fields": { + "timestamp": { + "density": 100, + "min_value_as_string": "2016-03-03T00:00:06.189Z", + "max_value": 1457049599152, + "max_doc": 415651, + "min_value": 1456963206189, + "doc_count": 415651, + "max_value_as_string": "2016-03-03T23:59:59.152Z", + "sum_total_term_freq": -1, + "sum_doc_freq": 1662604, + } + } + }, + "index-2016.03.04": { + "fields": { + "timestamp": { + "density": 100, + "min_value_as_string": "2016-03-04T00:00:00.812Z", + "max_value": 1457135999223, + "max_doc": 426762, + "min_value": 1457049600812, + "doc_count": 426762, + "max_value_as_string": "2016-03-04T23:59:59.223Z", + "sum_total_term_freq": -1, + "sum_doc_freq": 1673715, + } + } + }, } } fieldstats_four = { - 'indices': { - 'a-2016.03.03': { - 'fields': { - 'timestamp': { - 'density': 100, - 'min_value_as_string': '2016-03-03T00:00:06.189Z', - 'max_value': 1457049599152, 'max_doc': 415651, - 'min_value': 1456963206189, 'doc_count': 415651, - 'max_value_as_string': '2016-03-03T23:59:59.152Z', - 'sum_total_term_freq': -1, 'sum_doc_freq': 1662604}}}, - 'b-2016.03.04': { - 'fields': { - 'timestamp': { - 'density': 100, - 'min_value_as_string': '2016-03-04T00:00:00.812Z', - 'max_value': 1457135999223, 'max_doc': 426762, - 'min_value': 1457049600812, 'doc_count': 426762, - 'max_value_as_string': '2016-03-04T23:59:59.223Z', - 'sum_total_term_freq': -1, 'sum_doc_freq': 1673715}}}, - 'd-2016.03.06': { - 'fields': { - 'timestamp': { - 'density': 100, - 'min_value_as_string': '2016-03-04T00:00:00.812Z', - 'max_value': 1457308799223, 'max_doc': 426762, - 'min_value': 1457222400567, 'doc_count': 426762, - 'max_value_as_string': '2016-03-04T23:59:59.223Z', - 'sum_total_term_freq': -1, 'sum_doc_freq': 1673715}}}, + "indices": { + "a-2016.03.03": { + "fields": { + "timestamp": { + "density": 100, + "min_value_as_string": "2016-03-03T00:00:06.189Z", + "max_value": 1457049599152, + "max_doc": 415651, + "min_value": 1456963206189, + "doc_count": 415651, + "max_value_as_string": "2016-03-03T23:59:59.152Z", + "sum_total_term_freq": -1, + "sum_doc_freq": 1662604, + } + } + }, + "b-2016.03.04": { + "fields": { + "timestamp": { + "density": 100, + "min_value_as_string": "2016-03-04T00:00:00.812Z", + "max_value": 1457135999223, + "max_doc": 426762, + "min_value": 1457049600812, + "doc_count": 426762, + "max_value_as_string": "2016-03-04T23:59:59.223Z", + "sum_total_term_freq": -1, + "sum_doc_freq": 1673715, + } + } + }, + "d-2016.03.06": { + "fields": { + "timestamp": { + "density": 100, + "min_value_as_string": "2016-03-04T00:00:00.812Z", + "max_value": 1457308799223, + "max_doc": 426762, + "min_value": 1457222400567, + "doc_count": 426762, + "max_value_as_string": "2016-03-04T23:59:59.223Z", + "sum_total_term_freq": -1, + "sum_doc_freq": 1673715, + } + } + }, } } fieldstats_query = { - 'aggregations': { - 'min' : { - 'value_as_string': '2016-03-03T00:00:06.189Z', - 'value': 1456963206189, - }, - 'max' : { - 'value': 1457049599152, - 'value_as_string': '2016-03-03T23:59:59.152Z', - } + "aggregations": { + "min": { + "value_as_string": "2016-03-03T00:00:06.189Z", + "value": 1456963206189, + }, + "max": { + "value": 1457049599152, + "value_as_string": "2016-03-03T23:59:59.152Z", + }, } } -shards = { 'indices': { named_index: { 'shards': { - '0': [ { 'num_search_segments' : 15 }, { 'num_search_segments' : 21 } ], - '1': [ { 'num_search_segments' : 19 }, { 'num_search_segments' : 16 } ] }}}} -fm_shards = { 'indices': { named_index: { 'shards': { - '0': [ { 'num_search_segments' : 1 }, { 'num_search_segments' : 1 } ], - '1': [ { 'num_search_segments' : 1 }, { 'num_search_segments' : 1 } ] }}}} +shards = { + "indices": { + named_index: { + "shards": { + "0": [{"num_search_segments": 15}, {"num_search_segments": 21}], + "1": [{"num_search_segments": 19}, {"num_search_segments": 16}], + } + } + } +} +fm_shards = { + "indices": { + named_index: { + "shards": { + "0": [{"num_search_segments": 1}, {"num_search_segments": 1}], + "1": [{"num_search_segments": 1}, {"num_search_segments": 1}], + } + } + } +} -loginfo = { "loglevel": "INFO", - "logfile": None, - "logformat": "default" - } -default_format = '%(asctime)s %(levelname)-9s %(message)s' -debug_format = '%(asctime)s %(levelname)-9s %(name)22s %(funcName)22s:%(lineno)-4d %(message)s' +loginfo = {"loglevel": "INFO", "logfile": None, "logformat": "default"} +default_format = "%(asctime)s %(levelname)-9s %(message)s" +debug_format = ( + "%(asctime)s %(levelname)-9s %(name)22s %(funcName)22s:%(lineno)-4d %(message)s" +) -yamlconfig = ''' +yamlconfig = """ --- # Remember, leave a key empty to use the default value. None will be a string, # not a Python "NoneType" @@ -585,8 +795,8 @@ logfile: logformat: default quiet: False -''' -pattern_ft = ''' +""" +pattern_ft = """ --- actions: 1: @@ -600,8 +810,8 @@ kind: prefix value: a exclude: False -''' -age_ft = ''' +""" +age_ft = """ --- actions: 1: @@ -618,8 +828,8 @@ unit: seconds unit_count: 0 epoch: 1456963201 -''' -space_ft = ''' +""" +space_ft = """ --- actions: 1: @@ -634,8 +844,8 @@ source: name use_age: True timestring: '%Y.%m.%d' -''' -forcemerge_ft = ''' +""" +forcemerge_ft = """ --- actions: 1: @@ -647,8 +857,8 @@ filters: - filtertype: forcemerged max_num_segments: 2 -''' -allocated_ft = ''' +""" +allocated_ft = """ --- actions: 1: @@ -662,8 +872,8 @@ key: tag value: foo allocation_type: include -''' -kibana_ft = ''' +""" +kibana_ft = """ --- actions: 1: @@ -674,8 +884,8 @@ disable_action: False filters: - filtertype: kibana -''' -opened_ft = ''' +""" +opened_ft = """ --- actions: 1: @@ -686,8 +896,8 @@ disable_action: False filters: - filtertype: opened -''' -closed_ft = ''' +""" +closed_ft = """ --- actions: 1: @@ -698,8 +908,8 @@ disable_action: False filters: - filtertype: closed -''' -none_ft = ''' +""" +none_ft = """ --- actions: 1: @@ -710,8 +920,8 @@ disable_action: False filters: - filtertype: none -''' -invalid_ft = ''' +""" +invalid_ft = """ --- actions: 1: @@ -722,8 +932,8 @@ disable_action: False filters: - filtertype: sir_not_appearing_in_this_film -''' -snap_age_ft = ''' +""" +snap_age_ft = """ --- actions: 1: @@ -737,8 +947,8 @@ direction: older unit: days unit_count: 1 -''' -snap_pattern_ft= ''' +""" +snap_pattern_ft = """ --- actions: 1: @@ -751,8 +961,8 @@ - filtertype: pattern kind: prefix value: sna -''' -snap_none_ft = ''' +""" +snap_none_ft = """ --- actions: 1: @@ -763,8 +973,8 @@ disable_action: False filters: - filtertype: none -''' -size_ft = ''' +""" +size_ft = """ --- actions: 1: @@ -778,28 +988,836 @@ size_threshold: 1.04 size_behavior: total threshold_behavior: less_than -''' +""" -generic_task = {'task': 'I0ekFjMhSPCQz7FUs1zJOg:54510686'} -incomplete_task = {'completed': False, 'task': {'node': 'I0ekFjMhSPCQz7FUs1zJOg', 'status': {'retries': {'bulk': 0, 'search': 0}, 'updated': 0, 'batches': 3647, 'throttled_until_millis': 0, 'throttled_millis': 0, 'noops': 0, 'created': 3646581, 'deleted': 0, 'requests_per_second': -1.0, 'version_conflicts': 0, 'total': 3646581}, 'description': 'UNIT TEST', 'running_time_in_nanos': 1637039537721, 'cancellable': True, 'action': 'indices:data/write/reindex', 'type': 'transport', 'id': 54510686, 'start_time_in_millis': 1489695981997}, 'response': {'retries': {'bulk': 0, 'search': 0}, 'updated': 0, 'batches': 3647, 'throttled_until_millis': 0, 'throttled_millis': 0, 'noops': 0, 'created': 3646581, 'deleted': 0, 'took': 1636917, 'requests_per_second': -1.0, 'timed_out': False, 'failures': [], 'version_conflicts': 0, 'total': 3646581}} -completed_task = {'completed': True, 'task': {'node': 'I0ekFjMhSPCQz7FUs1zJOg', 'status': {'retries': {'bulk': 0, 'search': 0}, 'updated': 0, 'batches': 3647, 'throttled_until_millis': 0, 'throttled_millis': 0, 'noops': 0, 'created': 3646581, 'deleted': 0, 'requests_per_second': -1.0, 'version_conflicts': 0, 'total': 3646581}, 'description': 'UNIT TEST', 'running_time_in_nanos': 1637039537721, 'cancellable': True, 'action': 'indices:data/write/reindex', 'type': 'transport', 'id': 54510686, 'start_time_in_millis': 1489695981997}, 'response': {'retries': {'bulk': 0, 'search': 0}, 'updated': 0, 'batches': 3647, 'throttled_until_millis': 0, 'throttled_millis': 0, 'noops': 0, 'created': 3646581, 'deleted': 0, 'took': 1636917, 'requests_per_second': -1.0, 'timed_out': False, 'failures': [], 'version_conflicts': 0, 'total': 3646581}} -completed_task_zero_total = {'completed': True, 'task': {'node': 'I0ekFjMhSPCQz7FUs1zJOg', 'status': {'retries': {'bulk': 0, 'search': 0}, 'updated': 0, 'batches': 0, 'throttled_until_millis': 0, 'throttled_millis': 0, 'noops': 0, 'created': 0, 'deleted': 0, 'requests_per_second': -1.0, 'version_conflicts': 0, 'total': 0}, 'description': 'UNIT TEST', 'running_time_in_nanos': 1637039537721, 'cancellable': True, 'action': 'indices:data/write/reindex', 'type': 'transport', 'id': 54510686, 'start_time_in_millis': 1489695981997}, 'response': {'retries': {'bulk': 0, 'search': 0}, 'updated': 0, 'batches': 0, 'throttled_until_millis': 0, 'throttled_millis': 0, 'noops': 0, 'created': 0, 'deleted': 0, 'took': 1636917, 'requests_per_second': -1.0, 'timed_out': False, 'failures': [], 'version_conflicts': 0, 'total': 0}} -recovery_output = {'index-2015.01.01': {'shards' : [{'stage':'DONE'}]}, 'index-2015.02.01': {'shards' : [{'stage':'DONE'}]}} -unrecovered_output = {'index-2015.01.01': {'shards' : [{'stage':'INDEX'}]}, 'index-2015.02.01': {'shards' : [{'stage':'INDEX'}]}} -cluster_health = { "cluster_name": "unit_test", "status": "green", "timed_out": False, "number_of_nodes": 7, "number_of_data_nodes": 3, "active_primary_shards": 235, "active_shards": 471, "relocating_shards": 0, "initializing_shards": 0, "unassigned_shards": 0, "delayed_unassigned_shards": 0, "number_of_pending_tasks": 0, "task_max_waiting_in_queue_millis": 0, "active_shards_percent_as_number": 100} -reindex_basic = { 'source': { 'index': named_index }, 'dest': { 'index': 'other_index' } } -reindex_replace = { 'source': { 'index': 'REINDEX_SELECTION' }, 'dest': { 'index': 'other_index' } } -reindex_migration = { 'source': { 'index': named_index }, 'dest': { 'index': 'MIGRATION' } } -index_list_966 = ['indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d'] -recovery_966 = {'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d': {'shards': [{'total_time': '10.1m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10.1m', 'target_throttle_time': '-1', 'total_time_in_millis': 606577, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3171596177, 'reused': '0b', 'total_in_bytes': 3171596177, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '45ms', 'percent': '100.0%', 'total_time_in_millis': 45, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T11:54:48.183Z', 'primary': True, 'total_time_in_millis': 606631, 'stop_time_in_millis': 1494936294815, 'stop_time': '2017-05-16T12:04:54.815Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 1, 'start_time_in_millis': 1494935688183}, {'total_time': '10m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10m', 'target_throttle_time': '-1', 'total_time_in_millis': 602302, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3162299781, 'reused': '0b', 'total_in_bytes': 3162299781, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '389ms', 'percent': '100.0%', 'total_time_in_millis': 389, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T12:04:51.606Z', 'primary': True, 'total_time_in_millis': 602698, 'stop_time_in_millis': 1494936894305, 'stop_time': '2017-05-16T12:14:54.305Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 5, 'start_time_in_millis': 1494936291606}, {'total_time': '10.1m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10.1m', 'target_throttle_time': '-1', 'total_time_in_millis': 606692, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3156050994, 'reused': '0b', 'total_in_bytes': 3156050994, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '38ms', 'percent': '100.0%', 'total_time_in_millis': 38, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T11:54:48.166Z', 'primary': True, 'total_time_in_millis': 606737, 'stop_time_in_millis': 1494936294904, 'stop_time': '2017-05-16T12:04:54.904Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 3, 'start_time_in_millis': 1494935688166}, {'total_time': '10m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10m', 'target_throttle_time': '-1', 'total_time_in_millis': 602010, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3153017440, 'reused': '0b', 'total_in_bytes': 3153017440, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '558ms', 'percent': '100.0%', 'total_time_in_millis': 558, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T12:04:51.369Z', 'primary': True, 'total_time_in_millis': 602575, 'stop_time_in_millis': 1494936893944, 'stop_time': '2017-05-16T12:14:53.944Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 4, 'start_time_in_millis': 1494936291369}, {'total_time': '10m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10m', 'target_throttle_time': '-1', 'total_time_in_millis': 600492, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3153347402, 'reused': '0b', 'total_in_bytes': 3153347402, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '445ms', 'percent': '100.0%', 'total_time_in_millis': 445, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T12:04:54.817Z', 'primary': True, 'total_time_in_millis': 600946, 'stop_time_in_millis': 1494936895764, 'stop_time': '2017-05-16T12:14:55.764Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 6, 'start_time_in_millis': 1494936294817}, {'total_time': '10m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10m', 'target_throttle_time': '-1', 'total_time_in_millis': 603194, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3148003580, 'reused': '0b', 'total_in_bytes': 3148003580, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '225ms', 'percent': '100.0%', 'total_time_in_millis': 225, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T11:54:48.173Z', 'primary': True, 'total_time_in_millis': 603429, 'stop_time_in_millis': 1494936291602, 'stop_time': '2017-05-16T12:04:51.602Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 2, 'start_time_in_millis': 1494935688173}, {'total_time': '10m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10m', 'target_throttle_time': '-1', 'total_time_in_millis': 601453, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3168132171, 'reused': '0b', 'total_in_bytes': 3168132171, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '43ms', 'percent': '100.0%', 'total_time_in_millis': 43, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T12:04:54.905Z', 'primary': True, 'total_time_in_millis': 601503, 'stop_time_in_millis': 1494936896408, 'stop_time': '2017-05-16T12:14:56.408Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 7, 'start_time_in_millis': 1494936294905}, {'total_time': '10m', 'index': {'files': {'reused': 0, 'total': 15, 'percent': '100.0%', 'recovered': 15}, 'total_time': '10m', 'target_throttle_time': '-1', 'total_time_in_millis': 602897, 'source_throttle_time_in_millis': 0, 'source_throttle_time': '-1', 'target_throttle_time_in_millis': 0, 'size': {'recovered_in_bytes': 3153750393, 'reused': '0b', 'total_in_bytes': 3153750393, 'percent': '100.0%', 'reused_in_bytes': 0, 'total': '2.9gb', 'recovered': '2.9gb'}}, 'verify_index': {'total_time': '0s', 'total_time_in_millis': 0, 'check_index_time_in_millis': 0, 'check_index_time': '0s'}, 'target': {'ip': 'x.x.x.7', 'host': 'x.x.x.7', 'transport_address': 'x.x.x.7:9300', 'id': 'K4xQPaOFSWSPLwhb0P47aQ', 'name': 'staging-es5-forcem'}, 'source': {'index': 'indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d', 'version': '5.1.1', 'snapshot': 'force-merge', 'repository': 'force-merge'}, 'translog': {'total_time': '271ms', 'percent': '100.0%', 'total_time_in_millis': 271, 'total_on_start': 0, 'total': 0, 'recovered': 0}, 'start_time': '2017-05-16T11:54:48.191Z', 'primary': True, 'total_time_in_millis': 603174, 'stop_time_in_millis': 1494936291366, 'stop_time': '2017-05-16T12:04:51.366Z', 'stage': 'DONE', 'type': 'SNAPSHOT', 'id': 0, 'start_time_in_millis': 1494935688191}]}} -no_snap_tasks = {'nodes': {'node1': {'tasks': {'task1': {'action': 'cluster:monitor/tasks/lists[n]'}}}}} -snap_task = {'nodes': {'node1': {'tasks': {'task1': {'action': 'cluster:admin/snapshot/delete'}}}}} -watermark_persistent = {'persistent':{'cluster':{'routing':{'allocation':{'disk':{'watermark':{'low':'11%','high':'60gb'}}}}}}} -watermark_transient = {'transient':{'cluster':{'routing':{'allocation':{'disk':{'watermark':{'low':'9%','high':'50gb'}}}}}}} +generic_task = {"task": "I0ekFjMhSPCQz7FUs1zJOg:54510686"} +incomplete_task = { + "completed": False, + "task": { + "node": "I0ekFjMhSPCQz7FUs1zJOg", + "status": { + "retries": {"bulk": 0, "search": 0}, + "updated": 0, + "batches": 3647, + "throttled_until_millis": 0, + "throttled_millis": 0, + "noops": 0, + "created": 3646581, + "deleted": 0, + "requests_per_second": -1.0, + "version_conflicts": 0, + "total": 3646581, + }, + "description": "UNIT TEST", + "running_time_in_nanos": 1637039537721, + "cancellable": True, + "action": "indices:data/write/reindex", + "type": "transport", + "id": 54510686, + "start_time_in_millis": 1489695981997, + }, + "response": { + "retries": {"bulk": 0, "search": 0}, + "updated": 0, + "batches": 3647, + "throttled_until_millis": 0, + "throttled_millis": 0, + "noops": 0, + "created": 3646581, + "deleted": 0, + "took": 1636917, + "requests_per_second": -1.0, + "timed_out": False, + "failures": [], + "version_conflicts": 0, + "total": 3646581, + }, +} +completed_task = { + "completed": True, + "task": { + "node": "I0ekFjMhSPCQz7FUs1zJOg", + "status": { + "retries": {"bulk": 0, "search": 0}, + "updated": 0, + "batches": 3647, + "throttled_until_millis": 0, + "throttled_millis": 0, + "noops": 0, + "created": 3646581, + "deleted": 0, + "requests_per_second": -1.0, + "version_conflicts": 0, + "total": 3646581, + }, + "description": "UNIT TEST", + "running_time_in_nanos": 1637039537721, + "cancellable": True, + "action": "indices:data/write/reindex", + "type": "transport", + "id": 54510686, + "start_time_in_millis": 1489695981997, + }, + "response": { + "retries": {"bulk": 0, "search": 0}, + "updated": 0, + "batches": 3647, + "throttled_until_millis": 0, + "throttled_millis": 0, + "noops": 0, + "created": 3646581, + "deleted": 0, + "took": 1636917, + "requests_per_second": -1.0, + "timed_out": False, + "failures": [], + "version_conflicts": 0, + "total": 3646581, + }, +} +completed_task_zero_total = { + "completed": True, + "task": { + "node": "I0ekFjMhSPCQz7FUs1zJOg", + "status": { + "retries": {"bulk": 0, "search": 0}, + "updated": 0, + "batches": 0, + "throttled_until_millis": 0, + "throttled_millis": 0, + "noops": 0, + "created": 0, + "deleted": 0, + "requests_per_second": -1.0, + "version_conflicts": 0, + "total": 0, + }, + "description": "UNIT TEST", + "running_time_in_nanos": 1637039537721, + "cancellable": True, + "action": "indices:data/write/reindex", + "type": "transport", + "id": 54510686, + "start_time_in_millis": 1489695981997, + }, + "response": { + "retries": {"bulk": 0, "search": 0}, + "updated": 0, + "batches": 0, + "throttled_until_millis": 0, + "throttled_millis": 0, + "noops": 0, + "created": 0, + "deleted": 0, + "took": 1636917, + "requests_per_second": -1.0, + "timed_out": False, + "failures": [], + "version_conflicts": 0, + "total": 0, + }, +} +recovery_output = { + "index-2015.01.01": {"shards": [{"stage": "DONE"}]}, + "index-2015.02.01": {"shards": [{"stage": "DONE"}]}, +} +unrecovered_output = { + "index-2015.01.01": {"shards": [{"stage": "INDEX"}]}, + "index-2015.02.01": {"shards": [{"stage": "INDEX"}]}, +} +cluster_health = { + "cluster_name": "unit_test", + "status": "green", + "timed_out": False, + "number_of_nodes": 7, + "number_of_data_nodes": 3, + "active_primary_shards": 235, + "active_shards": 471, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100, +} +reindex_basic = {"source": {"index": named_index}, "dest": {"index": "other_index"}} +reindex_replace = { + "source": {"index": "REINDEX_SELECTION"}, + "dest": {"index": "other_index"}, +} +reindex_migration = {"source": {"index": named_index}, "dest": {"index": "MIGRATION"}} +index_list_966 = ["indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d"] +recovery_966 = { + "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d": { + "shards": [ + { + "total_time": "10.1m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10.1m", + "target_throttle_time": "-1", + "total_time_in_millis": 606577, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3171596177, + "reused": "0b", + "total_in_bytes": 3171596177, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "45ms", + "percent": "100.0%", + "total_time_in_millis": 45, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T11:54:48.183Z", + "primary": True, + "total_time_in_millis": 606631, + "stop_time_in_millis": 1494936294815, + "stop_time": "2017-05-16T12:04:54.815Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 1, + "start_time_in_millis": 1494935688183, + }, + { + "total_time": "10m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10m", + "target_throttle_time": "-1", + "total_time_in_millis": 602302, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3162299781, + "reused": "0b", + "total_in_bytes": 3162299781, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "389ms", + "percent": "100.0%", + "total_time_in_millis": 389, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T12:04:51.606Z", + "primary": True, + "total_time_in_millis": 602698, + "stop_time_in_millis": 1494936894305, + "stop_time": "2017-05-16T12:14:54.305Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 5, + "start_time_in_millis": 1494936291606, + }, + { + "total_time": "10.1m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10.1m", + "target_throttle_time": "-1", + "total_time_in_millis": 606692, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3156050994, + "reused": "0b", + "total_in_bytes": 3156050994, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "38ms", + "percent": "100.0%", + "total_time_in_millis": 38, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T11:54:48.166Z", + "primary": True, + "total_time_in_millis": 606737, + "stop_time_in_millis": 1494936294904, + "stop_time": "2017-05-16T12:04:54.904Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 3, + "start_time_in_millis": 1494935688166, + }, + { + "total_time": "10m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10m", + "target_throttle_time": "-1", + "total_time_in_millis": 602010, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3153017440, + "reused": "0b", + "total_in_bytes": 3153017440, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "558ms", + "percent": "100.0%", + "total_time_in_millis": 558, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T12:04:51.369Z", + "primary": True, + "total_time_in_millis": 602575, + "stop_time_in_millis": 1494936893944, + "stop_time": "2017-05-16T12:14:53.944Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 4, + "start_time_in_millis": 1494936291369, + }, + { + "total_time": "10m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10m", + "target_throttle_time": "-1", + "total_time_in_millis": 600492, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3153347402, + "reused": "0b", + "total_in_bytes": 3153347402, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "445ms", + "percent": "100.0%", + "total_time_in_millis": 445, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T12:04:54.817Z", + "primary": True, + "total_time_in_millis": 600946, + "stop_time_in_millis": 1494936895764, + "stop_time": "2017-05-16T12:14:55.764Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 6, + "start_time_in_millis": 1494936294817, + }, + { + "total_time": "10m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10m", + "target_throttle_time": "-1", + "total_time_in_millis": 603194, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3148003580, + "reused": "0b", + "total_in_bytes": 3148003580, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "225ms", + "percent": "100.0%", + "total_time_in_millis": 225, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T11:54:48.173Z", + "primary": True, + "total_time_in_millis": 603429, + "stop_time_in_millis": 1494936291602, + "stop_time": "2017-05-16T12:04:51.602Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 2, + "start_time_in_millis": 1494935688173, + }, + { + "total_time": "10m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10m", + "target_throttle_time": "-1", + "total_time_in_millis": 601453, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3168132171, + "reused": "0b", + "total_in_bytes": 3168132171, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "43ms", + "percent": "100.0%", + "total_time_in_millis": 43, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T12:04:54.905Z", + "primary": True, + "total_time_in_millis": 601503, + "stop_time_in_millis": 1494936896408, + "stop_time": "2017-05-16T12:14:56.408Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 7, + "start_time_in_millis": 1494936294905, + }, + { + "total_time": "10m", + "index": { + "files": { + "reused": 0, + "total": 15, + "percent": "100.0%", + "recovered": 15, + }, + "total_time": "10m", + "target_throttle_time": "-1", + "total_time_in_millis": 602897, + "source_throttle_time_in_millis": 0, + "source_throttle_time": "-1", + "target_throttle_time_in_millis": 0, + "size": { + "recovered_in_bytes": 3153750393, + "reused": "0b", + "total_in_bytes": 3153750393, + "percent": "100.0%", + "reused_in_bytes": 0, + "total": "2.9gb", + "recovered": "2.9gb", + }, + }, + "verify_index": { + "total_time": "0s", + "total_time_in_millis": 0, + "check_index_time_in_millis": 0, + "check_index_time": "0s", + }, + "target": { + "ip": "x.x.x.7", + "host": "x.x.x.7", + "transport_address": "x.x.x.7:9300", + "id": "K4xQPaOFSWSPLwhb0P47aQ", + "name": "staging-es5-forcem", + }, + "source": { + "index": "indexv0.2_2017-02-12_536a9247f9fa4fc7a7942ad46ea14e0d", + "version": "5.1.1", + "snapshot": "force-merge", + "repository": "force-merge", + }, + "translog": { + "total_time": "271ms", + "percent": "100.0%", + "total_time_in_millis": 271, + "total_on_start": 0, + "total": 0, + "recovered": 0, + }, + "start_time": "2017-05-16T11:54:48.191Z", + "primary": True, + "total_time_in_millis": 603174, + "stop_time_in_millis": 1494936291366, + "stop_time": "2017-05-16T12:04:51.366Z", + "stage": "DONE", + "type": "SNAPSHOT", + "id": 0, + "start_time_in_millis": 1494935688191, + }, + ] + } +} +no_snap_tasks = { + "nodes": { + "node1": {"tasks": {"task1": {"action": "cluster:monitor/tasks/lists[n]"}}} + } +} +snap_task = { + "nodes": { + "node1": {"tasks": {"task1": {"action": "cluster:admin/snapshot/delete"}}} + } +} +watermark_persistent = { + "persistent": { + "cluster": { + "routing": { + "allocation": {"disk": {"watermark": {"low": "11%", "high": "60gb"}}} + } + } + } +} +watermark_transient = { + "transient": { + "cluster": { + "routing": { + "allocation": {"disk": {"watermark": {"low": "9%", "high": "50gb"}}} + } + } + } +} watermark_both = { - 'persistent': {'cluster':{'routing':{'allocation':{'disk':{'watermark':{'low':'11%','high':'60gb'}}}}}}, - 'transient': {'cluster':{'routing':{'allocation':{'disk':{'watermark':{'low':'9%','high':'50gb'}}}}}}, + "persistent": { + "cluster": { + "routing": { + "allocation": {"disk": {"watermark": {"low": "11%", "high": "60gb"}}} + } + } + }, + "transient": { + "cluster": { + "routing": { + "allocation": {"disk": {"watermark": {"low": "9%", "high": "50gb"}}} + } + } + }, +} +empty_cluster_settings = {"persistent": {}, "transient": {}} +data_only_node_role = ["data"] +master_data_node_role = ["data", "master"] +# +# Deepfreeze values +# +repo_name_prefix = "deepfreeze-" +bucket_name_prefix = "deepfreeze-" +base_path = "snapshots" +canned_acl = "private" +storage_class = "intelligent_tiering" +keep = "6" +year = "2024" +month = "08" +month_exists = "06" +repositories = [ + "foo", + "deepfreeze-2024.01", + "deepfreeze-2024.02", + "deepfreeze-2024.03", + "deepfreeze-2024.04", + "deepfreeze-2024.05", + "deepfreeze-2024.06", + "deepfreeze-2024.07", +] +repositories_filtered = [ + "deepfreeze-2024.01", + "deepfreeze-2024.02", + "deepfreeze-2024.03", + "deepfreeze-2024.04", + "deepfreeze-2024.05", + "deepfreeze-2024.06", + "deepfreeze-2024.07", +] +ilm_policy_to_update = { + "deepfreeze-ilm-policy": { + "version": 3, + "modified_date": "2024-09-08T13:44:16.327Z", + "policy": { + "phases": { + "frozen": { + "min_age": "2d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "deepfreeze-2024.07", + "force_merge_index": True, + } + }, + }, + "delete": { + "min_age": "3d", + "actions": {"delete": {"delete_searchable_snapshot": False}}, + }, + "cold": { + "min_age": "1d", + "actions": { + "allocate": { + "number_of_replicas": 0, + "include": {}, + "exclude": {}, + "require": {}, + }, + "searchable_snapshot": { + "snapshot_repository": "deepfreeze-2024.07", + "force_merge_index": True, + }, + "set_priority": {"priority": 0}, + }, + }, + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb", + }, + "set_priority": {"priority": 100}, + }, + }, + } + }, + "in_use_by": {"indices": [], "data_streams": [], "composable_templates": []}, + } +} +ilm_policy_updated = { + "phases": { + "frozen": { + "min_age": "2d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "deepfreeze-2024.08", + "force_merge_index": True, + } + }, + }, + "delete": { + "min_age": "3d", + "actions": {"delete": {"delete_searchable_snapshot": False}}, + }, + "cold": { + "min_age": "1d", + "actions": { + "allocate": { + "number_of_replicas": 0, + "include": {}, + "exclude": {}, + "require": {}, + }, + "searchable_snapshot": { + "snapshot_repository": "deepfreeze-2024.08", + "force_merge_index": True, + }, + "set_priority": {"priority": 0}, + }, + }, + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_age": "30d", + "max_primary_shard_size": "50gb", + }, + "set_priority": {"priority": 100}, + }, + }, + } } -empty_cluster_settings = {'persistent':{},'transient':{}} -data_only_node_role = ['data'] -master_data_node_role = ['data','master']