Add step to backup stopped instances (#409)

Closes #405

Now it's possible to backup stopped instance by `backup_instance_dirs` step.
This makes it possible to transfer instances from one host to another.

Also, now it's possible to restore instances from local backup by `restore` step with `cartridge_restore_backup_path_local` variable.

Author: hackallcode

CHANGELOG.md

@@ -10,6 +10,11 @@ Please update `ansible-galaxy install` command in
 README.md to use the newest tag with new release
 -->
 
+### Added
+
+- Add `backup_instance_dirs` step to archive files of stopped instance
+- Add `cartridge_restore_backup_path_local` to restore instance from local backup
+
 ### Fixed
 
 - Remove old app configurations before uploading a new one

defaults/main.yml

@@ -163,6 +163,7 @@ cartridge_remote_backups_dir: /opt/tarantool/backups
 cartridge_fetch_backups: false
 cartridge_fetch_backups_dir: backups/
 cartridge_restore_backup_path: null
+cartridge_restore_backup_path_local: null
 cartridge_force_restore: false
 cartridge_allow_alien_backup: false
 cartridge_skip_cleanup_on_restore: false
@@ -313,6 +314,7 @@ cartridge_cached_fact_names_by_target:
     - cartridge_fetch_backups
     - cartridge_fetch_backups_dir
     - cartridge_restore_backup_path
+    - cartridge_restore_backup_path_local
     - cartridge_force_restore
     - cartridge_allow_alien_backup
     - cartridge_skip_cleanup_on_restore

doc/steps.md

@@ -43,6 +43,7 @@ but can be used in a custom one:
 - [backup](#step-backup)
 - [backup_start](#step-backup_start)
 - [backup_stop](#step-backup_stop)
+- [backup_instance_dirs](#step-backup_instance_dirs)
 - [restore](#step-restore)
 - [check_new_topology](#step-check_new_topology)
 
@@ -591,11 +592,32 @@ Input variables from config:
   (`.tarantool.cookie` will be kept independently of this variable);
   it's possible to use bash patterns, e.g. `*.control`.
 - `cartridge_restore_backup_path` - path to the instance backup archive on the remote machine;
+- `cartridge_restore_backup_path_local` - path to the instance backup archive on the local machine;
 - `cartridge_remote_backups_dir` - directory with backups on the remote;
 - `cartridge_force_restore` - flag indicates that conflicting files should be overwritten;
 - `cartridge_allow_alien_backup` - flag indicates that backup of instance with another name can be used;
 - `cartridge_skip_cleanup_on_restore` - flag indicates that cleanup before restoring should be skipped.
 
+### Step `backup_instance_dirs`
+
+Create a [backup](/doc/backups.md) archive for each **stopped** instance and fetch it on the local machine.
+
+Input variables from config:
+
+- `cartridge_remote_backups_dir` - directory to store backups on the remote;
+- `cartridge_fetch_backups` - flag indicates that backups should be fetched the local machine;
+- `cartridge_fetch_backups_dir` -  a directory on the local machine where backups should be fetched if `cartridge_fetch_backups` is `true`. This path is relative to the playbook path;
+- `cartridge_app_user` - user which will own the links;
+- `cartridge_app_group` - group which will own the links;
+- `stateboard` - indicates that the instance is a stateboard.
+
+Output facts:
+
+- `instance_backup_files` - list of instance files to back up;
+- `backup_files_from_machine` - list of files to back up for all instances on the same machine as a current one;
+- `backup_archive_path` - path to the instance backup archive on the remote machine;
+- `fetched_backup_archive_path` - path to the fetched backup file (is set only if `cartridge_fetch_backups` is `true`).
+
 ## Step `check_new_topology`
 
 Check for dangerous changes on edit topology

doc/variables.md

@@ -212,6 +212,8 @@ For more details see [scenario documentation](/doc/scenario.md).
   the playbook path.
 - `cartridge_restore_backup_path` (`string`): path to the instance backup archive on the remote
   machine;
+- `cartridge_restore_backup_path_local` (`string`): path to the instance backup archive on the local
+  machine;
 - `cartridge_force_restore` (`boolean`, default: `false`): flag indicates that conflicting files
   should be overwritten;
 - `cartridge_allow_alien_backup` (`boolean`, default: `false`): flag indicates that backup of

library/cartridge_backup_instance.py

@@ -5,22 +5,25 @@
 from ansible.module_utils.helpers import Helpers as helpers
 
 argument_spec = {
-    'console_sock': {'required': True, 'type': 'str'},
+    'console_sock': {'required': False, 'type': 'str'},
     'instance_id': {'required': False, 'type': 'str'},
     'stateboard': {'required': False, 'type': 'bool'},
     'backups_dir': {'required': False, 'type': 'str'},
     'instance_conf_file': {'required': False, 'type': 'str'},
     'app_conf_file': {'required': False, 'type': 'str'},
-
     'start_only': {'required': False, 'type': 'bool', 'default': False},
     'stop_only': {'required': False, 'type': 'bool', 'default': False},
+    'custom_backup_files': {'required': False, 'type': 'list'},
 }
 
 
 def backup_start(control_console, params):
-    stateboard = params['stateboard']
-    instance_conf_file = params['instance_conf_file']
-    app_conf_file = params['app_conf_file']
+    stateboard = params.get('stateboard')
+    assert stateboard is not None, 'Parameter "stateboard" is required'
+    instance_conf_file = params.get('instance_conf_file')
+    assert instance_conf_file is not None, 'Parameter "instance_conf_file" is required'
+    app_conf_file = params.get('app_conf_file')
+    assert app_conf_file is not None, 'Parameter "app_conf_file" is required'
 
     # create snapshot and start a backup
     backup_files, err = control_console.eval_res_err('''
@@ -81,7 +84,8 @@ def backup_pack(instance_id, backups_dir, backup_files):
 
     with tarfile.open(archive_path, "w:gz") as tar:
         for path in backup_files:
-            tar.add(path)
+            if os.path.exists(path):
+                tar.add(path)
 
     return archive_path
 
@@ -97,13 +101,27 @@ def backup_stop(control_console):
     return None
 
 
-def call_backup(params):
-    start_only = params['start_only']
-    stop_only = params['stop_only']
+def custom_backup(params):
+    custom_backup_files = helpers.get_required_param(params, 'custom_backup_files')
+
+    # ARCHIVE
+    instance_id = helpers.get_required_param(params, 'instance_id')
+    backups_dir = helpers.get_required_param(params, 'backups_dir')
+    backup_archive_path = backup_pack(instance_id, backups_dir, custom_backup_files)
+
+    return helpers.ModuleRes(changed=True, fact={
+        'backup_archive_path': backup_archive_path,
+        'backup_files': custom_backup_files,
+    })
+
+
+def tnt_backup(params):
+    start_only = params.get('start_only', False)
+    stop_only = params.get('stop_only', False)
 
     assert not (start_only and stop_only), "impossible to use 'start_only' with 'stop_only'"
 
-    console_sock = params['console_sock']
+    console_sock = helpers.get_required_param(params, 'console_sock')
     control_console = helpers.get_control_console(console_sock)
 
     if not helpers.box_cfg_was_called(control_console):
@@ -128,8 +146,8 @@ def call_backup(params):
         })
 
     # ARCHIVE
-    instance_id = params['instance_id']
-    backups_dir = params['backups_dir']
+    instance_id = helpers.get_required_param(params, 'instance_id')
+    backups_dir = helpers.get_required_param(params, 'backups_dir')
     backup_archive_path = backup_pack(instance_id, backups_dir, backup_files)
 
     # STOP
@@ -143,5 +161,12 @@ def call_backup(params):
     })
 
 
+def call_backup(params):
+    if params.get('custom_backup_files'):
+        return custom_backup(params)
+    else:
+        return tnt_backup(params)
+
+
 if __name__ == '__main__':
     helpers.execute_module(argument_spec, call_backup)

library/cartridge_get_instance_info.py

@@ -31,6 +31,12 @@ def get_instance_systemd_service(app_name, instance_name, stateboard=False):
     return '%s@%s' % (app_name, instance_name)
 
 
+def get_instance_systemd_service_template(app_name, stateboard=False):
+    if stateboard:
+        return '%s-stateboard.service' % app_name
+    return '%[email protected]' % app_name
+
+
 def get_instance_systemd_service_dir(systemd_dir, service_name):
     if not service_name.endswith('.service'):
         service_name += '.service'
@@ -74,18 +80,21 @@ def get_instance_info(params):
         'paths_to_remove_on_expel': set(),
         'files_to_remove_on_cleanup': set(),
         'dirs_to_remove_on_cleanup': set(),
+        'paths_to_backup_files': set(),
     }
 
     # app conf file
     instance_info['app_conf_file'] = get_app_conf_file(
         instance_vars['cartridge_conf_dir'], app_name
     )
+    instance_info['paths_to_backup_files'].add(instance_info['app_conf_file'])
 
     # instance conf file
     instance_info['conf_file'] = get_instance_conf_file(
         instance_vars['cartridge_conf_dir'], app_name, instance_name, instance_vars['stateboard'],
     )
     instance_info['paths_to_remove_on_expel'].add(instance_info['conf_file'])
+    instance_info['paths_to_backup_files'].add(instance_info['conf_file'])
 
     # instance id (e.g. used for conf section name)
     instance_info['instance_id'] = get_instance_conf_section(
@@ -111,6 +120,7 @@ def get_instance_info(params):
     )
     instance_info['paths_to_remove_on_expel'].add(instance_info['work_dir'])
     instance_info['dirs_to_remove_on_cleanup'].add(instance_info['work_dir'])
+    instance_info['paths_to_backup_files'].add(instance_info['work_dir'])
 
     # instance memtx dir
     instance_info['memtx_dir'] = None
@@ -120,6 +130,7 @@ def get_instance_info(params):
         )
         instance_info['paths_to_remove_on_expel'].add(instance_info['memtx_dir'])
         instance_info['dirs_to_remove_on_cleanup'].add(instance_info['memtx_dir'])
+        instance_info['paths_to_backup_files'].add(instance_info['memtx_dir'])
 
     # instance vinyl dir
     instance_info['vinyl_dir'] = None
@@ -129,6 +140,7 @@ def get_instance_info(params):
         )
         instance_info['paths_to_remove_on_expel'].add(instance_info['vinyl_dir'])
         instance_info['dirs_to_remove_on_cleanup'].add(instance_info['vinyl_dir'])
+        instance_info['paths_to_backup_files'].add(instance_info['vinyl_dir'])
 
     # instance wal dir
     instance_info['wal_dir'] = None
@@ -138,6 +150,7 @@ def get_instance_info(params):
         )
         instance_info['paths_to_remove_on_expel'].add(instance_info['wal_dir'])
         instance_info['dirs_to_remove_on_cleanup'].add(instance_info['wal_dir'])
+        instance_info['paths_to_backup_files'].add(instance_info['wal_dir'])
 
     # instance log dir
     instance_info['log_file'] = None
@@ -153,16 +166,22 @@ def get_instance_info(params):
     instance_info['systemd_service'] = get_instance_systemd_service(
         app_name, instance_name, instance_vars['stateboard']
     )
+    instance_info['paths_to_backup_files'].add(get_instance_systemd_service_template(
+        app_name, instance_vars['stateboard']
+    ))
+
     instance_info['systemd_service_dir'] = get_instance_systemd_service_dir(
         instance_vars['cartridge_systemd_dir'],
         instance_info['systemd_service'],
     )
     instance_info['systemd_service_env_file'] = get_systemd_service_env_file(instance_info['systemd_service_dir'])
+    instance_info['paths_to_backup_files'].add(instance_info['systemd_service_env_file'])
 
     # tmpfiles conf
     instance_info['tmpfiles_conf'] = os.path.join(
         instance_vars['cartridge_tmpfiles_dir'], '%s.conf' % app_name
     )
+    instance_info['paths_to_backup_files'].add(instance_info['tmpfiles_conf'])
 
     # code dirs
     if not instance_vars['cartridge_multiversion']:
@@ -182,6 +201,7 @@ def get_instance_info(params):
         )
 
     instance_info['paths_to_remove_on_expel'] = list(sorted(instance_info['paths_to_remove_on_expel']))
+    instance_info['paths_to_backup_files'] = list(sorted(instance_info['paths_to_backup_files']))
 
     instance_info['files_to_remove_on_cleanup'] = list(sorted(filter_paths_by_glob_list(
         instance_info['files_to_remove_on_cleanup'],

library/cartridge_validate_config.py

@@ -182,6 +182,7 @@
     'cartridge_fetch_backups': bool,
     'cartridge_fetch_backups_dir': str,
     'cartridge_restore_backup_path': str,
+    'cartridge_restore_backup_path_local': str,
     'cartridge_force_restore': bool,
     'cartridge_allow_alien_backup': bool,
     'cartridge_skip_cleanup_on_restore': bool,

module_utils/helpers.py

@@ -723,6 +723,11 @@ def glob_list_match(path, glob_list):
     return False
 
 
+def get_required_param(params, name):
+    assert params.get(name) is not None, 'Parameter "%s" is required' % name
+    return params[name]
+
+
 class Helpers:
     DYNAMIC_BOX_CFG_PARAMS = DYNAMIC_BOX_CFG_PARAMS
     MEMORY_SIZE_BOX_CFG_PARAMS = MEMORY_SIZE_BOX_CFG_PARAMS
@@ -766,3 +771,4 @@ class Helpers:
     get_disabled_instances = staticmethod(get_disabled_instances)
     get_topology_checksum = staticmethod(get_topology_checksum)
     glob_list_match = staticmethod(glob_list_match)
+    get_required_param = staticmethod(get_required_param)

molecule/move_instance/Dockerfile.j2

@@ -0,0 +1 @@
+../common/Dockerfile.j2