From 1972796f7d685e7f4cdaec90bf8c497fe2ab3f45 Mon Sep 17 00:00:00 2001 From: Kendrick Ren Date: Thu, 21 Aug 2025 13:37:54 -0400 Subject: [PATCH] Skip restoring system tables to resolve the issue of different table uuids on the target cluster Use Scylla Manager command "restore --restore-schema" (refer to https://manager.docs.scylladb.com/stable/sctool/restore.html for the detail of the command) to restore user's table instead of using the previous approach of through restoring system table data. This avoids the issue discussed in https://github.com/scylladb/scylla-manager/issues/3019. Refactor the task "Get names of the tables in the snapshot" with the new option --dump-manifest and also replace the previous JSON parsing logic by using Ansible built-in JSON parser to get more more reliable results. Refactor inventory file to have 2 separate sections, one for Scylla hosts and the other for Scylla Manager host which is required to run above Scylla Manager "restore --restore-schema" command. --- .../restore_scylla_manager_backup/README.md | 5 +- .../hosts.example | 4 ++ .../restore.yaml | 65 +++++++++---------- .../vars.yaml.example | 10 +++ 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/example-playbooks/restore_scylla_manager_backup/README.md b/example-playbooks/restore_scylla_manager_backup/README.md index e559611a..219da03f 100644 --- a/example-playbooks/restore_scylla_manager_backup/README.md +++ b/example-playbooks/restore_scylla_manager_backup/README.md @@ -8,6 +8,8 @@ It requires Scylla Manager 2.4 or newer. 1. New cluster with the same number of nodes as the source cluster. 1. Scylla Manager Agent installed on all the nodes. 1. Access to the backup location from all the nodes. +1. New cluster is registered in ScyllaDB Manager. + ## Parameters @@ -62,7 +64,8 @@ The snapshot ID must be put into `snapshot_tag` variable in `vars.yaml` file. ## Inventory -Put public IP addresses of all nodes to `hosts` file. +Put public IP addresses of all nodes to scylla_hosts section in `hosts` file. +Put public IP addresses of Scylla Manager host to scylla_manager section in `hosts` file. ## Running diff --git a/example-playbooks/restore_scylla_manager_backup/hosts.example b/example-playbooks/restore_scylla_manager_backup/hosts.example index 0c5580a9..5824c16a 100644 --- a/example-playbooks/restore_scylla_manager_backup/hosts.example +++ b/example-playbooks/restore_scylla_manager_backup/hosts.example @@ -1,3 +1,7 @@ +[scylla_manager] +3.65.230.31 + +[scylla_hosts] 3.65.230.34 3.66.75.209 3.65.102.133 diff --git a/example-playbooks/restore_scylla_manager_backup/restore.yaml b/example-playbooks/restore_scylla_manager_backup/restore.yaml index ef695a2e..2afecf7d 100644 --- a/example-playbooks/restore_scylla_manager_backup/restore.yaml +++ b/example-playbooks/restore_scylla_manager_backup/restore.yaml @@ -1,7 +1,7 @@ --- - name: Gather important facts about Scylla nodes and a snapshot - hosts: all + hosts: scylla_hosts become: true gather_facts: false tags: @@ -31,21 +31,27 @@ - name: Get names of the tables in the snapshot {{ snapshot_tag }} shell: | - scylla-manager-agent download-files -L {{ backup_location }} -n {{ host_id[inventory_hostname] }} -T {{ snapshot_tag }} -d /var/lib/scylla/data/ --dry-run | grep "^\s*\-" | cut -d"-" -f2 | cut -d"(" -f1 + scylla-manager-agent download-files -L {{ backup_location }} -n {{ host_id[inventory_hostname] }} -T {{ snapshot_tag }} --dump-manifest become_user: scylla - register: _tables_list + vars: + ansible_pipelining: true + register: _manifest_output - - name: Save tables names list as a fact + - name: Parse table names from manifest set_fact: - tables_to_restore: "{{ _tables_list.stdout.split('\n') }}" + _tables_list: + stdout_lines: "{{ (_manifest_output.stdout | from_json).index | map(attribute='keyspace') | zip((_manifest_output.stdout | from_json).index | map(attribute='table')) | map('join', '.') | list }}" - - name: Save system_schema tables names as a fact + - name: Save tables names list as a fact set_fact: - system_schema_tables: "{{ tables_to_restore | select('search', '^\\s*system_schema\\.') | list }}" + all_tables: "{{ _tables_list.stdout_lines }}" + - name: Save names of tables to restore as a fact + set_fact: + tables_to_restore: "{{ all_tables | reject('search', '^system(_|\\.)') | list }}" - name: Restore a token ring - cluster's data is going to be wiped - hosts: all + hosts: scylla_hosts become: true gather_facts: false tags: @@ -110,7 +116,7 @@ when: inventory_hostname != groups.all[0] - name: Start all nodes serially - hosts: all + hosts: scylla_hosts become: true gather_facts: false serial: 1 @@ -127,9 +133,8 @@ port: 9042 host: "{{ listen_address }}" - - name: Cleanup - restore the original cluster's configuration - hosts: all + hosts: scylla_hosts become: true gather_facts: false tags: @@ -149,38 +154,32 @@ \g<1> - seeds: {{ old_seeds }} backrefs: yes +- name: Restore schema from a backup snapshot {{ snapshot_tag }} + hosts: scylla_manager + become: true + gather_facts: false + tags: + - restore_schema_from_backup + tasks: + - name: Run sctool restore command to restore schema only + ansible.builtin.command: + cmd: "sctool restore --cluster {{ scylla_manager_cluster_name }} -L {{ backup_location }} --snapshot-tag {{ snapshot_tag }} --restore-schema" - name: Upload data from a backup snapshot {{ snapshot_tag }} - hosts: all + hosts: scylla_hosts become: true gather_facts: false tags: - upload_snapshot tasks: - - name: Download data from {{ snapshot_tag }} + - name: Download data shell: | - scylla-manager-agent download-files -L {{ backup_location }} -n {{ host_id[inventory_hostname] }} -T {{ snapshot_tag }} -d /var/lib/scylla/data/ --mode upload + scylla-manager-agent download-files -L {{ backup_location }} -n {{ host_id[inventory_hostname] }} -T {{ snapshot_tag }} -d {{ data_dir }} -K '*,!system_*,!!system.*' --mode upload become_user: scylla - async: 604800 # a week 7*24*60*60 seconds + async: 604800 poll: 5 - - name: Load system_schema tables data from the upload directory - shell: | - nodetool refresh {{ item.split('.') | join(' ') }} - with_items: "{{ system_schema_tables }}" - - - name: Restart Scylla service to force a new schema to be picked up - service: - name: scylla-server - state: restarted - - - name: Wait for CQL port - wait_for: - port: 9042 - host: "{{ listen_address }}" - - - name: Load the rest of tables data from the upload directory + - name: refresh nodes with the restored data shell: | - nodetool refresh {{ item.split('.') | join(' ') }} + nodetool refresh {{ item.split('.') | join(' ') }} with_items: "{{ tables_to_restore }}" - when: item not in system_schema_tables diff --git a/example-playbooks/restore_scylla_manager_backup/vars.yaml.example b/example-playbooks/restore_scylla_manager_backup/vars.yaml.example index 8d6bf0ab..ed8f26e9 100644 --- a/example-playbooks/restore_scylla_manager_backup/vars.yaml.example +++ b/example-playbooks/restore_scylla_manager_backup/vars.yaml.example @@ -1,3 +1,8 @@ +# Cluster name to restore schema to. This is the name of the target cluster which +# is registered in ScyllaDB Manager +# +scylla_manager_cluster_name: + # backup_location specifies the location parameter used in Scylla Manager # when scheduling a backup of a cluster. backup_location: s3:manager-test-demo1 @@ -5,6 +10,10 @@ backup_location: s3:manager-test-demo1 # snapshot_tag specifies the Scylla Manager snapshot tag you want to restore. snapshot_tag: sm_20210422133609UTC +# data_dir is the path to dump sstables during scylla-manager-agent download-files +# +data_dir: + # host_id specifies a mapping from the clone cluster node IP to the source # cluster host IDs. host_id: @@ -14,3 +23,4 @@ host_id: 3.125.251.43: 82f0f486-370d-4cfd-90ac-46464c8012cb 3.66.176.252: 9ee92c19-5f78-4865-a287-980218963d96 3.66.25.100: aff05f79-7c69-4ecf-a827-5ea790a0fdc6 + \ No newline at end of file