feat: Add mssql_ha_ag_is_contained and mssql_ha_ag_reuse_system_db

* Update configure_ag.j2 to drop AG in the case is_contained or
reuse_system_db is configured incorrecty on the existing AG
* Instead of checking if primary node is available, check if none of the
nodes failed
* Use ansible_play_hosts_all in configure_ag.j2
* Restore databases if in restoring state
* Only verify is_contained. There is no way to verify is_reuse_system_db
* Add cleanups for AG and DBs
* Split multi-host test and a single-node test
* Split tests_ha_single into versions
* Reset inventory_hostname after ha_cluster setup
* Verify if mssql_version is not bigger then the existing SQL Server version

Author: zhuatuzilo

README.md

@@ -864,6 +864,28 @@ Default: `[]`
 
 Type: `list`
 
+#### mssql_ha_ag_is_contained
+
+Whether to configure the availability group provided with [mssql_ha_ag_name](#mssql_ha_ag_name) as contained.
+
+This is supported since SQL Server version 2022.
+
+Default: `false`
+
+Type: `bool`
+
+#### mssql_ha_ag_reuse_system_db
+
+Only applicable when [mssql_ha_ag_is_contained](#mssql_ha_ag_is_contained) is true.
+
+Whether to configure the availability group provided with [mssql_ha_ag_name](#mssql_ha_ag_name) with the reuse_system_db setting.
+
+This is supported since SQL Server version 2022.
+
+Default: `false`
+
+Type: `bool`
+
 ### Configuring Pacemaker Variables
 
 With these variables, you can specify whether to configure Pacemaker HA solution and its configuration.

defaults/main.yml

@@ -52,6 +52,9 @@ mssql_ha_reset_cert: false
 mssql_ha_endpoint_name: null
 mssql_ha_ag_name: null
 mssql_ha_db_names: []
+mssql_ha_ag_is_contained: false
+# if mssql_ha_ag_is_contained=false, mssql_ha_ag_reuse_system_db will be false
+mssql_ha_ag_reuse_system_db: false
 
 mssql_ha_prep_for_pacemaker: "{{ mssql_ha_ag_cluster_type | lower != 'none' }}"
 mssql_ha_virtual_ip: null

tasks/main.yml

@@ -5,9 +5,10 @@
 
 # This is required to prevent the role from using a __mssql_sqlcmd_login_cmd
 # variable value from a previous role invocation
-- name: Unset the __mssql_sqlcmd_login_cmd fact
+- name: Unset internal role facts
   set_fact:
     __mssql_sqlcmd_login_cmd: null
+    __mssql_is_active: null
 
 - name: Link the deprecated mssql_input_sql_file fact
   when: mssql_input_sql_file is defined
@@ -165,6 +166,28 @@
       mssql_run_selinux_confined is set to true. You must either set both
       variables to true or to false.
 
+- name: Verify that mssql_ha_ag_is_contained is not provided with version 2017
+  when:
+    - mssql_ha_ag_is_contained | bool
+    - mssql_version is version('2022', '<')
+  fail:
+    msg: >-
+      mssql_ha_ag_is_contained is not supported on SQL Server < 2022.
+      SQL Server 2019 and greater supports mssql_ha_ag_is_contained.
+      You must either upgrade to a greater version or set
+      mssql_ha_ag_is_contained to false.
+
+- name: Verify fail reuse_system_db=true is not provided without ag_is_contained
+  when:
+    - not mssql_ha_ag_is_contained
+    - mssql_ha_ag_reuse_system_db | bool
+  fail:
+    msg: >-
+      mssql_ha_ag_reuse_system_db is not appilcable when
+      mssql_ha_ag_is_contained = false. You must either set
+      mssql_ha_ag_reuse_system_db to false or set mssql_ha_ag_is_contained
+      to true.
+
 - name: Gather package facts
   package_facts:
   no_log: "{{ __mssql_gather_facts_no_log | d(false) }}"
@@ -178,6 +201,19 @@
       | first | int == item.value
   with_dict: "{{ __mssql_version_package_mapping }}"
 
+- name: >-
+    Verify if mssql_version is not bigger then the existing SQL Server version
+  fail:
+    msg: >-
+      You set mssql_version to {{ mssql_version }}, but your SQL Server is
+      version {{ __mssql_current_version }}. You must either set mssql_upgrade
+      to true to upgrade or not set the mssql_version variable"
+  when:
+    - not mssql_upgrade
+    - mssql_version is not none
+    - __mssql_current_version is defined
+    - mssql_version | int > __mssql_current_version | int
+
 - name: Set mssql_version variable if user does not define it
   set_fact:
     mssql_version: "{{ __mssql_current_version }}"
@@ -1121,21 +1157,30 @@
         - "{{ ansible_failed_result.stdout_lines }}"
 
 - name: Configure for high availability
-  any_errors_fatal: true
   when: mssql_ha_configure | bool
   block:
-    - name: Verify that hosts with mssql_ha_replica_type=primary is available
-      assert:
-        that: ansible_play_hosts |
-            map('extract', hostvars, 'mssql_ha_replica_type') |
-            select('match', '^primary$') |
-            list |
-            length == 1
-        fail_msg:
-          - Host with mssql_ha_replica_type=primary failed earlier in the play.
-          - Therefore, it is not possible to configure for high availability.
-          - Fix the above error on the primary replica and re-run the role.
+    - name: Mark active hosts
+      set_fact:
+        __mssql_is_active: true
+
+    - name: Fail if any hosts are no longer active
       run_once: true
+      vars:
+        hostvars_list: "{{ ansible_play_hosts_all | map('extract', hostvars) | list }}"
+        active_hosts: >-
+          {{ hostvars_list
+            | selectattr('__mssql_is_active', 'defined')
+            | selectattr('__mssql_is_active')
+            | map(attribute='inventory_hostname')
+            | list }}
+        failed_hosts: "{{ ansible_play_hosts_all | difference(active_hosts) }}"
+      fail:
+        msg: "These hosts are no longer active: {{ failed_hosts | join(', ') }}"
+      when: failed_hosts | length > 0
+
+    - name: Unset __mssql_is_active
+      set_fact:
+        __mssql_is_active: null
 
     - name: Open required firewall ports and set required facts
       block:
@@ -1314,11 +1359,11 @@
 
         - name: Verify that primary instance hosts is available
           assert:
-            that: ansible_play_hosts |
-                map('extract', hostvars, '__mssql_ha_is_primary') |
-                select('match', '^[Tt]rue$') |
-                list |
-                length == 1
+            that: ansible_play_hosts
+              | map('extract', hostvars, '__mssql_ha_is_primary')
+              | select
+              | list
+              | length == 1
             fail_msg:
               - The role was not able to identify primary replica.
           run_once: true
@@ -1354,6 +1399,25 @@
               - replicate_db.j2
           include_tasks: input_sql_files.yml
 
+    - name: Mark active hosts
+      set_fact:
+        __mssql_is_active: true
+
+    - name: Fail if any hosts are no longer active
+      run_once: true
+      vars:
+        hostvars_list: "{{ ansible_play_hosts_all | map('extract', hostvars) | list }}"
+        active_hosts: >-
+          {{ hostvars_list
+            | selectattr('__mssql_is_active', 'defined')
+            | selectattr('__mssql_is_active')
+            | map(attribute='inventory_hostname')
+            | list }}
+        failed_hosts: "{{ ansible_play_hosts_all | difference(active_hosts) }}"
+      fail:
+        msg: "These hosts are no longer active: {{ failed_hosts | join(', ') }}"
+      when: failed_hosts | length > 0
+
     # In the case of fail over when primary is moved grant permission task above
     # is not run on primary above hence we need to run it separately
     - name: Grant permissions to HA login
@@ -1369,19 +1433,6 @@
     - name: Configure availability group on not primary replicas
       when: mssql_ha_replica_type in __mssql_ha_replica_types_secondary
       block:
-        - name: Verify that hosts with replica_type=primary is available
-          assert:
-            that: ansible_play_hosts |
-                map('extract', hostvars, 'mssql_ha_replica_type') |
-                select('match', '^primary$') |
-                list |
-                length == 1
-            fail_msg:
-              - Host with mssql_ha_replica_type=primary failed earlier
-              - Therefore, it is not possible to configure for high availability
-              - Fix the above error on the primary replica and re-run the role
-          run_once: true
-
         - name: Ensure the package {{ __mssql_server_ha_packages }}
           package:
             name: "{{ __mssql_server_ha_packages }}"

templates/configure_ag.j2

@@ -1,14 +1,38 @@
 IF EXISTS (
   SELECT name, cluster_type_desc
   FROM sys.availability_groups
-  WHERE name = '{{ mssql_ha_ag_name }}' AND
-        cluster_type_desc != '{{ mssql_ha_ag_cluster_type }}'
+  WHERE name = '{{ mssql_ha_ag_name }}'
 )
 BEGIN
-  PRINT 'The existing {{ mssql_ha_ag_name }} availability group has \
+  PRINT 'Verify existing availability group {{ mssql_ha_ag_name }}'
+  IF EXISTS (
+    SELECT name, cluster_type_desc
+    FROM sys.availability_groups
+    WHERE name = '{{ mssql_ha_ag_name }}'
+          AND cluster_type_desc != '{{ mssql_ha_ag_cluster_type }}'
+  )
+  BEGIN
+    PRINT 'The existing {{ mssql_ha_ag_name }} availability group has \
 incorrect cluster type set, dropping the group to re-create it';
-  DROP AVAILABILITY GROUP {{ mssql_ha_ag_name }};
-  PRINT 'The {{ mssql_ha_ag_name }} availability group dropped successfully';
+    DROP AVAILABILITY GROUP {{ mssql_ha_ag_name }};
+    PRINT 'The {{ mssql_ha_ag_name }} availability group dropped successfully';
+  END
+
+{# is_contained was added in SQL Server 2019 #}
+{% if mssql_version is version('2019', '>=') %}
+  IF EXISTS (
+    SELECT name, is_contained
+    FROM sys.availability_groups
+    WHERE name = '{{ mssql_ha_ag_name }}'
+          AND is_contained != {{ 1 if mssql_ha_ag_is_contained else 0 }}
+  )
+  BEGIN
+    PRINT 'The existing {{ mssql_ha_ag_name }} availability group has \
+incorrect is_contained setting, dropping the group to re-create it';
+    DROP AVAILABILITY GROUP {{ mssql_ha_ag_name }};
+    PRINT 'The {{ mssql_ha_ag_name }} availability group dropped successfully';
+  END
+{% endif %}
 END
 
 IF EXISTS (
@@ -18,6 +42,7 @@ IF EXISTS (
 )
 BEGIN
   PRINT 'Verifying the existing availability group {{ mssql_ha_ag_name }}'
+  PRINT 'Verifying {{ mssql_ha_ag_name }} DB_FAILOVER status'
   IF NOT EXISTS (
     SELECT name, db_failover
     FROM sys.availability_groups
@@ -27,14 +52,15 @@ BEGIN
   BEGIN
     ALTER AVAILABILITY GROUP {{ mssql_ha_ag_name }}
       SET (DB_FAILOVER = {{ __mssql_ha_db_failover }})
-    PRINT 'Set DB_FAILOVER to ON succesfully'
+    PRINT 'Set DB_FAILOVER to ON successfully'
   END
   ELSE
   BEGIN
     PRINT 'DB_FAILOVER = ON is already set, skipping'
   END
+
   PRINT 'Verifying replicas'
-{# Sort ansible_play_hosts #}
+{# Sort ansible_play_hosts_all #}
 {# Sort primary replica as the last in the list #}
 {# Sort witness replica as the next-to-last in the list #}
 {# Configure primary last because it has information about other replicas #}
@@ -43,15 +69,15 @@ BEGIN
 {# replica which has configuration-only availability mode.' #}
 {% set ag_replicas_primary_last = [] %}
 {# In the first loop, add primaries at the end and witness at the beginning #}
-{% for item in ansible_play_hosts %}
+{% for item in ansible_play_hosts_all %}
 {%   if hostvars[item]['mssql_ha_replica_type'] == 'primary' %}
 {{ ag_replicas_primary_last.append(item) -}}
 {%   elif hostvars[item]['mssql_ha_replica_type'] == 'witness' %}
 {{ ag_replicas_primary_last.insert(0, item) -}}
 {%   endif %}
 {% endfor %}
 {# In the second loop, add others at the beginning #}
-{% for item in ansible_play_hosts %}
+{% for item in ansible_play_hosts_all %}
 {%   if hostvars[item]['mssql_ha_replica_type']
      not in ag_replicas_primary_last %}
 {{ ag_replicas_primary_last.insert(0, item) -}}
@@ -209,31 +235,62 @@ successfully';
 {% endfor %}
 END
 
+{# is_contained was added in SQL Server 2019 #}
+{% if (not mssql_ha_ag_is_contained) or (mssql_version is version('2019', '<')) %}
+{%   set __contained_reuse_system_db = "" %}
+{% elif mssql_ha_ag_is_contained %}
+{%   set __contained_reuse_system_db = ",CONTAINED" %}
+{%   if mssql_ha_ag_reuse_system_db %}
+{%     set __contained_reuse_system_db = __contained_reuse_system_db + ",REUSE_SYSTEM_DATABASES" %}
+{%   endif %}
+{% endif %}
+
 IF NOT EXISTS (
   SELECT name, cluster_type_desc
   FROM sys.availability_groups
   WHERE name = '{{ mssql_ha_ag_name }}'
 )
 BEGIN
+{% if mssql_ha_ag_reuse_system_db %}
+{%   set sys_dbs = [mssql_ha_ag_name ~ '_master', mssql_ha_ag_name ~ '_msdb'] %}
+{%   for sys_db in sys_dbs %}
+  IF EXISTS (
+    SELECT name, state_desc
+    FROM sys.databases
+    WHERE state_desc = 'RESTORING'
+      AND name='{{ sys_db }}'
+  )
+  BEGIN
+    PRINT 'The system database {{ sys_db }} is in restoring state, running \
+restore to reuse it for the newly created AG {{ mssql_ha_ag_name }}';
+    RESTORE DATABASE [{{ sys_db }}];
+    PRINT 'The {{ sys_db }} database restored successfully';
+  END
+  ELSE
+  BEGIN
+    PRINT 'The {{ sys_db }} database is already online, skipping';
+  END
+{%   endfor %}
+{% endif %}
   PRINT 'Creating the {{ mssql_ha_ag_name }} availability group';
   CREATE AVAILABILITY GROUP {{ mssql_ha_ag_name }}
     WITH (
       DB_FAILOVER = {{ __mssql_ha_db_failover }},
-      CLUSTER_TYPE = {{ mssql_ha_ag_cluster_type }}
+      CLUSTER_TYPE = {{ mssql_ha_ag_cluster_type }}{{ __contained_reuse_system_db }}
     )
     FOR REPLICA ON
-{# Sort ansible_play_hosts #}
+{# Sort ansible_play_hosts_all #}
 {# Sort primary replica as the first in the list #}
 {# Configure AG with primary replica first because SQL Server requires this #}
 {% set ag_replicas_primary_first = [] %}
 {# In the first loop, add not primaries to a new list #}
-{% for item in ansible_play_hosts %}
+{% for item in ansible_play_hosts_all %}
 {%   if hostvars[item]['mssql_ha_replica_type'] != 'primary' %}
 {{ ag_replicas_primary_first.append(item) -}}
 {%   endif %}
 {% endfor %}
 {# In the second loop, add primary at the beginning #}
-{% for item in ansible_play_hosts %}
+{% for item in ansible_play_hosts_all %}
 {%   if hostvars[item]['mssql_ha_replica_type'] == 'primary' %}
 {{ ag_replicas_primary_first.insert(0, item) -}}
 {%   endif %}

templates/replicate_db.j2

@@ -17,6 +17,23 @@ BEGIN
   PRINT 'RECOVERY FULL on the {{ item }} database is set, skipping';
 END
 
+IF EXISTS (
+  SELECT name, state_desc
+  FROM sys.databases
+  WHERE state_desc = 'RESTORING'
+    AND name='{{ item }}'
+)
+BEGIN
+  PRINT 'The {{ item }} database is in restoring state, running restore with \
+recovery to add it to AG after';
+  RESTORE DATABASE [{{ item }}] WITH RECOVERY;
+  PRINT 'The {{ item }} database restored with recovery successfully';
+END
+ELSE
+BEGIN
+  PRINT 'The {{ item }} database is already online, skipping';
+END
+
 IF NOT EXISTS (
   SELECT [database_name], backup_start_date, backup_finish_date, [type]
   FROM msdb.dbo.backupset