[irods/irods#7229] Add support for iRODS 5

irods_testing_environment/irods_config.py

@@ -105,6 +105,16 @@ def get_irods_version_info(container, version_file_key):
     raise RuntimeError(f'[{container.name}]: No iRODS version file found')
 
 
+def server_version_is_irods_5(container):
+    """Returns True if iRODS 5 is installed in the container.
+
+    Arguments:
+    container -- the container to check
+    """
+    irods5_binary_exists = "[ -f /usr/sbin/irodsAgent ]"
+    return execute.execute_command(container, irods5_binary_exists, user='irods') == 0
+
+
 def configure_users_for_auth_tests(docker_client, compose_project):
     """Create Linux users and set passwords for authentication testing.
 
@@ -436,11 +446,31 @@ def configure_irods_federation_testing(ctx, remote_zone, zone_where_tests_will_r
     # add specific query to the local zone
     bug_3466_query = 'select alias, sqlStr from R_SPECIFIC_QUERY'
     asq = 'iadmin asq \'{}\' {}'.format(bug_3466_query, 'bug_3466_query')
-    logging.info('creating specific query[{}] [{}]'.format(asq, container.name))
+    logging.info('creating specific query [{}] [{}]'.format(asq, container.name))
     if execute.execute_command(container, asq, user='irods') is not 0:
         raise RuntimeError('failed to create specific query [{}] [{}]'
                            .format(bug_3466_query, container.name))
 
+    # reload configuration for both zones if iRODS 5 is installed
+    if server_version_is_irods_5(container):
+        reload_config = "python3 -c 'from scripts.irods.controller import IrodsController; IrodsController().reload_configuration()'"
+        logging.info('reloading configuration for [{}] [{}]'.format(reload_config, container.name))
+        if execute.execute_command(container, reload_config, user='irods', workdir=context.irods_home()) is not 0:
+            raise RuntimeError('failed to reload configuration [{}] [{}]'
+                               .format(reload_config, container.name))
+
+        container = ctx.docker_client.containers.get(
+            context.irods_catalog_provider_container(
+                ctx.compose_project.name,
+                service_instance=zone_where_tests_will_run.provider_service_instance
+            )
+        )
+
+        logging.info('reloading configuration for [{}] [{}]'.format(reload_config, container.name))
+        if execute.execute_command(container, reload_config, user='irods', workdir=context.irods_home()) is not 0:
+            raise RuntimeError('failed to reload configuration [{}] [{}]'
+                               .format(reload_config, container.name))
+
 
 def configure_pam_for_auth_plugin(docker_client, compose_project):
     """Add lines required for PAM legacy/pam_password auth plugin to work across all platforms.

irods_testing_environment/irods_setup.py

@@ -264,8 +264,23 @@ def build_input_for_catalog_consumer(self):
             '' # confirmation of inputs
         ]
 
+        # Handle the difference between 4.3 servers and 5.x servers.
+        if self.irods_version >= (4, 90, 0):
+            # Always remove the element at the highest index first to reduce
+            # complexity around indices.
+            del input_args[14] # Control plane key
+            del input_args[8]  # Control plane port
+            del input_args[8]  # Schema validation base URI
+
+            # Insert entries for iRODS 5.x.
+            input_args.insert(0, '') # Hostname
+
+            # Insert entries for iRODS 4.3.
+            input_args.insert(4, str(self.provides_local_storage))
+            input_args.insert(5, str(self.resource_name))
+            input_args.insert(6, str(self.vault_directory))
         # Handle the difference between 4.2 servers and 4.3 servers.
-        if self.irods_version >= (4, 3, 0):
+        elif self.irods_version >= (4, 3, 0):
             input_args.insert(3, str(self.provides_local_storage))
             input_args.insert(4, str(self.resource_name))
             input_args.insert(5, str(self.vault_directory))
@@ -312,8 +327,24 @@ def build_input_for_catalog_provider(self):
             '' # confirmation of inputs
         ]
 
+        # Handle the difference between 4.3 servers and 5.x servers.
+        if self.irods_version >= (4, 90, 0):
+            # Remove entries that do not apply to iRODS 5.
+            # Always remove the element at the highest index first to reduce
+            # complexity around indices.
+            del input_args[21] # Control plane key
+            del input_args[15] # Control plane port
+            del input_args[15] # Schema validation base URI
+
+            # Insert entries for iRODS 5.x.
+            input_args.insert(0, '') # Hostname
+
+            # Insert entries for iRODS 4.3.
+            input_args.insert(12, str(self.provides_local_storage))
+            input_args.insert(13, str(self.resource_name))
+            input_args.insert(14, str(self.vault_directory))
         # Handle the difference between 4.2 servers and 4.3 servers.
-        if self.irods_version >= (4, 3, 0):
+        elif self.irods_version >= (4, 3, 0):
             input_args.insert(11, str(self.provides_local_storage))
             input_args.insert(12, str(self.resource_name))
             input_args.insert(13, str(self.vault_directory))
@@ -384,6 +415,8 @@ def restart_rsyslog(container):
         \$template irods_format,\\"%msg%\\n\\"
         :programname,startswith,\\"irodsServer\\" /var/log/irods/irods.log;irods_format
         & stop
+        :programname,startswith,\\"irodsAgent\\" /var/log/irods/irods.log;irods_format
+        & stop
         :programname,startswith,\\"irodsDelayServer\\" /var/log/irods/irods.log;irods_format
         & stop''')
 
@@ -413,13 +446,13 @@ def restart_rsyslog(container):
 
 
 def stop_irods(container):
-    irodsctl = os.path.join(context.irods_home(), 'irodsctl')
-    return execute.execute_command(container, f'{irodsctl} stop', user='irods')
+    cmd = "python3 -c 'from scripts.irods.controller import IrodsController; IrodsController().stop()'"
+    return execute.execute_command(container, cmd, user='irods', workdir=context.irods_home())
 
 
 def restart_irods(container):
-    irodsctl = os.path.join(context.irods_home(), 'irodsctl')
-    return execute.execute_command(container, f'{irodsctl} restart', user='irods')
+    cmd = "python3 -c 'from scripts.irods.controller import IrodsController; IrodsController().restart()'"
+    return execute.execute_command(container, cmd, user='irods', workdir=context.irods_home())
 
 
 def setup_irods_server(container, setup_input):

irods_testing_environment/ssl_setup.py

@@ -104,8 +104,8 @@ def configure_ssl_on_server(container,
     chain_file = os.path.join(context.irods_config(), 'chain.pem')
     cert_file = os.path.join(context.irods_config(), 'server.crt')
 
-    irodsctl = os.path.join(context.irods_home(), 'irodsctl')
-    if execute.execute_command(container, '{} stop'.format(irodsctl), user='irods') is not 0:
+    stop_cmd = "python3 -c 'from scripts.irods.controller import IrodsController; IrodsController().stop()'"
+    if execute.execute_command(container, stop_cmd, user='irods', workdir=context.irods_home()) is not 0:
         raise RuntimeError(
             'failed to stop iRODS server before SSL configuration [{}]'
             .format(container.name))
@@ -142,7 +142,8 @@ def configure_ssl_on_server(container,
     negotiation_key.configure_ssl_in_server(container, 'CS_NEG_REQUIRE')
 
     # start the server again
-    if execute.execute_command(container, '{} start'.format(irodsctl), user='irods') is not 0:
+    start_cmd = "python3 -c 'from scripts.irods.controller import IrodsController; IrodsController().start()'"
+    if execute.execute_command(container, start_cmd, user='irods', workdir=context.irods_home()) is not 0:
         raise RuntimeError(
             'failed to start iRODS server after SSL configuration [{}]'
             .format(container.name))

projects/almalinux-8/almalinux-8-mariadb-10.11/docker-compose.yml

@@ -18,6 +18,11 @@ services:
       - catalog
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
   irods-catalog-consumer:
     build:
@@ -27,6 +32,11 @@ services:
       - irods-catalog-provider
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/almalinux-8/almalinux-8-mariadb-10.6/docker-compose.yml

@@ -18,6 +18,11 @@ services:
       - catalog
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
   irods-catalog-consumer:
     build:
@@ -27,6 +32,11 @@ services:
       - irods-catalog-provider
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/almalinux-8/almalinux-8-mariadb-11.4/docker-compose.yml

@@ -18,6 +18,11 @@ services:
       - catalog
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
   irods-catalog-consumer:
     build:
@@ -27,6 +32,11 @@ services:
       - irods-catalog-provider
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/almalinux-8/almalinux-8-mysql-8.0/docker-compose.yml

@@ -48,6 +48,11 @@ services:
             - catalog
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
     irods-catalog-consumer:
         build:
@@ -59,6 +64,11 @@ services:
             - irods-catalog-provider
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/almalinux-8/almalinux-8-mysql-8.4/docker-compose.yml

@@ -48,6 +48,11 @@ services:
             - catalog
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
     irods-catalog-consumer:
         build:
@@ -59,6 +64,11 @@ services:
             - irods-catalog-provider
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/almalinux-8/almalinux-8-postgres-14/docker-compose.yml

@@ -16,6 +16,11 @@ services:
             - catalog
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
     irods-catalog-consumer:
         build:
@@ -27,6 +32,11 @@ services:
             - irods-catalog-provider
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/almalinux-8/almalinux-8-postgres-16/docker-compose.yml

@@ -16,6 +16,11 @@ services:
             - catalog
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
     irods-catalog-consumer:
         build:
@@ -27,6 +32,11 @@ services:
             - irods-catalog-provider
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/debian-11/debian-11-mariadb-10.11/docker-compose.yml

@@ -18,6 +18,11 @@ services:
       - catalog
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
   irods-catalog-consumer:
     build:
@@ -27,6 +32,11 @@ services:
       - irods-catalog-provider
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/debian-11/debian-11-mariadb-10.6/docker-compose.yml

@@ -18,6 +18,11 @@ services:
       - catalog
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
   irods-catalog-consumer:
     build:
@@ -27,6 +32,11 @@ services:
       - irods-catalog-provider
     volumes:
       - shared_volume:/irods_testing_environment_mount_dir
+    # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+    # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+    # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+    # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+    shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.

projects/debian-11/debian-11-mysql-8.0/docker-compose.yml

@@ -48,6 +48,11 @@ services:
             - catalog
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
     irods-catalog-consumer:
         build:
@@ -59,6 +64,11 @@ services:
             - irods-catalog-provider
         volumes:
             - shared_volume:/irods_testing_environment_mount_dir
+        # The core test suite for iRODS uses SIGHUP to reload the server's configuration often.
+        # To keep the iRODS Rule Language from failing and producing RE_UNABLE_TO_READ_SESSION_VAR,
+        # we bump the maximum amount of shared memory from 64mb to 100mb. This is required because
+        # iRODS 5 can have two Agent Factories running simultaneously (due to the SIGHUP).
+        shm_size: 100mb
 
 # This volume is mounted on all test servers for detached mode testing which
 # requires a common vault.