Fixing node memory handling (#655)

* Attempt using TRES_CORE_MEMORY

* instead of //32 and capped at 2000 //4 +1000 sounds more reasonable

* fixed equation

* log warning if ram < 4096

* added unit

* line too long

Author: XaverStiensmeier

bibigrid/core/utility/validate_configuration.py

@@ -208,8 +208,8 @@ def validate(self):
         checks = [("master/vpn", self.check_master_vpn_worker), ("servergroup", self.check_server_group),
                   ("instances", self.check_instances), ("volumes", self.check_volumes), ("network", self.check_network),
                   ("quotas", self.check_quotas), ("sshPublicKeyFiles", self.check_ssh_public_key_files),
-                  ("cloudYamls", self.check_clouds_yamls), ("nfs", self.check_nfs), ("global security groups",
-                  self.check_configurations_security_groups)]
+                  ("cloudYamls", self.check_clouds_yamls), ("nfs", self.check_nfs),
+                  ("global security groups", self.check_configurations_security_groups)]
         if success:
             for check_name, check_function in checks:
                 success = evaluate(check_name, check_function(), self.log) and success
@@ -223,7 +223,7 @@ def _check_security_groups(self, provider, security_groups):
             security_group = provider.get_security_group(security_group_name)
             if not security_group:
                 self.log.warning(f"Couldn't find security group {security_group} on "
-                      f"cloud {provider.cloud_specification['identifier']}")
+                                 f"cloud {provider.cloud_specification['identifier']}")
                 success = False
             else:
                 self.log.debug(f"Found {security_group_name} on cloud {provider.cloud_specification['identifier']}")
@@ -349,6 +349,10 @@ def check_instance_type_image_combination(self, instance_type, instance_image, p
                                        (type_max_ram, image_min_ram, "ram")]:
             success = has_enough(maximum, needed, f"Type {instance_type}", thing, self.log) and success
         # prepare check quotas
+        if type_max_ram < 4096:
+            self.log.warning(
+                f"Flavor {instance_type} on {provider.cloud_specification['identifier']} has {type_max_ram} but should "
+                f"at least have 4096 MiB of RAM to efficiently run slurm and jobs!")
         self.required_resources_dict[provider.cloud_specification['identifier']]["total_ram"] += type_max_ram
         self.required_resources_dict[provider.cloud_specification['identifier']]["total_cores"] += flavor["vcpus"]
         return success

resources/defaults/slurm/slurm.j2

@@ -49,7 +49,7 @@ Waittime=0
 # SCHEDULING
 SchedulerType=sched/backfill
 SelectType=select/cons_tres
-SelectTypeParameters=CR_Core
+SelectTypeParameters=CR_Core_Memory
 
 # ACCOUNTING
 AccountingStorageType=accounting_storage/slurmdbd
@@ -75,8 +75,9 @@ SlurmdLogFile=/var/log/slurm/slurmd.log
 {% set _ = exclude_groups.append(node.name) %}
 {% endif %}
 {% set _ = node_groups.append(node.name) %}
-{% set mem = (node.flavor.ram // 1024) * 1000 %}
-NodeName={{ node.name }} SocketsPerBoard={{ node.flavor.vcpus }} CoresPerSocket=1 RealMemory={{ mem - [mem // 2, 16000] | min }} State={{node.state }} {{"Features=" + (node.features | join(",")) if node.features is defined }}# {{ node.cloud_identifier }}
+{# MiB to MB #}
+{% set mem = (node.flavor.ram * 1.048576) | int %}
+NodeName={{ node.name }} SocketsPerBoard={{ node.flavor.vcpus }} CoresPerSocket=1 RealMemory={{ mem }} MemSpecLimit={{ [mem//4 + 1000, 8000] | min }} State={{node.state }} {{"Features=" + (node.features | join(",")) if node.features is defined }}# {{ node.cloud_identifier }}
 {% for partition in node.partitions %}
 {% if partition not in partitions %}
 {% set _ = partitions.update({partition: []}) %}
@@ -118,4 +119,4 @@ ResumeFailProgram=/opt/slurm/fail.sh
 # job container
 # TO BE TESTED
 JobContainerType=job_container/tmpfs
-PrologFlags=Contain
+PrologFlags=Contain