FSx Lustre Deployment Types

We've added support for two new FSx Lustre deployment types
  * `SCRATCH_2` - The latest generation of scratch file systems that supports up to six times the baseline throughput for spiky workloads, and supports in-transit encryption of data for supported instance types. With this deployment type, the storage_capacity settings has possible values of 1200 and any multiple of 2400.
  * `PERSISTENT_1` - Designed for longer-term storage. The file servers are highly available and the data is replicated within the file systems AWS Availability Zone (AZ), and supports in-transit encryption of data for supported instance types. With this deployment type, the storage_capacity settings has possible values of 1200 and any multiple of 2400.

Added parameters:
* Add `deployment_type` parameter
* Add `per_unit_storage_throughput` parameter

Added IAM parameters:
* `fsx:DescribeFileSystems` This is needed to retreive the FSx MountName from the DescribeFileSystems API

Added Validatations:
* Validate `kms_key_id` and `per_unit_storage_throughput` are only set when `PERSISTENT_1`
* Add FSx Deployment Type Options to the integration tests
* Get FSx MountName from Boto3
  * For the new FSx FileSystem Types, the mountname changes from always /fsx to a value returned from the API. This is backwards compatible, so the API returns /fsx for `SCRATCH_1` (current default) filesystems.

Validate Storage Capacity

* If deployment_type == 'PERSISTENT_1' or 'SCRATCH_2' then

  storage_capacity can be 1200 GiB, 2400 GiB, or multiples of 2400

* If deployment_type == 'SCRATCH_1' then

  storage_capacity can be 1200 GiB, 2400 GiB, or multiples of 3600

Signed-off-by: Sean Smith <[email protected]>

Author: Sean Smith

CHANGELOG.rst

@@ -3,7 +3,14 @@ CHANGELOG
 =========
 
 2.6.0
-====
+=====
+
+**ENHANCEMENTS**
+* Support two new Lustre Features, Scratch 2 and Persistent filesystems
+  * This adds two parameters ``deployment_type`` and ``per_unit_storage_throughput`` to the ``fsx`` section.
+  * New storage sizes ``storage_capacity``, 1,200 GiB, 2,400 GiB and multiples of 2,400 are supported with ``SCRATCH_2``
+  * In transit encryption is available via ``kms_key_id`` parameter when ``deployment_type = PERSISTENT_1``.
+  * A new parameter, ``per_unit_storage_throughput`` is available when ``deployment_type = PERSISTENT_1``.
 
 **CHANGES**
 

cli/pcluster/config/mappings.py

@@ -108,6 +108,8 @@
     "volume_id": r"^vol-[0-9a-z]{8}$|^vol-[0-9a-z]{17}$",
     "volume_types": ["standard", "io1", "gp2", "st1", "sc1"],
     "vpc_id": r"^vpc-[0-9a-z]{8}$|^vpc-[0-9a-z]{17}$",
+    "deployment_type": ["SCRATCH_1", "SCRATCH_2", "PERSISTENT_1"],
+    "per_unit_storage_throughput": [50, 100, 200],
 }
 
 AWS = {
@@ -354,7 +356,7 @@
     "type": Section,
     "key": "fsx",
     "default_label": "default",
-    "validators": [fsx_validator],
+    "validators": [fsx_validator, fsx_storage_capacity_validator],
     "cfn_param_mapping": "FSXOptions",  # All the parameters in the section are converted into a single CFN parameter
     "params": OrderedDict(  # Use OrderedDict because the parameters must respect the order in the CFN parameter
         [
@@ -368,7 +370,6 @@
             }),
             ("storage_capacity", {
                 "type": IntParam,
-                "validators": [fsx_storage_capacity_validator]
             }),
             ("fsx_kms_key_id", {
                 "validators": [kms_key_validator],
@@ -386,6 +387,13 @@
             ("weekly_maintenance_start_time", {
                 "allowed_values": r"NONE|^[1-7]:([01]\d|2[0-3]):?([0-5]\d)$",
             }),
+            ("deployment_type", {
+                "allowed_values": ALLOWED_VALUES["deployment_type"],
+            }),
+            ("per_unit_storage_throughput", {
+                "type": IntParam,
+                "allowed_values": ALLOWED_VALUES["per_unit_storage_throughput"],
+            }),
         ]
     )
 }
@@ -412,6 +420,7 @@
         ]
     )
 }
+
 CW_LOG = {
     "type": Section,
     "key": "cw_log",

cli/pcluster/config/validators.py

@@ -147,8 +147,11 @@ def fsx_validator(section_key, section_label, pcluster_config):
     if fsx_export_path and not fsx_import_path:
         errors.append("When specifying 'export_path', the 'import_path' option must be specified")
 
-    if not fsx_section.get_param_value("storage_capacity") and not fsx_section.get_param_value("fsx_fs_id"):
-        errors.append("When specifying 'fsx' section, the 'storage_capacity' option must be specified")
+    if fsx_section.get_param_value("deployment_type") != "PERSISTENT_1":
+        if fsx_section.get_param_value("fsx_kms_key_id"):
+            errors.append("'fsx_kms_key_id' can only be used when 'deployment_type = PERSISTENT_1'")
+        if fsx_section.get_param_value("per_unit_storage_throughput"):
+            errors.append("'per_unit_storage_throughput' can only be used when 'deployment_type = PERSISTENT_1'")
 
     return errors, warnings
 
@@ -205,14 +208,28 @@ def fsx_id_validator(param_key, param_value, pcluster_config):
     return errors, warnings
 
 
-def fsx_storage_capacity_validator(param_key, param_value, pcluster_config):
+def fsx_storage_capacity_validator(section_key, section_label, pcluster_config):
     errors = []
     warnings = []
 
-    if int(param_value) > 0 and not (
-        int(param_value) == 1200 or int(param_value) == 2400 or int(param_value) % 3600 == 0
-    ):
-        errors.append("Capacity for FSx lustre filesystem, 1,200 GB, 2,400 GB or increments of 3,600 GB")
+    fsx_section = pcluster_config.get_section(section_key, section_label)
+    storage_capacity = fsx_section.get_param_value("storage_capacity")
+    deployment_type = fsx_section.get_param_value("deployment_type")
+
+    if fsx_section.get_param_value("fsx_fs_id"):
+        # if fsx_fs_id is provided, don't validate storage_capacity
+        return errors, warnings
+    elif not storage_capacity:
+        # if fsx_fs_id is not provided, storage_capacity must be provided
+        errors.append("When specifying 'fsx' section, the 'storage_capacity' option must be specified")
+    elif deployment_type == "SCRATCH_1":
+        if not (storage_capacity == 1200 or storage_capacity == 2400 or storage_capacity % 3600 == 0):
+            warnings.append("Capacity for FSx SCRATCH_1 filesystem is 1,200 GB, 2,400 GB or increments of 3,600 GB")
+    elif deployment_type in ["SCRATCH_2", "PERSISTENT_1"]:
+        if not (storage_capacity == 1200 or storage_capacity % 2400 == 0):
+            warnings.append(
+                "Capacity for FSx SCRATCH_2 and PERSISTENT_1 filesystems is 1,200 GB or increments of 2,400 GB"
+            )
 
     return errors, warnings
 

cli/pcluster/examples/config

@@ -260,3 +260,21 @@ master_subnet_id = subnet-12345678
 # This is used when AWS ParallelCluster creates the security group
 # (defaults to 0.0.0.0/0)
 #access_from = 0.0.0.0/0
+
+## FSx settings
+#[fsx fsx-custom]
+# Mount Point for Shared Dir
+#shared_dir = /fsx
+# Storage Capacity in GiB
+#storage_capacity = 1200
+# Deployment type allows you to launch filesystems with different characteristics
+# Valid options are SCRATCH_1, PERSISTENT_1, SCRATCH_2
+#deployment_type = PERSISTENT_1
+# When using SCRATCH_2 or PERSISTENT_1 you can set storage throughput to 50,100,200
+#per_unit_storage_throughput = 50
+# S3 Bucket to hydrate the fs from
+#import_path = s3://import-path
+# S3 Location to backup too
+#export_path = s3://import-path/export-dir
+# For SCRATCH_2 and PERSISTENT_1 you can provide a KMS Key for in transit encryption
+#fsx_kms_key_id = XXXX-XXXXX-XXX-XXXXX

cli/tests/pcluster/config/defaults.py

@@ -76,6 +76,8 @@
     "export_path": None,
     "import_path": None,
     "weekly_maintenance_start_time": None,
+    "deployment_type": None,
+    "per_unit_storage_throughput": None,
 }
 
 DEFAULT_DCV_DICT = {"enable": None, "port": 8443, "access_from": "0.0.0.0/0"}
@@ -192,7 +194,7 @@ class DefaultDict(Enum):
 
 DEFAULT_RAID_CFN_PARAMS = {"RAIDOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE"}
 
-DEFAULT_FSX_CFN_PARAMS = {"FSXOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE"}
+DEFAULT_FSX_CFN_PARAMS = {"FSXOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE"}
 
 DEFAULT_DCV_CFN_PARAMS = {"DCVOptions": "NONE,NONE,NONE"}
 DEFAULT_CW_LOG_CFN_PARAMS = {"CWLogOptions": "true,14"}
@@ -261,7 +263,7 @@ class DefaultDict(Enum):
     # raid
     "RAIDOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
     # fsx
-    "FSXOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
+    "FSXOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
     # dcv
     "DCVOptions": "NONE,NONE,NONE",
     # cw_log_settings

cli/tests/pcluster/config/test_section_cluster.py

@@ -1198,7 +1198,7 @@ def test_cluster_section_to_cfn(mocker, section_dict, expected_cfn_params):
                     # raid
                     "RAIDOptions": "raid,NONE,NONE,gp2,20,100,false,NONE",
                     # fsx
-                    "FSXOptions": "fsx,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
+                    "FSXOptions": "fsx,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
                     # dcv
                     "DCVOptions": "master,8555,10.0.0.0/0",
                 },
@@ -1264,7 +1264,7 @@ def test_cluster_section_to_cfn(mocker, section_dict, expected_cfn_params):
                     # raid
                     "RAIDOptions": "raid,NONE,NONE,gp2,20,100,false,NONE",
                     # fsx
-                    "FSXOptions": "fsx,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
+                    "FSXOptions": "fsx,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
                     # dcv
                     "DCVOptions": "master,8555,10.0.0.0/0",
                 },

cli/tests/pcluster/config/test_section_fsx.py

@@ -20,10 +20,10 @@
     [
         (DefaultCfnParams["fsx"].value, DefaultDict["fsx"].value),
         ({}, DefaultDict["fsx"].value),
-        ({"FSXOptions": "NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE"}, DefaultDict["fsx"].value),
-        ({"FSXOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE"}, DefaultDict["fsx"].value),
+        ({"FSXOptions": "NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE, NONE"}, DefaultDict["fsx"].value),
+        ({"FSXOptions": "NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE"}, DefaultDict["fsx"].value),
         (
-            {"FSXOptions": "test,NONE,NONE,NONE,NONE,NONE,NONE,NONE"},
+            {"FSXOptions": "test,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE"},
             {
                 "shared_dir": "test",
                 "fsx_fs_id": None,
@@ -33,10 +33,12 @@
                 "export_path": None,
                 "import_path": None,
                 "weekly_maintenance_start_time": None,
+                "deployment_type": None,
+                "per_unit_storage_throughput": None,
             },
         ),
         (
-            {"FSXOptions": "test,test1,10,test2,20,test3,test4,test5"},
+            {"FSXOptions": "test,test1,10,test2,20,test3,test4,test5,SCRATCH_1,50"},
             {
                 "shared_dir": "test",
                 "fsx_fs_id": "test1",
@@ -46,6 +48,8 @@
                 "export_path": "test3",
                 "import_path": "test4",
                 "weekly_maintenance_start_time": "test5",
+                "deployment_type": "SCRATCH_1",
+                "per_unit_storage_throughput": 50,
             },
         ),
     ],
@@ -65,6 +69,10 @@ def test_fsx_section_from_cfn(mocker, cfn_params_dict, expected_section_dict):
         ({"fsx default": {"storage_capacity": "wrong_value"}}, None, "must be an Integer"),
         # invalid key
         ({"fsx default": {"invalid_key": "fake_value"}}, None, "'invalid_key' is not allowed in the .* section"),
+        # invalid value
+        ({"fsx default": {"deployment_type": "BLAH"}}, None, "'deployment_type' has an invalid value 'BLAH'"),
+        # invalid value
+        ({"fsx default": {"per_unit_storage_throughput": 1000}}, None, "has an invalid value '1000'"),
     ],
 )
 def test_fsx_section_from_file(mocker, config_parser_dict, expected_dict_params, expected_message):
@@ -152,6 +160,19 @@ def test_fsx_section_to_cfn(mocker, section_dict, expected_cfn_params):
         ("weekly_maintenance_start_time", "10:00", "10:00", "has an invalid value"),
         ("weekly_maintenance_start_time", "1:10:00", "1:10:00", None),
         ("weekly_maintenance_start_time", "NONE", "NONE", None),
+        ("deployment_type", "SCRATCH_1", "SCRATCH_1", None),
+        ("deployment_type", "SCRATCH_2", "SCRATCH_2", None),
+        ("deployment_type", "PERSISTENT_1", "PERSISTENT_1", None),
+        (
+            "deployment_type",
+            "INVALID_VALUE",
+            "INVALID_VALUE",
+            " 'deployment_type' has an invalid value 'INVALID_VALUE'",
+        ),
+        ("per_unit_storage_throughput", "50", 50, None),
+        ("per_unit_storage_throughput", "100", 100, None),
+        ("per_unit_storage_throughput", "200", 200, None),
+        ("per_unit_storage_throughput", "101", 101, "'per_unit_storage_throughput' has an invalid value '101'"),
     ],
 )
 def test_fsx_param_from_file(mocker, param_key, param_value, expected_value, expected_message):
@@ -176,7 +197,7 @@ def test_fsx_param_from_file(mocker, param_key, param_value, expected_value, exp
                 {
                     "MasterSubnetId": "subnet-12345678",
                     "AvailabilityZone": "mocked_avail_zone",
-                    "FSXOptions": "fsx,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
+                    "FSXOptions": "fsx,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE,NONE",
                 },
             ),
         ),
@@ -187,7 +208,8 @@ def test_fsx_param_from_file(mocker, param_key, param_value, expected_value, exp
                 {
                     "MasterSubnetId": "subnet-12345678",
                     "AvailabilityZone": "mocked_avail_zone",
-                    "FSXOptions": "fsx,fs-12345678901234567,10,key1,1020,s3://test-export,s3://test-import,10",
+                    "FSXOptions": "fsx,fs-12345678901234567,10,key1,1020,s3://test-export,"
+                    "s3://test-import,10,SCRATCH_1,50",
                 },
             ),
         ),

cli/tests/pcluster/config/test_section_fsx/test_fsx_from_file_to_cfn/pcluster.config.ini

@@ -31,6 +31,8 @@ imported_file_chunk_size = 1020
 export_path = s3://test-export
 import_path = s3://test-import
 weekly_maintenance_start_time = 10
+deployment_type = SCRATCH_1
+per_unit_storage_throughput = 50
 
 [fsx test4]
 shared_dir = /fsx