Include storage record testing and fix errors

Author: alvarolopez

caso/record.py

@@ -289,7 +289,7 @@ class StorageRecord(_BaseRecord):
     EMI StAR
     """
 
-    version: str = "0.1"
+    version: str = pydantic.Field("0.1", exclude=True)
 
     uuid: m_uuid.UUID
     name: str
@@ -340,6 +340,7 @@ def map_fields(field: str) -> str:
                 "status": "Status",
                 "attached_to": "AttachedTo",
                 "attached_duration": "AttachedDuration",
+                "cloud_type": "CloudType",
                 "compute_service": "CloudComputeService",
             }
             return d.get(field, field)

caso/tests/conftest.py

@@ -236,6 +236,76 @@
     },
 ]
 
+valid_storage_records_fields = [
+    dict(
+        uuid="99cf5d02-a573-46a1-b90d-0f7327126876",
+        site_name="TEST-Site",
+        name="Test Volume 1",
+        user_id="63296dcd-b652-4039-b274-aaa70f9d57e5",
+        group_id="313c6f62-e05f-4ec7-b0f2-256612db18f5",
+        fqan="VO 1 FQAN",
+        compute_service="Fake Cloud Service",
+        status="in-use",
+        active_duration=400,
+        measure_time=now,
+        start_time=now - datetime.timedelta(days=5),
+        capacity=322122547200,
+        user_dn="[email protected]",
+    ),
+    dict(
+        uuid="99cf5d02-a573-46a1-b90d-0f7327126876",
+        site_name="TEST-Site",
+        name="Test Volume 1",
+        user_id="63296dcd-b652-4039-b274-aaa70f9d57e5",
+        group_id="313c6f62-e05f-4ec7-b0f2-256612db18f5",
+        fqan="VO 2 FQAN",
+        compute_service="Fake Cloud Service",
+        status="in-use",
+        active_duration=400,
+        measure_time=now,
+        start_time=now - datetime.timedelta(days=6),
+        capacity=122122547200,
+        user_dn="[email protected]",
+    ),
+]
+
+valid_storage_records_dict = [
+    {
+        "SiteName": "TEST-Site",
+        "CloudType": cloud_type,
+        "CloudComputeService": "Fake Cloud Service",
+        "VolumeUUID": "99cf5d02-a573-46a1-b90d-0f7327126876",
+        "RecordName": "Test Volume 1",
+        "LocalUser": "63296dcd-b652-4039-b274-aaa70f9d57e5",
+        "GlobalUserName": "[email protected]",
+        "LocalGroup": "313c6f62-e05f-4ec7-b0f2-256612db18f5",
+        "FQAN": "VO 1 FQAN",
+        "ActiveDuration": 400,
+        "CreateTime": "2023-05-25T21:59:06+00:00",
+        "StartTime": "2023-05-20T21:59:06+00:00",
+        "Type": "Block Storage (cinder)",
+        "Status": "in-use",
+        "Capacity": 322122547200,
+    },
+    {
+        "SiteName": "TEST-Site",
+        "CloudType": cloud_type,
+        "CloudComputeService": "Fake Cloud Service",
+        "VolumeUUID": "99cf5d02-a573-46a1-b90d-0f7327126876",
+        "RecordName": "Test Volume 2",
+        "LocalUser": "63296dcd-b652-4039-b274-aaa70f9d57e5",
+        "GlobalUserName": "[email protected]",
+        "LocalGroup": "313c6f62-e05f-4ec7-b0f2-256612db18f5",
+        "FQAN": "VO 2 FQAN",
+        "ActiveDuration": 400,
+        "CreateTime": "2023-05-25T21:59:06+00:00",
+        "StartTime": "2023-05-20T21:59:06+00:00",
+        "Type": "Block Storage (cinder)",
+        "Status": "in-use",
+        "Capacity": 122122547200,
+    },
+]
+
 # Cloud Record fixtures
 
 
@@ -362,6 +432,43 @@ def accelerator_record_list(
     return [accelerator_record, another_accelerator_record]
 
 
+# Storage records
+
+
+@pytest.fixture(scope="module")
+def storage_record() -> caso.record.StorageRecord:
+    """Get a fixture for the StorageRecord."""
+    record = caso.record.StorageRecord(**valid_storage_records_fields[0])
+    return record
+
+
+@pytest.fixture(scope="module")
+def another_storage_record() -> caso.record.StorageRecord:
+    """Get another fixture for the StorageRecord."""
+    record = caso.record.StorageRecord(**valid_storage_records_fields[1])
+    return record
+
+
+@pytest.fixture(scope="module")
+def valid_storage_record() -> dict:
+    """Get a fixture for a valid record."""
+    return valid_storage_records_dict[0]
+
+
+@pytest.fixture(scope="module")
+def valid_storage_records() -> typing.List[dict]:
+    """Get a fixture for valid records as a dict."""
+    return valid_storage_records_dict
+
+
+@pytest.fixture(scope="module")
+def storage_record_list(
+    storage_record, another_storage_record
+) -> typing.List[caso.record.StorageRecord]:
+    """Get a fixture for a list of Storage records."""
+    return [storage_record, another_storage_record]
+
+
 # SSM entries
 
 
@@ -571,3 +678,48 @@ def expected_message_accelerator() -> str:
         "]}"
     )
     return message
+
+
+@pytest.fixture
+def expected_entries_storage(storage_record_list) -> list[caso.record.StorageRecord]:
+    """Get a fixture for all Storage entries."""
+    ssm_entries = storage_record_list.copy()
+    return ssm_entries
+
+
+@pytest.fixture
+def expected_message_storage() -> str:
+    """Get a fixture for a complete Storage message."""
+    message = (
+        '<sr:StorageUsageRecords xmlns:sr="http://eu-emi.eu/namespaces/2011/02/storagerecord">'  # noqa
+        "<sr:StorageUsageRecord>"
+        '<sr:RecordIdentity sr:createTime="2023-05-25T21:59:06+00:00" sr:recordId="99cf5d02-a573-46a1-b90d-0f7327126876" />'  # noqa
+        "<sr:StorageSystem>Fake Cloud Service</sr:StorageSystem>"
+        "<sr:Site>TEST-Site</sr:Site>"
+        "<sr:SubjectIdentity>"
+        "<sr:LocalUser>63296dcd-b652-4039-b274-aaa70f9d57e5</sr:LocalUser>"
+        "<sr:LocalGroup>313c6f62-e05f-4ec7-b0f2-256612db18f5</sr:LocalGroup>"
+        "<sr:UserIdentity>[email protected]</sr:UserIdentity>"
+        "<sr:Group>VO 1 FQAN</sr:Group>"
+        "</sr:SubjectIdentity>"
+        "<sr:StartTime>2023-05-20T21:59:06+00:00</sr:StartTime>"
+        "<sr:EndTime>2023-05-25T21:59:06+00:00</sr:EndTime>"
+        "<sr:ResourceCapacityUsed>345876451382054092800</sr:ResourceCapacityUsed>"
+        "</sr:StorageUsageRecord>"
+        "<sr:StorageUsageRecord>"
+        '<sr:RecordIdentity sr:createTime="2023-05-25T21:59:06+00:00" sr:recordId="99cf5d02-a573-46a1-b90d-0f7327126876" />'  # noqa
+        "<sr:StorageSystem>Fake Cloud Service</sr:StorageSystem>"
+        "<sr:Site>TEST-Site</sr:Site>"
+        "<sr:SubjectIdentity>"
+        "<sr:LocalUser>63296dcd-b652-4039-b274-aaa70f9d57e5</sr:LocalUser>"
+        "<sr:LocalGroup>313c6f62-e05f-4ec7-b0f2-256612db18f5</sr:LocalGroup>"
+        "<sr:UserIdentity>[email protected]</sr:UserIdentity>"
+        "<sr:Group>VO 2 FQAN</sr:Group>"
+        "</sr:SubjectIdentity>"
+        "<sr:StartTime>2023-05-19T21:59:06+00:00</sr:StartTime>"
+        "<sr:EndTime>2023-05-25T21:59:06+00:00</sr:EndTime>"
+        "<sr:ResourceCapacityUsed>131128086582054092800</sr:ResourceCapacityUsed>"
+        "</sr:StorageUsageRecord>"
+        "</sr:StorageUsageRecords>"
+    )
+    return message.encode("utf-8")

caso/tests/test_record.py

@@ -119,3 +119,17 @@ def test_accelerator_record_map_opts(accelerator_record, valid_accelerator_recor
         "exclude_none": True,
     }
     assert json.loads(accelerator_record.json(**opts)) == valid_accelerator_record
+
+
+def test_storage_record(storage_record):
+    """Test that an IP record is correctly generated."""
+    assert isinstance(storage_record.active_duration, int)
+
+
+def test_storage_record_map_opts(storage_record, valid_storage_record):
+    """Test that an IP record is correctly generated."""
+    opts = {
+        "by_alias": True,
+        "exclude_none": True,
+    }
+    assert json.loads(storage_record.json(**opts)) == valid_storage_record

caso/tests/test_ssm.py

@@ -93,6 +93,23 @@ def mock_push(entries_cloud, entries_ip, entries_accelerator, entries_storage):
         messenger.push(accelerator_record_list)
 
 
+def test_storage_records_pushed(
+    monkeypatch, storage_record_list, expected_entries_storage
+):
+    """Test that Storage records are correctly rendered."""
+
+    def mock_push(entries_cloud, entries_ip, entries_accelerator, entries_storage):
+        assert entries_storage == expected_entries_storage
+
+    with monkeypatch.context() as m:
+        m.setattr("caso.utils.makedirs", lambda x: None)
+        m.setattr("dirq.QueueSimple.QueueSimple", lambda x: None)
+        messenger = ssm.SSMMessenger()
+
+        m.setattr(messenger, "_push", mock_push)
+        messenger.push(storage_record_list)
+
+
 def test_cloud_ip_records_pushed(
     monkeypatch,
     cloud_record_list,
@@ -173,3 +190,22 @@ def mock_add(message):
 
         m.setattr(messenger.queue, "add", mock_add)
         messenger._push_message_accelerator(expected_entries_accelerator)
+
+
+def test_complete_storage_message(
+    monkeypatch, expected_entries_storage, expected_message_storage
+):
+    """Test a complete cloud message."""
+
+    def mock_add(message):
+        print(message)
+        print(expected_message_storage)
+        assert message == expected_message_storage
+
+    with monkeypatch.context() as m:
+        m.setattr("caso.utils.makedirs", lambda x: None)
+        m.setattr("dirq.QueueSimple.QueueSimple", lambda x: _MockQueue())
+        messenger = ssm.SSMMessenger()
+
+        m.setattr(messenger.queue, "add", mock_add)
+        messenger._push_message_storage(expected_entries_storage)