Do not render records using .dict() but usin JSOn

The `dict` method of the records are, by design, meant to just conver
the record to a dictionary, without converting data types. If we want to
serialize the records correcty, as we are doing when generating the SSM
records, we need to rely on serialization. See [1] for more context.

[1]: pydantic/pydantic#5155 (comment)

Fixes: #113
SemVer: bugfix

Author: alvarolopez

caso/messenger/ssm.py

@@ -16,7 +16,6 @@
 
 """Module containing the APEL SSM Messenger."""
 
-import datetime
 import json
 import typing
 import warnings
@@ -27,7 +26,6 @@
 import dirq.QueueSimple
 from oslo_config import cfg
 from oslo_log import log
-import six
 
 import caso.exception
 import caso.messenger
@@ -191,13 +189,13 @@ def push(self, records):
         for record in records:
             if isinstance(record, caso.record.CloudRecord):
                 aux = []
-                for k, v in six.iteritems(record.dict(**opts)):
-                    if v is not None:
-                        # FIXME(aloga): to be handled at record level.
-                        if isinstance(v, datetime.datetime):
-                            aux.append(f"{k}: {int(v.timestamp())}")
-                        else:
-                            aux.append(f"{k}: {v}")
+                # NOTE(aloga): do not iter over the dictionary returned by record.dict()
+                # as this is just a dictionary representation of the object, where no
+                # serialization is done. In order to get objects correctly serialized
+                # we need to convert to JSON, then reload the model
+                serialized_record = json.loads(record.json(**opts))
+                for k, v in serialized_record.items():
+                    aux.append(f"{k}: {v}")
                 entries_cloud.append("\n".join(aux))
             elif isinstance(record, caso.record.IPRecord):
                 entries_ip.append(record.json(**opts))

caso/record.py

@@ -143,6 +143,7 @@ def map_fields(value: str) -> str:
             }
             return d.get(value, value)
 
+        json_encoders = {datetime.datetime: lambda v: int(v.timestamp())}
         alias_generator = map_fields
         allow_population_by_field_name = True
         underscore_attrs_are_private = True
@@ -189,7 +190,7 @@ def map_fields(field: str) -> str:
             }
             return d.get(field, field)
 
-        json_encoders = {datetime: lambda v: int(v.timestamp())}
+        json_encoders = {datetime.datetime: lambda v: int(v.timestamp())}
         alias_generator = map_fields
         allow_population_by_field_name = True
         underscore_attrs_are_private = True
@@ -262,6 +263,7 @@ def map_fields(field: str) -> str:
             }
             return d.get(field, field)
 
+        json_encoders = {datetime.datetime: lambda v: int(v.timestamp())}
         alias_generator = map_fields
         allow_population_by_field_name = True
         underscore_attrs_are_private = True

caso/tests/test_record.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Spanish National Research Council (CSIC)
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Test for cASO records."""
+
+import datetime
+import json
+import uuid
+
+import pytest
+
+import caso.record
+
+valid_record_fields = dict(
+    uuid=str(uuid.uuid4()),
+    site_name="TEST-Site",
+    name="VM Name",
+    user_id=str(uuid.uuid4()),
+    group_id=str(uuid.uuid4()),
+    fqan="VO FQAN",
+    start_time=datetime.datetime.now() - datetime.timedelta(days=5),
+    end_time=datetime.datetime.now(),
+    compute_service="Fake Cloud Service",
+    status="ACTIVE",
+    image_id=str(uuid.uuid4()),
+    user_dn="User DN",
+    benchmark_type=None,
+    benchmark_value=None,
+    memory=16,
+    cpu_count=8,
+    disk=250,
+    public_ip_count=7,
+)
+
+valid_record = {
+    "CloudComputeService": "Fake Cloud Service",
+    "CpuCount": 8,
+    "CpuDuration": 3456000,
+    "Disk": 250,
+    "StartTime": int(valid_record_fields["start_time"].timestamp()),
+    "EndTime": int(valid_record_fields["end_time"].timestamp()),
+    "FQAN": "VO FQAN",
+    "GlobalUserName": "User DN",
+    "ImageId": valid_record_fields["image_id"],
+    "LocalGroupId": valid_record_fields["group_id"],
+    "LocalUserId": valid_record_fields["user_id"],
+    "MachineName": "VM Name",
+    "Memory": 16,
+    "PublicIPCount": 7,
+    "SiteName": "TEST-Site",
+    "Status": "ACTIVE",
+    "VMUUID": valid_record_fields["uuid"],
+    "WallDuration": 432000,
+}
+
+
+@pytest.mark.skip(reason="Pydantic 1 does not support computed fields")
+def test_cloud_record():
+    """Test a cloud record is correctly generated."""
+    record = caso.record.CloudRecord(**valid_record_fields)
+
+    wall = datetime.timedelta(days=5).total_seconds()
+    cpu = wall * record.cpu_count
+
+    assert record.wall_duration == wall
+    assert record.cpu_duration == cpu
+
+    assert isinstance(record.start_time, datetime.datetime)
+    assert isinstance(record.end_time, datetime.datetime)
+
+
+@pytest.mark.skip(reason="Pydantic 1 does not support computed fields")
+def test_cloud_record_map_opts():
+    """Test a cloud record is correctly rendered."""
+    record = caso.record.CloudRecord(**valid_record_fields)
+
+    opts = {
+        "by_alias": True,
+        "exclude_unset": True,
+        "exclude_none": True,
+    }
+
+    assert json.loads(record.json(**opts)) == valid_record
+
+
+def test_cloud_record_map_opts_custom_wall_cpu():
+    """Test a cloud record is correctly rendered with custom wall and cpu times."""
+    record = caso.record.CloudRecord(**valid_record_fields)
+
+    wall = datetime.timedelta(days=5).total_seconds()
+    cpu = wall * record.cpu_count
+    record.wall_duration = wall
+    record.cpu_duration = cpu
+
+    opts = {
+        "by_alias": True,
+        "exclude_unset": True,
+        "exclude_none": True,
+    }
+
+    assert json.loads(record.json(**opts)) == valid_record
+
+
+@pytest.mark.skip(reason="Pydantic 1 does not support computed fields")
+def test_cloud_record_custom_wall():
+    """Test a cloud record is correctly rendered with custom wall time."""
+    record = caso.record.CloudRecord(**valid_record_fields)
+
+    wall = 200
+    cpu = wall * record.cpu_count
+    record.wall_duration = wall
+    assert record.wall_duration == wall
+    assert record.cpu_duration == cpu
+
+
+def test_cloud_record_custom_wall_cpu():
+    """Test a cloud record is correctly generated with custom wall and cpu time."""
+    record = caso.record.CloudRecord(**valid_record_fields)
+
+    wall = 200
+    cpu = wall * record.cpu_count
+    record.wall_duration = wall
+    record.cpu_duration = cpu
+
+    assert record.wall_duration == wall
+    assert record.cpu_duration == cpu
+
+
+@pytest.mark.skip(reason="Pydantic 1 does not support computed fields")
+def test_cloud_record_custom_cpu():
+    """Test a cloud record is correctly rendered with custom cpu time."""
+    record = caso.record.CloudRecord(**valid_record_fields)
+
+    wall = datetime.timedelta(days=5).total_seconds()
+    cpu = wall * record.cpu_count * 10
+    record.cpu_duration = cpu
+    assert record.wall_duration == wall
+    assert record.cpu_duration == cpu