Add loading of custom MATLAB datetime structures

mat73/core.py

@@ -9,6 +9,7 @@
 import os
 import numpy as np
 import h5py
+from datetime import datetime, timedelta
 import logging
 from typing import Iterable
 from mat73.version import __version__
@@ -300,6 +301,141 @@ def convert_mat(self, dataset, depth, MATLAB_class=None):
             if arr.size==1: arr=bool(arr)
             return arr
 
+        elif MATLAB_class == 'datetime':
+            # MATLAB datetime objects can be encoded in different ways:
+            # 1. With MATLAB_object_decode=3: MCOS objects (object-oriented encoding)
+            # 2. Without it: Simple numeric date arrays
+
+            if 'MATLAB_object_decode' in dataset.attrs and dataset.attrs['MATLAB_object_decode'] == 3:
+                # This is an MCOS-encoded datetime object
+                # The dataset contains reference metadata, not the actual datetime values
+                # We need to navigate to the MCOS subsystem to get the actual timestamps
+
+                # Get the file handle from the dataset
+                file_handle = dataset.file
+
+                # Check if MCOS subsystem exists
+                if '#subsystem#' in file_handle and 'MCOS' in file_handle['#subsystem#']:
+                    mcos_refs = file_handle['#subsystem#/MCOS'][0]
+
+                    # The 5th element (index 4) in the dataset appears to be the MCOS reference index
+                    # Extract reference indices from the dataset
+                    raw_data = np.array(dataset)
+
+                    # Handle different dataset shapes
+                    if raw_data.ndim == 2:
+                        # Shape is (n, 6) where n is the number of datetime values
+                        # The 5th column (index 4) contains a 0-based index into datetime objects
+                        # The actual datetime data starts at MCOS[2] (first two are metadata)
+                        # So the MCOS index is field[4] + 2
+                        mcos_indices = raw_data[:, 4] + 1
+                    else:
+                        # Fallback for unexpected shapes
+                        logger.warning(f"Unexpected shape for MCOS datetime: {raw_data.shape}")
+                        return None
+
+                    # Decode each datetime value from MCOS
+                    # Each index can point to either a scalar or array of datetime values
+                    all_datetimes = []
+                    for idx in mcos_indices:
+                        try:
+                            idx = int(idx)
+                            if 0 <= idx < len(mcos_refs):
+                                mcos_ref = mcos_refs[idx]
+                                mcos_obj = file_handle[mcos_ref]
+
+                                if isinstance(mcos_obj, h5py.Dataset) and mcos_obj.dtype == np.float64:
+                                    # The MCOS object contains Unix timestamp(s) in milliseconds
+                                    # It can be a scalar or an array of values
+                                    timestamps_ms = np.array(mcos_obj)
+
+                                    # Preserve the original shape
+                                    original_shape = timestamps_ms.shape
+
+                                    # Flatten to 1D array for processing
+                                    flat_timestamps = timestamps_ms.flatten()
+
+                                    # Convert each timestamp
+                                    datetime_values = []
+                                    for ts_ms in flat_timestamps:
+                                        if np.isnan(ts_ms):
+                                            datetime_values.append(None)  # NaT (Not-a-Time)
+                                        else:
+                                            # Convert milliseconds to seconds
+                                            timestamp_sec = ts_ms / 1000.0
+                                            # Convert to datetime
+                                            dt = datetime(1970, 1, 1) + timedelta(seconds=timestamp_sec)
+                                            datetime_values.append(dt)
+
+                                    # Reshape back to original shape and apply squeeze
+                                    if len(datetime_values) == 1:
+                                        all_datetimes.append(datetime_values[0])
+                                    else:
+                                        result = np.array(datetime_values, dtype=object).reshape(original_shape)
+                                        # Apply MATLAB-style transpose and squeeze
+                                        result = result.T
+                                        all_datetimes.append(squeeze(result))
+                                else:
+                                    logger.warning(f"Unexpected MCOS object type for datetime at index {idx}")
+                                    all_datetimes.append(None)
+                            else:
+                                logger.warning(f"MCOS index {idx} out of range")
+                                all_datetimes.append(None)
+                        except Exception as e:
+                            logger.warning(f"Error decoding MCOS datetime at index {idx}: {e}")
+                            all_datetimes.append(None)
+
+                    # Return the result appropriately
+                    if len(all_datetimes) == 1:
+                        # Single datetime (scalar or array)
+                        return all_datetimes[0]
+                    else:
+                        # Multiple datetime values/arrays
+                        return np.array(all_datetimes, dtype=object)
+                else:
+                    logger.warning("MCOS subsystem not found for MATLAB_object_decode=3 datetime")
+                    return None
+            else:
+                # Legacy format: numeric date arrays (datenums)
+                datenums_raw = np.array(dataset, dtype=np.float64)
+                datenums_transposed = datenums_raw.T
+
+                original_shape = datenums_transposed.shape
+                flat_datenums = datenums_transposed.flatten()
+                py_datetimes = []
+
+                # MATLAB uses days since January 0, 0000, Python uses days since January 1, 0001
+                MATLAB_TO_PYTHON_OFFSET = 366
+
+                for mdn in flat_datenums:
+                    if np.isnan(mdn):
+                        py_datetimes.append(None)  # Represent MATLAB NaT (Not-a-Time) as None
+                        continue
+
+                    # Convert MATLAB datenum to Python ordinal
+                    python_ordinal = mdn - MATLAB_TO_PYTHON_OFFSET
+                    dt_ordinal_day = int(np.floor(python_ordinal))
+                    time_fraction = python_ordinal - np.floor(python_ordinal)
+
+                    if dt_ordinal_day < 1:
+                        logging.warning(
+                            f"MATLAB datetime value {mdn} is before 0001-01-01. Storing as None."
+                        )
+                        py_datetimes.append(None)
+                        continue
+
+                    try:
+                        current_dt = datetime.fromordinal(dt_ordinal_day) + timedelta(days=time_fraction)
+                        py_datetimes.append(current_dt)
+                    except (ValueError, OverflowError) as e:
+                        logging.warning(
+                            f"Could not convert MATLAB datetime value {mdn}: {e}. Storing as None."
+                        )
+                        py_datetimes.append(None)
+
+                result_array = np.array(py_datetimes, dtype=object).reshape(original_shape)
+                return squeeze(result_array)
+
         elif mtype=='canonical empty':
             return None
 

tests/create_mat.m

@@ -82,3 +82,52 @@
 
 save('testfile16.mat','char_arr_1d', 'char_arr_2d', 'char_arr_3d', '-v7.3')
 clear all
+
+%% file to test datetime loading
+% Scalar datetime
+dt_scalar = datetime(2023, 10, 26);
+
+% Row vector of datetimes
+dt_row_vector = [datetime(2023, 11, 1), datetime(2023, 11, 15), datetime(2023, 12, 25)];
+
+% Column vector of datetimes
+dt_column_vector = [datetime(2024, 1, 10); datetime(2024, 2, 20); datetime(2024, 3, 30)];
+
+% 2x2 matrix of datetimes
+dt_matrix = [datetime(2025, 1, 1), datetime(2025, 2, 1); ...
+             datetime(2025, 3, 1), datetime(2025, 4, 1)];
+
+% Datetime with specific time including milliseconds
+dt_specific_time = datetime(2023, 3, 15, 14, 30, 45, 678);
+
+% NaT (Not-a-Time)
+dt_nat = NaT;
+
+% Array with NaT values
+dt_nat_array = [datetime(2023, 1, 1), NaT, datetime(2023, 1, 3)];
+
+% Datetime with timezone (will be converted to UTC when stored)
+dt_with_timezone = datetime(2023, 10, 26, 10, 0, 0, 'TimeZone', 'America/New_York');
+
+% Past date (before Unix epoch)
+dt_past_scalar = datetime(1500, 1, 1);
+
+% Future date
+dt_future_scalar = datetime(2500, 1, 1);
+
+% Struct containing datetime fields
+dt_struct.scalar = datetime(2026, 7, 4);
+dt_struct.array = [datetime(2026, 8, 1); datetime(2026, 9, 15)];
+dt_struct.mixed_datetime_in_struct_array.d = [datetime(2026, 10, 1); datetime(2026, 11, 1)];
+
+% Cell array with datetime values
+dt_cell_array = {datetime(2027, 5, 18), [datetime(2027, 6, 1), datetime(2027, 7, 1)]};
+
+% Cell array column with datetime values
+dt_cell_array_column = {[datetime(2027, 8, 1)]; [datetime(2027, 9, 1)]};
+
+save('testfile_datetime.mat', 'dt_scalar', 'dt_row_vector', 'dt_column_vector', ...
+     'dt_matrix', 'dt_specific_time', 'dt_nat', 'dt_nat_array', 'dt_with_timezone', ...
+     'dt_past_scalar', 'dt_future_scalar', 'dt_struct', 'dt_cell_array', ...
+     'dt_cell_array_column', '-v7.3')
+clear all

tests/test_mat73.py

@@ -7,6 +7,7 @@
 import os
 import numpy as np
 import mat73
+from datetime import datetime
 import unittest
 import time
 try:
@@ -525,6 +526,139 @@ def test_file16_2d_char_array(self):
         self.assertEqual(data['char_arr_2d'], expected)
         self.assertEqual(data['char_arr_3d'], [['abcd', 'defg'], ['ghij', 'jklm'], ['mnöp', 'pqrs']])
 
+    def test_datetime_loading(self):
+        """Test loading of MATLAB datetime objects."""
+        test_file_path = os.path.join('tests', 'testfile_datetime.mat')
+        if not os.path.exists(test_file_path):
+            self.skipTest(f"{test_file_path} not found. User needs to provide this file. Skipping datetime tests.")
+
+        d = mat73.loadmat(test_file_path, use_attrdict=True)
+
+        # Helper function for datetime comparison
+        def assert_datetime_equal(dt_obj, year, month, day, hour=0, minute=0, second=0, microsecond=0):
+            self.assertIsInstance(dt_obj, datetime)
+            self.assertEqual(dt_obj.year, year)
+            self.assertEqual(dt_obj.month, month)
+            self.assertEqual(dt_obj.day, day)
+            self.assertEqual(dt_obj.hour, hour)
+            self.assertEqual(dt_obj.minute, minute)
+            self.assertEqual(dt_obj.second, second)
+            self.assertEqual(dt_obj.microsecond, microsecond)
+
+        # dt_scalar: datetime(2023, 10, 26)
+        self.assertIn('dt_scalar', d)
+        assert_datetime_equal(d.dt_scalar, 2023, 10, 26)
+
+        # dt_row_vector: [datetime(2023, 11, 1), datetime(2023, 11, 15), datetime(2023, 12, 25)]
+        self.assertIn('dt_row_vector', d)
+        dt_row_vector = d.dt_row_vector
+        self.assertTrue(isinstance(dt_row_vector, (list, np.ndarray)))
+        self.assertEqual(len(dt_row_vector), 3)
+        assert_datetime_equal(dt_row_vector[0], 2023, 11, 1)
+        assert_datetime_equal(dt_row_vector[1], 2023, 11, 15)
+        assert_datetime_equal(dt_row_vector[2], 2023, 12, 25)
+
+        # dt_column_vector: [datetime(2024, 1, 10), datetime(2024, 2, 20), datetime(2024, 3, 30)]
+        # Expecting a flat list or 1D array due to squeeze
+        self.assertIn('dt_column_vector', d)
+        dt_column_vector = d.dt_column_vector
+        self.assertTrue(isinstance(dt_column_vector, (list, np.ndarray)))
+        self.assertEqual(len(dt_column_vector), 3)
+        assert_datetime_equal(dt_column_vector[0], 2024, 1, 10)
+        assert_datetime_equal(dt_column_vector[1], 2024, 2, 20)
+        assert_datetime_equal(dt_column_vector[2], 2024, 3, 30)
+
+        # dt_matrix: [[datetime(2025, 1, 1), datetime(2025, 2, 1)], [datetime(2025, 3, 1), datetime(2025, 4, 1)]]
+        # Note: MATLAB is column-major, Python (NumPy) is row-major.
+        # mat73 transposes arrays, so d.dt_matrix[row_idx, col_idx] should correspond to MATLAB(row_idx+1, col_idx+1)
+        self.assertIn('dt_matrix', d)
+        dt_matrix = d.dt_matrix
+        self.assertIsInstance(dt_matrix, np.ndarray) # Usually loaded as numpy array
+        self.assertEqual(dt_matrix.shape, (2, 2))
+        assert_datetime_equal(dt_matrix[0, 0], 2025, 1, 1)
+        assert_datetime_equal(dt_matrix[0, 1], 2025, 2, 1) # MATLAB: (1,2)
+        assert_datetime_equal(dt_matrix[1, 0], 2025, 3, 1) # MATLAB: (2,1)
+        assert_datetime_equal(dt_matrix[1, 1], 2025, 4, 1) # MATLAB: (2,2)
+
+        # dt_specific_time: datetime(2023, 3, 15, 14, 30, 45, 678000)
+        self.assertIn('dt_specific_time', d)
+        assert_datetime_equal(d.dt_specific_time, 2023, 3, 15, 14, 30, 45, 678000)
+
+        # dt_nat: None
+        self.assertIn('dt_nat', d)
+        self.assertIsNone(d.dt_nat)
+
+        # dt_nat_array: [datetime(2023, 1, 1), None, datetime(2023, 1, 3)]
+        self.assertIn('dt_nat_array', d)
+        dt_nat_array = d.dt_nat_array
+        self.assertTrue(isinstance(dt_nat_array, (list, np.ndarray)))
+        self.assertEqual(len(dt_nat_array), 3)
+        assert_datetime_equal(dt_nat_array[0], 2023, 1, 1)
+        self.assertIsNone(dt_nat_array[1])
+        assert_datetime_equal(dt_nat_array[2], 2023, 1, 3)
+
+        # dt_with_timezone: datetime(2023, 10, 26, 10, 0, 0) (naive)
+        self.assertIn('dt_with_timezone', d)
+        # Timezone information is expected to be lost, testing for naive datetime
+        assert_datetime_equal(d.dt_with_timezone, 2023, 10, 26, 10, 0, 0)
+
+        # dt_past_scalar: datetime(1500, 1, 1)
+        self.assertIn('dt_past_scalar', d)
+        assert_datetime_equal(d.dt_past_scalar, 1500, 1, 1)
+
+        # dt_future_scalar: datetime(2500, 1, 1)
+        self.assertIn('dt_future_scalar', d)
+        assert_datetime_equal(d.dt_future_scalar, 2500, 1, 1)
+
+        # dt_struct:
+        self.assertIn('dt_struct', d)
+        dt_struct = d.dt_struct
+        self.assertTrue(hasattr(dt_struct, 'scalar')) # AttrDict access
+        assert_datetime_equal(dt_struct.scalar, 2026, 7, 4)
+
+        self.assertTrue(hasattr(dt_struct, 'array'))
+        struct_array_field = dt_struct.array
+        self.assertTrue(isinstance(struct_array_field, (list, np.ndarray)))
+        self.assertEqual(len(struct_array_field), 2)
+        assert_datetime_equal(struct_array_field[0], 2026, 8, 1)
+        assert_datetime_equal(struct_array_field[1], 2026, 9, 15)
+
+        self.assertTrue(hasattr(dt_struct, 'mixed_datetime_in_struct_array'))
+        mixed_struct_array = dt_struct.mixed_datetime_in_struct_array
+        self.assertIsInstance(mixed_struct_array, list) # Struct arrays become lists of AttrDicts
+        self.assertEqual(len(mixed_struct_array), 2)
+        self.assertTrue(hasattr(mixed_struct_array[0], 'd'))
+        assert_datetime_equal(mixed_struct_array[0].d, 2026, 10, 1)
+        self.assertTrue(hasattr(mixed_struct_array[1], 'd'))
+        assert_datetime_equal(mixed_struct_array[1].d, 2026, 11, 1)
+
+        # dt_cell_array:
+        # d['dt_cell_array'][0]: datetime(2027, 5, 18)
+        # d['dt_cell_array'][1][0]: datetime(2027, 6, 1)
+        # d['dt_cell_array'][1][1]: datetime(2027, 7, 1)
+        self.assertIn('dt_cell_array', d)
+        dt_cell_array = d.dt_cell_array # This will be a list
+        self.assertIsInstance(dt_cell_array, list)
+        self.assertEqual(len(dt_cell_array), 2)
+        assert_datetime_equal(dt_cell_array[0], 2027, 5, 18)
+
+        cell_inner_array = dt_cell_array[1]
+        self.assertTrue(isinstance(cell_inner_array, (list, np.ndarray)))
+        self.assertEqual(len(cell_inner_array), 2)
+        assert_datetime_equal(cell_inner_array[0], 2027, 6, 1)
+        assert_datetime_equal(cell_inner_array[1], 2027, 7, 1)
+
+        # dt_cell_array_column:
+        # d['dt_cell_array_column'][0]: datetime(2027, 8, 1)
+        # d['dt_cell_array_column'][1]: datetime(2027, 9, 1)
+        self.assertIn('dt_cell_array_column', d)
+        dt_cell_array_column = d.dt_cell_array_column # This will be a list
+        self.assertIsInstance(dt_cell_array_column, list)
+        self.assertEqual(len(dt_cell_array_column), 2) # MATLAB {{dt1};{dt2}} results in a 2x1 cell array
+                                                       # which mat73 typically converts to a list of 2 elements.
+        assert_datetime_equal(dt_cell_array_column[0], 2027, 8, 1)
+        assert_datetime_equal(dt_cell_array_column[1], 2027, 9, 1)
+
 if __name__ == '__main__':
 
     unittest.main()