diff --git a/xblock/fields.py b/xblock/fields.py index 9f814ce06..6ad3ca1bb 100644 --- a/xblock/fields.py +++ b/xblock/fields.py @@ -11,7 +11,9 @@ import hashlib import itertools import json +import logging import re +import time import traceback import warnings @@ -19,6 +21,11 @@ from lxml import etree import pytz import yaml +from pytz import UTC + +from xblock.scorable import Score + +log = logging.getLogger(__name__) # __all__ controls what classes end up in the docs, and in what order. @@ -26,6 +33,7 @@ 'BlockScope', 'UserScope', 'Scope', 'ScopeIds', 'Field', 'Boolean', 'Dict', 'Float', 'Integer', 'List', 'Set', 'String', 'XMLString', + "Date", "DateTime", "Timedelta", "RelativeTime", "ScoreField", "ListScoreField", ] @@ -939,6 +947,89 @@ def enforce_type(self, value): return value +class Date(JSONField): + """ + Date fields know how to parse and produce json (iso) compatible formats. Converts to tz aware datetimes. + """ + + # See note below about not defaulting these + CURRENT_YEAR = datetime.datetime.now(UTC).year + PREVENT_DEFAULT_DAY_MON_SEED1 = datetime.datetime(CURRENT_YEAR, 1, 1, tzinfo=UTC) + PREVENT_DEFAULT_DAY_MON_SEED2 = datetime.datetime(CURRENT_YEAR, 2, 2, tzinfo=UTC) + + MUTABLE = False + + def _parse_date_wo_default_month_day(self, field): + """ + Parse the field as an iso string but prevent dateutils from defaulting the day or month while + allowing it to default the other fields. + """ + # It's not trivial to replace dateutil b/c parsing timezones as Z, +03:30, -400 is hard in python + # however, we don't want dateutil to default the month or day (but some tests at least expect + # us to default year); so, we'll see if dateutil uses the defaults for these the hard way + result = dateutil.parser.parse(field, default=self.PREVENT_DEFAULT_DAY_MON_SEED1) + result_other = dateutil.parser.parse(field, default=self.PREVENT_DEFAULT_DAY_MON_SEED2) + if result != result_other: + log.warning("Field %s is missing month or day", self.name) + return None + if result.tzinfo is None: + result = result.replace(tzinfo=UTC) + return result + + def from_json(self, value): + """ + Parse an optional metadata key containing a time: if present, complain + if it doesn't parse. + Return None if not present or invalid. + """ + if value is None: + return value + + if value == "": + return None + + if isinstance(value, str): + return self._parse_date_wo_default_month_day(value) + + if isinstance(value, (int, float)): + return datetime.datetime.fromtimestamp(value / 1000, UTC) + + if isinstance(value, time.struct_time): + return datetime.datetime.fromtimestamp(time.mktime(value), UTC) + + if isinstance(value, datetime.datetime): + return value + + msg = f"value {self.name} has bad value '{value}'" + raise TypeError(msg) + + def to_json(self, value): + """ + Convert a time struct to a string + """ + if value is None: + return None + + if isinstance(value, time.struct_time): + # struct_times are always utc + return time.strftime("%Y-%m-%dT%H:%M:%SZ", value) + + if isinstance(value, datetime.datetime): + if value.tzinfo is None or value.utcoffset().total_seconds() == 0: + if value.year < 1900: + # strftime doesn't work for pre-1900 dates, so use + # isoformat instead + return value.isoformat() + # isoformat adds +00:00 rather than Z + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + return value.isoformat() + + raise TypeError(f"Cannot convert {value!r} to json") + + enforce_type = from_json + + class DateTime(JSONField): """ A field for representing a datetime. @@ -1010,6 +1101,178 @@ def to_string(self, value): enforce_type = from_json +TIMEDELTA_REGEX = re.compile( + r"^((?P\d+?) day(?:s?))?(\s)?" + r"((?P\d+?) hour(?:s?))?(\s)?" + r"((?P\d+?) minute(?:s)?)?(\s)?" + r"((?P\d+?) second(?:s)?)?$" +) + + +class Timedelta(JSONField): + """Field type for serializing/deserializing timedelta values to/from human-readable strings.""" + + # Timedeltas are immutable, see https://docs.python.org/3/library/datetime.html#available-types + MUTABLE = False + + def from_json(self, value): + """ + value: A string with the following components: + day[s] (optional) + hour[s] (optional) + minute[s] (optional) + second[s] (optional) + + Returns a datetime.timedelta parsed from the string + """ + if value is None: + return None + + if isinstance(value, datetime.timedelta): + return value + + parts = TIMEDELTA_REGEX.match(value) + if not parts: + return None + parts = parts.groupdict() + time_params = {} + for name, param in parts.items(): + if param: + time_params[name] = int(param) + return datetime.timedelta(**time_params) + + def to_json(self, value): + """Serialize a datetime.timedelta object into a human-readable string.""" + if value is None: + return None + + values = [] + for attr in ("days", "hours", "minutes", "seconds"): + cur_value = getattr(value, attr, 0) + if cur_value > 0: + values.append(f"{cur_value} {attr}") + return " ".join(values) + + def enforce_type(self, value): + """ + Ensure that when set explicitly the Field is set to a timedelta + """ + if isinstance(value, datetime.timedelta) or value is None: + return value + + return self.from_json(value) + + +class RelativeTime(JSONField): + """ + Field for start_time and end_time video block properties. + + It was decided, that python representation of start_time and end_time + should be python datetime.timedelta object, to be consistent with + common time representation. + + At the same time, serialized representation should be "HH:MM:SS" + This format is convenient to use in XML (and it is used now), + and also it is used in frond-end studio editor of video block as format + for start and end time fields. + + In database we previously had float type for start_time and end_time fields, + so we are checking it also. + + Python object of RelativeTime is datetime.timedelta. + JSONed representation of RelativeTime is "HH:MM:SS" + """ + + # Timedeltas are immutable, see https://docs.python.org/3/library/datetime.html#available-types + MUTABLE = False + + @classmethod + def isotime_to_timedelta(cls, value): + """ + Validate that value in "HH:MM:SS" format and convert to timedelta. + + Validate that user, that edits XML, sets proper format, and + that max value that can be used by user is "23:59:59". + """ + try: + obj_time = time.strptime(value, "%H:%M:%S") + except ValueError as e: + raise ValueError( + f"Incorrect RelativeTime value {value!r} was set in XML or serialized. Original parse message is {e}" + ) from e + return datetime.timedelta(hours=obj_time.tm_hour, minutes=obj_time.tm_min, seconds=obj_time.tm_sec) + + def from_json(self, value): + """ + Convert value is in 'HH:MM:SS' format to datetime.timedelta. + + If not value, returns 0. + If value is float (backward compatibility issue), convert to timedelta. + """ + if not value: + return datetime.timedelta(seconds=0) + + if isinstance(value, datetime.timedelta): + return value + + # We've seen serialized versions of float in this field + if isinstance(value, float): + return datetime.timedelta(seconds=value) + + if isinstance(value, str): + return self.isotime_to_timedelta(value) + + msg = f"RelativeTime Field {self.name} has bad value '{value!r}'" + raise TypeError(msg) + + def to_json(self, value): + """ + Convert datetime.timedelta to "HH:MM:SS" format. + + If not value, return "00:00:00" + + Backward compatibility: check if value is float, and convert it. No exceptions here. + + If value is not float, but is exceed 23:59:59, raise exception. + """ + if not value: + return "00:00:00" + + if isinstance(value, float): # backward compatibility + value = min(value, 86400) + return self.timedelta_to_string(datetime.timedelta(seconds=value)) + + if isinstance(value, datetime.timedelta): + if value.total_seconds() > 86400: # sanity check + raise ValueError( + f"RelativeTime max value is 23:59:59=86400.0 seconds, but {value.total_seconds()} seconds is passed" + ) + return self.timedelta_to_string(value) + + raise TypeError(f"RelativeTime: cannot convert {value!r} to json") + + def timedelta_to_string(self, value): + """ + Makes first 'H' in str representation non-optional. + + str(timedelta) has [H]H:MM:SS format, which is not suitable + for front-end (and ISO time standard), so we force HH:MM:SS format. + """ + stringified = str(value) + if len(stringified) == 7: + stringified = "0" + stringified + return stringified + + def enforce_type(self, value): + """ + Ensure that when set explicitly the Field is set to a timedelta + """ + if isinstance(value, datetime.timedelta) or value is None: + return value + + return self.from_json(value) + + class Any(JSONField): """ A field class for representing any piece of data; type is not enforced. @@ -1051,6 +1314,69 @@ class ReferenceValueDict(Dict): # but since Reference doesn't stipulate a definition for from/to, that seems unnecessary at this time. +class ScoreField(JSONField): + """ + Field for blocks that need to store a Score. XBlocks that implement + the ScorableXBlockMixin may need to store their score separately + from their problem state, specifically for use in staff override + of problem scores. + """ + + MUTABLE = False + + def from_json(self, value): + """Deserialize a dict with 'raw_earned' and 'raw_possible' into a Score object, validating values.""" + if value is None: + return value + if isinstance(value, Score): + return value + + if set(value) != {"raw_earned", "raw_possible"}: + raise TypeError(f"Scores must contain only a raw earned and raw possible value. Got {set(value)}") + + raw_earned = value["raw_earned"] + raw_possible = value["raw_possible"] + + if raw_possible < 0: + raise ValueError( + f"Error deserializing field of type {self.display_name}: " + f"Expected a positive number for raw_possible, got {raw_possible}." + ) + + if not 0 <= raw_earned <= raw_possible: + raise ValueError( + f"Error deserializing field of type {self.display_name}: " + f"Expected raw_earned between 0 and {raw_possible}, got {raw_earned}." + ) + + return Score(raw_earned, raw_possible) + + enforce_type = from_json + + +class ListScoreField(ScoreField, List): + """ + Field for blocks that need to store a list of Scores. + """ + + MUTABLE = True + _default = [] + + def from_json(self, value): + if value is None: + return value + if isinstance(value, list): + scores = [] + for score_json in value: + score = super().from_json(score_json) + scores.append(score) + return scores + + raise TypeError(f"Value must be a list of Scores. Got {type(value)}") + + enforce_type = from_json + + def scope_key(instance, xblock): """Generate a unique key for a scope that can be used as a filename, in a URL, or in a KVS. diff --git a/xblock/test/test_fields.py b/xblock/test/test_fields.py index 6f67f71eb..1941e7410 100644 --- a/xblock/test/test_fields.py +++ b/xblock/test/test_fields.py @@ -19,9 +19,9 @@ from xblock.field_data import DictFieldData from xblock.fields import ( Any, Boolean, Dict, Field, Float, Integer, List, Set, String, XMLString, DateTime, Reference, ReferenceList, - ScopeIds, Sentinel, UNIQUE_ID, scope_key, + ScopeIds, Sentinel, UNIQUE_ID, scope_key, Date, Timedelta, RelativeTime, ScoreField, ListScoreField ) - +from xblock.scorable import Score from xblock.test.tools import TestRuntime @@ -275,10 +275,104 @@ def test_bad_xml(self, input_text): self.assertEqual(unchecked_xml_string.to_json(input_text), input_text) +class DateTest(unittest.TestCase): + """Tests JSON conversion and type enforcement for Date fields.""" + + date = Date() + + def compare_dates(self, dt1, dt2, expected_delta): + """Assert that two datetime objects differ by the expected timedelta.""" + assert (dt1 - dt2) == expected_delta, (((str(dt1) + "-") + str(dt2)) + "!=") + str(expected_delta) + + def test_from_json(self): + """Test conversion from iso compatible date strings to struct_time""" + self.compare_dates( + DateTest.date.from_json("2013-01-01"), DateTest.date.from_json("2012-12-31"), dt.timedelta(days=1) + ) + self.compare_dates( + DateTest.date.from_json("2013-01-01T00"), + DateTest.date.from_json("2012-12-31T23"), + dt.timedelta(hours=1), + ) + self.compare_dates( + DateTest.date.from_json("2013-01-01T00:00"), + DateTest.date.from_json("2012-12-31T23:59"), + dt.timedelta(minutes=1), + ) + self.compare_dates( + DateTest.date.from_json("2013-01-01T00:00:00"), + DateTest.date.from_json("2012-12-31T23:59:59"), + dt.timedelta(seconds=1), + ) + self.compare_dates( + DateTest.date.from_json("2013-01-01T00:00:00Z"), + DateTest.date.from_json("2012-12-31T23:59:59Z"), + dt.timedelta(seconds=1), + ) + self.compare_dates( + DateTest.date.from_json("2012-12-31T23:00:01-01:00"), + DateTest.date.from_json("2013-01-01T00:00:00+01:00"), + dt.timedelta(hours=1, seconds=1), + ) + + def test_enforce_type(self): + """Test enforcement of input types for Date field.""" + assert DateTest.date.enforce_type(None) is None + assert DateTest.date.enforce_type("") is None + assert DateTest.date.enforce_type("2012-12-31T23:00:01") == dt.datetime(2012, 12, 31, 23, 0, 1, tzinfo=pytz.UTC) + assert DateTest.date.enforce_type(1234567890000) == dt.datetime(2009, 2, 13, 23, 31, 30, tzinfo=pytz.UTC) + assert DateTest.date.enforce_type(dt.datetime(2014, 5, 9, 21, 1, 27, tzinfo=pytz.UTC)) == dt.datetime( + 2014, 5, 9, 21, 1, 27, tzinfo=pytz.UTC + ) + with self.assertRaises(TypeError): + DateTest.date.enforce_type([1]) + + def test_return_none(self): + """Test that invalid or empty inputs return None for Date field.""" + assert DateTest.date.from_json("") is None + assert DateTest.date.from_json(None) is None + with self.assertRaises(TypeError): + DateTest.date.from_json(["unknown value"]) + + def test_old_due_date_format(self): + """Test parsing of non-standard human-readable date formats.""" + current = dt.datetime.today() + assert dt.datetime(current.year, 3, 12, 12, tzinfo=pytz.UTC) == DateTest.date.from_json("March 12 12:00") + assert dt.datetime(current.year, 12, 4, 16, 30, tzinfo=pytz.UTC) == DateTest.date.from_json("December 4 16:30") + assert DateTest.date.from_json("12 12:00") is None + + def test_non_std_from_json(self): + """ + Test the non-standard args being passed to from_json + """ + now = dt.datetime.now(pytz.UTC) + delta = now - dt.datetime.fromtimestamp(0, pytz.UTC) + assert DateTest.date.from_json(delta.total_seconds() * 1000) == now + yesterday = dt.datetime.now(pytz.UTC) - dt.timedelta(days=-1) + assert DateTest.date.from_json(yesterday) == yesterday + + def test_to_json(self): + """ + Test converting time reprs to iso dates + """ + assert ( + DateTest.date.to_json( + dt.datetime.strptime("2012-12-31T23:59:59Z", "%Y-%m-%dT%H:%M:%SZ") + ) == "2012-12-31T23:59:59Z" + ) + + assert DateTest.date.to_json(DateTest.date.from_json("2012-12-31T23:59:59Z")) == "2012-12-31T23:59:59Z" + assert ( + DateTest.date.to_json(DateTest.date.from_json("2012-12-31T23:00:01-01:00")) == "2012-12-31T23:00:01-01:00" + ) + with self.assertRaises(TypeError): + DateTest.date.to_json("2012-12-31T23:00:01-01:00") + + @ddt.ddt -class DateTest(FieldTest): +class DateTimeTest(FieldTest): """ - Tests of the Date field. + Tests of the DateTime field. """ FIELD_TO_TEST = DateTime @@ -384,6 +478,83 @@ def test_serialize_error(self): DateTime().to_json('not a datetime') +class TimedeltaTest(unittest.TestCase): + """Tests JSON conversion and type enforcement for Timedelta fields.""" + + delta = Timedelta() + + def test_from_json(self): + """Test conversion from string representations to timedelta objects.""" + assert TimedeltaTest.delta.from_json("1 day 12 hours 59 minutes 59 seconds") == dt.timedelta( + days=1, hours=12, minutes=59, seconds=59 + ) + + assert TimedeltaTest.delta.from_json("1 day 46799 seconds") == dt.timedelta(days=1, seconds=46799) + + def test_enforce_type(self): + """Test enforcement of input types for Timedelta field.""" + assert TimedeltaTest.delta.enforce_type(None) is None + assert TimedeltaTest.delta.enforce_type(dt.timedelta(days=1, seconds=46799)) == dt.timedelta( + days=1, seconds=46799 + ) + assert TimedeltaTest.delta.enforce_type("1 day 46799 seconds") == dt.timedelta(days=1, seconds=46799) + with self.assertRaises(TypeError): + TimedeltaTest.delta.enforce_type([1]) + + def test_to_json(self): + """Test converting timedelta objects to string representations.""" + assert "1 days 46799 seconds" == TimedeltaTest.delta.to_json( + dt.timedelta(days=1, hours=12, minutes=59, seconds=59) + ) + + +class RelativeTimeTest(unittest.TestCase): + """Tests JSON conversion and type enforcement for RelativeTime fields.""" + + delta = RelativeTime() + + def test_from_json(self): + """Test conversion from string or numeric values to timedelta objects.""" + assert RelativeTimeTest.delta.from_json("0:05:07") == dt.timedelta(seconds=307) + + assert RelativeTimeTest.delta.from_json(100.0) == dt.timedelta(seconds=100) + assert RelativeTimeTest.delta.from_json(None) == dt.timedelta(seconds=0) + + with self.assertRaises(TypeError): + RelativeTimeTest.delta.from_json(1234) # int + + with self.assertRaises(ValueError): + RelativeTimeTest.delta.from_json("77:77:77") + + def test_enforce_type(self): + """Test enforcement of input types for RelativeTime field.""" + assert RelativeTimeTest.delta.enforce_type(None) is None + assert RelativeTimeTest.delta.enforce_type(dt.timedelta(days=1, seconds=46799)) == dt.timedelta( + days=1, seconds=46799 + ) + assert RelativeTimeTest.delta.enforce_type("0:05:07") == dt.timedelta(seconds=307) + with self.assertRaises(TypeError): + RelativeTimeTest.delta.enforce_type([1]) + + def test_to_json(self): + """Test converting timedelta objects to HH:MM:SS string format.""" + assert "01:02:03" == RelativeTimeTest.delta.to_json(dt.timedelta(seconds=3723)) + assert "00:00:00" == RelativeTimeTest.delta.to_json(None) + assert "00:01:40" == RelativeTimeTest.delta.to_json(100.0) + + error_msg = "RelativeTime max value is 23:59:59=86400.0 seconds, but 90000.0 seconds is passed" + with self.assertRaisesRegex(ValueError, error_msg): + RelativeTimeTest.delta.to_json(dt.timedelta(seconds=90000)) + + with self.assertRaises(TypeError): + RelativeTimeTest.delta.to_json("123") + + def test_str(self): + """Test that RelativeTime outputs correct HH:MM:SS string representations.""" + assert "01:02:03" == RelativeTimeTest.delta.to_json(dt.timedelta(seconds=3723)) + assert "11:02:03" == RelativeTimeTest.delta.to_json(dt.timedelta(seconds=39723)) + + class AnyTest(FieldTest): """ Tests the Any Field. @@ -929,3 +1100,101 @@ def test_from_string_errors(self, _type, string): """ Cases that raises various exceptions.""" with self.assertRaises(Exception): _type().from_string(string) + + +@ddt.ddt +class ScoreFieldTest(unittest.TestCase): + """ + Tests for ScoreField and ListScoreField. + """ + + FIELD_TO_TEST = ScoreField + + def test_score_field_basic_serialization(self): + """Test basic JSON -> Score object conversion.""" + field = ScoreField() + data = {"raw_earned": 5, "raw_possible": 10} + + result = field.from_json(data) + + self.assertIsInstance(result, Score) + self.assertEqual(result.raw_earned, 5) + self.assertEqual(result.raw_possible, 10) + + def test_score_field_idempotency(self): + """Test that passing a Score object returns it unchanged.""" + field = ScoreField() + score = Score(raw_earned=5, raw_possible=10) + self.assertEqual(field.from_json(score), score) + + def test_score_field_none(self): + """Test that None is handled correctly.""" + field = ScoreField() + self.assertIsNone(field.from_json(None)) + + @ddt.data( + {"raw_earned": 5}, # Missing key + {"raw_possible": 10}, # Missing key + {"raw_earned": 5, "raw_possible": 10, "extra": 1}, # Extra key + ) + def test_score_field_invalid_structure(self, data): + """Test TypeError is raised for incorrect dictionary structure.""" + field = ScoreField() + with self.assertRaises(TypeError): + field.from_json(data) + + @ddt.data( + {"raw_earned": -1, "raw_possible": 10}, # Negative earned + {"raw_earned": 5, "raw_possible": -1}, # Negative possible + {"raw_earned": 11, "raw_possible": 10}, # Earned > Possible + ) + def test_score_field_validation_logic(self, data): + """Test ValueError is raised for logic violations.""" + field = ScoreField(display_name="Test Score") + # The error message relies on display_name, so we set it to ensure formatting works + with self.assertRaises(ValueError): + field.from_json(data) + + def test_list_score_field_success(self): + """Test deserializing a list of score dictionaries.""" + field = ListScoreField() + data = [{"raw_earned": 5, "raw_possible": 10}, {"raw_earned": 0, "raw_possible": 5}] + + results = field.from_json(data) + + self.assertIsInstance(results, list) + self.assertEqual(len(results), 2) + self.assertIsInstance(results[0], Score) + self.assertIsInstance(results[1], Score) + self.assertEqual(results[0].raw_earned, 5) + self.assertEqual(results[1].raw_possible, 5) + + def test_list_score_field_empty(self): + """Test handling of empty lists.""" + field = ListScoreField() + self.assertEqual(field.from_json([]), []) + + def test_list_score_field_none(self): + """Test handling of None for the list field.""" + field = ListScoreField() + self.assertIsNone(field.from_json(None)) + + def test_list_score_field_type_error(self): + """Test that passing a non-list raises TypeError.""" + field = ListScoreField() + with self.assertRaisesRegex(TypeError, "Value must be a list"): + field.from_json({"not": "a list"}) + + def test_list_score_field_propagates_validation_error(self): + """ + Test that if a single item in the list is invalid, + the specific validation error from ScoreField bubbles up. + """ + field = ListScoreField() + data = [ + {"raw_earned": 5, "raw_possible": 10}, # Valid + {"raw_earned": 15, "raw_possible": 10}, # Invalid (Earned > Possible) + ] + + with self.assertRaises(ValueError): + field.from_json(data)