Merge pull request #461 from The-Strategy-Unit/remove_data_modification_steps

removes data modification steps

Author: tomjemmett

src/nhp/model/aae.py

@@ -67,10 +67,6 @@ def __init__(
     def _get_data(self, data_loader: Data) -> pd.DataFrame:
         return data_loader.get_aae()
 
-    def _add_pod_to_data(self) -> None:
-        """Adds the POD column to data."""
-        self.data["pod"] = "aae_type-" + self.data["aedepttype"]
-
     def get_data_counts(self, data: pd.DataFrame) -> np.ndarray:
         """Get row counts of data.
 

src/nhp/model/helpers.py

@@ -41,38 +41,6 @@ def rnorm(rng: np.random.Generator, low: float, high: float) -> float:
     return rng.normal(mean, stdev)
 
 
-def age_groups(age: pd.Series) -> pd.Series:
-    """Cut age into groups.
-
-    Takes a pandas Series of age's and cut's into discrete intervals
-
-    :param age: a Series of ages
-    :type age: pandas.Series
-
-    :returns: a Series of age groups
-    :rtype: pandas.Series
-    """
-    return pd.cut(
-        age.fillna(-1),
-        [-1, 0, 1, 5, 10, 16, 18, 35, 50, 65, 75, 85, 1000],
-        right=False,
-        labels=[
-            "Unknown",
-            "0",
-            "1-4",
-            "5-9",
-            "10-15",
-            "16-17",
-            "18-34",
-            "35-49",
-            "50-64",
-            "65-74",
-            "75-84",
-            "85+",
-        ],
-    ).astype(str)
-
-
 def load_params(filename: str) -> dict:
     """Load a params file.
 

src/nhp/model/inpatients.py

@@ -65,24 +65,6 @@ def __init__(
             save_full_model_results,
         )
 
-    def _add_pod_to_data(self) -> None:
-        """Adds the POD column to data."""
-        self.data["pod"] = "ip_" + self.data["group"] + "_admission"
-        # update pod for daycases/regular attenders
-        classpat = self.data["classpat"]
-        self.data.loc[classpat == "2", "pod"] = "ip_elective_daycase"
-
-        # handle regular attenders
-        self.data.loc[classpat == "3", "pod"], self.data.loc[classpat == "4", "pod"] = (
-            ("ip_regular_day_attender", "ip_regular_night_attender")
-            if self.params.get("separate_regular_attenders", True)
-            else ("ip_elective_daycase", "ip_elective_admission")
-        )
-
-    def _add_ndggrp_to_data(self) -> None:
-        super()._add_ndggrp_to_data()
-        self.data.loc[self.data["admimeth"].isin(["82", "83"]), "ndggrp"] = "maternity"
-
     def _get_data(self, data_loader: Data) -> pd.DataFrame:
         return data_loader.get_ip()
 

src/nhp/model/model.py

@@ -21,7 +21,7 @@
     HealthStatusAdjustment,
     HealthStatusAdjustmentInterpolated,
 )
-from nhp.model.helpers import age_groups, inrange, load_params, rnorm
+from nhp.model.helpers import inrange, load_params, rnorm
 from nhp.model.model_iteration import ModelIteration, ModelRunResult
 
 
@@ -136,10 +136,7 @@ def _get_data(self, data_loader: Data) -> pd.DataFrame:
 
     def _load_data(self, data_loader: Data) -> None:
         self.data = self._get_data(data_loader).sort_values("rn")
-        self.data["age_group"] = age_groups(self.data["age"])
         self.baseline_counts = self.get_data_counts(self.data)
-        self._add_pod_to_data()
-        self._add_ndggrp_to_data()
 
         self.baseline_step_counts = (
             pd.DataFrame(

src/nhp/model/outpatients.py

@@ -67,12 +67,6 @@ def __init__(
     def _get_data(self, data_loader: Data) -> pd.DataFrame:
         return data_loader.get_op()
 
-    def _add_pod_to_data(self) -> None:
-        """Adds the POD column to data."""
-        self.data.loc[self.data["is_first"], "pod"] = "op_first"
-        self.data.loc[~self.data["is_first"], "pod"] = "op_follow-up"
-        self.data.loc[self.data["has_procedures"], "pod"] = "op_procedure"
-
     def get_data_counts(self, data: pd.DataFrame) -> np.ndarray:
         """Get row counts of data.
 

tests/unit/nhp/model/test_aae.py

@@ -95,21 +95,6 @@ def test_get_data(mock_model):
     data_loader.get_aae.assert_called_once_with()
 
 
-def test_add_pod_to_data(mock_model):
-    # arrange
-    mock_model.data = pd.DataFrame({"aedepttype": ["01", "02", "03", "04"]})
-    # act
-    mock_model._add_pod_to_data()
-
-    # assert
-    assert mock_model.data["pod"].to_list() == [
-        "aae_type-01",
-        "aae_type-02",
-        "aae_type-03",
-        "aae_type-04",
-    ]
-
-
 def test_get_data_counts(mock_model):
     # arrange
     mdl = mock_model

tests/unit/nhp/model/test_helpers.py

@@ -6,7 +6,7 @@
 import pandas as pd
 import pytest
 
-from nhp.model.helpers import age_groups, inrange, load_params, rnorm
+from nhp.model.helpers import inrange, load_params, rnorm
 
 
 @pytest.mark.parametrize("value, expected", [(-1.1, 0), (1.1, 1), (0, 0), (1, 1), (0.5, 0.5)])
@@ -31,30 +31,6 @@ def test_rnorm():
     rng.normal.assert_called_once_with(1.5, 0.3901520929904105)
 
 
-def test_age_groups():
-    """Test that the age_groups function returns expected values."""
-    ages = pd.Series([np.nan] + list(range(0, 90)))
-    expected = list(
-        np.concatenate(
-            [
-                ["Unknown"],
-                ["0"],
-                ["1-4"] * 4,
-                ["5-9"] * 5,
-                ["10-15"] * 6,
-                ["16-17"] * 2,
-                ["18-34"] * 17,
-                ["35-49"] * 15,
-                ["50-64"] * 15,
-                ["65-74"] * 10,
-                ["75-84"] * 10,
-                ["85+"] * 5,
-            ]
-        ).flat
-    )
-    assert np.array_equal(age_groups(ages), expected)
-
-
 def test_load_params():
     """Test that load_params opens the params file."""
     with patch("builtins.open", mock_open(read_data='{"params": 0}')) as mock_file:

tests/unit/nhp/model/test_inpatients.py

@@ -81,92 +81,6 @@ def test_init_calls_super_init(mocker):
     super_mock.assert_called_once()
 
 
-def test_add_ndggrp_to_data(mock_model):
-    # arrange
-    mdl = mock_model
-    mdl.data = pd.DataFrame({"group": ["a", "b", "c"], "admimeth": ["80", "82", "83"]})
-
-    # act
-    mock_model._add_ndggrp_to_data()
-
-    # assert
-    assert mdl.data["ndggrp"].to_list() == ["a", "maternity", "maternity"]
-
-
-def test_add_pod_to_data(mock_model):
-    # arrange
-    mock_model.data = pd.DataFrame(
-        {
-            "group": [
-                "elective",
-                "elective",
-                "elective",
-                "non-elective",
-                "maternity",
-                "elective",
-                "maternity",
-            ],
-            "classpat": ["1", "2", "3", "1", "1", "4", "5"],
-        }
-    )
-    # act
-    mock_model._add_pod_to_data()
-
-    # assert
-    assert mock_model.data["pod"].to_list() == [
-        "ip_elective_admission",
-        "ip_elective_daycase",
-        "ip_regular_day_attender",
-        "ip_non-elective_admission",
-        "ip_maternity_admission",
-        "ip_regular_night_attender",
-        "ip_maternity_admission",
-    ]
-
-
-@pytest.mark.parametrize(
-    "test, expected",
-    [
-        (True, ["ip_regular_day_attender", "ip_regular_night_attender"]),
-        (False, ["ip_elective_daycase", "ip_elective_admission"]),
-    ],
-)
-def test_add_pod_to_data_separate_regular_day_attenders_param(mock_model, test, expected):
-    # arrange
-    mock_model.data = pd.DataFrame(
-        {
-            "group": ["elective", "elective"],
-            "classpat": ["3", "4"],
-        }
-    )
-    mock_model.params["separate_regular_attenders"] = test
-
-    # act
-    mock_model._add_pod_to_data()
-
-    # assert
-    assert mock_model.data["pod"].to_list() == expected
-
-
-def test_add_pod_to_data_no_regular_attenders(mock_model):
-    # arrange
-    mock_model.data = pd.DataFrame(
-        {
-            "group": ["elective", "elective"],
-            "classpat": ["1", "2"],
-        }
-    )
-
-    # act
-    mock_model._add_pod_to_data()
-
-    # assert
-    assert mock_model.data["pod"].to_list() == [
-        "ip_elective_admission",
-        "ip_elective_daycase",
-    ]
-
-
 def test_get_data(mock_model):
     # arrange
     mdl = mock_model

tests/unit/nhp/model/test_model.py

@@ -303,40 +303,26 @@ def test_load_data(mocker, mock_model):
     # arrange
     mdl = mock_model
     mdl._measures = ["x", "y"]
-    mocker.patch("nhp.model.model.age_groups", return_value="age_groups")
     mocker.patch("nhp.model.model.Model.get_data_counts", return_value=np.array([[1, 2], [3, 4]]))
-    mocker.patch("nhp.model.model.Model._add_pod_to_data")
-    mocker.patch("nhp.model.model.Model._add_ndggrp_to_data")
-
     data_loader = Mock()
 
-    mdl._get_data = Mock(
-        return_value=pd.DataFrame(
-            {"rn": [2, 1], "age": [2, 1], "pod": ["a", "b"], "sitetret": ["c", "d"]}
-        )
-    )
+    data = {"rn": [1, 2], "age": [1, 2], "pod": ["a", "b"], "sitetret": ["c", "d"]}
+
+    mdl._get_data = Mock(return_value=pd.DataFrame(data))
 
     # act
     mdl._load_data(data_loader)
 
     # assert
-    assert mdl.data.to_dict(orient="list") == {
-        "rn": [1, 2],
-        "age": [1, 2],
-        "pod": ["b", "a"],
-        "sitetret": ["d", "c"],
-        "age_group": ["age_groups"] * 2,
-    }
+    assert mdl.data.to_dict(orient="list") == data
     assert mdl.baseline_counts.tolist() == [[1, 2], [3, 4]]
     mdl.get_data_counts.call_args_list[0][0][0].equals(mdl.data)
-    mdl._add_pod_to_data.assert_called_once_with()
-    mdl._add_ndggrp_to_data.assert_called_once_with()
 
     assert mdl.baseline_step_counts.to_dict("list") == {
         "pod": ["a", "b"],
         "sitetret": ["c", "d"],
-        "x": [2, 1],
-        "y": [4, 3],
+        "x": [1, 2],
+        "y": [3, 4],
         "change_factor": ["baseline", "baseline"],
         "strategy": ["-", "-"],
     }

tests/unit/nhp/model/test_outpatients.py

@@ -93,26 +93,6 @@ def test_get_data(mock_model):
     data_loader.get_op.assert_called_once_with()
 
 
-def test_add_pod_to_data(mock_model):
-    # arrange
-    mock_model.data = pd.DataFrame(
-        {
-            "is_first": [True, True, False, False],
-            "has_procedures": [True, False, True, False],
-        }
-    )
-    # act
-    mock_model._add_pod_to_data()
-
-    # assert
-    assert mock_model.data["pod"].to_list() == [
-        "op_procedure",
-        "op_first",
-        "op_procedure",
-        "op_follow-up",
-    ]
-
-
 def test_get_data_counts(mock_model):
     # arrange
     mdl = mock_model