deprecates load_data_frame

Author: hirosassa

gokart/task.py

@@ -323,32 +323,6 @@ def _load(targets):
 
         return _load(self._get_input_targets(target))
 
-    @deprecated("""This function is deprecated. use `load` instead.
-If you want to specify `required_columns` and `drop_columns`, please extract the columns after loading. ex: `load()[['colA', 'colB']]`
-""")
-    def load_data_frame(
-        self, target: Union[None, str, TargetOnKart] = None, required_columns: Optional[Set[str]] = None, drop_columns: bool = False
-    ) -> pd.DataFrame:
-        def _flatten_recursively(dfs):
-            if isinstance(dfs, list):
-                return pd.concat([_flatten_recursively(df) for df in dfs])
-            else:
-                return dfs
-
-        dfs = self.load(target=target)
-        if isinstance(dfs, dict) and len(dfs) == 1:
-            dfs = list(dfs.values())[0]
-
-        data = _flatten_recursively(dfs)
-
-        required_columns = required_columns or set()
-        if data.empty and len(data.index) == 0 and len(required_columns - set(data.columns)) > 0:
-            return pd.DataFrame(columns=list(required_columns))
-        assert required_columns.issubset(set(data.columns)), f'data must have columns {required_columns}, but actually have only {data.columns}.'
-        if drop_columns:
-            data = data[list(required_columns)]
-        return data
-
     @overload
     def dump(self, obj: T, target: None = None) -> None: ...
 

test/test_task_on_kart.py

@@ -442,52 +442,6 @@ class DummyTaskAddConfiguration(gokart.TaskOnKart):
         mock_cmdline.return_value = luigi.cmdline_parser.CmdlineParser(['DummyTaskAddConfiguration', '--DummyTaskAddConfiguration-aa', '2'])
         self.assertEqual(DummyTaskAddConfiguration().aa, 2)
 
-    def test_load_list_of_list_pandas(self):
-        task = _DummyTask()
-        task.load = Mock(return_value=[pd.DataFrame(dict(a=[1])), [pd.DataFrame(dict(a=[2])), pd.DataFrame(dict(a=[3]))]])  # type: ignore
-
-        df = task.load_data_frame()
-        self.assertIsInstance(df, pd.DataFrame)
-        self.assertEqual(3, df.shape[0])
-
-    def test_load_single_value_dict_of_dataframe(self):
-        task = _DummyTask()
-        task.load = Mock(return_value={'a': pd.DataFrame(dict(a=[1]))})  # type: ignore
-
-        df = task.load_data_frame()
-        self.assertIsInstance(df, pd.DataFrame)
-        self.assertEqual(1, df.shape[0])
-
-    def test_load_data_frame_drop_columns(self):
-        task = _DummyTask()
-        task.load = Mock(return_value=pd.DataFrame(dict(a=[1], b=[2], c=[3])))  # type: ignore
-
-        df = task.load_data_frame(required_columns={'a', 'c'}, drop_columns=True)
-        self.assertIsInstance(df, pd.DataFrame)
-        self.assertEqual(1, df.shape[0])
-        self.assertSetEqual({'a', 'c'}, set(df.columns))
-
-    def test_load_data_frame_empty_input(self):
-        task = _DummyTask()
-        task.load = Mock(return_value=pd.DataFrame(dict(a=[], b=[], c=[])))  # type: ignore
-
-        df = task.load_data_frame(required_columns={'a', 'c'})
-        self.assertIsInstance(df, pd.DataFrame)
-        self.assertEqual(0, df.shape[0])
-        self.assertSetEqual({'a', 'b', 'c'}, set(df.columns))
-
-    def test_load_index_only_dataframe(self):
-        task = _DummyTask()
-        task.load = Mock(return_value=pd.DataFrame(index=range(3)))  # type: ignore
-
-        # connnot load index only frame with required_columns
-        self.assertRaises(AssertionError, lambda: task.load_data_frame(required_columns={'a', 'c'}))
-
-        df: pd.DataFrame = task.load_data_frame()
-        self.assertIsInstance(df, pd.DataFrame)
-        self.assertTrue(df.empty)
-        self.assertListEqual(list(range(3)), list(df.index))
-
     def test_use_rerun_with_inherits(self):
         # All tasks are completed.
         task_c = _DummyTaskC()