Rename arguments of list_evaluations (#933)

* list evals name change

* list evals - update

Author: Bilgecelik

examples/30_extended/fetch_evaluations_tutorial.py

@@ -63,7 +63,7 @@
 
 metric = "predictive_accuracy"
 evals = openml.evaluations.list_evaluations(
-    function=metric, task=[task_id], output_format="dataframe"
+    function=metric, tasks=[task_id], output_format="dataframe"
 )
 # Displaying the first 10 rows
 print(evals.head(n=10))
@@ -162,7 +162,7 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
 # List evaluations in descending order based on predictive_accuracy with
 # hyperparameters
 evals_setups = openml.evaluations.list_evaluations_setups(
-    function="predictive_accuracy", task=[31], size=100, sort_order="desc"
+    function="predictive_accuracy", tasks=[31], size=100, sort_order="desc"
 )
 
 ""
@@ -173,7 +173,7 @@ def plot_flow_compare(evaluations, top_n=10, metric="predictive_accuracy"):
 # with hyperparameters. parameters_in_separate_columns returns parameters in
 # separate columns
 evals_setups = openml.evaluations.list_evaluations_setups(
-    function="predictive_accuracy", flow=[6767], size=100, parameters_in_separate_columns=True
+    function="predictive_accuracy", flows=[6767], size=100, parameters_in_separate_columns=True
 )
 
 ""

examples/30_extended/plot_svm_hyperparameters_tutorial.py

@@ -20,8 +20,8 @@
 # uploaded runs (called *setup*).
 df = openml.evaluations.list_evaluations_setups(
     function="predictive_accuracy",
-    flow=[8353],
-    task=[6],
+    flows=[8353],
+    tasks=[6],
     output_format="dataframe",
     # Using this flag incorporates the hyperparameters into the returned dataframe. Otherwise,
     # the dataframe would contain a field ``paramaters`` containing an unparsed dictionary.

examples/40_paper/2018_ida_strang_example.py

@@ -47,7 +47,7 @@
 
 # Downloads all evaluation records related to this study
 evaluations = openml.evaluations.list_evaluations(
-    measure, flow=flow_ids, study=study_id, output_format="dataframe"
+    measure, flows=flow_ids, study=study_id, output_format="dataframe"
 )
 # gives us a table with columns data_id, flow1_value, flow2_value
 evaluations = evaluations.pivot(index="data_id", columns="flow_id", values="value").dropna()

examples/40_paper/2018_kdd_rijn_example.py

@@ -88,8 +88,8 @@
     # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)
     evals = openml.evaluations.list_evaluations_setups(
         evaluation_measure,
-        flow=[flow_id],
-        task=[task_id],
+        flows=[flow_id],
+        tasks=[task_id],
         size=limit_per_task,
         output_format="dataframe",
     )

examples/40_paper/2018_neurips_perrone_example.py

@@ -91,9 +91,9 @@ def fetch_evaluations(run_full=False, flow_type="svm", metric="area_under_roc_cu
     # Fetching evaluations
     eval_df = openml.evaluations.list_evaluations_setups(
         function=metric,
-        task=task_ids,
-        flow=[flow_id],
-        uploader=[2702],
+        tasks=task_ids,
+        flows=[flow_id],
+        uploaders=[2702],
         output_format="dataframe",
         parameters_in_separate_columns=True,
     )

openml/evaluations/functions.py

@@ -17,11 +17,11 @@ def list_evaluations(
     function: str,
     offset: Optional[int] = None,
     size: Optional[int] = None,
-    task: Optional[List] = None,
-    setup: Optional[List] = None,
-    flow: Optional[List] = None,
-    run: Optional[List] = None,
-    uploader: Optional[List] = None,
+    tasks: Optional[List[Union[str, int]]] = None,
+    setups: Optional[List[Union[str, int]]] = None,
+    flows: Optional[List[Union[str, int]]] = None,
+    runs: Optional[List[Union[str, int]]] = None,
+    uploaders: Optional[List[Union[str, int]]] = None,
     tag: Optional[str] = None,
     study: Optional[int] = None,
     per_fold: Optional[bool] = None,
@@ -41,17 +41,18 @@ def list_evaluations(
     size : int, optional
         the maximum number of runs to show
 
-    task : list, optional
-
-    setup: list, optional
-
-    flow : list, optional
-
-    run : list, optional
-
-    uploader : list, optional
-
+    tasks : list[int,str], optional
+        the list of task IDs
+    setups: list[int,str], optional
+        the list of setup IDs
+    flows : list[int,str], optional
+        the list of flow IDs
+    runs :list[int,str], optional
+        the list of run IDs
+    uploaders : list[int,str], optional
+        the list of uploader IDs
     tag : str, optional
+        filter evaluation based on given tag
 
     study : int, optional
 
@@ -85,11 +86,11 @@ def list_evaluations(
         function=function,
         offset=offset,
         size=size,
-        task=task,
-        setup=setup,
-        flow=flow,
-        run=run,
-        uploader=uploader,
+        tasks=tasks,
+        setups=setups,
+        flows=flows,
+        runs=runs,
+        uploaders=uploaders,
         tag=tag,
         study=study,
         sort_order=sort_order,
@@ -99,11 +100,11 @@ def list_evaluations(
 
 def _list_evaluations(
     function: str,
-    task: Optional[List] = None,
-    setup: Optional[List] = None,
-    flow: Optional[List] = None,
-    run: Optional[List] = None,
-    uploader: Optional[List] = None,
+    tasks: Optional[List] = None,
+    setups: Optional[List] = None,
+    flows: Optional[List] = None,
+    runs: Optional[List] = None,
+    uploaders: Optional[List] = None,
     study: Optional[int] = None,
     sort_order: Optional[str] = None,
     output_format: str = "object",
@@ -120,15 +121,16 @@ def _list_evaluations(
     function : str
         the evaluation function. e.g., predictive_accuracy
 
-    task : list, optional
-
-    setup: list, optional
-
-    flow : list, optional
-
-    run : list, optional
-
-    uploader : list, optional
+    tasks : list[int,str], optional
+        the list of task IDs
+    setups: list[int,str], optional
+        the list of setup IDs
+    flows : list[int,str], optional
+        the list of flow IDs
+    runs :list[int,str], optional
+        the list of run IDs
+    uploaders : list[int,str], optional
+        the list of uploader IDs
 
     study : int, optional
 
@@ -155,16 +157,16 @@ def _list_evaluations(
     if kwargs is not None:
         for operator, value in kwargs.items():
             api_call += "/%s/%s" % (operator, value)
-    if task is not None:
-        api_call += "/task/%s" % ",".join([str(int(i)) for i in task])
-    if setup is not None:
-        api_call += "/setup/%s" % ",".join([str(int(i)) for i in setup])
-    if flow is not None:
-        api_call += "/flow/%s" % ",".join([str(int(i)) for i in flow])
-    if run is not None:
-        api_call += "/run/%s" % ",".join([str(int(i)) for i in run])
-    if uploader is not None:
-        api_call += "/uploader/%s" % ",".join([str(int(i)) for i in uploader])
+    if tasks is not None:
+        api_call += "/task/%s" % ",".join([str(int(i)) for i in tasks])
+    if setups is not None:
+        api_call += "/setup/%s" % ",".join([str(int(i)) for i in setups])
+    if flows is not None:
+        api_call += "/flow/%s" % ",".join([str(int(i)) for i in flows])
+    if runs is not None:
+        api_call += "/run/%s" % ",".join([str(int(i)) for i in runs])
+    if uploaders is not None:
+        api_call += "/uploader/%s" % ",".join([str(int(i)) for i in uploaders])
     if study is not None:
         api_call += "/study/%d" % study
     if sort_order is not None:
@@ -276,11 +278,11 @@ def list_evaluations_setups(
     function: str,
     offset: Optional[int] = None,
     size: Optional[int] = None,
-    task: Optional[List] = None,
-    setup: Optional[List] = None,
-    flow: Optional[List] = None,
-    run: Optional[List] = None,
-    uploader: Optional[List] = None,
+    tasks: Optional[List] = None,
+    setups: Optional[List] = None,
+    flows: Optional[List] = None,
+    runs: Optional[List] = None,
+    uploaders: Optional[List] = None,
     tag: Optional[str] = None,
     per_fold: Optional[bool] = None,
     sort_order: Optional[str] = None,
@@ -299,15 +301,15 @@ def list_evaluations_setups(
         the number of runs to skip, starting from the first
     size : int, optional
         the maximum number of runs to show
-    task : list[int], optional
+    tasks : list[int], optional
         the list of task IDs
-    setup: list[int], optional
+    setups: list[int], optional
         the list of setup IDs
-    flow : list[int], optional
+    flows : list[int], optional
         the list of flow IDs
-    run : list[int], optional
+    runs : list[int], optional
         the list of run IDs
-    uploader : list[int], optional
+    uploaders : list[int], optional
         the list of uploader IDs
     tag : str, optional
         filter evaluation based on given tag
@@ -327,7 +329,7 @@ def list_evaluations_setups(
     -------
     dict or dataframe with hyperparameter settings as a list of tuples.
     """
-    if parameters_in_separate_columns and (flow is None or len(flow) != 1):
+    if parameters_in_separate_columns and (flows is None or len(flows) != 1):
         raise ValueError(
             "Can set parameters_in_separate_columns to true " "only for single flow_id"
         )
@@ -337,11 +339,11 @@ def list_evaluations_setups(
         function=function,
         offset=offset,
         size=size,
-        run=run,
-        task=task,
-        setup=setup,
-        flow=flow,
-        uploader=uploader,
+        runs=runs,
+        tasks=tasks,
+        setups=setups,
+        flows=flows,
+        uploaders=uploaders,
         tag=tag,
         per_fold=per_fold,
         sort_order=sort_order,
@@ -359,24 +361,26 @@ def list_evaluations_setups(
         setup_chunks = np.array_split(
             ary=evals["setup_id"].unique(), indices_or_sections=((length - 1) // N) + 1
         )
-        setups = pd.DataFrame()
-        for setup in setup_chunks:
-            result = pd.DataFrame(openml.setups.list_setups(setup=setup, output_format="dataframe"))
+        setup_data = pd.DataFrame()
+        for setups in setup_chunks:
+            result = pd.DataFrame(
+                openml.setups.list_setups(setup=setups, output_format="dataframe")
+            )
             result.drop("flow_id", axis=1, inplace=True)
             # concat resulting setup chunks into single datframe
-            setups = pd.concat([setups, result], ignore_index=True)
+            setup_data = pd.concat([setup_data, result], ignore_index=True)
         parameters = []
         # Convert parameters of setup into list of tuples of (hyperparameter, value)
-        for parameter_dict in setups["parameters"]:
+        for parameter_dict in setup_data["parameters"]:
             if parameter_dict is not None:
                 parameters.append(
                     {param["full_name"]: param["value"] for param in parameter_dict.values()}
                 )
             else:
                 parameters.append({})
-        setups["parameters"] = parameters
+        setup_data["parameters"] = parameters
         # Merge setups with evaluations
-        df = pd.merge(evals, setups, on="setup_id", how="left")
+        df = pd.merge(evals, setup_data, on="setup_id", how="left")
 
     if parameters_in_separate_columns:
         df = pd.concat([df.drop("parameters", axis=1), df["parameters"].apply(pd.Series)], axis=1)