Merge branch 'main' into copilot/fix-training-test-set-overlap

thinkall · web-flow · commit 11787cecef42 · 2026-01-20T13:30:25.000+08:00
diff --git a/flaml/automl/automl.py b/flaml/automl/automl.py
@@ -118,6 +118,8 @@ def __init__(self, **settings):
                 e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_weighted',
                 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted', 'f1', 'micro_f1', 'macro_f1',
                 'log_loss', 'mae', 'mse', 'r2', 'mape'. Default is 'auto'.
+                For a full list of supported built-in metrics, please refer to
+                https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#optimization-metric
                 If passing a customized metric function, the function needs to
                 have the following input arguments:
 
@@ -1765,6 +1767,8 @@ def fit(
                 e.g., 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_weighted',
                 'roc_auc_ovo_weighted', 'roc_auc_ovr_weighted', 'f1', 'micro_f1', 'macro_f1',
                 'log_loss', 'mae', 'mse', 'r2', 'mape'. Default is 'auto'.
+                For a full list of supported built-in metrics, please refer to
+                https://microsoft.github.io/FLAML/docs/Use-Cases/Task-Oriented-AutoML#optimization-metric
                 If passing a customized metric function, the function needs to
                 have the following input arguments:
 
diff --git a/flaml/version.py b/flaml/version.py
@@ -1 +1 @@
-__version__ = "2.4.2"
+__version__ = "2.5.0"
diff --git a/website/docs/Examples/Default-Flamlized.md b/website/docs/Examples/Default-Flamlized.md
@@ -67,6 +67,82 @@ X_test.shape: (5160, 8), y_test.shape: (5160,)
 
 [Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/zeroshot_lightgbm.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/zeroshot_lightgbm.ipynb)
 
+## Flamlized LGBMClassifier
+
+### Prerequisites
+
+This example requires the [autozero] option.
+
+```bash
+pip install flaml[autozero] lightgbm openml
+```
+
+### Zero-shot AutoML
+
+```python
+from flaml.automl.data import load_openml_dataset
+from flaml.default import LGBMClassifier
+from flaml.automl.ml import sklearn_metric_loss_score
+
+X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
+lgbm = LGBMClassifier()
+lgbm.fit(X_train, y_train)
+y_pred = lgbm.predict(X_test)
+print(
+    "flamlized lgbm accuracy",
+    "=",
+    1 - sklearn_metric_loss_score("accuracy", y_pred, y_test),
+)
+print(lgbm)
+```
+
+#### Sample output
+
+```
+load dataset from ./openml_ds1169.pkl
+Dataset name: airlines
+X_train.shape: (404537, 7), y_train.shape: (404537,);
+X_test.shape: (134846, 7), y_test.shape: (134846,)
+flamlized lgbm accuracy = 0.6745
+LGBMClassifier(colsample_bytree=0.85, learning_rate=0.05, max_bin=255,
+               min_child_samples=20, n_estimators=500, num_leaves=31,
+               reg_alpha=0.01, reg_lambda=0.1, verbose=-1)
+```
+
+## Flamlized XGBRegressor
+
+### Prerequisites
+
+This example requires xgboost, sklearn, openml==0.10.2.
+
+### Zero-shot AutoML
+
+```python
+from flaml.automl.data import load_openml_dataset
+from flaml.default import XGBRegressor
+from flaml.automl.ml import sklearn_metric_loss_score
+
+X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")
+xgb = XGBRegressor()
+xgb.fit(X_train, y_train)
+y_pred = xgb.predict(X_test)
+print("flamlized xgb r2", "=", 1 - sklearn_metric_loss_score("r2", y_pred, y_test))
+print(xgb)
+```
+
+#### Sample output
+
+```
+load dataset from ./openml_ds537.pkl
+Dataset name: houses
+X_train.shape: (15480, 8), y_train.shape: (15480,);
+X_test.shape: (5160, 8), y_test.shape: (5160,)
+flamlized xgb r2 = 0.8542
+XGBRegressor(colsample_bylevel=1, colsample_bytree=0.85, learning_rate=0.05,
+             max_depth=6, n_estimators=500, reg_alpha=0.01, reg_lambda=1.0,
+             subsample=0.9)
+```
+
 ## Flamlized XGBClassifier
 
 ### Prerequisites
@@ -112,3 +188,159 @@ XGBClassifier(base_score=0.5, booster='gbtree',
               scale_pos_weight=1, subsample=1.0, tree_method='hist',
               use_label_encoder=False, validate_parameters=1, verbosity=0)
 ```
+
+## Flamlized RandomForestRegressor
+
+### Prerequisites
+
+This example requires the [autozero] option.
+
+```bash
+pip install flaml[autozero] scikit-learn openml
+```
+
+### Zero-shot AutoML
+
+```python
+from flaml.automl.data import load_openml_dataset
+from flaml.default import RandomForestRegressor
+from flaml.automl.ml import sklearn_metric_loss_score
+
+X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")
+rf = RandomForestRegressor()
+rf.fit(X_train, y_train)
+y_pred = rf.predict(X_test)
+print("flamlized rf r2", "=", 1 - sklearn_metric_loss_score("r2", y_pred, y_test))
+print(rf)
+```
+
+#### Sample output
+
+```
+load dataset from ./openml_ds537.pkl
+Dataset name: houses
+X_train.shape: (15480, 8), y_train.shape: (15480,);
+X_test.shape: (5160, 8), y_test.shape: (5160,)
+flamlized rf r2 = 0.8521
+RandomForestRegressor(max_features=0.8, min_samples_leaf=2, min_samples_split=5,
+                      n_estimators=500)
+```
+
+## Flamlized RandomForestClassifier
+
+### Prerequisites
+
+This example requires the [autozero] option.
+
+```bash
+pip install flaml[autozero] scikit-learn openml
+```
+
+### Zero-shot AutoML
+
+```python
+from flaml.automl.data import load_openml_dataset
+from flaml.default import RandomForestClassifier
+from flaml.automl.ml import sklearn_metric_loss_score
+
+X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
+rf = RandomForestClassifier()
+rf.fit(X_train, y_train)
+y_pred = rf.predict(X_test)
+print(
+    "flamlized rf accuracy",
+    "=",
+    1 - sklearn_metric_loss_score("accuracy", y_pred, y_test),
+)
+print(rf)
+```
+
+#### Sample output
+
+```
+load dataset from ./openml_ds1169.pkl
+Dataset name: airlines
+X_train.shape: (404537, 7), y_train.shape: (404537,);
+X_test.shape: (134846, 7), y_test.shape: (134846,)
+flamlized rf accuracy = 0.6701
+RandomForestClassifier(max_features=0.7, min_samples_leaf=3, min_samples_split=5,
+                       n_estimators=500)
+```
+
+## Flamlized ExtraTreesRegressor
+
+### Prerequisites
+
+This example requires the [autozero] option.
+
+```bash
+pip install flaml[autozero] scikit-learn openml
+```
+
+### Zero-shot AutoML
+
+```python
+from flaml.automl.data import load_openml_dataset
+from flaml.default import ExtraTreesRegressor
+from flaml.automl.ml import sklearn_metric_loss_score
+
+X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=537, data_dir="./")
+et = ExtraTreesRegressor()
+et.fit(X_train, y_train)
+y_pred = et.predict(X_test)
+print("flamlized et r2", "=", 1 - sklearn_metric_loss_score("r2", y_pred, y_test))
+print(et)
+```
+
+#### Sample output
+
+```
+load dataset from ./openml_ds537.pkl
+Dataset name: houses
+X_train.shape: (15480, 8), y_train.shape: (15480,);
+X_test.shape: (5160, 8), y_test.shape: (5160,)
+flamlized et r2 = 0.8534
+ExtraTreesRegressor(max_features=0.75, min_samples_leaf=2, min_samples_split=5,
+                    n_estimators=500)
+```
+
+## Flamlized ExtraTreesClassifier
+
+### Prerequisites
+
+This example requires the [autozero] option.
+
+```bash
+pip install flaml[autozero] scikit-learn openml
+```
+
+### Zero-shot AutoML
+
+```python
+from flaml.automl.data import load_openml_dataset
+from flaml.default import ExtraTreesClassifier
+from flaml.automl.ml import sklearn_metric_loss_score
+
+X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id=1169, data_dir="./")
+et = ExtraTreesClassifier()
+et.fit(X_train, y_train)
+y_pred = et.predict(X_test)
+print(
+    "flamlized et accuracy",
+    "=",
+    1 - sklearn_metric_loss_score("accuracy", y_pred, y_test),
+)
+print(et)
+```
+
+#### Sample output
+
+```
+load dataset from ./openml_ds1169.pkl
+Dataset name: airlines
+X_train.shape: (404537, 7), y_train.shape: (404537,);
+X_test.shape: (134846, 7), y_test.shape: (134846,)
+flamlized et accuracy = 0.6698
+ExtraTreesClassifier(max_features=0.7, min_samples_leaf=3, min_samples_split=5,
+                     n_estimators=500)
+```
diff --git a/website/docs/Use-Cases/Task-Oriented-AutoML.md b/website/docs/Use-Cases/Task-Oriented-AutoML.md
@@ -51,6 +51,7 @@ If users provide the minimal inputs only, `AutoML` uses the default settings for
 The optimization metric is specified via the `metric` argument. It can be either a string which refers to a built-in metric, or a user-defined function.
 
 - Built-in metric.
+
   - 'accuracy': 1 - accuracy as the corresponding metric to minimize.
   - 'log_loss': default metric for multiclass classification.
   - 'r2': 1 - r2_score as the corresponding metric to minimize. Default metric for regression.
@@ -70,6 +71,40 @@ The optimization metric is specified via the `metric` argument. It can be either
   - 'ap': minimize 1 - average_precision_score.
   - 'ndcg': minimize 1 - ndcg_score.
   - 'ndcg@k': minimize 1 - ndcg_score@k. k is an integer.
+  - 'pr_auc': minimize 1 - precision-recall AUC score. (Spark-specific)
+  - 'var': minimize variance. (Spark-specific)
+
+- Built-in HuggingFace metrics (for NLP tasks).
+
+  - 'accuracy': minimize 1 - accuracy.
+  - 'bertscore': minimize 1 - BERTScore.
+  - 'bleu': minimize 1 - BLEU score.
+  - 'bleurt': minimize 1 - BLEURT score.
+  - 'cer': minimize character error rate.
+  - 'chrf': minimize ChrF score.
+  - 'code_eval': minimize 1 - code evaluation score.
+  - 'comet': minimize 1 - COMET score.
+  - 'competition_math': minimize 1 - competition math score.
+  - 'coval': minimize 1 - CoVal score.
+  - 'cuad': minimize 1 - CUAD score.
+  - 'f1': minimize 1 - F1 score.
+  - 'gleu': minimize 1 - GLEU score.
+  - 'google_bleu': minimize 1 - Google BLEU score.
+  - 'matthews_correlation': minimize 1 - Matthews correlation coefficient.
+  - 'meteor': minimize 1 - METEOR score.
+  - 'pearsonr': minimize 1 - Pearson correlation coefficient.
+  - 'precision': minimize 1 - precision.
+  - 'recall': minimize 1 - recall.
+  - 'rouge': minimize 1 - ROUGE score.
+  - 'rouge1': minimize 1 - ROUGE-1 score.
+  - 'rouge2': minimize 1 - ROUGE-2 score.
+  - 'sacrebleu': minimize 1 - SacreBLEU score.
+  - 'sari': minimize 1 - SARI score.
+  - 'seqeval': minimize 1 - SeqEval score.
+  - 'spearmanr': minimize 1 - Spearman correlation coefficient.
+  - 'ter': minimize translation error rate.
+  - 'wer': minimize word error rate.
+
 - User-defined function.
   A customized metric function that requires the following (input) signature, and returns the input config’s value in terms of the metric you want to minimize, and a dictionary of auxiliary information at your choice:
 
@@ -207,6 +242,7 @@ To tune a custom estimator that is not built-in, you need to:
 
 ```python
 from flaml.automl.model import SKLearnEstimator
+
 # SKLearnEstimator is derived from BaseEstimator
 import rgf
 
@@ -215,31 +251,44 @@ class MyRegularizedGreedyForest(SKLearnEstimator):
     def __init__(self, task="binary", **config):
         super().__init__(task, **config)
 
-        if task in CLASSIFICATION:
-        from rgf.sklearn import RGFClassifier
+        if isinstance(task, str):
+            from flaml.automl.task.factory import task_factory
+
+            task = task_factory(task)
 
-        self.estimator_class = RGFClassifier
+        if task.is_classification():
+            from rgf.sklearn import RGFClassifier
+
+            self.estimator_class = RGFClassifier
         else:
-        from rgf.sklearn import RGFRegressor
+            from rgf.sklearn import RGFRegressor
 
-        self.estimator_class = RGFRegressor
+            self.estimator_class = RGFRegressor
 
     @classmethod
     def search_space(cls, data_size, task):
         space = {
-        "max_leaf": {
-            "domain": tune.lograndint(lower=4, upper=data_size),
-            "low_cost_init_value": 4,
-        },
-        "n_iter": {
-            "domain": tune.lograndint(lower=1, upper=data_size),
-            "low_cost_init_value": 1,
-        },
-        "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)},
-        "min_samples_leaf": {
-            "domain": tune.lograndint(lower=1, upper=20),
-            "init_value": 20,
-        },
+            "max_leaf": {
+                "domain": tune.lograndint(lower=4, upper=data_size[0]),
+                "init_value": 4,
+            },
+            "n_iter": {
+                "domain": tune.lograndint(lower=1, upper=data_size[0]),
+                "init_value": 1,
+            },
+            "n_tree_search": {
+                "domain": tune.lograndint(lower=1, upper=32768),
+                "init_value": 1,
+            },
+            "opt_interval": {
+                "domain": tune.lograndint(lower=1, upper=10000),
+                "init_value": 100,
+            },
+            "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)},
+            "min_samples_leaf": {
+                "domain": tune.lograndint(lower=1, upper=20),
+                "init_value": 20,
+            },
         }
         return space
 ```

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "2.4.2"`
	`1`	`+__version__ = "2.5.0"`