Add optimizer comparison example: HillClimbing vs BayesianOptimizer

examples/gfo/Optimizer comparison random forest.py

@@ -0,0 +1,114 @@
+"""
+Optimizer Comparison: HillClimbing vs BayesianOptimizer
+========================================================
+This example demonstrates one of Hyperactive's core strengths:
+you can swap optimizers without changing any experiment code.
+
+The same RandomForest experiment and search space is run with two
+different optimizers. Only the optimizer line changes — everything
+else stays identical.
+
+Dataset : Iris (150 samples, 3 classes, 4 features)
+Model   : RandomForestClassifier
+Metric  : 5-fold cross-validation accuracy (mean)
+"""
+
+import numpy as np
+from sklearn.datasets import load_iris
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.metrics import accuracy_score
+from sklearn.model_selection import KFold
+
+from hyperactive.experiment.integrations import SklearnCvExperiment
+from hyperactive.opt.gfo import BayesianOptimizer, HillClimbing
+
+# ── 1. Data ──────────────────────────────────────────────────────────────────
+
+X, y = load_iris(return_X_y=True)
+
+# ── 2. Experiment (shared by both optimizers) ─────────────────────────────────
+# SklearnCvExperiment wraps cross-validation so you don't have to write
+# the CV loop manually. The optimizer will call this internally on each
+# candidate set of hyperparameters.
+
+experiment = SklearnCvExperiment(
+    estimator=RandomForestClassifier(random_state=42),
+    scoring=accuracy_score,
+    cv=KFold(n_splits=5, shuffle=True, random_state=42),
+    X=X,
+    y=y,
+)
+
+# ── 3. Search space (shared by both optimizers) ───────────────────────────────
+
+search_space = {
+    "n_estimators": list(range(10, 200, 10)),  # 10, 20, ..., 190 trees
+    "max_depth": list(range(1, 20)),            # tree depth: 1 to 19
+    "min_samples_split": list(range(2, 10)),    # min samples to split: 2 to 9
+}
+
+# ── 4. Run with HillClimbing ──────────────────────────────────────────────────
+# HillClimbing starts from a random point and moves to neighbouring
+# parameter values that improve the score. Fast but can get stuck in
+# local optima.
+
+print("=" * 55)
+print("Optimizer 1: HillClimbing")
+print("=" * 55)
+
+optimizer_hc = HillClimbing(
+    search_space=search_space,
+    n_iter=30,
+    experiment=experiment,
+)
+best_params_hc = optimizer_hc.solve()
+
+print(f"Best parameters : {best_params_hc}")
+print(f"Best CV score   : {optimizer_hc.best_score:.4f}\n")
+
+# ── 5. Run with BayesianOptimizer ─────────────────────────────────────────────
+# BayesianOptimizer builds a probabilistic model of the search space
+# and uses it to pick the most promising candidates next. Smarter
+# exploration — especially useful when n_iter is limited.
+#
+# Notice: the experiment and search_space are identical to above.
+# Only this line changes compared to the HillClimbing run.
+
+print("=" * 55)
+print("Optimizer 2: BayesianOptimizer")
+print("=" * 55)
+
+optimizer_bo = BayesianOptimizer(
+    search_space=search_space,
+    n_iter=30,
+    experiment=experiment,
+)
+best_params_bo = optimizer_bo.solve()
+
+print(f"Best parameters : {best_params_bo}")
+print(f"Best CV score   : {optimizer_bo.best_score:.4f}\n")
+
+# ── 6. Side-by-side comparison ────────────────────────────────────────────────
+
+print("=" * 55)
+print("Comparison")
+print("=" * 55)
+print(f"{'Optimizer':<25} {'n_estimators':<15} {'max_depth':<12} {'CV Score'}")
+print("-" * 55)
+print(
+    f"{'HillClimbing':<25}"
+    f"{best_params_hc['n_estimators']:<15}"
+    f"{best_params_hc['max_depth']:<12}"
+    f"{optimizer_hc.best_score:.4f}"
+)
+print(
+    f"{'BayesianOptimizer':<25}"
+    f"{best_params_bo['n_estimators']:<15}"
+    f"{best_params_bo['max_depth']:<12}"
+    f"{optimizer_bo.best_score:.4f}"
+)
+
+# ── Key takeaway ──────────────────────────────────────────────────────────────
+# The experiment and search_space definitions are completely reused.
+# Swapping optimizers in Hyperactive is a one-line change — this is the
+# unified interface the library is built around.