C. Benjamins: [MF] Determine number of trials for a total optimizatio…

…n budget (#1121)

development/10_experimental.html

@@ -223,6 +223,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -629,6 +634,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/1_installation.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/2_package_overview.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/3_getting_started.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/4_minimal_example.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/5_api.html

@@ -226,6 +226,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -632,6 +637,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/6_references.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/7_glossary.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/8_faq.html

@@ -225,6 +225,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -631,6 +636,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/9_license.html

@@ -224,6 +224,11 @@
        Stochastic Gradient Descent On Multiple Datasets
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="examples/2_multi_fidelity/3_specify_HB_via_total_budget.html">
+       Specify Number of Trials via a Total Budget in Hyperband
+      </a>
+     </li>
     </ul>
    </li>
    <li class="toctree-l2 has-children">
@@ -630,6 +635,11 @@
        smac.intensifier.hyperband
       </a>
      </li>
+     <li class="toctree-l3">
+      <a class="reference internal" href="api/smac.intensifier.hyperband_utils.html">
+       smac.intensifier.hyperband_utils
+      </a>
+     </li>
      <li class="toctree-l3">
       <a class="reference internal" href="api/smac.intensifier.intensifier.html">
        smac.intensifier.intensifier

development/_downloads/60791edaa81ce9889d1de4a67940b6d4/3_specify_HB_via_total_budget.ipynb

@@ -0,0 +1,43 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\n# Specify Number of Trials via a Total Budget in Hyperband\nThis example uses a dummy function but illustrates how to setup Hyperband if you \nwant to specify a total optimization budget in terms of fidelity units.\n\nIn Hyperband, normally SMAC calculates a typical Hyperband round.\nIf the number of trials is not used up by one single round, the next round is started.\nInstead of specifying the number of trial beforehand, specify the total budget\nin terms of the fidelity units and let SMAC calculate how many trials that would be.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "from __future__ import annotations\n\nimport numpy as np\nfrom ConfigSpace import Configuration, ConfigurationSpace, Float\nfrom matplotlib import pyplot as plt\n\nfrom smac import MultiFidelityFacade, RunHistory, Scenario\nfrom smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity\n\n__copyright__ = \"Copyright 2021, AutoML.org Freiburg-Hannover\"\n__license__ = \"3-clause BSD\"\n\n\nclass QuadraticFunction:\n    max_budget = 500\n\n    @property\n    def configspace(self) -> ConfigurationSpace:\n        cs = ConfigurationSpace(seed=0)\n        x = Float(\"x\", (-5, 5), default=-5)\n        cs.add_hyperparameters([x])\n\n        return cs\n\n    def train(self, config: Configuration, seed: int = 0, budget: float | None = None) -> float:\n        \"\"\"Returns the y value of a quadratic function with a minimum we know to be at x=0.\"\"\"\n        x = config[\"x\"]\n\n        if budget is None:\n            multiplier = 1\n        else:\n            multiplier = 1 + budget / self.max_budget\n\n        return x**2 * multiplier\n\n\ndef plot(runhistory: RunHistory, incumbent: Configuration) -> None:\n    plt.figure()\n\n    # Plot ground truth\n    x = list(np.linspace(-5, 5, 100))\n    y = [xi * xi for xi in x]\n    plt.plot(x, y)\n\n    # Plot all trials\n    for k, v in runhistory.items():\n        config = runhistory.get_config(k.config_id)\n        x = config[\"x\"]\n        y = v.cost  # type: ignore\n        plt.scatter(x, y, c=\"blue\", alpha=0.1, zorder=9999, marker=\"o\")\n\n    # Plot incumbent\n    plt.scatter(incumbent[\"x\"], incumbent[\"x\"] * incumbent[\"x\"], c=\"red\", zorder=10000, marker=\"x\")\n\n    plt.show()\n\n\nif __name__ == \"__main__\":\n    model = QuadraticFunction()\n\n    min_budget = 10  # minimum budget per trial\n    max_budget = 500  # maximum budget per trial\n    eta = 3  # standard HB parameter influencing the number of stages\n\n    # Let's calculate how many trials we need to exhaust the total optimization budget (in terms of\n    # fidelity units)\n    n_trials = get_n_trials_for_hyperband_multifidelity(\n        total_budget=10000,  # this is the total optimization budget we specify in terms of fidelity units\n        min_budget=min_budget,  # This influences the Hyperband rounds, minimum budget per trial\n        max_budget=max_budget,  # This influences the Hyperband rounds, maximum budget per trial\n        eta=eta,  # This influences the Hyperband rounds\n        print_summary=True,\n    )\n\n    # Scenario object specifying the optimization \"environment\"\n    scenario = Scenario(\n        model.configspace, deterministic=True, n_trials=n_trials, min_budget=min_budget, max_budget=max_budget\n    )\n\n    # Now we use SMAC to find the best hyperparameters\n    smac = MultiFidelityFacade(\n        scenario,\n        model.train,  # We pass the target function here\n        overwrite=True,  # Overrides any previous results that are found that are inconsistent with the meta-data\n        intensifier=MultiFidelityFacade.get_intensifier(scenario=scenario, eta=eta),\n    )\n\n    incumbent = smac.optimize()\n\n    # Get cost of default configuration\n    default_cost = smac.validate(model.configspace.get_default_configuration())\n    print(f\"Default cost: {default_cost}\")\n\n    # Let's calculate the cost of the incumbent\n    incumbent_cost = smac.validate(incumbent)\n    print(f\"Incumbent cost: {incumbent_cost}\")\n\n    # Let's plot it too\n    plot(smac.runhistory, incumbent)"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.14"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

development/_downloads/a94f3a529918fc1008873056ca54f5d0/3_specify_HB_via_total_budget.py

@@ -0,0 +1,112 @@
+"""
+Specify Number of Trials via a Total Budget in Hyperband
+^^^^^^^^^^^^^^^^^^
+This example uses a dummy function but illustrates how to setup Hyperband if you 
+want to specify a total optimization budget in terms of fidelity units.
+
+In Hyperband, normally SMAC calculates a typical Hyperband round.
+If the number of trials is not used up by one single round, the next round is started.
+Instead of specifying the number of trial beforehand, specify the total budget
+in terms of the fidelity units and let SMAC calculate how many trials that would be.
+
+
+"""
+from __future__ import annotations
+
+import numpy as np
+from ConfigSpace import Configuration, ConfigurationSpace, Float
+from matplotlib import pyplot as plt
+
+from smac import MultiFidelityFacade, RunHistory, Scenario
+from smac.intensifier.hyperband_utils import get_n_trials_for_hyperband_multifidelity
+
+__copyright__ = "Copyright 2021, AutoML.org Freiburg-Hannover"
+__license__ = "3-clause BSD"
+
+
+class QuadraticFunction:
+    max_budget = 500
+
+    @property
+    def configspace(self) -> ConfigurationSpace:
+        cs = ConfigurationSpace(seed=0)
+        x = Float("x", (-5, 5), default=-5)
+        cs.add_hyperparameters([x])
+
+        return cs
+
+    def train(self, config: Configuration, seed: int = 0, budget: float | None = None) -> float:
+        """Returns the y value of a quadratic function with a minimum we know to be at x=0."""
+        x = config["x"]
+
+        if budget is None:
+            multiplier = 1
+        else:
+            multiplier = 1 + budget / self.max_budget
+
+        return x**2 * multiplier
+
+
+def plot(runhistory: RunHistory, incumbent: Configuration) -> None:
+    plt.figure()
+
+    # Plot ground truth
+    x = list(np.linspace(-5, 5, 100))
+    y = [xi * xi for xi in x]
+    plt.plot(x, y)
+
+    # Plot all trials
+    for k, v in runhistory.items():
+        config = runhistory.get_config(k.config_id)
+        x = config["x"]
+        y = v.cost  # type: ignore
+        plt.scatter(x, y, c="blue", alpha=0.1, zorder=9999, marker="o")
+
+    # Plot incumbent
+    plt.scatter(incumbent["x"], incumbent["x"] * incumbent["x"], c="red", zorder=10000, marker="x")
+
+    plt.show()
+
+
+if __name__ == "__main__":
+    model = QuadraticFunction()
+
+    min_budget = 10  # minimum budget per trial
+    max_budget = 500  # maximum budget per trial
+    eta = 3  # standard HB parameter influencing the number of stages
+
+    # Let's calculate how many trials we need to exhaust the total optimization budget (in terms of
+    # fidelity units)
+    n_trials = get_n_trials_for_hyperband_multifidelity(
+        total_budget=10000,  # this is the total optimization budget we specify in terms of fidelity units
+        min_budget=min_budget,  # This influences the Hyperband rounds, minimum budget per trial
+        max_budget=max_budget,  # This influences the Hyperband rounds, maximum budget per trial
+        eta=eta,  # This influences the Hyperband rounds
+        print_summary=True,
+    )
+
+    # Scenario object specifying the optimization "environment"
+    scenario = Scenario(
+        model.configspace, deterministic=True, n_trials=n_trials, min_budget=min_budget, max_budget=max_budget
+    )
+
+    # Now we use SMAC to find the best hyperparameters
+    smac = MultiFidelityFacade(
+        scenario,
+        model.train,  # We pass the target function here
+        overwrite=True,  # Overrides any previous results that are found that are inconsistent with the meta-data
+        intensifier=MultiFidelityFacade.get_intensifier(scenario=scenario, eta=eta),
+    )
+
+    incumbent = smac.optimize()
+
+    # Get cost of default configuration
+    default_cost = smac.validate(model.configspace.get_default_configuration())
+    print(f"Default cost: {default_cost}")
+
+    # Let's calculate the cost of the incumbent
+    incumbent_cost = smac.validate(incumbent)
+    print(f"Incumbent cost: {incumbent_cost}")
+
+    # Let's plot it too
+    plot(smac.runhistory, incumbent)