diff --git a/README.md b/README.md index 4f004602..53e22a71 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ -# HPOlib2 +# HPOBench -HPOlib2 is a library for hyperparameter optimization and black-box optimization benchmark with a focus on reproducibility. +HPOBench is a library for hyperparameter optimization and black-box optimization benchmark with a focus on reproducibility. -**Note:** Hpolib2 is under active construction. Stay tuned for more benchmarks. Information on how to contribute a new benchmark will follow shortly. +**Note:** HPOBench is under active construction. Stay tuned for more benchmarks. Information on how to contribute a new benchmark will follow shortly. -**Note:** If you are looking for a different version of HPOlib2, you might be looking for [HPOlib1.5](https://github.com/automl/HPOlib1.5) +**Note:** "If you are looking for a different or older version of our benchmarking library, you might be looking for + [HPOlib1.5](https://github.com/automl/HPOlib1.5) ## In 4 lines of code Run a random configuration within a singularity container ```python -from hpolib.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark +from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark b = XGBoostBenchmark(task_id=167149, container_source='library://phmueller/automl', rng=1) config = b.get_configuration_space(seed=1).sample_configuration() result_dict = b.objective_function(configuration=config, fidelity={"n_estimators": 128, "subsample": 0.5}, rng=1) @@ -19,7 +20,7 @@ result_dict = b.objective_function(configuration=config, fidelity={"n_estimators All benchmarks can also be queried with fewer or no fidelities: ```python -from hpolib.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark +from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark b = XGBoostBenchmark(task_id=167149, container_source='library://phmueller/automl', rng=1) config = b.get_configuration_space(seed=1).sample_configuration() result_dict = b.objective_function(configuration=config, fidelity={"n_estimators": 128,}, rng=1) @@ -35,7 +36,7 @@ Further requirements are: [ConfigSpace](https://github.com/automl/ConfigSpace), A simple example is the XGBoost benchmark which can be installed with `pip install .[xgboost]` ```python -from hpolib.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark +from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark b = XGBoostBenchmark(task_id=167149) config = b.get_configuration_space(seed=1).sample_configuration() result_dict = b.objective_function(configuration=config, fidelity={"n_estimators": 128, "subsample": 0.5}, rng=1) @@ -47,8 +48,8 @@ result_dict = b.objective_function(configuration=config, fidelity={"n_estimators Before we start, we recommend using a virtual environment. To run any benchmark using its singularity container, run the following: ``` -git clone https://github.com/automl/HPOlib2.git -cd HPOlib2 +git clone https://github.com/automl/HPOBench.git +cd HPOBench pip install . ``` @@ -78,29 +79,29 @@ pip install . With singularity installed run the following to built the xgboost container ```bash -cd hpolib/container/recipes/ml +cd hpobench/container/recipes/ml sudo singularity build xgboost_benchmark Singularity.XGBoostBenchmark ``` You can use this local image with: ```python -from hpolib.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark +from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark b = XGBoostBenchmark(task_id=167149, container_name="xgboost_benchmark", - container_source='./') # path to hpolib/container/recipes/ml + container_source='./') # path to hpobench/container/recipes/ml config = b.get_configuration_space(seed=1).sample_configuration() result_dict = b.objective_function(config, fidelity={"n_estimators": 128, "subsample": 0.5}) ``` ### Remove all caches -#### HPOlib data -HPOlib stores downloaded containers and datasets at the following locations: +#### HPOBench data +HPOBench stores downloaded containers and datasets at the following locations: ```bash -$XDG_CONFIG_HOME # ~/.config/hpolib2 -$XDG_CACHE_HOME # ~/.cache/hpolib2 -$XDG_DATA_HOME # ~/.local/share/hpolib2 +$XDG_CONFIG_HOME # ~/.config/hpobench +$XDG_CACHE_HOME # ~/.cache/hpobench +$XDG_DATA_HOME # ~/.local/share/hpobench ``` For crashes or when not properly shutting down containers, there might be socket files left under `/tmp/`. @@ -127,10 +128,10 @@ See whether in `~/.singularity/instances/sing/$HOSTNAME/*/` there is a file that Status for Master Branch: -[![Build Status](https://travis-ci.org/automl/HPOlib2.svg?branch=master)](https://travis-ci.org/automl/HPOlib2) -[![codecov](https://codecov.io/gh/automl/HPOlib2/branch/master/graph/badge.svg)](https://codecov.io/gh/automl/HPOlib2) +[![Build Status](https://travis-ci.org/automl/HPOBench.svg?branch=master)](https://travis-ci.org/automl/HPOBench) +[![codecov](https://codecov.io/gh/automl/HPOBench/branch/master/graph/badge.svg)](https://codecov.io/gh/automl/HPOBench) Status for Development Branch: -[![Build Status](https://travis-ci.org/automl/HPOlib2.svg?branch=development)](https://travis-ci.org/automl/HPOlib2) -[![codecov](https://codecov.io/gh/automl/HPOlib2/branch/development/graph/badge.svg)](https://codecov.io/gh/automl/HPOlib2) +[![Build Status](https://travis-ci.org/automl/HPOBench.svg?branch=development)](https://travis-ci.org/automl/HPOBench) +[![codecov](https://codecov.io/gh/automl/HPOBench/branch/development/graph/badge.svg)](https://codecov.io/gh/automl/HPOBench) diff --git a/changelog.md b/changelog.md index 41f88b92..ef75377f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +# 0.0.5 + * Rename package to HPOBench + * Add BNN (pybnn) benchmark + * Update returned loss values for nasbench101 and tabular benchmarks + * Updat returned loss values for nasbench201 as well its data + * Nasbench201 is now 1 indexed instead of 0. + * Add MinMaxScaler to SVM Benchmark's preprocessing + * Add further tests + # 0.0.4 * improve test coverage * update HowToAddANewBenchmark.md diff --git a/ci_scripts/codestyle.sh b/ci_scripts/codestyle.sh index 085ea3e5..01e47fa7 100644 --- a/ci_scripts/codestyle.sh +++ b/ci_scripts/codestyle.sh @@ -3,7 +3,7 @@ if [[ "$RUN_CODESTYLE" == "true" ]]; then echo "Performing codestyle checking" - test_codestyle=$(pycodestyle --max-line-length=120 ./hpolib) + test_codestyle=$(pycodestyle --max-line-length=120 ./hpobench) if [[ $test_codestyle ]]; then echo $test_codestyle exit 1 @@ -11,7 +11,7 @@ if [[ "$RUN_CODESTYLE" == "true" ]]; then echo "Codesytle: No errors found" fi - test_flake=$(flake8 --max-line-length=120 ./hpolib) + test_flake=$(flake8 --max-line-length=120 ./hpobench) if [[ $test_flake ]]; then echo $test_flake exit 1 diff --git a/ci_scripts/install.sh b/ci_scripts/install.sh index f6780ae8..640f61ff 100644 --- a/ci_scripts/install.sh +++ b/ci_scripts/install.sh @@ -68,5 +68,5 @@ fi # remove the trailing comma install_packages="$(echo ${install_packages} | sed 's/,*\r*$//')" -echo "Install HPOlib2 with options: ${install_packages}" +echo "Install HPOBench with options: ${install_packages}" pip install .["${install_packages}"] diff --git a/ci_scripts/script.sh b/ci_scripts/script.sh index bde23b1a..803cbddc 100644 --- a/ci_scripts/script.sh +++ b/ci_scripts/script.sh @@ -3,7 +3,7 @@ if [[ "$RUN_TESTS" == "true" ]]; then if [[ "$RUN_CODECOV" == "true" ]]; then echo "Run tests with code coverage" - pytest -sv --cov=hpolib tests/ + pytest -sv --cov=hpobench tests/ exit_code=$? echo "Run code coverage" diff --git a/codecov.yml b/codecov.yml index 0d4c689b..19ba4fa7 100644 --- a/codecov.yml +++ b/codecov.yml @@ -20,6 +20,6 @@ comment: require_changes: no ignore: - - "hpolib/benchmarks" - - "hpolib/util/dependencies.py" - - "hpolib/util/example_utils.py" \ No newline at end of file + - "hpobench/benchmarks" + - "hpobench/util/dependencies.py" + - "hpobench/util/example_utils.py" \ No newline at end of file diff --git a/examples/container/tabular_benchmark_example.py b/examples/container/tabular_benchmark_example.py index 7ad5c8d0..1b9ddc52 100644 --- a/examples/container/tabular_benchmark_example.py +++ b/examples/container/tabular_benchmark_example.py @@ -8,7 +8,7 @@ container_source can be either a path to a registry (e.g. sylabs.io, singularity_hub.org) or a local path on your local file system. If it is a link to a registry, the container will be downloaded to the default data dir, set in the -hpolibrc. A second call, will first look into the data directory, if the container is already available, so it will not +hpobenchrc. A second call, will first look into the data directory, if the container is already available, so it will not be downloaded twice. Please install the necessary dependencies via ``pip install .`` and singularity (v3.5). @@ -17,7 +17,7 @@ import argparse -from hpolib.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark as TabBenchmarkContainer +from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark as TabBenchmarkContainer def run_experiment(on_travis=False): diff --git a/examples/container/xgboost_with_container.py b/examples/container/xgboost_with_container.py index a5964f91..a374aebe 100644 --- a/examples/container/xgboost_with_container.py +++ b/examples/container/xgboost_with_container.py @@ -3,7 +3,7 @@ ================================ In this example, we show how to use a benchmark with a container. We provide container for some benchmarks. -They are hosted on https://cloud.sylabs.io/library/keggensperger/automl. +They are hosted on https://cloud.sylabs.io/library/muelleph/automl. Furthermore, we use different fidelities to train the xgboost model - the number of estimators as well as the fraction of training data points. @@ -11,7 +11,7 @@ To use the container-example, you have to have singulartiy (>3.5) installed. Follow the official installation guide on https://sylabs.io/guides/3.1/user-guide/quick_start.html#quick-installation-steps -Furthermore, make sure to install the right dependencies for the hpolib via: +Furthermore, make sure to install the right dependencies for the hpobench via: ``pip3 install .``. """ @@ -19,8 +19,8 @@ import logging from time import time -from hpolib.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark -from hpolib.util.openml_data_manager import get_openmlcc18_taskids +from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark +from hpobench.util.openml_data_manager import get_openmlcc18_taskids logging.basicConfig(level=logging.INFO) @@ -66,7 +66,7 @@ def run_experiment(on_travis: bool = False): if __name__ == '__main__': - parser = argparse.ArgumentParser(prog='HPOlib CC Datasets', description='HPOlib2 on the CC18 data sets.', + parser = argparse.ArgumentParser(prog='HPOBench CC Datasets', description='HPOBench on the CC18 data sets.', usage='%(prog)s --array_id ') parser.add_argument('--on_travis', action='store_true', diff --git a/examples/local/xgboost_local.py b/examples/local/xgboost_local.py index dadd91ec..90c95029 100644 --- a/examples/local/xgboost_local.py +++ b/examples/local/xgboost_local.py @@ -10,8 +10,8 @@ import argparse from time import time -from hpolib.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark -from hpolib.util.openml_data_manager import get_openmlcc18_taskids +from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark +from hpobench.util.openml_data_manager import get_openmlcc18_taskids def run_experiment(on_travis: bool = False): @@ -51,7 +51,7 @@ def run_experiment(on_travis: bool = False): if __name__ == '__main__': - parser = argparse.ArgumentParser(prog='HPOlib CC Datasets', description='HPOlib2 on the CC18 data sets.', + parser = argparse.ArgumentParser(prog='HPOBench CC Datasets', description='HPOBench on the CC18 data sets.', usage='%(prog)s --array_id ') parser.add_argument('--on_travis', action='store_true', diff --git a/examples/w_optimizer/cartpole_bohb.py b/examples/w_optimizer/cartpole_bohb.py index f9128d31..8ba9ebb8 100644 --- a/examples/w_optimizer/cartpole_bohb.py +++ b/examples/w_optimizer/cartpole_bohb.py @@ -18,9 +18,9 @@ from hpbandster.core.worker import Worker from hpbandster.optimizers import BOHB -from hpolib.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark -from hpolib.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core -from hpolib.util.rng_helper import get_rng +from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark +from hpobench.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core +from hpobench.util.rng_helper import get_rng logger = logging.getLogger('BOHB on cartpole') set_env_variables_to_use_only_one_core() @@ -116,8 +116,8 @@ def run_experiment(out_path, on_travis): if __name__ == '__main__': import argparse - parser = argparse.ArgumentParser(prog='HPOlib - BOHB', - description='HPOlib2 with BOHB on Cartpole', + parser = argparse.ArgumentParser(prog='HPOBench - BOHB', + description='HPOBench with BOHB on Cartpole', usage='%(prog)s --out_path ') parser.add_argument('--out_path', default='./cartpole_smac_hb', type=str) parser.add_argument('--on_travis', action='store_true', diff --git a/examples/w_optimizer/cartpole_hyperband.py b/examples/w_optimizer/cartpole_hyperband.py index 8cbd465e..887ca303 100644 --- a/examples/w_optimizer/cartpole_hyperband.py +++ b/examples/w_optimizer/cartpole_hyperband.py @@ -17,8 +17,8 @@ from smac.intensification.hyperband import Hyperband from smac.scenario.scenario import Scenario -from hpolib.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark -from hpolib.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core +from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark +from hpobench.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core logger = logging.getLogger("HB on cartpole") logging.basicConfig(level=logging.INFO) @@ -93,8 +93,8 @@ def optimization_function_wrapper(cfg, seed, instance, budget): if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(prog='HPOlib - Hyperband', - description='HPOlib2 with HB on Cartpole', + parser = argparse.ArgumentParser(prog='HPOBench - Hyperband', + description='HPOBench with HB on Cartpole', usage='%(prog)s --out_path ') parser.add_argument('--out_path', default='./cartpole_smac_hb', type=str) parser.add_argument('--on_travis', action='store_true', diff --git a/examples/w_optimizer/cartpole_succesive_halving.py b/examples/w_optimizer/cartpole_succesive_halving.py index 0ea255b6..0dad5cdf 100644 --- a/examples/w_optimizer/cartpole_succesive_halving.py +++ b/examples/w_optimizer/cartpole_succesive_halving.py @@ -17,8 +17,8 @@ from smac.intensification.hyperband import SuccessiveHalving from smac.scenario.scenario import Scenario -from hpolib.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark -from hpolib.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core +from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark +from hpobench.util.example_utils import get_travis_settings, set_env_variables_to_use_only_one_core logger = logging.getLogger("SH on cartpole") logging.basicConfig(level=logging.INFO) @@ -96,8 +96,8 @@ def optimization_function_wrapper(cfg, seed, instance, budget): if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(prog='HPOlib - Successive Halving', - description='HPOlib2 with SH on Cartpole', + parser = argparse.ArgumentParser(prog='HPOBench - Successive Halving', + description='HPOBench with SH on Cartpole', usage='%(prog)s --out_path ') parser.add_argument('--out_path', default='./cartpole_smac_sh', type=str) parser.add_argument('--on_travis', action='store_true', diff --git a/extra_requirements/pybnn.json b/extra_requirements/pybnn.json new file mode 100644 index 00000000..c83a22fd --- /dev/null +++ b/extra_requirements/pybnn.json @@ -0,0 +1,4 @@ +{ + "pybnn": ["Theano==1.0.5","sgmcmc@git+https://github.com/automl/sgmcmc#egg=sgmcmc-0.0.1", + "Lasagne@git+https://github.com/Lasagne/Lasagne#egg=Lasagne-0.0.1"] +} \ No newline at end of file diff --git a/hpolib/HowToAddANewBenchmark.md b/hpobench/HowToAddANewBenchmark.md similarity index 58% rename from hpolib/HowToAddANewBenchmark.md rename to hpobench/HowToAddANewBenchmark.md index 9b9384f4..b0f86de0 100644 --- a/hpolib/HowToAddANewBenchmark.md +++ b/hpobench/HowToAddANewBenchmark.md @@ -2,11 +2,11 @@ ## Create a local benchmark -Clone Hpolib2, switch to the development branch and create your own branch, then install hpolib2. +Clone HPOBench, switch to the development branch and create your own branch, then install hpobench. with `pip install .` ```bash -git clone https://github.com/automl/HPOlib2.git -cd HPOlib2 +git clone https://github.com/automl/HPOBench.git +cd HPOBench git checkout development git branch newBenchmark git checkout newBenchmark @@ -15,21 +15,21 @@ pip install . Then: - 1. Implement your new benchmark `hpolib/benchmarks//` inheriting from the base class - `AbstractBenchmark` in `hpolib.abstract_benchmark`. Your benchmark should implement `__init__()`, + 1. Implement your new benchmark `hpobench/benchmarks//` inheriting from the base class + `AbstractBenchmark` in `hpobench.abstract_benchmark`. Your benchmark should implement `__init__()`, `get_configuration_space()`, `get_fidelity_space()`, `objective_function()` and `objective_function_test()`. - A good example for this can be found in `hpolib/benchmarks/ml/xgboost_benchmark.py` + A good example for this can be found in `hpobench/benchmarks/ml/xgboost_benchmark.py` 3. If your benchmarks needs a dataset (e.g. for training a ml model), please also implement a DataManager, see e.g. - `hpolib/util/openml_data_manager.py` with a `load()` method that downloads data once and reuses it for further calls. + `hpobench/util/openml_data_manager.py` with a `load()` method that downloads data once and reuses it for further calls. 4. Collect all **additional Python** and **non-Python** dependencies while doing this. Consider fixing the version of each dependency to maintain reproducibility. 5. Add dependencies to PIPy in a new file to `/extra_requirements` 6. Add the remaining dependencies or steps necessary to run your benchmark in the docstring of your benchmark class - (see, e.g. `hpolib/benchmarks/nas/nasbench_101.py`). + (see, e.g. `hpobench/benchmarks/nas/nasbench_101.py`). 7. Verify that everything works with, e.g. ```python -from hpolib.benchmarks.. import +from hpobench.benchmarks.. import b = (, rng=1) config = b.get_configuration_space(seed=1).sample_configuration() result_dict = b.objective_function(configuration=config, rng=1) @@ -42,10 +42,10 @@ Now, you can create a PR marked as [WIP] and proceed with building a containeriz ## Create a containerized benchmark - 1. Create a container benchmark class in `hpolib/container/benchmarks//` inheriting from the - base class `AbstractBenchmarkClient` in `hpolib.container.client_abstract_benchmark`. - Note: this are just a few lines of code, see, e.g. `hpolib/container/benchmarks/ml/xgboost_benchmark.py`) - 2. Copy `hpolib/container/recipes/Singularity.template` to `hpolib/container/recipes//name` + 1. Create a container benchmark class in `hpobench/container/benchmarks//` inheriting from the + base class `AbstractBenchmarkClient` in `hpobench.container.client_abstract_benchmark`. + Note: this are just a few lines of code, see, e.g. `hpobench/container/benchmarks/ml/xgboost_benchmark.py`) + 2. Copy `hpobench/container/recipes/Singularity.template` to `hpobench/container/recipes//name` 3. Modify the recipe and add your **additional Python** and **non-Python** dependencies collected above. 3. Test your container locally (see below) @@ -53,10 +53,10 @@ Now, you can update your PR and let us know, so we can upload the container to S ## How to test your container locally - 1. `cd hpolib/container/benchmarks/recipes/` and change to following lines in the recipe: + 1. `cd hpobench/container/benchmarks/recipes/` and change to following lines in the recipe: ```bash - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout development \ ``` to point to the branch/repo where your fork is on, e.g. `newBenchmark`. @@ -65,7 +65,7 @@ Now, you can update your PR and let us know, so we can upload the container to S 3. Verify that everything works with: ```python -from hpolib.container.benchmarks.. import +from hpobench.container.benchmarks.. import b = (container_source="./", container_name="newBenchmark") res = b.objective_function(configuration=b.get_configuration_space(seed=1).sample_configuration()) ``` diff --git a/hpobench/__init__.py b/hpobench/__init__.py new file mode 100644 index 00000000..de4fbba8 --- /dev/null +++ b/hpobench/__init__.py @@ -0,0 +1,4 @@ +__contact__ = "automl.org" + +from hpobench.__version__ import __version__ # noqa: F401 +from hpobench.config import config_file # noqa: F401 diff --git a/hpobench/__version__.py b/hpobench/__version__.py new file mode 100644 index 00000000..eead3198 --- /dev/null +++ b/hpobench/__version__.py @@ -0,0 +1 @@ +__version__ = '0.0.5' diff --git a/hpolib/abstract_benchmark.py b/hpobench/abstract_benchmark.py similarity index 98% rename from hpolib/abstract_benchmark.py rename to hpobench/abstract_benchmark.py index 6a09c4fb..fcba9e9d 100644 --- a/hpolib/abstract_benchmark.py +++ b/hpobench/abstract_benchmark.py @@ -7,7 +7,7 @@ import ConfigSpace import numpy as np -from hpolib.util import rng_helper +from hpobench.util import rng_helper logger = logging.getLogger('AbstractBenchmark') @@ -84,7 +84,7 @@ def objective_function_test(self, configuration: Dict, fidelity: Union[Dict, Non fidelity: Dict, None Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None - see :py:func:`~HPOlib2.abstract_benchmark.objective_function` + see :py:func:`~HPOBench.abstract_benchmark.objective_function` Returns ------- diff --git a/hpolib/benchmarks/__init__.py b/hpobench/benchmarks/__init__.py similarity index 100% rename from hpolib/benchmarks/__init__.py rename to hpobench/benchmarks/__init__.py diff --git a/hpolib/benchmarks/ml/__init__.py b/hpobench/benchmarks/ml/__init__.py similarity index 100% rename from hpolib/benchmarks/ml/__init__.py rename to hpobench/benchmarks/ml/__init__.py diff --git a/hpobench/benchmarks/ml/pybnn.py b/hpobench/benchmarks/ml/pybnn.py new file mode 100644 index 00000000..26de83f4 --- /dev/null +++ b/hpobench/benchmarks/ml/pybnn.py @@ -0,0 +1,343 @@ +""" +How to use this benchmark: +-------------------------- + +We recommend using the containerized version of this benchmark. +If you want to use this benchmark locally (without running it via the corresponding container), +you need to install the following packages besides installing the hpobench with + +```pip install .[pybnn]``` + +""" +from functools import partial +import logging +import os +import time +import tempfile +from typing import Union, Dict, Any + +import numpy as np +from scipy import stats + +import ConfigSpace as CS + +from hpobench.util.data_manager import BostonHousingData, ProteinStructureData, YearPredictionMSDData +from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.util import rng_helper + +# This has to happen before any other imports +if "TMPDIR" not in os.environ: + tmpdir = tempfile.TemporaryDirectory() + tmpdir_name = tmpdir.name +else: + tmpdir_name = os.environ["TMPDIR"] +os.environ["THEANO_FLAGS"] = f"base_compiledir={tmpdir_name}" + +import lasagne # noqa: E402 +from sgmcmc.bnn.model import BayesianNeuralNetwork # noqa: E402 +from sgmcmc.bnn.lasagne_layers import AppendLayer # noqa: E402 + + +__version__ = '0.0.2' +logger = logging.getLogger('PyBnnBenchmark') + + +class BayesianNeuralNetworkBenchmark(AbstractBenchmark): + + def __init__(self, + rng: Union[np.random.RandomState, int, None] = None): + """ + + Parameters + ---------- + rng : np.random.RandomState, int, None + """ + super(BayesianNeuralNetworkBenchmark, self).__init__(rng=rng) + + self.n_calls = 0 + self.max_iters = 10000 + + self.train, self.train_targets, self.valid, self.valid_targets, \ + self.test, self.test_targets = self.get_data() + + def get_data(self): + raise NotImplementedError() + + @AbstractBenchmark._configuration_as_dict + @AbstractBenchmark._check_configuration + @AbstractBenchmark._check_fidelity + def objective_function(self, configuration: Union[Dict, CS.Configuration], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: + """ + Trains a bayesian neural network with 3 layers on the defined data set and evaluates the trained model on + the validation split. + + Parameters + ---------- + configuration : Dict, CS.Configuration + Configuration for the pyBNN model + fidelity: Dict, None + budget : int [100 - 10000] + number of epochs to train the model + Fidelity parameters for the pyBNN model, check get_fidelity_space(). Uses default (max) value if None. + rng : np.random.RandomState, int, None, + Random seed for benchmark. By default the class level random seed. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : validation loss + cost : time to train and evaluate the model + info : Dict + fidelity : used fidelities in this evaluation + """ + start = time.time() + + self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) + np.random.seed(self.rng.randint(1, 10000)) + + burn_in_steps = int(configuration['burn_in'] * fidelity['budget']) + + net = partial(_get_net, + n_units_1=configuration['n_units_1'], + n_units_2=configuration['n_units_2']) + + model = BayesianNeuralNetwork(sampling_method="sghmc", + get_net=net, + l_rate=configuration['l_rate'], + mdecay=configuration['mdecay'], + burn_in=burn_in_steps, + n_iters=fidelity['budget'], + precondition=True, + normalize_input=True, + normalize_output=True, + rng=self.rng) + + model.train(self.train, self.train_targets, + valid=self.valid, valid_targets=self.valid_targets, + valid_after_n_steps=100) + + mean_pred, var_pred = model.predict(self.valid) + + # Negative log-likelihood + valid_loss = self._neg_log_likelihood(self.valid_targets, mean_pred, var_pred) + + cost = time.time() - start + + return {'function_value': float(valid_loss), + 'cost': cost, + 'info': {'fidelity': fidelity}} + + @AbstractBenchmark._configuration_as_dict + @AbstractBenchmark._check_configuration + @AbstractBenchmark._check_fidelity + def objective_function_test(self, configuration: Union[Dict, CS.Configuration], + fidelity: Union[Dict, CS.Configuration, None] = None, + rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: + """ + Trains a bayesian neural network with 3 layers on the train and valid data split and evaluates it on the test + split. + + Parameters + ---------- + configuration : Dict, CS.Configuration + Configuration for the pyBNN model + fidelity: Dict, None + budget : int [100 - 10000] + number of epochs to train the model + Fidelity parameters for the pyBNN model, check get_fidelity_space(). Uses default (max) value if None. + + Note: The fidelity should be here the max budget (= 10000). By leaving this field empty, the maximum budget + will be used by default. + rng : np.random.RandomState, int, None, + Random seed for benchmark. By default the class level random seed. + + To prevent overfitting on a single seed, it is possible to pass a + parameter ``rng`` as 'int' or 'np.random.RandomState' to this function. + If this parameter is not given, the default random state is used. + kwargs + + Returns + ------- + Dict - + function_value : validation loss + cost : time to train and evaluate the model + info : Dict + fidelity : used fidelities in this evaluation + """ + start = time.time() + + self.rng = rng_helper.get_rng(rng=rng, self_rng=self.rng) + np.random.seed(self.rng.randint(1, 10000)) + + burn_in_steps = int(configuration['burn_in'] * fidelity['budget']) + + net = partial(_get_net, + n_units_1=configuration['n_units_1'], + n_units_2=configuration['n_units_2']) + + model = BayesianNeuralNetwork(sampling_method="sghmc", + get_net=net, + l_rate=configuration['l_rate'], + mdecay=configuration['mdecay'], + burn_in=burn_in_steps, + n_iters=fidelity['budget'], + precondition=True, + normalize_input=True, + normalize_output=True, + rng=self.rng) + + train = np.concatenate((self.train, self.valid)) + train_targets = np.concatenate((self.train_targets, self.valid_targets)) + model.train(train, train_targets) + + mean_pred, var_pred = model.predict(self.test) + test_loss = self._neg_log_likelihood(self.test_targets, mean_pred, var_pred) + + cost = time.time() - start + return {'function_value': float(test_loss), + 'cost': cost, + 'info': {'fidelity': fidelity}} + + @staticmethod + def _neg_log_likelihood(targets: np.ndarray, mean_pred: np.ndarray, var_pred: np.ndarray) -> np.ndarray: + """ Compute the negative log likelihood for normal distributions. """ + nll = [stats.norm.logpdf(targets[i], loc=mean_pred[i], scale=np.sqrt(var_pred[i])) + for i in range(targets.shape[0])] + return -np.mean(nll) + + @staticmethod + def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: + """ + Creates a ConfigSpace.ConfigurationSpace containing all parameters for the pybnn benchmark. + + Parameters + ---------- + seed : int, None + Fixing the seed for the ConfigSpace.ConfigurationSpace + + Returns + ------- + ConfigSpace.ConfigurationSpace + """ + seed = seed if seed is not None else np.random.randint(1, 100000) + cs = CS.ConfigurationSpace(seed=seed) + + cs.add_hyperparameters([ + CS.UniformFloatHyperparameter('l_rate', lower=1e-6, upper=1e-1, default_value=1e-2, log=True), + CS.UniformFloatHyperparameter('burn_in', lower=0.0, upper=0.8, default_value=0.3), + CS.UniformIntegerHyperparameter('n_units_1', lower=16, upper=512, default_value=64, log=True), + CS.UniformIntegerHyperparameter('n_units_2', lower=16, upper=512, default_value=64, log=True), + CS.UniformFloatHyperparameter('mdecay', lower=0, upper=1, default_value=0.05) + ]) + + return cs + + @staticmethod + def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: + """ + Creates a ConfigSpace.ConfigurationSpace containing all fidelity parameters for pyBNN benchmark. + Fidelities + ---------- + budget : int : [100, 10000] + number of epochs to train the network + + Parameters + ---------- + seed : int, None + Fixing the seed for the ConfigSpace.ConfigurationSpace + + Returns + ------- + ConfigSpace.ConfigurationSpace + """ + seed = seed if seed is not None else np.random.randint(1, 100000) + fidel_space = CS.ConfigurationSpace(seed=seed) + + fidel_space.add_hyperparameters([ + CS.UniformIntegerHyperparameter("budget", lower=500, upper=10000, default_value=10000, log=False) + ]) + + return fidel_space + + @staticmethod + def get_meta_information(): + return {'name': 'BNN Benchmark', + 'references': ['@InProceedings{falkner-icml-18,' + 'title = {{BOHB}: Robust and Efficient Hyperparameter Optimization at Scale},' + 'url = http://proceedings.mlr.press/v80/falkner18a.html' + 'author = {Falkner, Stefan and Klein, Aaron and Hutter, Frank}, ' + 'booktitle = {Proceedings of the 35th International Conference on Machine Learning},' + 'pages = {1436 - -1445},' + 'year = {2018}}'], + } + + +def _get_net(n_inputs: int, n_units_1: int, n_units_2: int) -> Any: + """ Create a Network with theano """ + l_in = lasagne.layers.InputLayer(shape=(None, n_inputs)) + + fc_layer_1 = lasagne.layers.DenseLayer(l_in, + num_units=n_units_1, + W=lasagne.init.HeNormal(), + b=lasagne.init.Constant(val=0.0), + nonlinearity=lasagne.nonlinearities.tanh) + fc_layer_2 = lasagne.layers.DenseLayer(fc_layer_1, + num_units=n_units_2, + W=lasagne.init.HeNormal(), + b=lasagne.init.Constant(val=0.0), + nonlinearity=lasagne.nonlinearities.tanh) + l_out = lasagne.layers.DenseLayer(fc_layer_2, + num_units=1, + W=lasagne.init.HeNormal(), + b=lasagne.init.Constant(val=0.0), + nonlinearity=lasagne.nonlinearities.linear) + + network = AppendLayer(l_out, num_units=1, b=lasagne.init.Constant(np.log(1e-3))) + return network + + +class BNNOnToyFunction(BayesianNeuralNetworkBenchmark): + """ Test Benchmark for the pyBNN Benchmark """ + def get_data(self): + rng = np.random.RandomState(42) + + def f(x): + eps = rng.normal() * 0.02 + y = x + 0.3 * np.sin(2 * np.pi * (x + eps)) + 0.3 * np.sin(4 * np.pi * (x + eps)) + eps + return y + + X = rng.rand(1000, 1) + y = np.array([f(xi) for xi in X])[:, 0] + + train = X[:600] + train_targets = y[:600] + valid = X[600:800] + valid_targets = y[600:800] + test = X[800:] + test_targets = y[800:] + return train, train_targets, valid, valid_targets, test, test_targets + + +class BNNOnBostonHousing(BayesianNeuralNetworkBenchmark): + def get_data(self): + dm = BostonHousingData() + return dm.load() + + +class BNNOnProteinStructure(BayesianNeuralNetworkBenchmark): + def get_data(self): + dm = ProteinStructureData() + return dm.load() + + +class BNNOnYearPrediction(BayesianNeuralNetworkBenchmark): + def get_data(self): + dm = YearPredictionMSDData() + return dm.load() diff --git a/hpolib/benchmarks/ml/svm_benchmark.py b/hpobench/benchmarks/ml/svm_benchmark.py similarity index 97% rename from hpolib/benchmarks/ml/svm_benchmark.py rename to hpobench/benchmarks/ml/svm_benchmark.py index 37f5c3c0..b4454627 100644 --- a/hpolib/benchmarks/ml/svm_benchmark.py +++ b/hpobench/benchmarks/ml/svm_benchmark.py @@ -1,3 +1,4 @@ +import logging import time from typing import Union, Tuple, Dict, List @@ -9,16 +10,14 @@ from sklearn.compose import ColumnTransformer from sklearn.impute import SimpleImputer from sklearn.metrics import accuracy_score, make_scorer -from sklearn.preprocessing import OneHotEncoder +from sklearn.preprocessing import OneHotEncoder, MinMaxScaler -import hpolib.util.rng_helper as rng_helper -from hpolib.abstract_benchmark import AbstractBenchmark -from hpolib.util.openml_data_manager import OpenMLHoldoutDataManager +import hpobench.util.rng_helper as rng_helper +from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.util.openml_data_manager import OpenMLHoldoutDataManager __version__ = '0.0.1' -import logging - logger = logging.getLogger('SVMBenchmark') @@ -244,7 +243,7 @@ def get_pipeline(self, C: float, gamma: float) -> pipeline.Pipeline: ('preprocess_one_hot', ColumnTransformer([ ("categorical", OneHotEncoder(categories=self.categories, sparse=False), self.categorical_data), - ("continuous", "passthrough", ~self.categorical_data)])), + ("continuous", MinMaxScaler(feature_range=(0, 1)), ~self.categorical_data)])), ('svm', svm.SVC(gamma=gamma, C=C, random_state=self.rng, cache_size=self.cache_size)) ]) diff --git a/hpolib/benchmarks/ml/xgboost_benchmark.py b/hpobench/benchmarks/ml/xgboost_benchmark.py similarity index 98% rename from hpolib/benchmarks/ml/xgboost_benchmark.py rename to hpobench/benchmarks/ml/xgboost_benchmark.py index 488087e3..c9a9a70e 100644 --- a/hpolib/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/benchmarks/ml/xgboost_benchmark.py @@ -10,9 +10,9 @@ from sklearn.metrics import accuracy_score, make_scorer from sklearn.preprocessing import OneHotEncoder -import hpolib.util.rng_helper as rng_helper -from hpolib.abstract_benchmark import AbstractBenchmark -from hpolib.util.openml_data_manager import OpenMLHoldoutDataManager +import hpobench.util.rng_helper as rng_helper +from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.util.openml_data_manager import OpenMLHoldoutDataManager __version__ = '0.0.1' diff --git a/hpolib/benchmarks/nas/__init__.py b/hpobench/benchmarks/nas/__init__.py similarity index 100% rename from hpolib/benchmarks/nas/__init__.py rename to hpobench/benchmarks/nas/__init__.py diff --git a/hpolib/benchmarks/nas/nasbench_101.py b/hpobench/benchmarks/nas/nasbench_101.py similarity index 57% rename from hpolib/benchmarks/nas/nasbench_101.py rename to hpobench/benchmarks/nas/nasbench_101.py index 17614833..d29c5ba3 100644 --- a/hpolib/benchmarks/nas/nasbench_101.py +++ b/hpobench/benchmarks/nas/nasbench_101.py @@ -21,7 +21,7 @@ 2. Clone and install ==================== ``` -cd /path/to/HPOlib2 +cd /path/to/HPOBench pip install .[nasbench_101] pip install git+https://github.com/google-research/nasbench.git@master @@ -41,11 +41,13 @@ import ConfigSpace as CS import numpy as np from tabular_benchmarks.nas_cifar10 import NASCifar10 +from nasbench import api +from nasbench.lib import graph_util -import hpolib.util.rng_helper as rng_helper -from hpolib.abstract_benchmark import AbstractBenchmark +import hpobench.util.rng_helper as rng_helper +from hpobench.abstract_benchmark import AbstractBenchmark -__version__ = '0.0.1' +__version__ = '0.0.2' MAX_EDGES = 9 VERTICES = 7 @@ -73,11 +75,14 @@ def __init__(self, benchmark: NASCifar10, data_path: Union[Path, str, None] = ". self.benchmark = benchmark self.data_path = data_path + def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict: + raise NotImplementedError + @AbstractBenchmark._configuration_as_dict @AbstractBenchmark._check_configuration @AbstractBenchmark._check_fidelity def objective_function(self, configuration: Union[CS.Configuration, Dict], - fidelity: Union[Dict, None] = None, + fidelity: Union[CS.Configuration, Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ @@ -108,20 +113,20 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], self.rng = rng_helper.get_rng(rng, self_rng=self.rng) - # Returns (valid_error_rate: 1, runtime: 0) if it is invalid, e.g. config not valid or + # Returns (valid_accuracy: 0, runtime: 0) if it is invalid, e.g. config not valid or # budget not in 4 12 36 108 - valid_error_rate, runtime = self.benchmark.objective_function(config=configuration, - budget=fidelity["budget"]) - return {'function_value': float(valid_error_rate), - 'cost': float(runtime), - 'info': {'fidelity': fidelity} + data = self._query_benchmark(config=configuration, budget=fidelity['budget']) + return {'function_value': 1 - data['validation_accuracy'], + 'cost': data['training_time'], + 'info': {'fidelity': fidelity, + 'data': data} } @AbstractBenchmark._configuration_as_dict @AbstractBenchmark._check_configuration @AbstractBenchmark._check_fidelity def objective_function_test(self, configuration: Union[Dict, CS.Configuration], - fidelity: Union[Dict, None] = None, + fidelity: Union[CS.Configuration, Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ @@ -146,8 +151,14 @@ def objective_function_test(self, configuration: Union[Dict, CS.Configuration], info : Dict fidelity : used fidelities in this evaluation """ - return self.objective_function(configuration=configuration, fidelity=fidelity, rng=rng, - **kwargs) + + data = self._query_benchmark(config=configuration, budget=fidelity['budget']) + + return {'function_value': 1 - data['test_accuracy'], + 'cost': data['training_time'], + 'info': {'fidelity': fidelity, + 'data': data} + } @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: @@ -220,11 +231,62 @@ def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSp from tabular_benchmarks.nas_cifar10 import NASCifar10A return NASCifar10BBenchmark._get_configuration_space(NASCifar10A, seed) + def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict: + """ + Copy of the 'objective_function' from nas_cifar10.py + We adapted the file in such a way, that the complete result is returned. The original implementation returns + only the validation error. Now, it can also return the test loss for a given configuration. + + Parameters + ---------- + config : Dict + budget : int + The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned. + + Returns + ------- + Dict + """ + + failure = {"test_accuracy": 0, "validation_accuracy": 0, "training_time": 0, "info": "failure"} + + if self.benchmark.multi_fidelity is False: + assert budget == 108 + + matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) + idx = np.triu_indices(matrix.shape[0], k=1) + for i in range(VERTICES * (VERTICES - 1) // 2): + row = idx[0][i] + col = idx[1][i] + matrix[row, col] = config["edge_%d" % i] + + # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: + if graph_util.num_edges(matrix) > MAX_EDGES: + self.benchmark.record_invalid(config, 1, 1, 0) + return failure + + labeling = [config["op_node_%d" % i] for i in range(5)] + labeling = ['input'] + list(labeling) + ['output'] + model_spec = api.ModelSpec(matrix, labeling) + try: + data = self.benchmark.dataset.query(model_spec, epochs=budget) + except api.OutOfDomainError: + self.benchmark.record_invalid(config, 1, 1, 0) + return failure + + self.benchmark.record_valid(config, data, model_spec) + + # We dont need this field. + data.pop('module_adjacency') + + return data + class NASCifar10BBenchmark(NASCifar10BaseBenchmark): def __init__(self, data_path: Union[Path, str, None] = './fcnet_tabular_benchmarks/', rng: Union[np.random.RandomState, int, None] = None, **kwargs): from tabular_benchmarks.nas_cifar10 import NASCifar10B + benchmark = NASCifar10B(data_dir=str(data_path), multi_fidelity=True) super(NASCifar10BBenchmark, self).__init__(benchmark=benchmark, data_path=data_path, rng=rng, **kwargs) @@ -245,10 +307,63 @@ def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSp from tabular_benchmarks.nas_cifar10 import NASCifar10B return NASCifar10BBenchmark._get_configuration_space(NASCifar10B, seed) + def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict: + """ + Copy of the 'objective_function' from nas_cifar10.py + We adapted the file in such a way, that the complete result is returned. The original implementation returns + only the validation error. Now, it can also return the test loss for a given configuration. + + Parameters + ---------- + config : Dict + budget : int + The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned. + + Returns + ------- + Dict + """ + failure = {"test_accuracy": 0, "validation_accuracy": 0, "training_time": 0, "info": "failure"} + + if self.benchmark.multi_fidelity is False: + assert budget == 108 + + bitlist = [0] * (VERTICES * (VERTICES - 1) // 2) + for i in range(MAX_EDGES): + bitlist[config["edge_%d" % i]] = 1 + out = 0 + for bit in bitlist: + out = (out << 1) | bit + + matrix = np.fromfunction(graph_util.gen_is_edge_fn(out), + (VERTICES, VERTICES), + dtype=np.int8) + # if not graph_util.is_full_dag(matrix) or graph_util.num_edges(matrix) > MAX_EDGES: + if graph_util.num_edges(matrix) > MAX_EDGES: + self.benchmark.record_invalid(config, 1, 1, 0) + return failure + + labeling = [config["op_node_%d" % i] for i in range(5)] + labeling = ['input'] + list(labeling) + ['output'] + model_spec = api.ModelSpec(matrix, labeling) + try: + data = self.benchmark.dataset.query(model_spec, epochs=budget) + except api.OutOfDomainError: + self.benchmark.record_invalid(config, 1, 1, 0) + return failure + + self.benchmark.record_valid(config, data, model_spec) + + # We dont need this field. + data.pop('module_adjacency') + + return data + class NASCifar10CBenchmark(NASCifar10BaseBenchmark): def __init__(self, data_path: Union[Path, str, None] = './fcnet_tabular_benchmarks/', rng: Union[np.random.RandomState, int, None] = None, **kwargs): + from tabular_benchmarks.nas_cifar10 import NASCifar10C benchmark = NASCifar10C(data_dir=str(data_path), multi_fidelity=True) super(NASCifar10CBenchmark, self).__init__(benchmark=benchmark, data_path=data_path, rng=rng, **kwargs) @@ -269,3 +384,59 @@ def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSp from tabular_benchmarks.nas_cifar10 import NASCifar10C return NASCifar10BBenchmark._get_configuration_space(NASCifar10C, seed) + + def _query_benchmark(self, config: Dict, budget: int = 108) -> Dict: + """ + Copy of the 'objective_function' from nas_cifar10.py + We adapted the file in such a way, that the complete result is returned. The original implementation returns + only the validation error. Now, it can also return the test loss for a given configuration. + + Parameters + ---------- + config : Dict + budget : int + The number of epochs. Must be one of: 4 12 36 108. Otherwise a accuracy of 0 is returned. + + Returns + ------- + Dict + """ + # Unify the return value to a dictionary. + failure = {"test_accuracy": 0, "validation_accuracy": 0, "training_time": 0, "info": "failure"} + + if self.benchmark.multi_fidelity is False: + assert budget == 108 + + edge_prob = [] + for i in range(VERTICES * (VERTICES - 1) // 2): + edge_prob.append(config["edge_%d" % i]) + + idx = np.argsort(edge_prob)[::-1][:config["num_edges"]] + binay_encoding = np.zeros(len(edge_prob)) + binay_encoding[idx] = 1 + matrix = np.zeros([VERTICES, VERTICES], dtype=np.int8) + idx = np.triu_indices(matrix.shape[0], k=1) + for i in range(VERTICES * (VERTICES - 1) // 2): + row = idx[0][i] + col = idx[1][i] + matrix[row, col] = binay_encoding[i] + + if graph_util.num_edges(matrix) > MAX_EDGES: + self.benchmark.record_invalid(config, 1, 1, 0) + return failure + + labeling = [config["op_node_%d" % i] for i in range(5)] + labeling = ['input'] + list(labeling) + ['output'] + model_spec = api.ModelSpec(matrix, labeling) + try: + data = self.benchmark.dataset.query(model_spec, epochs=budget) + except api.OutOfDomainError: + self.benchmark.record_invalid(config, 1, 1, 0) + return failure + + self.benchmark.record_valid(config, data, model_spec) + + # We dont need this field. + data.pop('module_adjacency') + + return data diff --git a/hpolib/benchmarks/nas/nasbench_201.py b/hpobench/benchmarks/nas/nasbench_201.py similarity index 88% rename from hpolib/benchmarks/nas/nasbench_201.py rename to hpobench/benchmarks/nas/nasbench_201.py index 92443bb8..dcd0f0ee 100644 --- a/hpolib/benchmarks/nas/nasbench_201.py +++ b/hpobench/benchmarks/nas/nasbench_201.py @@ -13,12 +13,12 @@ 1. Clone and install ==================== -Since the data is downloaded automatically, you dont have to do anything but installing the hpolib. +Since the data is downloaded automatically, you dont have to do anything but installing the hpobench. Recommend: ``Python >= 3.6.0`` ``` -cd /path/to/HPOlib3 +cd /path/to/HPOBench pip install . ``` @@ -32,11 +32,11 @@ import ConfigSpace as CS import numpy as np -import hpolib.util.rng_helper as rng_helper -from hpolib.abstract_benchmark import AbstractBenchmark -from hpolib.util.data_manager import NASBench_201Data +import hpobench.util.rng_helper as rng_helper +from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.util.data_manager import NASBench_201Data -__version__ = '0.0.1' +__version__ = '0.0.2' MAX_NODES = 4 logger = logging.getLogger('NASBENCH201') @@ -152,9 +152,9 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], ---------- configuration fidelity: Dict, None - epoch: int - Values: [0, 199] + epoch: int - Values: [1, 200] Number of epochs an architecture was trained. - Note: the number of epoch is 0 indexed! (Results after the first epoch: epoch = 0) + Note: the number of epoch is 1 indexed! (Results after the first epoch: epoch = 1) Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None @@ -193,7 +193,6 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], fidelity : Dict used fidelities in this evaluation """ - # Check if the data set seeds are valid assert isinstance(data_seed, List) or isinstance(data_seed, Tuple) or isinstance(data_seed, int), \ f'data seed has unknown data type {type(data_seed)}, but should be tuple or int (777,888,999)' @@ -212,47 +211,56 @@ def objective_function(self, configuration: Union[CS.Configuration, Dict], structure = self.config_to_structure(configuration) structure_str = structure.tostr() - epoch = fidelity['epoch'] + epoch = fidelity['epoch'] - 1 train_accuracies = [self.data[(seed, 'train_acc1es')][structure_str][epoch] for seed in data_seed] train_losses = [self.data[(seed, 'train_losses')][structure_str][epoch] for seed in data_seed] train_times = [np.sum(self.data[(seed, 'train_times')][structure_str][:epoch + 1]) for seed in data_seed] - eval_accuracies = [self.data[(seed, 'eval_acc1es')][structure_str][epoch] for seed in data_seed] - eval_losses = [self.data[(seed, 'eval_losses')][structure_str][epoch] for seed in data_seed] - eval_times = [np.sum(self.data[(seed, 'eval_times')][structure_str][:epoch + 1]) for seed in data_seed] + valid_accuracies = [self.data[(seed, 'valid_acc1es')][structure_str][epoch] for seed in data_seed] + valid_losses = [self.data[(seed, 'valid_losses')][structure_str][epoch] for seed in data_seed] + valid_times = [np.sum(self.data[(seed, 'valid_times')][structure_str][:epoch + 1]) for seed in data_seed] + + # There is a single value for the eval data per seed. (only epoch 200) + test_accuracies = [self.data[(seed, 'test_acc1es')][structure_str] for seed in data_seed] + test_losses = [self.data[(seed, 'test_losses')][structure_str] for seed in data_seed] + test_times = [np.sum(self.data[(seed, 'test_times')][structure_str]) for seed in data_seed] - return {'function_value': float(100 - np.mean(train_accuracies)), - 'cost': float(np.sum(train_times)), + return {'function_value': float(100 - np.mean(valid_accuracies)), + 'cost': float(np.sum(valid_times) + np.sum(train_times)), 'info': {'train_precision': float(100 - np.mean(train_accuracies)), 'train_losses': float(np.mean(train_losses)), 'train_cost': float(np.sum(train_times)), - 'eval_precision': float(100 - np.mean(eval_accuracies)), - 'eval_losses': float(np.mean(eval_losses)), - 'eval_cost': float(np.sum(train_times)) + float(np.sum(eval_times)), + 'valid_precision': float(100 - np.mean(valid_accuracies)), + 'valid_losses': float(np.mean(valid_losses)), + 'valid_cost': float(np.sum(valid_times) + np.sum(train_times)), + 'test_precision': float(100 - np.mean(test_accuracies)), + 'test_losses': float(np.mean(test_losses)), + 'test_cost': float(np.sum(train_times)) + float(np.sum(test_times)), 'fidelity': fidelity } } @AbstractBenchmark._configuration_as_dict @AbstractBenchmark._check_configuration + @AbstractBenchmark._check_fidelity def objective_function_test(self, configuration: Union[CS.Configuration, Dict], fidelity: Union[Dict, None] = None, rng: Union[np.random.RandomState, int, None] = None, **kwargs) -> Dict: """ - Get the validated results from the NASBench201. Runs a given configuration on the largest budget (here: 199). + Get the validated results from the NASBench201. Runs a given configuration on the largest budget (here: 200). The test function uses all data set seeds (777, 888, 999). - See also :py:meth:`~hpolib.benchmarks.nas.nasbench_201.objective_function` + See also :py:meth:`~hpobench.benchmarks.nas.nasbench_201.objective_function` Parameters ---------- configuration fidelity: Dict, None - epoch: int - Values: [0, 199] + epoch: int - Values: [1, 200] Number of epochs an architecture was trained. - Note: the number of epoch is 0 indexed! (Results after the first epoch: epoch = 0) + Note: the number of epoch is 1 indexed. (Results after the first epoch: epoch = 1) Fidelity parameters, check get_fidelity_space(). Uses default (max) value if None. rng : np.random.RandomState, int, None @@ -281,10 +289,13 @@ def objective_function_test(self, configuration: Union[CS.Configuration, Dict], # The result dict should contain already all necessary information -> Just swap the function value from valid # to test and the corresponding time cost - result = self.objective_function(configuration=configuration, fidelity=fidelity, data_seed=(777, 888, 999), + assert fidelity['epoch'] == 200, 'Only test data for the 200. epoch is available. ' + + result = self.objective_function(configuration=configuration, fidelity=fidelity, + data_seed=(777, 888, 999), rng=rng, **kwargs) - result['function_value'] = result['info']['eval_precision'] - result['cost'] = result['info']['eval_cost'] + result['function_value'] = result['info']['test_precision'] + result['cost'] = result['info']['test_cost'] return result @staticmethod @@ -364,7 +375,7 @@ def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: fidel_space = CS.ConfigurationSpace(seed=seed) fidel_space.add_hyperparameters([ - CS.UniformIntegerHyperparameter('epoch', lower=0, upper=199, default_value=199) + CS.UniformIntegerHyperparameter('epoch', lower=1, upper=200, default_value=200) ]) return fidel_space diff --git a/hpolib/benchmarks/nas/tabular_benchmarks.py b/hpobench/benchmarks/nas/tabular_benchmarks.py similarity index 94% rename from hpolib/benchmarks/nas/tabular_benchmarks.py rename to hpobench/benchmarks/nas/tabular_benchmarks.py index b38829bc..978bf9cb 100644 --- a/hpolib/benchmarks/nas/tabular_benchmarks.py +++ b/hpobench/benchmarks/nas/tabular_benchmarks.py @@ -19,7 +19,7 @@ 2. Clone and install ==================== ``` -cd /path/to/HPOlib2 +cd /path/to/HPOBench pip install .[tabular_benchmarks] pip install git+https://github.com/google-research/nasbench.git@master @@ -37,10 +37,10 @@ import numpy as np from tabular_benchmarks.fcnet_benchmark import FCNetBenchmark -import hpolib.util.rng_helper as rng_helper -from hpolib.abstract_benchmark import AbstractBenchmark +import hpobench.util.rng_helper as rng_helper +from hpobench.abstract_benchmark import AbstractBenchmark -__version__ = '0.0.1' +__version__ = '0.0.2' class FCNetBaseBenchmark(AbstractBenchmark): @@ -153,8 +153,14 @@ def objective_function_test(self, configuration: Union[Dict, CS.Configuration], runtime_per_run fidelity : used fidelities in this evaluation """ - return self.objective_function(configuration=configuration, fidelity=fidelity, rng=rng, - **kwargs) + default_fidelity = self.get_fidelity_space().get_default_configuration().get_dictionary() + assert fidelity == default_fidelity, 'Test function works only on the highest budget.' + result = self.benchmark.objective_function_test(configuration) + + return {'function_value': float(result[0]), + 'cost': float(result[1]), + 'info': {'fidelity': fidelity}, + } @staticmethod def get_configuration_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: diff --git a/hpolib/benchmarks/rl/__init__.py b/hpobench/benchmarks/rl/__init__.py similarity index 100% rename from hpolib/benchmarks/rl/__init__.py rename to hpobench/benchmarks/rl/__init__.py diff --git a/hpolib/benchmarks/rl/cartpole.py b/hpobench/benchmarks/rl/cartpole.py similarity index 95% rename from hpolib/benchmarks/rl/cartpole.py rename to hpobench/benchmarks/rl/cartpole.py index f0c1f454..abb2ebf6 100644 --- a/hpolib/benchmarks/rl/cartpole.py +++ b/hpobench/benchmarks/rl/cartpole.py @@ -9,8 +9,8 @@ from tensorforce.contrib.openai_gym import OpenAIGym from tensorforce.execution import Runner -from hpolib.abstract_benchmark import AbstractBenchmark -from hpolib.util import rng_helper +from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.util import rng_helper __version__ = '0.0.1' @@ -220,7 +220,13 @@ def objective_function_test(self, configuration: Union[Dict, CS.Configuration], @staticmethod def get_meta_information() -> Dict: return {'name': 'Cartpole', - 'references': ['http://proceedings.mlr.press/v80/falkner18a.html'], + 'references': ['@InProceedings{falkner-icml-18,' + 'title = {{BOHB}: Robust and Efficient Hyperparameter Optimization at Scale},' + 'url = http://proceedings.mlr.press/v80/falkner18a.html' + 'author = {Falkner, Stefan and Klein, Aaron and Hutter, Frank}, ' + 'booktitle = {Proceedings of the 35th International Conference on Machine Learning},' + 'pages = {1436 - -1445},' + 'year = {2018}}'], 'note': 'This benchmark is not deterministic, since the gym environment is not deterministic.' ' Also, often the benchmark is already converged after 1000 episodes.' ' Increasing the budget \"max_episodes\" may lead to the same results.'} diff --git a/hpolib/benchmarks/rl/learna_benchmark.py b/hpobench/benchmarks/rl/learna_benchmark.py similarity index 98% rename from hpolib/benchmarks/rl/learna_benchmark.py rename to hpobench/benchmarks/rl/learna_benchmark.py index 8a5744ec..75831a50 100644 --- a/hpolib/benchmarks/rl/learna_benchmark.py +++ b/hpobench/benchmarks/rl/learna_benchmark.py @@ -8,9 +8,9 @@ Prerequisites: ============== -Conda environment in which the HPOlib is installed (pip install .). Activate your environment. +Conda environment in which the HPOBench is installed (pip install .). Activate your environment. ``` -conda activate +conda activate ``` 1. Clone from github: @@ -27,7 +27,7 @@ Install the necessary requirements defined in the environment.yml, which is in the downloaded learna repository. ``` -conda env update --name --file ./environment.yml +conda env update --name --file ./environment.yml ``` 3. Get the data: @@ -63,9 +63,9 @@ from learna.learna.environment import RnaDesignEnvironmentConfig from learna.learna.learn_to_design_rna import learn_to_design_rna -import hpolib.config -from hpolib.abstract_benchmark import AbstractBenchmark -from hpolib.util import rng_helper +import hpobench.config +from hpobench.abstract_benchmark import AbstractBenchmark +from hpobench.util import rng_helper __version__ = '0.0.1' @@ -387,7 +387,7 @@ def objective_function_test(self, configuration: Union[Dict, CS.Configuration], class MetaLearna(BaseLearna): def __init__(self, data_path: Union[str, Path]): super(MetaLearna, self).__init__(data_path) - self.config = hpolib.config.config_file + self.config = hpobench.config.config_file @staticmethod def get_fidelity_space(seed: Union[int, None] = None) -> CS.ConfigurationSpace: diff --git a/hpolib/config.py b/hpobench/config.py similarity index 91% rename from hpolib/config.py rename to hpobench/config.py index a35358ee..82746eb1 100644 --- a/hpolib/config.py +++ b/hpobench/config.py @@ -7,10 +7,10 @@ from typing import Union, Any -class HPOlibConfig: +class HPOBenchConfig: def __init__(self): - """ Class holding the configuration for the HPOlib. When initialized, it reads (or creates) + """ Class holding the configuration for the HPOBench. When initialized, it reads (or creates) the config file and data directory accordingly. Parameters: @@ -19,13 +19,13 @@ def __init__(self): config_file : str Path to config file """ - self.logger = logging.getLogger('HPOlibConfig') + self.logger = logging.getLogger('HPOBenchConfig') # According to https://github.com/openml/openml-python/issues/884, try to set default directories. - self.config_base_dir = Path(os.environ.get('XDG_CONFIG_HOME', '~/.config/hpolib2')).expanduser() - self.config_file = self.config_base_dir / '.hpolibrc' - self.cache_dir = Path(os.environ.get('XDG_CACHE_HOME', '~/.cache/hpolib2')).expanduser() - self.global_data_dir = Path(os.environ.get('XDG_DATA_HOME', '~/.local/share/hpolib2')).expanduser() + self.config_base_dir = Path(os.environ.get('XDG_CONFIG_HOME', '~/.config/hpobench')).expanduser() + self.config_file = self.config_base_dir / '.hpobenchrc' + self.cache_dir = Path(os.environ.get('XDG_CACHE_HOME', '~/.cache/hpobench')).expanduser() + self.global_data_dir = Path(os.environ.get('XDG_DATA_HOME', '~/.local/share/hpobench')).expanduser() self.config = None self.data_dir = None @@ -37,7 +37,7 @@ def __init__(self): self.defaults = {'verbosity': 0, 'data_dir': self.config_base_dir, 'socket_dir': self.socket_dir, - 'container_dir': self.cache_dir / f'hpolib2-{os.getuid()}', + 'container_dir': self.cache_dir / f'hpobench-{os.getuid()}', # Find all hosted container on: # https://cloud.sylabs.io/library/phmueller/automl 'container_source': 'library://phmueller/automl', @@ -134,5 +134,5 @@ def __check_dir(self, path: Path): raise e -config_file = HPOlibConfig() -__all__ = ['config_file', 'HPOlibConfig'] +config_file = HPOBenchConfig() +__all__ = ['config_file', 'HPOBenchConfig'] diff --git a/hpolib/container/__init__.py b/hpobench/container/__init__.py similarity index 100% rename from hpolib/container/__init__.py rename to hpobench/container/__init__.py diff --git a/hpolib/container/benchmarks/__init__.py b/hpobench/container/benchmarks/__init__.py similarity index 100% rename from hpolib/container/benchmarks/__init__.py rename to hpobench/container/benchmarks/__init__.py diff --git a/hpolib/container/benchmarks/ml/__init__.py b/hpobench/container/benchmarks/ml/__init__.py similarity index 100% rename from hpolib/container/benchmarks/ml/__init__.py rename to hpobench/container/benchmarks/ml/__init__.py diff --git a/hpobench/container/benchmarks/ml/pybnn.py b/hpobench/container/benchmarks/ml/pybnn.py new file mode 100644 index 00000000..d32f30a2 --- /dev/null +++ b/hpobench/container/benchmarks/ml/pybnn.py @@ -0,0 +1,34 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +""" Benchmark for the pybnn Benchmark from hpobench/benchmarks/ml/pybnn.py """ + +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient + + +class BNNOnToyFunction(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnToyFunction') + kwargs['container_name'] = kwargs.get('container_name', 'pybnn') + super(BNNOnToyFunction, self).__init__(**kwargs) + + +class BNNOnBostonHousing(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnBostonHousing') + kwargs['container_name'] = kwargs.get('container_name', 'pybnn') + super(BNNOnBostonHousing, self).__init__(**kwargs) + + +class BNNOnProteinStructure(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnProteinStructure') + kwargs['container_name'] = kwargs.get('container_name', 'pybnn') + super(BNNOnProteinStructure, self).__init__(**kwargs) + + +class BNNOnYearPrediction(AbstractBenchmarkClient): + def __init__(self, **kwargs): + kwargs['benchmark_name'] = kwargs.get('benchmark_name', 'BNNOnYearPrediction') + kwargs['container_name'] = kwargs.get('container_name', 'pybnn') + super(BNNOnYearPrediction, self).__init__(**kwargs) diff --git a/hpolib/container/benchmarks/ml/svm_benchmark.py b/hpobench/container/benchmarks/ml/svm_benchmark.py similarity index 68% rename from hpolib/container/benchmarks/ml/svm_benchmark.py rename to hpobench/container/benchmarks/ml/svm_benchmark.py index 198b2f26..ff5246b0 100644 --- a/hpolib/container/benchmarks/ml/svm_benchmark.py +++ b/hpobench/container/benchmarks/ml/svm_benchmark.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the XGBoost Benchmark from hpolib/benchmarks/ml/xgboost_benchmark """ +""" Benchmark for the XGBoost Benchmark from hpobench/benchmarks/ml/xgboost_benchmark """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class SupportVectorMachine(AbstractBenchmarkClient): diff --git a/hpolib/container/benchmarks/ml/xgboost_benchmark.py b/hpobench/container/benchmarks/ml/xgboost_benchmark.py similarity index 67% rename from hpolib/container/benchmarks/ml/xgboost_benchmark.py rename to hpobench/container/benchmarks/ml/xgboost_benchmark.py index d10af09f..e600dd0c 100644 --- a/hpolib/container/benchmarks/ml/xgboost_benchmark.py +++ b/hpobench/container/benchmarks/ml/xgboost_benchmark.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the XGBoost Benchmark from hpolib/benchmarks/ml/xgboost_benchmark """ +""" Benchmark for the XGBoost Benchmark from hpobench/benchmarks/ml/xgboost_benchmark """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class XGBoostBenchmark(AbstractBenchmarkClient): diff --git a/hpolib/container/benchmarks/nas/__init__.py b/hpobench/container/benchmarks/nas/__init__.py similarity index 100% rename from hpolib/container/benchmarks/nas/__init__.py rename to hpobench/container/benchmarks/nas/__init__.py diff --git a/hpolib/container/benchmarks/nas/nasbench_101.py b/hpobench/container/benchmarks/nas/nasbench_101.py similarity index 86% rename from hpolib/container/benchmarks/nas/nasbench_101.py rename to hpobench/container/benchmarks/nas/nasbench_101.py index 9a851d32..a436617b 100644 --- a/hpolib/container/benchmarks/nas/nasbench_101.py +++ b/hpobench/container/benchmarks/nas/nasbench_101.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the Tabular Benchmark from hpolib/benchmarks/nas/nasbench_101.py """ +""" Benchmark for the Tabular Benchmark from hpobench/benchmarks/nas/nasbench_101.py """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class NASCifar10ABenchmark(AbstractBenchmarkClient): diff --git a/hpolib/container/benchmarks/nas/nasbench_201.py b/hpobench/container/benchmarks/nas/nasbench_201.py similarity index 89% rename from hpolib/container/benchmarks/nas/nasbench_201.py rename to hpobench/container/benchmarks/nas/nasbench_201.py index befc7473..5f6d96f3 100644 --- a/hpolib/container/benchmarks/nas/nasbench_201.py +++ b/hpobench/container/benchmarks/nas/nasbench_201.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the NasBench201 Benchmark from hpolib/benchmarks/nas/nasbench_201.py """ +""" Benchmark for the NasBench201 Benchmark from hpobench/benchmarks/nas/nasbench_201.py """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class Cifar10NasBench201Benchmark(AbstractBenchmarkClient): diff --git a/hpolib/container/benchmarks/nas/tabular_benchmarks.py b/hpobench/container/benchmarks/nas/tabular_benchmarks.py similarity index 90% rename from hpolib/container/benchmarks/nas/tabular_benchmarks.py rename to hpobench/container/benchmarks/nas/tabular_benchmarks.py index d0f28d2f..ae59e143 100644 --- a/hpolib/container/benchmarks/nas/tabular_benchmarks.py +++ b/hpobench/container/benchmarks/nas/tabular_benchmarks.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the Tabular Benchmark from hpolib/benchmarks/nas/tabular_benchmarks.py """ +""" Benchmark for the Tabular Benchmark from hpobench/benchmarks/nas/tabular_benchmarks.py """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class SliceLocalizationBenchmark(AbstractBenchmarkClient): diff --git a/hpolib/container/benchmarks/rl/__init__.py b/hpobench/container/benchmarks/rl/__init__.py similarity index 100% rename from hpolib/container/benchmarks/rl/__init__.py rename to hpobench/container/benchmarks/rl/__init__.py diff --git a/hpolib/container/benchmarks/rl/cartpole.py b/hpobench/container/benchmarks/rl/cartpole.py similarity index 78% rename from hpolib/container/benchmarks/rl/cartpole.py rename to hpobench/container/benchmarks/rl/cartpole.py index 11c08b85..cad214ed 100644 --- a/hpolib/container/benchmarks/rl/cartpole.py +++ b/hpobench/container/benchmarks/rl/cartpole.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the Cartpole Benchmark from hpolib/benchmarks/rl/cartpole_hyperband.py """ +""" Benchmark for the Cartpole Benchmark from hpobench/benchmarks/rl/cartpole_hyperband.py """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class CartpoleReduced(AbstractBenchmarkClient): diff --git a/hpolib/container/benchmarks/rl/learna_benchmark.py b/hpobench/container/benchmarks/rl/learna_benchmark.py similarity index 80% rename from hpolib/container/benchmarks/rl/learna_benchmark.py rename to hpobench/container/benchmarks/rl/learna_benchmark.py index 6d1681cb..b70432fd 100644 --- a/hpolib/container/benchmarks/rl/learna_benchmark.py +++ b/hpobench/container/benchmarks/rl/learna_benchmark.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -""" Benchmark for the learna benchmark from hpolib/benchmarks/rl/learna_benchmarks.py """ +""" Benchmark for the learna benchmark from hpobench/benchmarks/rl/learna_benchmarks.py """ -from hpolib.container.client_abstract_benchmark import AbstractBenchmarkClient +from hpobench.container.client_abstract_benchmark import AbstractBenchmarkClient class Learna(AbstractBenchmarkClient): diff --git a/hpolib/container/client_abstract_benchmark.py b/hpobench/container/client_abstract_benchmark.py similarity index 96% rename from hpolib/container/client_abstract_benchmark.py rename to hpobench/container/client_abstract_benchmark.py index 00ba9307..fd81b4e9 100644 --- a/hpolib/container/client_abstract_benchmark.py +++ b/hpobench/container/client_abstract_benchmark.py @@ -9,10 +9,10 @@ To reduce download traffic, firstly, it checks if the container is already downloaded. The container source as well as the path, where it should be stored, -are defined in the ~/.hpolibrc - file. +are defined in the ~/.hpobenchrc - file. The name of the container (``container_name``) is defined either in its belonging -container-benchmark definition. (hpolib/container// or via ``container_name``. +container-benchmark definition. (hpobench/container// or via ``container_name``. """ import os import abc @@ -33,10 +33,10 @@ from ConfigSpace.read_and_write import json as csjson from oslo_concurrency import lockutils -import hpolib.config +import hpobench.config -# Read in the verbosity level from the environment variable HPOLIB_DEBUG -log_level_str = os.environ.get('HPOLIB_DEBUG', 'false') +# Read in the verbosity level from the environment variable HPOBENCH_DEBUG +log_level_str = os.environ.get('HPOBENCH_DEBUG', 'false') log_level = logging.DEBUG if log_level_str == 'true' else logging.INFO root = logging.getLogger() @@ -71,7 +71,7 @@ def _setup(self, benchmark_name: str, container_name: str, container_source: Opt """ Initialization of the benchmark using container. This setup function downloads the container from a defined source. The source is defined either in the - .hpolibrc or in the its benchmark definition (hpolib/container/benchmarks//). If an container + .hpobenchrc or in the its benchmark definition (hpobench/container/benchmarks//). If an container is already locally available, the local container is used. Then, the container is started and a connection between the container and the client is established. @@ -79,7 +79,7 @@ def _setup(self, benchmark_name: str, container_name: str, container_source: Opt ---------- benchmark_name: str Class name of the benchmark to use. For example XGBoostBenchmark. This value is defined in the benchmark - definition (hpolib/container/benchmarks//) + definition (hpobench/container/benchmarks//) container_source : Optional[str] Path to the container. Either local path or url to a hosting platform, e.g. singularity hub. @@ -94,7 +94,7 @@ def _setup(self, benchmark_name: str, container_name: str, container_source: Opt (Not tested) """ # Create unique ID - self.config = hpolib.config.config_file + self.config = hpobench.config.config_file # We can point to a different container source. See below. container_source = container_source or self.config.container_source @@ -144,7 +144,7 @@ def download_container(container_dir, container_name, container_source): f'{container_dir / container_name}' logger.debug('Image found on the local file system.') - bind_options = f'--bind /var/lib/,{self.config.global_data_dir}:/var/lib/' + bind_options = f'--bind /var/lib/,{self.config.global_data_dir}:/var/lib/,{self.config.container_dir}' if self.config.global_data_dir != self.config.data_dir: bind_options += f',{self.config.data_dir}:/var/lib/' bind_options += ' ' @@ -152,7 +152,7 @@ def download_container(container_dir, container_name, container_source): gpu_opt = '--nv ' if gpu else '' # Option for enabling GPU support container_options = f'{container_dir / container_name}' - log_str = f'SINGULARITYENV_HPOLIB_DEBUG={log_level_str}' + log_str = f'SINGULARITYENV_HPOBENCH_DEBUG={log_level_str}' cmd = f'{log_str} singularity instance start {bind_options}{gpu_opt}{container_options} {self.socket_id}' logger.debug(cmd) diff --git a/hpolib/container/recipes/Singularity.template b/hpobench/container/recipes/Singularity.template similarity index 73% rename from hpolib/container/recipes/Singularity.template rename to hpobench/container/recipes/Singularity.template index 702a7d0f..538dddfb 100644 --- a/hpolib/container/recipes/Singularity.template +++ b/hpobench/container/recipes/Singularity.template @@ -18,19 +18,19 @@ VERSION v0.0.1 cd /home \ && echo "Here you can install everything you need, e.g. dependencies not available on pypi" \ - && echo "Next, we clone and install HPOlib2" \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && echo "Next, we clone and install HPOBench" \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && echo "Please never push a recipe that checks out any other branch than development or master" \ && git checkout development \ && echo "Here you can install extra requirements additional to singularity" \ && pip install . \ && echo "Please don't touch the following lines" && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ echo "Finally, please change the benchmark in the runscript to point to your benchmark" %runscript - python -s /home/HPOlib2/hpolib/container/server_abstract_benchmark.py .new_benchmark_file $@ \ No newline at end of file + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py .new_benchmark_file $@ \ No newline at end of file diff --git a/hpobench/container/recipes/ml/Singularity.PyBNN b/hpobench/container/recipes/ml/Singularity.PyBNN new file mode 100644 index 00000000..890de220 --- /dev/null +++ b/hpobench/container/recipes/ml/Singularity.PyBNN @@ -0,0 +1,21 @@ +Bootstrap: docker +From: python:3.7-slim + +%labels +MAINTAINER muelleph@cs.uni-freiburg.de +VERSION v0.0.1 + +%post + apt update -y \ + && apt install build-essential git -y \ + && cd /home \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ + && git checkout development \ + && pip install .[pybnn] \ + && cd / \ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ + +%runscript + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml.pybnn $@ diff --git a/hpolib/container/recipes/ml/Singularity.SupportVectorMachine b/hpobench/container/recipes/ml/Singularity.SupportVectorMachine similarity index 52% rename from hpolib/container/recipes/ml/Singularity.SupportVectorMachine rename to hpobench/container/recipes/ml/Singularity.SupportVectorMachine index 3ca6c876..fe2e6ddd 100644 --- a/hpolib/container/recipes/ml/Singularity.SupportVectorMachine +++ b/hpobench/container/recipes/ml/Singularity.SupportVectorMachine @@ -9,13 +9,13 @@ VERSION v0.0.1 apt update -y \ && apt install build-essential git -y \ && cd /home \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && pip install .[svm] \ && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ %runscript - python -s /home/HPOlib2/hpolib/container/server_abstract_benchmark.py ml.svm_benchmark $@ + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml.svm_benchmark $@ diff --git a/hpolib/container/recipes/ml/Singularity.XGBoostBenchmark b/hpobench/container/recipes/ml/Singularity.XGBoostBenchmark similarity index 52% rename from hpolib/container/recipes/ml/Singularity.XGBoostBenchmark rename to hpobench/container/recipes/ml/Singularity.XGBoostBenchmark index c089fd50..e763156a 100644 --- a/hpolib/container/recipes/ml/Singularity.XGBoostBenchmark +++ b/hpobench/container/recipes/ml/Singularity.XGBoostBenchmark @@ -9,13 +9,13 @@ VERSION v0.0.1 apt update -y \ && apt install build-essential git -y \ && cd /home \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && pip install .[xgboost] \ && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ %runscript - python -s /home/HPOlib2/hpolib/container/server_abstract_benchmark.py ml.xgboost_benchmark $@ + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py ml.xgboost_benchmark $@ diff --git a/hpolib/container/recipes/nas/Singularity.TabularBenchmarks b/hpobench/container/recipes/nas/Singularity.TabularBenchmarks similarity index 69% rename from hpolib/container/recipes/nas/Singularity.TabularBenchmarks rename to hpobench/container/recipes/nas/Singularity.TabularBenchmarks index 64970ee0..6736b011 100644 --- a/hpolib/container/recipes/nas/Singularity.TabularBenchmarks +++ b/hpobench/container/recipes/nas/Singularity.TabularBenchmarks @@ -16,13 +16,13 @@ VERSION v0.0.1 cd /home \ && pip install git+https://github.com/google-research/nasbench.git@master \ && pip install git+https://github.com/automl/nas_benchmarks.git \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && pip install .[tabular_benchmarks] \ && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ %runscript - python -s /home/HPOlib2/hpolib/container/server_abstract_benchmark.py nas.tabular_benchmarks $@ + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.tabular_benchmarks $@ diff --git a/hpolib/container/recipes/nas/Singularity.nasbench_101 b/hpobench/container/recipes/nas/Singularity.nasbench_101 similarity index 68% rename from hpolib/container/recipes/nas/Singularity.nasbench_101 rename to hpobench/container/recipes/nas/Singularity.nasbench_101 index 3033d0a4..a976b502 100644 --- a/hpolib/container/recipes/nas/Singularity.nasbench_101 +++ b/hpobench/container/recipes/nas/Singularity.nasbench_101 @@ -16,13 +16,13 @@ VERSION v0.0.1 cd /home \ && pip install git+https://github.com/google-research/nasbench.git@master \ && pip install git+https://github.com/automl/nas_benchmarks.git \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && pip install .[nasbench_101] \ && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ %runscript - python -s /home/HPOlib2/hpolib/container/server_abstract_benchmark.py nas.nasbench_101 $@ + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.nasbench_101 $@ diff --git a/hpolib/container/recipes/nas/Singularity.nasbench_201 b/hpobench/container/recipes/nas/Singularity.nasbench_201 similarity index 52% rename from hpolib/container/recipes/nas/Singularity.nasbench_201 rename to hpobench/container/recipes/nas/Singularity.nasbench_201 index 55767424..5e503e0c 100644 --- a/hpolib/container/recipes/nas/Singularity.nasbench_201 +++ b/hpobench/container/recipes/nas/Singularity.nasbench_201 @@ -10,13 +10,13 @@ VERSION v0.0.1 apt install build-essential git wget curl -y cd /home \ - && git clone https://github.com/automl/HPOlib3.git \ - && cd HPOlib3 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && pip install . \ && cd / \ - && mkdir /var/lib/hpolib3/ \ - && chmod -R 777 /var/lib/hpolib3/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ %runscript - python -s /home/HPOlib3/hpolib/container/server_abstract_benchmark.py nas.nasbench_201 $@ + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py nas.nasbench_201 $@ diff --git a/hpolib/container/recipes/rl/Singularity.Cartpole b/hpobench/container/recipes/rl/Singularity.Cartpole similarity index 56% rename from hpolib/container/recipes/rl/Singularity.Cartpole rename to hpobench/container/recipes/rl/Singularity.Cartpole index 569b76fb..a059bdbd 100644 --- a/hpolib/container/recipes/rl/Singularity.Cartpole +++ b/hpobench/container/recipes/rl/Singularity.Cartpole @@ -11,13 +11,13 @@ VERSION v0.0.1 pip install numpy==1.18.1 cython==0.29.14 cd /home \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && pip install .[cartpole] \ && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/bench/hpobench/ %runscript - python -s /home/HPOlib2/hpolib/container/server_abstract_benchmark.py rl.cartpole $@ + python -s /home/HPOBench/hpobench/container/server_abstract_benchmark.py rl.cartpole $@ diff --git a/hpolib/container/recipes/rl/Singularity.learnaBenchmark b/hpobench/container/recipes/rl/Singularity.learnaBenchmark similarity index 84% rename from hpolib/container/recipes/rl/Singularity.learnaBenchmark rename to hpobench/container/recipes/rl/Singularity.learnaBenchmark index 42fcb1ad..db117b91 100644 --- a/hpolib/container/recipes/rl/Singularity.learnaBenchmark +++ b/hpobench/container/recipes/rl/Singularity.learnaBenchmark @@ -24,14 +24,14 @@ VERSION v0.0.1 && rm -rf data/rfam_learn \ && chmod -R 755 data/ \ && cd /home \ - && git clone https://github.com/automl/HPOlib2.git \ - && cd HPOlib2 \ + && git clone https://github.com/automl/HPOBench.git \ + && cd HPOBench \ && git checkout master \ && ../learna/thirdparty/miniconda/miniconda/envs/learna/bin/python -m pip install . \ && cd / \ - && mkdir /var/lib/hpolib2/ \ - && chmod -R 777 /var/lib/hpolib2/ + && mkdir /var/lib/hpobench/ \ + && chmod -R 777 /var/lib/hpobench/ %runscript /home/learna/thirdparty/miniconda/miniconda/envs/learna/bin/python -s \ - /home/HPOlib2/hpolib/container/server_abstract_benchmark.py rl.learna_benchmark $@ + /home/HPOBench/hpobench/container/server_abstract_benchmark.py rl.learna_benchmark $@ diff --git a/hpolib/container/server_abstract_benchmark.py b/hpobench/container/server_abstract_benchmark.py similarity index 93% rename from hpolib/container/server_abstract_benchmark.py rename to hpobench/container/server_abstract_benchmark.py index bd4ccb9f..8f7cd95c 100644 --- a/hpolib/container/server_abstract_benchmark.py +++ b/hpobench/container/server_abstract_benchmark.py @@ -19,10 +19,10 @@ import numpy as np from ConfigSpace.read_and_write import json as csjson -from hpolib.config import HPOlibConfig +from hpobench.config import HPOBenchConfig -# Read in the verbosity level from the environment variable HPOLIB_DEBUG -log_level_str = os.environ.get('HPOLIB_DEBUG', 'false') +# Read in the verbosity level from the environment variable HPOBENCH_DEBUG +log_level_str = os.environ.get('HPOBENCH_DEBUG', 'false') log_level = logging.DEBUG if log_level_str == 'true' else logging.INFO console = logging.StreamHandler() @@ -47,7 +47,7 @@ def default(self, obj): class BenchmarkServer: def __init__(self, socket_id): self.pyroRunning = True - config = HPOlibConfig() + config = HPOBenchConfig() self.benchmark = None self.socket_id = socket_id @@ -149,10 +149,10 @@ def shutdown(self): Pyro4.config.REQUIRE_EXPOSE = False parser = argparse.ArgumentParser(prog='server_abstract_benchmark.py', - description='HPOlib2 Container Server', + description='HPOBench Container Server', usage='%(prog)s ') parser.add_argument('importBase', type=str, - help='Relative path to benchmark file in hpolib/benchmarks, e.g. ml.xgboost_benchmark') + help='Relative path to benchmark file in hpobench/benchmarks, e.g. ml.xgboost_benchmark') parser.add_argument('benchmark', type=str, help='Classname of the benchmark, e.g. XGBoostBenchmark') parser.add_argument('socket_id', type=str, @@ -160,5 +160,5 @@ def shutdown(self): args = parser.parse_args() - exec(f"from hpolib.benchmarks.{args.importBase} import {args.benchmark} as Benchmark") + exec(f"from hpobench.benchmarks.{args.importBase} import {args.benchmark} as Benchmark") bp = BenchmarkServer(args.socket_id) diff --git a/hpolib/util/__init__.py b/hpobench/util/__init__.py similarity index 100% rename from hpolib/util/__init__.py rename to hpobench/util/__init__.py diff --git a/hpolib/util/container_utils.py b/hpobench/util/container_utils.py similarity index 62% rename from hpolib/util/container_utils.py rename to hpobench/util/container_utils.py index 7a554275..d204b305 100644 --- a/hpolib/util/container_utils.py +++ b/hpobench/util/container_utils.py @@ -7,17 +7,17 @@ def __reload_module(): The env variable which enables the debug level is read in during the import of the client module. Reloading the module, re-reads the env variable and therefore changes the level. """ - import hpolib.container.client_abstract_benchmark as client + import hpobench.container.client_abstract_benchmark as client importlib.reload(client) def enable_container_debug(): - """ Sets the environment variable "HPOLIB_DEBUG" to true. The container checks this variable and if set to true, + """ Sets the environment variable "HPOBENCH_DEBUG" to true. The container checks this variable and if set to true, enables debugging on the container side. """ - os.environ['HPOLIB_DEBUG'] = 'true' + os.environ['HPOBENCH_DEBUG'] = 'true' __reload_module() def disable_container_debug(): - os.environ['HPOLIB_DEBUG'] = 'false' + os.environ['HPOBENCH_DEBUG'] = 'false' __reload_module() diff --git a/hpolib/util/data_manager.py b/hpobench/util/data_manager.py similarity index 61% rename from hpolib/util/data_manager.py rename to hpobench/util/data_manager.py index c2ca2d4d..aec81a48 100644 --- a/hpolib/util/data_manager.py +++ b/hpobench/util/data_manager.py @@ -6,7 +6,7 @@ splits. For OpenML data sets (defined by task id or similar) please use the -hpolib.util.openml_data_manager. +hpobench.util.openml_data_manager. """ import abc @@ -28,7 +28,7 @@ except ImportError: print("oslo_concurrency not installed, can't download datasets for nasbench201 (not needed for containers)") -import hpolib +import hpobench class DataManager(object, metaclass=abc.ABCMeta): @@ -115,7 +115,7 @@ def __init__(self): super(MNISTData, self).__init__() self._url_source = 'http://yann.lecun.com/exdb/mnist/' - self._save_to = hpolib.config_file.data_dir / "MNIST" + self._save_to = hpobench.config_file.data_dir / "MNIST" self.create_save_directory(self._save_to) @@ -235,7 +235,7 @@ def __init__(self): self._url_source = 'https://www.cs.toronto.edu/~kriz/' \ 'cifar-10-python.tar.gz' - self._save_to = hpolib.config_file.data_dir / "cifar10" + self._save_to = hpobench.config_file.data_dir / "cifar10" self.create_save_directory(self._save_to) @@ -339,7 +339,7 @@ def __init__(self): super(SVHNData, self).__init__() self._url_source = 'http://ufldl.stanford.edu/housenumbers/' - self._save_to = hpolib.config_file.data_dir / "svhn" + self._save_to = hpobench.config_file.data_dir / "svhn" self.n_train_all = 73257 self.n_valid = 6000 @@ -458,8 +458,8 @@ def __init__(self, dataset: str): super(NASBench_201Data, self).__init__() self.files = self.get_files_per_dataset(dataset) - self._save_dir = hpolib.config_file.data_dir / "nasbench_201" - self._url_source = 'https://www.automl.org/wp-content/uploads/2020/08/nasbench_201_data_v1.1.zip' + self._save_dir = hpobench.config_file.data_dir / "nasbench_201" + self._url_source = 'https://www.automl.org/wp-content/uploads/2020/08/nasbench_201_data_v1.2.zip' self.data = {} self.create_save_directory(self._save_dir) @@ -474,7 +474,8 @@ def get_seeds_metrics(): @staticmethod def get_metrics(): return ['train_acc1es', 'train_losses', 'train_times', - 'eval_acc1es', 'eval_times', 'eval_losses'] + 'valid_acc1es', 'valid_times', 'valid_losses', + 'test_acc1es', 'test_times', 'test_losses'] @staticmethod def get_files_per_dataset(dataset): @@ -483,11 +484,11 @@ def get_files_per_dataset(dataset): return files @lockutils.synchronized('not_thread_process_safe', external=True, - lock_path=f'{hpolib.config_file.cache_dir}/lock_nasbench_201_data', delay=0.5) + lock_path=f'{hpobench.config_file.cache_dir}/lock_nasbench_201_data', delay=0.5) def _download(self): # Check if data is already downloaded. If a single file is missing, we have to download the complete zip again. # Use a file lock to ensure that no two processes try to download the same files at the same time. - file_is_missing = not all([(self._save_dir / 'data' / file).exists() for file in self.files]) + file_is_missing = not all([(self._save_dir / file).exists() for file in self.files]) if not file_is_missing: self.logger.debug('NasBench201DataManager: Data already downloaded') @@ -504,7 +505,7 @@ def _load(self) -> Dict: import pickle data = {} for (seed, metric_name), file in zip(NASBench_201Data.get_seeds_metrics(), self.files): - with (self._save_dir / 'data' / file).open('rb') as fh: + with (self._save_dir / file).open('rb') as fh: metric = pickle.load(fh) data[(seed, metric_name)] = metric @@ -520,3 +521,231 @@ def load(self) -> Dict: self.logger.info(f'NasBench201DataManager: Data successfully loaded after {time() - t:.2f}') return self.data + + +class BostonHousingData(HoldoutDataManager): + + def __init__(self): + + super(BostonHousingData, self).__init__() + self.url_source = 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' + self._save_dir = hpobench.config_file.data_dir / "BostonHousing" + self.create_save_directory(self._save_dir) + + def load(self): + """ + Loads BostonHousing from data directory as defined in hpobenchrc.data_directory. + Downloads data if necessary. + + Returns + ------- + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + """ + self.logger.debug('BostonHousingDataManager: Starting to load data') + t = time() + + self._download() + + X_trn, y_trn, X_val, y_val, X_tst, y_tst = self._load() + self.logger.info(f'BostonHousingDataManager: Data successfully loaded after {time() - t:.2f}') + + return X_trn, y_trn, X_val, y_val, X_tst, y_tst + + @lockutils.synchronized('not_thread_process_safe', external=True, + lock_path=f'{hpobench.config_file.cache_dir}/lock_protein_structure_data', delay=0.5) + def _download(self): + """ + Loads data from UCI website + https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data + If necessary downloads data, otherwise loads data from data_directory + """ + # Check if data is already downloaded. + # Use a file lock to ensure that no two processes try to download the same files at the same time. + if (self._save_dir / 'housing.data').exists(): + self.logger.debug('BostonHousingDataManager: Data already downloaded') + else: + self.logger.info(f'BostonHousingDataManager: Start downloading data from {self.url_source} ' + f'to {self._save_dir}') + urlretrieve(self.url_source, self._save_dir / 'housing.data') + + def _load(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """ + Load the data from file and split it into train, test and validation split. + + Returns + ------- + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + """ + data = np.loadtxt(self._save_dir / 'housing.data') + + N = data.shape[0] + + n_train = int(N * 0.6) + n_val = int(N * 0.2) + + X_train, y_train = data[:n_train, :-1], data[:n_train, -1] + X_val, y_val = data[n_train:n_train + n_val, :-1], data[n_train:n_train + n_val, -1] + X_test, y_test = data[n_train + n_val:, :-1], data[n_train + n_val:, -1] + + return X_train, y_train, X_val, y_val, X_test, y_test + + +class ProteinStructureData(HoldoutDataManager): + + def __init__(self): + super(ProteinStructureData, self).__init__() + self.url_source = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00265/CASP.csv' + self._save_dir = hpobench.config_file.data_dir / "ProteinStructure" + self.create_save_directory(self._save_dir) + + def load(self): + """ + Loads Physicochemical Properties of Protein Tertiary Structure Data Set + from data directory as defined in _config.data_directory. + Downloads data if necessary from UCI. + Returns + ------- + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + """ + self.logger.debug('ProteinStructureDataManager: Starting to load data') + t = time() + self._download() + + X_train, y_train, X_val, y_val, X_test, y_test = self._load() + self.logger.info(f'ProteinStructureDataManager: Data successfully loaded after {time() - t:.2f}') + + return X_train, y_train, X_val, y_val, X_test, y_test + + @lockutils.synchronized('not_thread_process_safe', external=True, + lock_path=f'{hpobench.config_file.cache_dir}/lock_protein_structure_data', delay=0.5) + def _download(self): + """ + Loads data from UCI website + https://archive.ics.uci.edu/ml/machine-learning-databases/00265/CASP.csv + If necessary downloads data, otherwise loads data from data_directory + """ + # Check if data is already downloaded. + # Use a file lock to ensure that no two processes try to download the same files at the same time. + if (self._save_dir / 'CASP.csv').exists(): + self.logger.debug('ProteinStructureDataManager: Data already downloaded') + else: + self.logger.info(f'ProteinStructureDataManager: Start downloading data from {self.url_source} ' + f'to {self._save_dir}') + urlretrieve(self.url_source, self._save_dir / 'CASP.csv') + + def _load(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """ + Load the data from file and split it into train, test and validation split. + + Returns + ------- + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + """ + data = np.loadtxt(self._save_dir / 'CASP.csv', delimiter=',', skiprows=1) + + n_train = int(data.shape[0] * 0.6) + n_val = int(data.shape[0] * 0.2) + + # note the target value is the first column for this dataset! + X_train, y_train = data[:n_train, 1:], data[:n_train, 0] + X_val, y_val = data[n_train:n_train + n_val, 1:], data[n_train:n_train + n_val, 0] + X_test, y_test = data[n_train + n_val:, 1:], data[n_train + n_val:, 0] + + return X_train, y_train, X_val, y_val, X_test, y_test + + +class YearPredictionMSDData(HoldoutDataManager): + + def __init__(self): + super(YearPredictionMSDData, self).__init__() + self.url_source = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00203/YearPredictionMSD.txt.zip' + self._save_dir = hpobench.config_file.data_dir / "YearPredictionMSD" + self.create_save_directory(self._save_dir) + + def load(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """ + Loads Physicochemical Properties of Protein Tertiary Structure Data Set from data directory as + defined in _config.data_directory. Downloads data if necessary from UCI. + + Returns + ------- + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + """ + self.logger.debug('YearPredictionMSDDataManager: Starting to load data') + t = time() + + self._download() + + X_trn, y_trn, X_val, y_val, X_tst, y_tst = self._load() + self.logger.info(f'YearPredictionMSDDataManager: Data successfully loaded after {time() - t:.2f}') + + return X_trn, y_trn, X_val, y_val, X_tst, y_tst + + @lockutils.synchronized('not_thread_process_safe', external=True, + lock_path=f'{hpobench.config_file.cache_dir}/lock_year_prediction_data', delay=0.5) + def _download(self): + # Check if data is already downloaded. + # Use a file lock to ensure that no two processes try to download the same files at the same time. + + if (self._save_dir / 'YearPredictionMSD.txt').exists(): + self.logger.debug('YearPredictionMSDDataManager: Data already downloaded') + else: + self.logger.info(f'YearPredictionMSDDataManager: Start downloading data from {self.url_source} ' + f'to {self._save_dir}') + + with urlopen(self.url_source) as zip_archive: + with ZipFile(BytesIO(zip_archive.read())) as zip_file: + zip_file.extractall(self._save_dir) + + def _load(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """ + Load the data from file and split it into train, test and validation split. + + Returns + ------- + X_train: np.ndarray + y_train: np.ndarray + X_val: np.ndarray + y_val: np.ndarray + X_test: np.ndarray + y_test: np.ndarray + """ + + with (self._save_dir / 'YearPredictionMSD.txt').open('r') as fh: + data = np.loadtxt(fh, delimiter=',') + + # Use 70% of the data as train split, 20% as validation split and 10% as test split + n_trn = int(data.shape[0] * 0.7) + n_val = int(data.shape[0] * 0.2) + + # Note the target value is the first column for this dataset! + X_trn, y_trn = data[:n_trn, 1:], data[:n_trn, 0] + X_val, y_val = data[n_trn:n_trn + n_val, 1:], data[n_trn:n_trn + n_val, 0] + X_tst, y_tst = data[n_trn + n_val:, 1:], data[n_trn + n_val:, 0] + + return X_trn, y_trn, X_val, y_val, X_tst, y_tst diff --git a/hpolib/util/dependencies.py b/hpobench/util/dependencies.py similarity index 100% rename from hpolib/util/dependencies.py rename to hpobench/util/dependencies.py diff --git a/hpolib/util/example_utils.py b/hpobench/util/example_utils.py similarity index 100% rename from hpolib/util/example_utils.py rename to hpobench/util/example_utils.py diff --git a/hpolib/util/openml_data_manager.py b/hpobench/util/openml_data_manager.py similarity index 96% rename from hpolib/util/openml_data_manager.py rename to hpobench/util/openml_data_manager.py index 1bc414d3..c77144ed 100644 --- a/hpolib/util/openml_data_manager.py +++ b/hpobench/util/openml_data_manager.py @@ -6,7 +6,7 @@ identifier. It splits the data in train, test and optional validation splits. It can be distinguished between holdout and cross-validation data sets. -For Non-OpenML data sets please use the hpolib.util.data_manager. +For Non-OpenML data sets please use the hpobench.util.data_manager. """ from typing import Tuple, Union, List @@ -24,9 +24,9 @@ print("scikit-learn not installed, can't download datasets (not needed for containers)") -import hpolib -from hpolib.util.data_manager import HoldoutDataManager, CrossvalidationDataManager -from hpolib.util.rng_helper import get_rng +import hpobench +from hpobench.util.data_manager import HoldoutDataManager, CrossvalidationDataManager +from hpobench.util.rng_helper import get_rng def get_openml100_taskids(): @@ -128,7 +128,7 @@ class OpenMLHoldoutDataManager(HoldoutDataManager): def __init__(self, openml_task_id: int, rng: Union[int, np.random.RandomState, None] = None): super(OpenMLHoldoutDataManager, self).__init__() - self._save_to = hpolib.config_file.data_dir / 'OpenML' + self._save_to = hpobench.config_file.data_dir / 'OpenML' self.task_id = openml_task_id self.rng = get_rng(rng=rng) self.name = None @@ -215,7 +215,7 @@ class OpenMLCrossvalidationDataManager(CrossvalidationDataManager): def __init__(self, openml_task_id: int, rng: Union[int, np.random.RandomState, None] = None): super(OpenMLCrossvalidationDataManager, self).__init__() - self._save_to = hpolib.config_file.data_dir / 'OpenML' + self._save_to = hpobench.config_file.data_dir / 'OpenML' self.task_id = openml_task_id self.rng = get_rng(rng=rng) self.name = None diff --git a/hpolib/util/rng_helper.py b/hpobench/util/rng_helper.py similarity index 100% rename from hpolib/util/rng_helper.py rename to hpobench/util/rng_helper.py diff --git a/hpolib/__init__.py b/hpolib/__init__.py deleted file mode 100644 index 078a8be7..00000000 --- a/hpolib/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -__contact__ = "automl.org" - -from hpolib.__version__ import __version__ # noqa: F401 -from hpolib.config import config_file # noqa: F401 diff --git a/hpolib/__version__.py b/hpolib/__version__.py deleted file mode 100644 index 156d6f9a..00000000 --- a/hpolib/__version__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '0.0.4' diff --git a/setup.py b/setup.py index adb0983b..cbfa1145 100644 --- a/setup.py +++ b/setup.py @@ -22,18 +22,18 @@ def read_file(file_name): setuptools.setup( - name='hpolib2', + name='hpobench', author_email='eggenspk@informatik.uni-freiburg.de', description='Benchmark-Suite for Hyperparameter Optimization', long_description=read_file('README.md'), long_description_content_type='text/markdown', license='Apache-2.0', - url='https://www.automl.org/automl/hpolib/', + url='https://www.automl.org/automl/hpobench/', project_urls={ - 'Documentation': 'https://automl.github.io/HPOlib2/', - 'Source Code': 'https://github.com/automl/HPOlib2' + 'Documentation': 'https://automl.github.io/HPOBench/', + 'Source Code': 'https://github.com/automl/HPOBench' }, - version=read_file('hpolib/__version__.py').split()[-1].strip('\''), + version=read_file('hpobench/__version__.py').split()[-1].strip('\''), packages=setuptools.find_packages(exclude=['*.tests', '*.tests.*', 'tests.*', 'tests'],), python_requires='>=3.6, <3.8', diff --git a/tests/test_abstract_benchmark.py b/tests/test_abstract_benchmark.py index 8f2b0a9b..22a26790 100644 --- a/tests/test_abstract_benchmark.py +++ b/tests/test_abstract_benchmark.py @@ -1,6 +1,6 @@ import pytest -from hpolib.abstract_benchmark import AbstractBenchmark +from hpobench.abstract_benchmark import AbstractBenchmark with pytest.raises(NotImplementedError): AbstractBenchmark.get_configuration_space() diff --git a/tests/test_check_configuration.py b/tests/test_check_configuration.py index e8f8038e..33745342 100644 --- a/tests/test_check_configuration.py +++ b/tests/test_check_configuration.py @@ -7,7 +7,7 @@ from ConfigSpace import UniformFloatHyperparameter, UniformIntegerHyperparameter, \ CategoricalHyperparameter -from hpolib.abstract_benchmark import AbstractBenchmark +from hpobench.abstract_benchmark import AbstractBenchmark class TestCheckUnittest(unittest.TestCase): diff --git a/tests/test_container_availbable.py b/tests/test_container_availbable.py new file mode 100644 index 00000000..d39444c8 --- /dev/null +++ b/tests/test_container_availbable.py @@ -0,0 +1,37 @@ +import subprocess +from hpobench import config_file +import logging + + +library = config_file.container_source + + +def search_container(container_name): + out = subprocess.getoutput(f'singularity search {container_name}') + logging.debug(out) + + out = out.split('\n\n') + container_available = any((f'{library}/{container_name}' in line for line in out)) + return container_available + + +def test_availability(): + container_names = ['pybnn', + 'svm_benchmark', + 'xgboost_benchmark', + 'nasbench_101', + 'nasbench_201', + 'tabular_benchmarks', + 'cartpole', + 'learna_benchmark' + ] + + all_available = True + for container in container_names: + container_available = search_container(container) + + if not container_available: + logging.warning(f'Container for {container} is not found in {library}') + all_available = False + + assert all_available, 'Some containers are not online available' diff --git a/tests/test_data_manager.py b/tests/test_data_manager.py index 2219935f..b10d23ec 100644 --- a/tests/test_data_manager.py +++ b/tests/test_data_manager.py @@ -1,12 +1,14 @@ -import pytest -import hpolib -from hpolib.util.data_manager import NASBench_201Data import shutil from multiprocessing import Pool +import pytest + +import hpobench +from hpobench.util.data_manager import NASBench_201Data, YearPredictionMSDData, ProteinStructureData, BostonHousingData + def test_nasbench_201_load_thread_safe(): - shutil.rmtree(hpolib.config_file.data_dir / "nasbench_201", ignore_errors=True) + shutil.rmtree(hpobench.config_file.data_dir / "nasbench_201", ignore_errors=True) function = lambda: NASBench_201Data(dataset='cifar100').load() with Pool(3) as pool: pool.map(function, []) @@ -15,7 +17,7 @@ def test_nasbench_201_load_thread_safe(): def test_nasbench_201_get_files(): files = NASBench_201Data.get_files_per_dataset(dataset='cifar10') - assert len(files) == 18 + assert len(files) == 27 assert all([file.startswith('nb201_cifar10') for file in files]) @@ -23,37 +25,89 @@ def test_nasbench_201_get_metrics(): metrics = NASBench_201Data.get_metrics() assert metrics == ['train_acc1es', 'train_losses', 'train_times', - 'eval_acc1es', 'eval_times', 'eval_losses'] + 'valid_acc1es', 'valid_times', 'valid_losses', + 'test_acc1es', 'test_times', 'test_losses'] def test_nasbench_201_init(): data_manager = NASBench_201Data(dataset='cifar100') - assert len(data_manager.files) == 18 + assert len(data_manager.files) == 27 assert all([file.startswith('nb201_cifar10') for file in data_manager.files]) with pytest.raises(AssertionError): NASBench_201Data(dataset='Non_existing_dataset') - assert data_manager._save_dir == hpolib.config_file.data_dir / "nasbench_201" + assert data_manager._save_dir == hpobench.config_file.data_dir / "nasbench_201" assert data_manager._save_dir.exists() def test_nasbench_201_load(): - shutil.rmtree(hpolib.config_file.data_dir / "nasbench_201", ignore_errors=True) + shutil.rmtree(hpobench.config_file.data_dir / "nasbench_201", ignore_errors=True) data_manager = NASBench_201Data(dataset='cifar100') data = data_manager.load() assert len(data) == len(list(NASBench_201Data.get_seeds_metrics())) assert len(data) == 3 * len(NASBench_201Data.get_metrics()) - assert (hpolib.config_file.data_dir / "nasbench_201").exists() - assert len(list((hpolib.config_file.data_dir / "nasbench_201" / "data").glob('*.pkl'))) == 72 - assert not (hpolib.config_file.data_dir / "nasbench_201_data_v1.1.zip").exists() + assert (hpobench.config_file.data_dir / "nasbench_201").exists() + assert len(list((hpobench.config_file.data_dir / "nasbench_201").glob('*.pkl'))) == 108 + assert not (hpobench.config_file.data_dir / "nasbench_201_data_v1.2.zip").exists() data_manager.data = None data_manager = NASBench_201Data(dataset='cifar100') data = data_manager.load() assert len(data) == 3 * len(NASBench_201Data.get_metrics()) + + +def test_year_prediction_msd_data(): + dm = YearPredictionMSDData() + assert dm.url_source == 'https://archive.ics.uci.edu/ml/machine-learning-databases/00203/YearPredictionMSD.txt.zip' + assert dm._save_dir.exists() + + # First one downloads the data + x_train, y_train, x_valid, y_valid, x_test, y_test = dm.load() + + # second call should check the 'if exists' branch + _ = dm.load() + + # train = 70%, valid = 20%, test = 10% + assert 0 < len(x_test) == len(y_test) + assert len(y_test) < len(x_valid) == len(y_valid) + assert len(y_valid) < len(x_train) == len(y_train) + + +def test_protein_structure_data(): + dm = ProteinStructureData() + assert dm.url_source == 'https://archive.ics.uci.edu/ml/machine-learning-databases/00265/CASP.csv' + assert dm._save_dir.exists() + + # First one downloads the data + x_train, y_train, x_valid, y_valid, x_test, y_test = dm.load() + + # second call should check the 'if exists' branch + _ = dm.load() + + # train = 60%, valid = 20%, test = 20% + assert 0 < len(x_test) == len(y_test) + assert 0 < len(x_valid) == len(y_valid) + assert len(y_valid) < len(x_train) == len(y_train) + + +def test_boston_data(): + dm = BostonHousingData() + assert dm.url_source == 'https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data' + assert dm._save_dir.exists() + + # First one downloads the data + x_train, y_train, x_valid, y_valid, x_test, y_test = dm.load() + + # second call should check the 'if exists' branch + _ = dm.load() + + # train = 60%, valid = 20%, test = 20% + assert 0 < len(x_test) == len(y_test) + assert 0 < len(x_valid) == len(y_valid) + assert len(y_valid) < len(x_train) == len(y_train) diff --git a/tests/test_nasbench_201.py b/tests/test_nasbench_201.py index 3cdfb431..57512a80 100644 --- a/tests/test_nasbench_201.py +++ b/tests/test_nasbench_201.py @@ -3,12 +3,12 @@ import pytest -from hpolib.benchmarks.nas.nasbench_201 import ImageNetNasBench201Benchmark, Cifar100NasBench201Benchmark, \ - Cifar10ValidNasBench201Benchmark, Cifar10NasBench201Benchmark as Cifar10NasBench201BenchmarkContainer +from hpobench.benchmarks.nas.nasbench_201 import ImageNetNasBench201Benchmark, Cifar100NasBench201Benchmark, \ + Cifar10ValidNasBench201Benchmark, Cifar10NasBench201Benchmark -from hpolib.benchmarks.nas.nasbench_201 import Cifar10NasBench201Benchmark -from hpolib.util.container_utils import disable_container_debug, enable_container_debug +from hpobench.util.container_utils import disable_container_debug, enable_container_debug +skip_message = 'We currently skip this test because it takes too much time.' @pytest.fixture(scope='module') def enable_debug(): @@ -17,6 +17,8 @@ def enable_debug(): disable_container_debug() + +@pytest.mark.skip(reason=skip_message) def test_nasbench201_cifar10valid(enable_debug): b = Cifar10ValidNasBench201Benchmark(rng=0) @@ -32,7 +34,12 @@ def test_nasbench201_cifar10valid(enable_debug): assert result['info']['train_precision'] == result['function_value'] assert result['info']['train_cost'] == result['cost'] + result = b.objective_function_test(configuration=config, fidelity=fidelity, data_seed=(777, 888, 999)) + + with pytest.raises(AssertionError): + result = b.objective_function_test(configuration=config, fidelity={'epoch': 10}) +@pytest.mark.skip(reason=skip_message) def test_nasbench201_cifar100(enable_debug): b = Cifar100NasBench201Benchmark(rng=0) @@ -49,6 +56,7 @@ def test_nasbench201_cifar100(enable_debug): assert result['info']['train_cost'] == result['cost'] +@pytest.mark.skip(reason=skip_message) def test_nasbench201_Image(enable_debug): b = ImageNetNasBench201Benchmark(rng=0) @@ -65,8 +73,9 @@ def test_nasbench201_Image(enable_debug): assert result['info']['train_cost'] == result['cost'] +@pytest.mark.skip(reason=skip_message) def test_nasbench201_cifar10_container(enable_debug): - b = Cifar10NasBench201BenchmarkContainer(rng=0) + b = Cifar10NasBench201Benchmark(rng=0) cs = b.get_configuration_space(seed=0) config = cs.sample_configuration() @@ -80,6 +89,7 @@ def test_nasbench201_cifar10_container(enable_debug): assert result['info']['train_precision'] == result['function_value'] +@pytest.mark.skip(reason=skip_message) def test_nasbench201_cifar10(): b = Cifar10NasBench201Benchmark(rng=0) @@ -97,7 +107,7 @@ def test_nasbench201_cifar10(): assert result['cost'] == pytest.approx(13301.76, abs=0.1) assert result['info']['train_precision'] == result['function_value'] - result_test = b.objective_function_test(configuration=config, fidelity=fidelity) + result_test = b.objective_function_test(configuration=config) assert result['info']['train_precision'] == result_test['info']['train_precision'] assert result['info']['train_cost'] == result_test['info']['train_cost'] assert result['info']['train_losses'] == result_test['info']['train_losses'] @@ -128,12 +138,12 @@ def test_nasbench201_cifar10(): def test_nasbench201_fidelity_space(): - fs = Cifar10NasBench201Benchmark(rng=0).get_fidelity_space() + fs = Cifar10NasBench201Benchmark.get_fidelity_space() assert len(fs.get_hyperparameters()) == 1 def test_nasbench201_config(): - cs = Cifar10NasBench201Benchmark(rng=0).get_configuration_space(seed=0) + cs = Cifar10NasBench201Benchmark.get_configuration_space(seed=0) c = cs.sample_configuration() func = Cifar10NasBench201Benchmark.config_to_structure_func(4) struct = func(c) diff --git a/tests/test_openml_datamanager.py b/tests/test_openml_datamanager.py index 4d416e9d..31b291fb 100644 --- a/tests/test_openml_datamanager.py +++ b/tests/test_openml_datamanager.py @@ -1,6 +1,7 @@ -from hpolib.util.openml_data_manager import OpenMLHoldoutDataManager import numpy as np +from hpobench.util.openml_data_manager import OpenMLHoldoutDataManager + def test_convert_nan_values_in_cat_columns(): x = np.array([[1, np.nan, 3, 4], diff --git a/tests/test_pybnn.py b/tests/test_pybnn.py new file mode 100644 index 00000000..0e749457 --- /dev/null +++ b/tests/test_pybnn.py @@ -0,0 +1,74 @@ +import pytest + +from hpobench.container.benchmarks.ml.pybnn import BNNOnToyFunction, BNNOnBostonHousing, BNNOnProteinStructure, \ + BNNOnYearPrediction + +import logging +logging.basicConfig(level=logging.DEBUG) +from hpobench.util.container_utils import enable_container_debug +enable_container_debug() + + +def test_bnn_init(): + benchmark = BNNOnToyFunction(rng=1) + + fs = benchmark.get_fidelity_space(seed=0) + fidelity = fs.sample_configuration().get_dictionary() + assert fidelity['budget'] == 5714 + + meta = benchmark.get_meta_information() + assert meta is not None + + cs = benchmark.get_configuration_space(seed=0) + config = cs.sample_configuration().get_dictionary() + + assert config['l_rate'] == pytest.approx(0.0037, abs=0.001) + assert config['burn_in'] == pytest.approx(0.43905, abs=0.001) + assert config['n_units_1'] == 104 + assert config['n_units_2'] == 68 + assert config['mdecay'] == pytest.approx(0.6027, abs=0.001) + + result = benchmark.objective_function(configuration=config, fidelity=fidelity, rng=1) + assert result['function_value'] == pytest.approx(380.08, abs=0.1) + assert result['cost'] > 1 + assert result['info']['fidelity']['budget'] == 5714 + + result = benchmark.objective_function_test(configuration=config) + assert result['function_value'] == pytest.approx(183.6146, abs=0.1) + assert result['cost'] is not None + # test if budget is maximal: + assert result['info']['fidelity']['budget'] == 10000 + + +def simple_call(benchmark): + cs = benchmark.get_configuration_space(seed=0) + config = cs.sample_configuration().get_dictionary() + + fidelity = {'budget': 1000} + + result = benchmark.objective_function(configuration=config, fidelity=fidelity, rng=1) + return result + + +def test_bnn_boston_housing(): + benchmark = BNNOnBostonHousing(rng=1) + test_result = simple_call(benchmark) + assert test_result['function_value'] == pytest.approx(1262.0869, abs=0.1) + assert test_result['cost'] > 0 + assert test_result['info']['fidelity']['budget'] == 1000 + + +def test_bnn_protein(): + benchmark = BNNOnProteinStructure(rng=1) + test_result = simple_call(benchmark) + assert test_result['function_value'] == pytest.approx(1050.5733, abs=0.1) + assert test_result['cost'] > 0 + assert test_result['info']['fidelity']['budget'] == 1000 + + +def test_year_pred(): + benchmark = BNNOnYearPrediction(rng=1) + test_result = simple_call(benchmark) + assert test_result['function_value'] == pytest.approx(2105.2726, abs=0.1) + assert test_result['cost'] > 0 + assert test_result['info']['fidelity']['budget'] == 1000 diff --git a/tests/test_server.py b/tests/test_server.py index 7dc66324..6533c441 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -4,18 +4,18 @@ def set_log_level(debug): - os.environ['HPOLIB_DEBUG'] = 'true' if debug else 'false' - import hpolib.container.client_abstract_benchmark as client + os.environ['HPOBENCH_DEBUG'] = 'true' if debug else 'false' + import hpobench.container.client_abstract_benchmark as client importlib.reload(client) def test_debug_env_variable_1(): set_log_level(False) - from hpolib.container.client_abstract_benchmark import log_level + from hpobench.container.client_abstract_benchmark import log_level assert log_level == logging.INFO set_log_level(True) - from hpolib.container.client_abstract_benchmark import log_level + from hpobench.container.client_abstract_benchmark import log_level assert log_level == logging.DEBUG @@ -24,8 +24,8 @@ def test_debug_container(): set_log_level(True) - from hpolib.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark - from hpolib.util.openml_data_manager import get_openmlcc18_taskids + from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark + from hpobench.util.openml_data_manager import get_openmlcc18_taskids task_id = get_openmlcc18_taskids()[0] @@ -46,7 +46,7 @@ class test_enum(Enum): def __str__(self): return str(self.value) - from hpolib.container.server_abstract_benchmark import BenchmarkEncoder + from hpobench.container.server_abstract_benchmark import BenchmarkEncoder import json import numpy as np diff --git a/tests/test_svm.py b/tests/test_svm.py index 6c072a7b..609b4d5b 100644 --- a/tests/test_svm.py +++ b/tests/test_svm.py @@ -1,7 +1,7 @@ import pytest -from hpolib.container.benchmarks.ml.svm_benchmark import SupportVectorMachine -from hpolib.util.openml_data_manager import get_openmlcc18_taskids +from hpobench.container.benchmarks.ml.svm_benchmark import SupportVectorMachine +from hpobench.util.openml_data_manager import get_openmlcc18_taskids task_ids = get_openmlcc18_taskids() @@ -34,7 +34,3 @@ def test_svm_init(): result = benchmark.objective_function_test(configuration=config) assert result['function_value'] == pytest.approx(0.4648, abs=0.1) assert result['cost'] is not None - - -if __name__ == "__main__": - test_svm_init() diff --git a/tests/test_tabular_benchmarks.py b/tests/test_tabular_benchmarks.py index b89eebe0..139b4108 100644 --- a/tests/test_tabular_benchmarks.py +++ b/tests/test_tabular_benchmarks.py @@ -6,9 +6,9 @@ import os -os.environ['HPOLIB_DEBUG'] = 'true' +os.environ['HPOBENCH_DEBUG'] = 'true' -from hpolib.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark, \ +from hpobench.container.benchmarks.nas.tabular_benchmarks import SliceLocalizationBenchmark, \ NavalPropulsionBenchmark, ParkinsonsTelemonitoringBenchmark, ProteinStructureBenchmark @@ -43,6 +43,9 @@ def test_tabular_benchmark_wrong_input(): with pytest.raises(ValueError): benchmark.objective_function(configuration=default_config, fidelity=dict(budget=101), run_index=3) + with pytest.raises((AssertionError, ValueError)): + benchmark.objective_function_test(configuration=default_config, fidelity=dict(budget=107)) + benchmark = None @@ -121,14 +124,12 @@ def test_parkinson_benchmark(): mean = 0.7425 assert result['function_value'] == pytest.approx(mean, abs=0.0001) - runs = result['info']['valid_rmse_per_run'] - calculated_mean = sum(runs) / len(runs) - assert calculated_mean == pytest.approx(mean, abs=0.0001) + with pytest.raises(AssertionError): + benchmark.objective_function_test(default_config, fidelity=dict(budget=1,)) - runtime = 0.6272 - assert result['cost'] == pytest.approx(runtime, abs=0.0001) + result = benchmark.objective_function_test(configuration=default_config, fidelity=dict(budget=100)) + assert pytest.approx(0.15010187, result['function_value'], abs=0.001) - runtimes = result['info']['runtime_per_run'] - calculated_runtime = sum(runtimes) / len(runtimes) - assert calculated_runtime == pytest.approx(runtime, abs=0.0001) + runtime = 62.7268 + assert result['cost'] == pytest.approx(runtime, abs=0.0001) benchmark = None diff --git a/tests/test_utils.py b/tests/test_utils.py index b33ca02b..53a44cd5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,7 @@ def test_example_utils(): - from hpolib.util.example_utils import get_travis_settings + from hpobench.util.example_utils import get_travis_settings res = get_travis_settings('smac') assert res['runcount-limit'] == 5 @@ -16,7 +16,7 @@ def test_example_utils(): def test_example_utils_2(): - from hpolib.util.example_utils import set_env_variables_to_use_only_one_core + from hpobench.util.example_utils import set_env_variables_to_use_only_one_core import os set_env_variables_to_use_only_one_core() assert os.environ['OMP_NUM_THREADS'] == '1' @@ -28,7 +28,7 @@ def test_example_utils_2(): def test_rng_helper(): - from hpolib.util.rng_helper import _cast_int_to_random_state + from hpobench.util.rng_helper import _cast_int_to_random_state rng = np.random.RandomState(123) @@ -42,7 +42,7 @@ def test_rng_helper(): def test_rng_helper_2(): - from hpolib.util.rng_helper import get_rng + from hpobench.util.rng_helper import get_rng rng = get_rng(None, None) assert isinstance(rng, np.random.RandomState) @@ -53,10 +53,10 @@ def test_rng_helper_2(): def test_debug_level(): - from hpolib.util.container_utils import enable_container_debug, disable_container_debug + from hpobench.util.container_utils import enable_container_debug, disable_container_debug import os enable_container_debug() - assert os.environ['HPOLIB_DEBUG'] == 'true' + assert os.environ['HPOBENCH_DEBUG'] == 'true' disable_container_debug() - assert os.environ['HPOLIB_DEBUG'] == 'false' + assert os.environ['HPOBENCH_DEBUG'] == 'false' diff --git a/tests/test_whitebox.py b/tests/test_whitebox.py index d22d1155..b7422fa7 100644 --- a/tests/test_whitebox.py +++ b/tests/test_whitebox.py @@ -14,7 +14,7 @@ def test_whitebox_without_container_xgb(): - from hpolib.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark + from hpobench.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark b = Benchmark(task_id=167199, rng=0) cs = b.get_configuration_space(seed=0) @@ -39,7 +39,7 @@ def test_whitebox_without_container_xgb(): @pytest.mark.skipif(skip_container_test, reason="Requires singularity and flask") def test_whitebox_with_container(): - from hpolib.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark + from hpobench.container.benchmarks.ml.xgboost_benchmark import XGBoostBenchmark as Benchmark b = Benchmark(container_name='xgboost_benchmark', task_id=167199, rng=0) @@ -64,13 +64,13 @@ def test_whitebox_with_container(): def test_cartpole(): - from hpolib.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark + from hpobench.container.benchmarks.rl.cartpole import CartpoleReduced as Benchmark b = Benchmark(container_name='cartpole', rng=1) cs = b.get_configuration_space(seed=1) print(cs.get_default_configuration()) - from hpolib.container.benchmarks.rl.cartpole import CartpoleFull as Benchmark + from hpobench.container.benchmarks.rl.cartpole import CartpoleFull as Benchmark b = Benchmark(container_name='cartpole', rng=1) cs = b.get_configuration_space(seed=1)