From fd6bf3bb1a17557d34d606cc706b9b4e49152b61 Mon Sep 17 00:00:00 2001 From: Eddie Bergman Date: Wed, 15 Jan 2025 17:39:05 +0100 Subject: [PATCH] refactor: a lot (#167) --- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 47 - README.md | 33 +- docs/_code/api_generator.py | 1 - docs/doc_yamls/architecture_search_space.py | 97 -- .../doc_yamls/customizing_neps_optimizer.yaml | 21 - docs/doc_yamls/defining_hooks.yaml | 24 - .../full_configuration_template.yaml | 42 - docs/doc_yamls/loading_own_optimizer.yaml | 21 - .../loading_pipeline_space_dict.yaml | 11 - docs/doc_yamls/outsourcing_optimizer.yaml | 18 - .../doc_yamls/outsourcing_pipeline_space.yaml | 10 - docs/doc_yamls/pipeline_space.yaml | 21 - docs/doc_yamls/run_pipeline.py | 7 - docs/doc_yamls/run_pipeline_architecture.py | 24 - .../run_pipeline_big_search_space.py | 6 - docs/doc_yamls/run_pipeline_extended.py | 6 - docs/doc_yamls/set_up_optimizer.yaml | 5 - docs/doc_yamls/simple_example.yaml | 16 - ...simple_example_including_run_pipeline.yaml | 20 - docs/getting_started.md | 106 +- docs/index.md | 30 +- docs/reference/analyse.md | 4 +- docs/reference/cli.md | 24 +- docs/reference/declarative_usage.md | 328 ++---- .../{run_pipeline.md => evaluate_pipeline.md} | 28 +- docs/reference/neps_run.md | 192 ++-- docs/reference/optimizers.md | 80 +- docs/reference/pipeline_space.md | 38 +- docs/stylesheets/custom.css | 4 +- mkdocs.yml | 6 +- neps/__init__.py | 30 +- neps/api.py | 776 +++++++------- .../bayesian_optimization => cli}/__init__.py | 0 neps/optimizers/__init__.py | 124 ++- neps/optimizers/acquisition/__init__.py | 5 + .../cost_cooling.py | 6 +- .../pibo.py | 9 +- .../weighted_acquisition.py | 6 +- neps/optimizers/algorithms.py | 856 +++++++++++++++ neps/optimizers/base_optimizer.py | 173 --- .../optimizer.py => bayesian_optimization.py} | 130 +-- .../acquisition_functions/__init__.py | 45 - .../acquisition_functions/base_acquisition.py | 30 - .../acquisition_functions/ei.py | 128 --- .../acquisition_functions/ucb.py | 57 - .../bayesian_optimization/models/__init__.py | 4 - neps/optimizers/bracket_optimizer.py | 184 ++++ neps/optimizers/default_searchers/asha.yaml | 13 - .../default_searchers/asha_prior.yaml | 13 - .../bayesian_optimization.yaml | 7 - .../default_searchers/hyperband.yaml | 12 - neps/optimizers/default_searchers/ifbo.yaml | 11 - .../optimizers/default_searchers/mobster.yaml | 24 - neps/optimizers/default_searchers/pibo.yaml | 7 - .../default_searchers/priorband.yaml | 20 - .../default_searchers/priorband_bo.yaml | 32 - .../default_searchers/random_search.yaml | 4 - .../default_searchers/successive_halving.yaml | 13 - .../successive_halving_prior.yaml | 13 - neps/optimizers/grid_search.py | 41 + neps/optimizers/grid_search/optimizer.py | 115 -- neps/optimizers/ifbo.py | 266 +++++ neps/optimizers/info.py | 100 -- neps/optimizers/models/__init__.py | 4 + .../models/ftpfn.py | 30 +- .../{bayesian_optimization => }/models/gp.py | 40 +- neps/optimizers/multi_fidelity/__init__.py | 29 - neps/optimizers/multi_fidelity/hyperband.py | 564 ---------- neps/optimizers/multi_fidelity/ifbo.py | 293 ----- neps/optimizers/multi_fidelity/mf_bo.py | 211 ---- .../multi_fidelity/promotion_policy.py | 111 -- .../multi_fidelity/sampling_policy.py | 430 -------- .../multi_fidelity/successive_halving.py | 677 ------------ neps/optimizers/multi_fidelity/utils.py | 251 ----- .../multi_fidelity_prior/__init__.py | 11 - .../multi_fidelity_prior/async_priorband.py | 320 ------ .../multi_fidelity_prior/priorband.py | 407 ------- neps/optimizers/multi_fidelity_prior/utils.py | 179 ---- neps/optimizers/optimizer.py | 66 ++ neps/optimizers/priorband.py | 212 ++++ neps/optimizers/random_search.py | 46 + neps/optimizers/random_search/optimizer.py | 77 -- .../{grid_search => utils}/__init__.py | 0 neps/optimizers/utils/brackets.py | 566 ++++++++++ neps/optimizers/utils/grid.py | 60 ++ neps/optimizers/{ => utils}/initial_design.py | 16 +- neps/plot/plot.py | 3 +- neps/plot/plot3D.py | 20 +- neps/plot/plotting.py | 3 +- neps/runtime.py | 16 +- neps/sampling/__init__.py | 14 +- neps/sampling/distributions.py | 30 +- neps/sampling/priors.py | 101 +- neps/sampling/samplers.py | 76 +- neps/search_spaces/__init__.py | 48 - neps/search_spaces/architecture/__init__.py | 0 neps/search_spaces/architecture/api.py | 205 ---- neps/search_spaces/architecture/cfg.py | 320 ------ .../architecture/cfg_variants/__init__.py | 0 .../cfg_variants/constrained_cfg.py | 496 --------- .../architecture/core_graph_grammar.py | 998 ------------------ neps/search_spaces/architecture/graph.py | 690 ------------ .../architecture/graph_grammar.py | 308 ------ neps/search_spaces/architecture/mutations.py | 70 -- neps/search_spaces/architecture/primitives.py | 499 --------- neps/search_spaces/architecture/topologies.py | 187 ---- .../search_spaces/hyperparameters/__init__.py | 21 - .../hyperparameters/categorical.py | 212 ---- .../search_spaces/hyperparameters/constant.py | 137 --- neps/search_spaces/hyperparameters/float.py | 209 ---- neps/search_spaces/hyperparameters/integer.py | 202 ---- .../hyperparameters/numerical.py | 238 ----- neps/search_spaces/parameter.py | 216 ---- neps/search_spaces/search_space.py | 301 ------ neps/search_spaces/yaml_search_space_utils.py | 385 ------- neps/space/__init__.py | 15 + neps/{search_spaces => space}/domain.py | 29 +- neps/{search_spaces => space}/encoding.py | 95 +- neps/{search_spaces => space}/functions.py | 58 +- neps/space/parameters.py | 237 +++++ neps/space/parsing.py | 293 +++++ neps/space/search_space.py | 80 ++ neps/state/__init__.py | 6 + neps/state/_eval.py | 2 +- neps/state/filebased.py | 10 +- neps/state/neps_state.py | 43 +- neps/state/settings.py | 2 +- neps/state/trial.py | 57 +- neps/status/status.py | 52 +- neps/utils/cli.py | 180 +--- neps/utils/common.py | 202 ++-- neps/utils/files.py | 26 + neps/utils/run_args.py | 636 ----------- neps/utils/types.py | 74 -- neps_examples/README.md | 2 +- neps_examples/basic_usage/architecture.py | 130 --- .../architecture_and_hyperparameters.py | 127 --- neps_examples/basic_usage/hyperparameters.py | 10 +- .../convenience/declarative_usage/config.yaml | 5 +- .../convenience/logging_additional_info.py | 7 +- .../convenience/neps_tblogger_tutorial.py | 15 +- neps_examples/convenience/neps_x_lightning.py | 38 +- .../convenience/running_on_slurm_scripts.py | 9 +- .../working_directory_per_pipeline.py | 6 +- .../expert_priors_for_hyperparameters.py | 18 +- neps_examples/efficiency/multi_fidelity.py | 19 +- .../multi_fidelity_and_expert_priors.py | 26 +- ...rs_for_architecture_and_hyperparameters.py | 139 --- neps_examples/experimental/freeze_thaw.py | 35 +- .../experimental/hierarchical_architecture.py | 103 -- .../template}/__init__.py | 0 neps_examples/template/basic.py | 78 ++ neps_examples/template/ifbo.py | 125 +++ neps_examples/template/priorband.py | 154 +++ neps_examples/template/pytorch-lightning.py | 176 +++ pyproject.toml | 33 +- tests/joint_config_space.py | 113 -- tests/losses.json | 1 - tests/regression_objectives.py | 295 ------ tests/regression_runner.py | 276 ----- tests/settings.py | 20 - tests/test_config_encoder.py | 179 +--- tests/test_domain.py | 4 +- tests/test_neps_api/__init__.py | 0 .../solution_yamls/bo_custom_created.yaml | 5 - .../solution_yamls/bo_neps_decided.yaml | 10 - .../hyperband_custom_created.yaml | 5 - .../hyperband_neps_decided.yaml | 11 - .../solution_yamls/pibo_neps_decided.yaml | 10 - .../priorband_bo_user_decided.yaml | 22 - .../priorband_neps_decided.yaml | 17 - .../solution_yamls/user_yaml_bo.yaml | 8 - tests/test_neps_api/test_api.py | 59 -- .../testing_scripts/baseoptimizer_neps.py | 61 -- .../testing_scripts/default_neps.py | 91 -- .../testing_scripts/user_yaml_neps.py | 41 - .../testing_yaml/optimizer_test.yaml | 5 - tests/test_regression.py | 62 -- .../test_default_report_values.py | 31 +- .../test_error_handling_strategies.py | 37 +- tests/test_runtime/test_stopping_criterion.py | 75 +- tests/test_samplers.py | 13 +- tests/test_search_space_functions.py | 77 -- tests/test_search_space_parsing.py | 95 ++ tests/test_settings/__init__.py | 0 tests/test_settings/overwrite_run_args.yaml | 43 - .../run_args_optimizer_outside.yaml | 16 - .../run_args_optimizer_settings.yaml | 50 - tests/test_settings/run_args_optional.yaml | 14 - tests/test_settings/run_args_required.yaml | 8 - tests/test_settings/test_settings.py | 360 ------- tests/test_state/test_filebased_neps_state.py | 1 + tests/test_state/test_neps_state.py | 109 +- tests/test_yaml_run_args/__init__.py | 0 tests/test_yaml_run_args/pipeline_space.yaml | 9 - tests/test_yaml_run_args/run_args_empty.yaml | 23 - tests/test_yaml_run_args/run_args_full.yaml | 34 - .../run_args_full_same_level.yaml | 22 - .../run_args_invalid_key.yaml | 34 - .../run_args_invalid_type.yaml | 29 - .../run_args_key_missing.yaml | 22 - .../run_args_optional_loading_format.yaml | 24 - .../test_yaml_run_args/run_args_partial.yaml | 27 - .../run_args_partial_same_level.yaml | 14 - .../run_args_wrong_name.yaml | 34 - .../run_args_wrong_path.yaml | 34 - .../test_declarative_usage_docs/__init__.py | 0 .../customizing_neps_optimizer.yaml | 23 - .../defining_hooks.yaml | 25 - .../evaluate_pipeline.py | 29 - .../full_configuration_template.yaml | 42 - .../test_declarative_usage_docs/hooks.py | 11 - .../loading_own_optimizer.yaml | 22 - .../loading_pipeline_space_dict.yaml | 11 - .../test_declarative_usage_docs/neps_run.py | 33 - .../outsourcing_optimizer.yaml | 18 - .../outsourcing_pipeline_space.yaml | 10 - .../pipeline_space.py | 10 - .../pipeline_space.yaml | 17 - .../set_up_optimizer.yaml | 5 - .../simple_example.yaml | 19 - ...simple_example_including_run_pipeline.yaml | 22 - .../test_declarative_usage_docs.py | 54 - .../test_run_args_by_neps_run/__init__.py | 0 .../test_run_args_by_neps_run/config.yaml | 25 - .../config_hyperband_mixed_args.yaml | 29 - .../config_priorband_with_args.yaml | 38 - .../config_select_bo.yaml | 23 - .../loading_optimizer.yaml | 27 - .../loading_pipeline_space.yaml | 29 - .../test_run_args_by_neps_run/neps_run.py | 48 - .../hyperband_searcher_kwargs_yaml_args.yaml | 11 - .../priorband_args_run_args.yaml | 22 - .../optimizer_yamls/select_bo_run_args.yaml | 10 - .../search_space.yaml | 11 - .../search_space_with_fidelity.yaml | 11 - .../search_space_with_priors.yaml | 15 - .../test_neps_run.py | 84 -- .../test_yaml_run_args/test_yaml_run_args.py | 245 ----- tests/test_yaml_search_space/__init__.py | 0 .../config_including_unknown_types.yaml | 17 - .../config_including_wrong_types.yaml | 17 - .../correct_config.yaml | 28 - .../correct_config_including_priors.yml | 18 - .../correct_config_including_types.yaml | 31 - .../default_not_in_range_config.yaml | 6 - .../default_value_not_in_choices_config.yaml | 3 - .../inconsistent_types_config.yml | 14 - .../inconsistent_types_config2.yml | 15 - .../incorrect_config.txt | 4 - .../incorrect_fidelity_bounds_config.yaml | 22 - .../missing_key_config.yml | 13 - .../not_allowed_key_config.yml | 25 - ...t_boolean_type_is_fidelity_cat_config.yaml | 4 - ...boolean_type_is_fidelity_float_config.yaml | 6 - .../not_boolean_type_log_config.yaml | 6 - .../test_search_space.py | 166 --- 258 files changed, 5167 insertions(+), 17220 deletions(-) delete mode 100644 docs/doc_yamls/architecture_search_space.py delete mode 100644 docs/doc_yamls/customizing_neps_optimizer.yaml delete mode 100644 docs/doc_yamls/defining_hooks.yaml delete mode 100644 docs/doc_yamls/full_configuration_template.yaml delete mode 100644 docs/doc_yamls/loading_own_optimizer.yaml delete mode 100644 docs/doc_yamls/loading_pipeline_space_dict.yaml delete mode 100644 docs/doc_yamls/outsourcing_optimizer.yaml delete mode 100644 docs/doc_yamls/outsourcing_pipeline_space.yaml delete mode 100644 docs/doc_yamls/pipeline_space.yaml delete mode 100644 docs/doc_yamls/run_pipeline.py delete mode 100644 docs/doc_yamls/run_pipeline_architecture.py delete mode 100644 docs/doc_yamls/run_pipeline_big_search_space.py delete mode 100644 docs/doc_yamls/run_pipeline_extended.py delete mode 100644 docs/doc_yamls/set_up_optimizer.yaml delete mode 100644 docs/doc_yamls/simple_example.yaml delete mode 100644 docs/doc_yamls/simple_example_including_run_pipeline.yaml rename docs/reference/{run_pipeline.md => evaluate_pipeline.md} (75%) rename neps/{optimizers/bayesian_optimization => cli}/__init__.py (100%) create mode 100644 neps/optimizers/acquisition/__init__.py rename neps/optimizers/{bayesian_optimization/acquisition_functions => acquisition}/cost_cooling.py (89%) rename neps/optimizers/{bayesian_optimization/acquisition_functions => acquisition}/pibo.py (84%) rename neps/optimizers/{bayesian_optimization/acquisition_functions => acquisition}/weighted_acquisition.py (97%) create mode 100644 neps/optimizers/algorithms.py delete mode 100644 neps/optimizers/base_optimizer.py rename neps/optimizers/{bayesian_optimization/optimizer.py => bayesian_optimization.py} (57%) delete mode 100644 neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py delete mode 100644 neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py delete mode 100644 neps/optimizers/bayesian_optimization/acquisition_functions/ei.py delete mode 100644 neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py delete mode 100755 neps/optimizers/bayesian_optimization/models/__init__.py create mode 100644 neps/optimizers/bracket_optimizer.py delete mode 100644 neps/optimizers/default_searchers/asha.yaml delete mode 100644 neps/optimizers/default_searchers/asha_prior.yaml delete mode 100644 neps/optimizers/default_searchers/bayesian_optimization.yaml delete mode 100644 neps/optimizers/default_searchers/hyperband.yaml delete mode 100644 neps/optimizers/default_searchers/ifbo.yaml delete mode 100644 neps/optimizers/default_searchers/mobster.yaml delete mode 100644 neps/optimizers/default_searchers/pibo.yaml delete mode 100644 neps/optimizers/default_searchers/priorband.yaml delete mode 100644 neps/optimizers/default_searchers/priorband_bo.yaml delete mode 100644 neps/optimizers/default_searchers/random_search.yaml delete mode 100644 neps/optimizers/default_searchers/successive_halving.yaml delete mode 100644 neps/optimizers/default_searchers/successive_halving_prior.yaml create mode 100644 neps/optimizers/grid_search.py delete mode 100644 neps/optimizers/grid_search/optimizer.py create mode 100755 neps/optimizers/ifbo.py delete mode 100644 neps/optimizers/info.py create mode 100755 neps/optimizers/models/__init__.py rename neps/optimizers/{bayesian_optimization => }/models/ftpfn.py (95%) rename neps/optimizers/{bayesian_optimization => }/models/gp.py (94%) delete mode 100644 neps/optimizers/multi_fidelity/__init__.py delete mode 100644 neps/optimizers/multi_fidelity/hyperband.py delete mode 100755 neps/optimizers/multi_fidelity/ifbo.py delete mode 100755 neps/optimizers/multi_fidelity/mf_bo.py delete mode 100644 neps/optimizers/multi_fidelity/promotion_policy.py delete mode 100644 neps/optimizers/multi_fidelity/sampling_policy.py delete mode 100644 neps/optimizers/multi_fidelity/successive_halving.py delete mode 100644 neps/optimizers/multi_fidelity/utils.py delete mode 100644 neps/optimizers/multi_fidelity_prior/__init__.py delete mode 100644 neps/optimizers/multi_fidelity_prior/async_priorband.py delete mode 100644 neps/optimizers/multi_fidelity_prior/priorband.py delete mode 100644 neps/optimizers/multi_fidelity_prior/utils.py create mode 100644 neps/optimizers/optimizer.py create mode 100644 neps/optimizers/priorband.py create mode 100644 neps/optimizers/random_search.py delete mode 100644 neps/optimizers/random_search/optimizer.py rename neps/optimizers/{grid_search => utils}/__init__.py (100%) create mode 100644 neps/optimizers/utils/brackets.py create mode 100644 neps/optimizers/utils/grid.py rename neps/optimizers/{ => utils}/initial_design.py (92%) delete mode 100644 neps/search_spaces/__init__.py delete mode 100644 neps/search_spaces/architecture/__init__.py delete mode 100644 neps/search_spaces/architecture/api.py delete mode 100644 neps/search_spaces/architecture/cfg.py delete mode 100644 neps/search_spaces/architecture/cfg_variants/__init__.py delete mode 100644 neps/search_spaces/architecture/cfg_variants/constrained_cfg.py delete mode 100644 neps/search_spaces/architecture/core_graph_grammar.py delete mode 100644 neps/search_spaces/architecture/graph.py delete mode 100644 neps/search_spaces/architecture/graph_grammar.py delete mode 100644 neps/search_spaces/architecture/mutations.py delete mode 100644 neps/search_spaces/architecture/primitives.py delete mode 100644 neps/search_spaces/architecture/topologies.py delete mode 100644 neps/search_spaces/hyperparameters/__init__.py delete mode 100644 neps/search_spaces/hyperparameters/categorical.py delete mode 100644 neps/search_spaces/hyperparameters/constant.py delete mode 100644 neps/search_spaces/hyperparameters/float.py delete mode 100644 neps/search_spaces/hyperparameters/integer.py delete mode 100644 neps/search_spaces/hyperparameters/numerical.py delete mode 100644 neps/search_spaces/parameter.py delete mode 100644 neps/search_spaces/search_space.py delete mode 100644 neps/search_spaces/yaml_search_space_utils.py create mode 100644 neps/space/__init__.py rename neps/{search_spaces => space}/domain.py (93%) rename neps/{search_spaces => space}/encoding.py (82%) rename neps/{search_spaces => space}/functions.py (55%) create mode 100644 neps/space/parameters.py create mode 100644 neps/space/parsing.py create mode 100644 neps/space/search_space.py delete mode 100644 neps/utils/run_args.py delete mode 100644 neps/utils/types.py delete mode 100644 neps_examples/basic_usage/architecture.py delete mode 100644 neps_examples/basic_usage/architecture_and_hyperparameters.py delete mode 100644 neps_examples/experimental/expert_priors_for_architecture_and_hyperparameters.py delete mode 100644 neps_examples/experimental/hierarchical_architecture.py rename {neps/optimizers/random_search => neps_examples/template}/__init__.py (100%) create mode 100644 neps_examples/template/basic.py create mode 100644 neps_examples/template/ifbo.py create mode 100644 neps_examples/template/priorband.py create mode 100644 neps_examples/template/pytorch-lightning.py delete mode 100644 tests/joint_config_space.py delete mode 100644 tests/losses.json delete mode 100644 tests/regression_objectives.py delete mode 100644 tests/regression_runner.py delete mode 100644 tests/settings.py delete mode 100644 tests/test_neps_api/__init__.py delete mode 100644 tests/test_neps_api/solution_yamls/bo_custom_created.yaml delete mode 100644 tests/test_neps_api/solution_yamls/bo_neps_decided.yaml delete mode 100644 tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml delete mode 100644 tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml delete mode 100644 tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml delete mode 100644 tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml delete mode 100644 tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml delete mode 100644 tests/test_neps_api/solution_yamls/user_yaml_bo.yaml delete mode 100644 tests/test_neps_api/test_api.py delete mode 100644 tests/test_neps_api/testing_scripts/baseoptimizer_neps.py delete mode 100644 tests/test_neps_api/testing_scripts/default_neps.py delete mode 100644 tests/test_neps_api/testing_scripts/user_yaml_neps.py delete mode 100644 tests/test_neps_api/testing_yaml/optimizer_test.yaml delete mode 100644 tests/test_regression.py delete mode 100644 tests/test_search_space_functions.py create mode 100644 tests/test_search_space_parsing.py delete mode 100644 tests/test_settings/__init__.py delete mode 100644 tests/test_settings/overwrite_run_args.yaml delete mode 100644 tests/test_settings/run_args_optimizer_outside.yaml delete mode 100644 tests/test_settings/run_args_optimizer_settings.yaml delete mode 100644 tests/test_settings/run_args_optional.yaml delete mode 100644 tests/test_settings/run_args_required.yaml delete mode 100644 tests/test_settings/test_settings.py delete mode 100644 tests/test_yaml_run_args/__init__.py delete mode 100644 tests/test_yaml_run_args/pipeline_space.yaml delete mode 100644 tests/test_yaml_run_args/run_args_empty.yaml delete mode 100644 tests/test_yaml_run_args/run_args_full.yaml delete mode 100644 tests/test_yaml_run_args/run_args_full_same_level.yaml delete mode 100644 tests/test_yaml_run_args/run_args_invalid_key.yaml delete mode 100644 tests/test_yaml_run_args/run_args_invalid_type.yaml delete mode 100644 tests/test_yaml_run_args/run_args_key_missing.yaml delete mode 100644 tests/test_yaml_run_args/run_args_optional_loading_format.yaml delete mode 100644 tests/test_yaml_run_args/run_args_partial.yaml delete mode 100644 tests/test_yaml_run_args/run_args_partial_same_level.yaml delete mode 100644 tests/test_yaml_run_args/run_args_wrong_name.yaml delete mode 100644 tests/test_yaml_run_args/run_args_wrong_path.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/__init__.py delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/hooks.py delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/neps_run.py delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml delete mode 100644 tests/test_yaml_run_args/test_declarative_usage_docs/test_declarative_usage_docs.py delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/__init__.py delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml delete mode 100644 tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py delete mode 100644 tests/test_yaml_run_args/test_yaml_run_args.py delete mode 100644 tests/test_yaml_search_space/__init__.py delete mode 100644 tests/test_yaml_search_space/config_including_unknown_types.yaml delete mode 100644 tests/test_yaml_search_space/config_including_wrong_types.yaml delete mode 100644 tests/test_yaml_search_space/correct_config.yaml delete mode 100644 tests/test_yaml_search_space/correct_config_including_priors.yml delete mode 100644 tests/test_yaml_search_space/correct_config_including_types.yaml delete mode 100644 tests/test_yaml_search_space/default_not_in_range_config.yaml delete mode 100644 tests/test_yaml_search_space/default_value_not_in_choices_config.yaml delete mode 100644 tests/test_yaml_search_space/inconsistent_types_config.yml delete mode 100644 tests/test_yaml_search_space/inconsistent_types_config2.yml delete mode 100644 tests/test_yaml_search_space/incorrect_config.txt delete mode 100644 tests/test_yaml_search_space/incorrect_fidelity_bounds_config.yaml delete mode 100644 tests/test_yaml_search_space/missing_key_config.yml delete mode 100644 tests/test_yaml_search_space/not_allowed_key_config.yml delete mode 100644 tests/test_yaml_search_space/not_boolean_type_is_fidelity_cat_config.yaml delete mode 100644 tests/test_yaml_search_space/not_boolean_type_is_fidelity_float_config.yaml delete mode 100644 tests/test_yaml_search_space/not_boolean_type_log_config.yaml delete mode 100644 tests/test_yaml_search_space/test_search_space.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7916aee9..d2a75b86d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,7 +52,7 @@ repos: files: '^\.github/dependabot\.ya?ml$' - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.1 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix, --no-cache] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c3d08ced..bce83b9fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,6 @@ We have setup checks and tests at several points in the development flow: This is setup during our [installation process](https://automl.github.io/neps/contributing/installation/). - At every commit / push locally running a minimal suite of integration tests is encouraged. The tests correspond directly to examples in [neps_examples](https://github.com/automl/neps/tree/master/neps_examples) and only check for crash-causing errors. -- At every push all integration tests and regression tests are run automatically using [github actions](https://github.com/automl/neps/actions). ## Checks and tests @@ -151,54 +150,8 @@ pytest If tests fail for you on the master, please raise an issue on github, preferably with some information on the error, traceback and the environment in which you are running, i.e. python version, OS, etc. -## Regression Tests - -Regression tests are run on each push to the repository to assure the performance of the optimizers don't degrade. - -Currently, regression runs are recorded on JAHS-Bench-201 data for 2 tasks: `cifar10` and `fashion_mnist` and only for optimizers: `random_search`, `bayesian_optimization`, `mf_bayesian_optimization`. -This information is stored in the `tests/regression_runner.py` as two lists: `TASKS`, `OPTIMIZERS`. -The recorded results are stored as a json dictionary in the `tests/losses.json` file. - -### Adding new optimizer algorithms - -Once a new algorithm is added to NEPS library, we need to first record the performance of the algorithm for 100 optimization runs. - -- If the algorithm expects standard loss function (pipeline) and accepts fidelity hyperparameters in pipeline space, then recording results only requires adding the optimizer name into `OPTIMIZERS` list in `tests/regression_runner.py` and running `tests/regression_runner.py` - -- In case your algorithm requires custom pipeline and/or pipeline space you can modify the `runner.run_pipeline` and `runner.pipeline_space` attributes of the `RegressionRunner` after initialization (around line `#322` in `tests/regression_runner.py`) - -You can verify the optimizer is recorded by rerunning the `regression_runner.py`. -Now regression test will be run on your new optimizer as well on every push. - -### Regression test metrics - -For each regression test the algorithm is run 10 times to sample its performance, then they are statistically compared to the 100 recorded runs. We use these 3 boolean metrics to define the performance of the algorithm on any task: - -1. [Kolmogorov-Smirnov test for goodness of fit](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.kstest.html) - `pvalue` >= 10% -1. Absolute median distance - bounded within 92.5% confidence range of the expected median distance -1. Median improvement - Median improvement over the recorded median - -Test metrics are run for each `(optimizer, task)` combination separately and then collected. -The collected metrics are then further combined into 2 metrics - -1. Task pass - either both `Kolmogorov-Smirnov test` and `Absolute median distance` test passes or just `Median improvement` -1. Test aggregate - Sum_over_tasks(`Kolmogorov-Smirnov test` + `Absolute median distance` + 2 * `Median improvement`) - -Finally, a test for an optimizer only passes when at least for one of the tasks `Task pass` is true, and `Test aggregate` is higher than 1 + `number of tasks` - -### On regression test failures - -Regression tests are stochastic by nature, so they might fail occasionally even the algorithm performance didn't degrade. -In the case of regression test failure, try running it again first, if the problem still persists, then you can contact [Danny Stoll](mailto:stolld@cs.uni-freiburg.de) or [Samir](mailto:garibovs@cs.uni-freiburg.de). -You can also run tests locally by running: - -``` -uv run pytest -m regression_all -``` - ## Disabling and Skipping Checks etc. - ### Pre-commit: How to not run hooks? To commit without running `pre-commit` use `git commit --no-verify -m `. diff --git a/README.md b/README.md index 462587abd..4e549326f 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,10 @@ pip install neural-pipeline-search Using `neps` always follows the same pattern: -1. Define a `run_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations +1. Define a `evaluate_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations for your problem. 1. Define a search space named `pipeline_space` of those Parameters e.g. via a dictionary -1. Call `neps.run` to optimize `run_pipeline` over `pipeline_space` +1. Call `neps.run(evaluate_pipeline, pipeline_space)` In code, the usage pattern can look like this: @@ -50,34 +50,33 @@ In code, the usage pattern can look like this: import neps import logging +logging.basicConfig(level=logging.INFO) # 1. Define a function that accepts hyperparameters and computes the validation error -def run_pipeline( - hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str -) -> dict: +def evaluate_pipeline(lr: float, alpha: int, optimizer: str) -> float: # Create your model - model = MyModel(architecture_parameter) + model = MyModel(lr=lr, alpha=alpha, optimizer=optimizer) # Train and evaluate the model with your training pipeline - validation_error = train_and_eval( - model, hyperparameter_a, hyperparameter_b - ) + validation_error = train_and_eval(model) return validation_error -# 2. Define a search space of parameters; use the same parameter names as in run_pipeline +# 2. Define a search space of parameters; use the same parameter names as in evaluate_pipeline pipeline_space = dict( - hyperparameter_a=neps.Float( - lower=0.001, upper=0.1, log=True # The search space is sampled in log space + lr=neps.Float( + lower=1e-5, + upper=1e-1, + log=True, # Log spaces + prior=1e-3, # Incorporate you knowledge to help optimization ), - hyperparameter_b=neps.Integer(lower=1, upper=42), - architecture_parameter=neps.Categorical(["option_a", "option_b"]), + alpha=neps.Integer(lower=1, upper=42), + optimizer=neps.Categorical(choices=["sgd", "adam"]) ) # 3. Run the NePS optimization -logging.basicConfig(level=logging.INFO) neps.run( - run_pipeline=run_pipeline, + evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, root_directory="path/to/save/results", # Replace with the actual path. max_evaluations_total=100, @@ -94,8 +93,6 @@ Discover how NePS works through these examples: - **[Utilizing Expert Priors for Hyperparameters](neps_examples/efficiency/expert_priors_for_hyperparameters.py)**: Learn how to incorporate expert priors for more efficient hyperparameter selection. -- **[Architecture Search](neps_examples/basic_usage/architecture.py)**: Dive into (hierarchical) architecture search in NePS. - - **[Additional NePS Examples](neps_examples/)**: Explore more examples, including various use cases and advanced configurations in NePS. ## Contributing diff --git a/docs/_code/api_generator.py b/docs/_code/api_generator.py index b19f40a25..8a696a674 100644 --- a/docs/_code/api_generator.py +++ b/docs/_code/api_generator.py @@ -3,7 +3,6 @@ # https://mkdocstrings.github.io/recipes/ """ - import logging from pathlib import Path diff --git a/docs/doc_yamls/architecture_search_space.py b/docs/doc_yamls/architecture_search_space.py deleted file mode 100644 index 66771cb3b..000000000 --- a/docs/doc_yamls/architecture_search_space.py +++ /dev/null @@ -1,97 +0,0 @@ - -from torch import nn -import neps -from neps.search_spaces.architecture import primitives as ops -from neps.search_spaces.architecture import topologies as topos -from neps.search_spaces.architecture.primitives import AbstractPrimitive - - -class DownSampleBlock(AbstractPrimitive): - def __init__(self, in_channels: int, out_channels: int): - super().__init__(locals()) - self.conv_a = ReLUConvBN( - in_channels, out_channels, kernel_size=3, stride=2, padding=1 - ) - self.conv_b = ReLUConvBN( - out_channels, out_channels, kernel_size=3, stride=1, padding=1 - ) - self.downsample = nn.Sequential( - nn.AvgPool2d(kernel_size=2, stride=2, padding=0), - nn.Conv2d( - in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False - ), - ) - - def forward(self, inputs): - basicblock = self.conv_a(inputs) - basicblock = self.conv_b(basicblock) - residual = self.downsample(inputs) - return residual + basicblock - - -class ReLUConvBN(AbstractPrimitive): - def __init__(self, in_channels, out_channels, kernel_size, stride, padding): - super().__init__(locals()) - - self.kernel_size = kernel_size - self.op = nn.Sequential( - nn.ReLU(inplace=False), - nn.Conv2d( - in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=1, - bias=False, - ), - nn.BatchNorm2d(out_channels, affine=True, track_running_stats=True), - ) - - def forward(self, x): - return self.op(x) - - -class AvgPool(AbstractPrimitive): - def __init__(self, **kwargs): - super().__init__(kwargs) - self.op = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False) - - def forward(self, x): - return self.op(x) - - -primitives = { - "Sequential15": topos.get_sequential_n_edge(15), - "DenseCell": topos.get_dense_n_node_dag(4), - "down": {"op": DownSampleBlock}, - "avg_pool": {"op": AvgPool}, - "id": {"op": ops.Identity}, - "conv3x3": {"op": ReLUConvBN, "kernel_size": 3, "stride": 1, "padding": 1}, - "conv1x1": {"op": ReLUConvBN, "kernel_size": 1, "stride": 1, "padding": 0}, -} - - -structure = { - "S": ["Sequential15(C, C, C, C, C, down, C, C, C, C, C, down, C, C, C, C, C)"], - "C": ["DenseCell(OPS, OPS, OPS, OPS, OPS, OPS)"], - "OPS": ["id", "conv3x3", "conv1x1", "avg_pool"], -} - - -def set_recursive_attribute(op_name, predecessor_values): - in_channels = 16 if predecessor_values is None else predecessor_values["out_channels"] - out_channels = in_channels * 2 if op_name == "DownSampleBlock" else in_channels - return dict(in_channels=in_channels, out_channels=out_channels) - - -pipeline_space = dict( - architecture=neps.Architecture( - set_recursive_attribute=set_recursive_attribute, - structure=structure, - primitives=primitives, - ), - optimizer=neps.Categorical(choices=["sgd", "adam"]), - learning_rate=neps.Float(lower=10e-7, upper=10e-3, log=True), -) - diff --git a/docs/doc_yamls/customizing_neps_optimizer.yaml b/docs/doc_yamls/customizing_neps_optimizer.yaml deleted file mode 100644 index 5d98140f1..000000000 --- a/docs/doc_yamls/customizing_neps_optimizer.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Customizing NePS Searcher -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - optimizer: - choices: [adam, sgd, adamw] - epochs: 50 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget -searcher: - strategy: bayesian_optimization # key for neps searcher - name: "my_bayesian" # optional; changing the searcher_name for better recognition - # Specific arguments depending on the searcher - initial_design_size: 7 diff --git a/docs/doc_yamls/defining_hooks.yaml b/docs/doc_yamls/defining_hooks.yaml deleted file mode 100644 index 67032ab69..000000000 --- a/docs/doc_yamls/defining_hooks.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Hooks -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: true - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget - -pre_load_hooks: - hook1: path/to/your/hooks.py # (function_name: Path to the function's file) - hook2: path/to/your/hooks.py # Different function name 'hook2' from the same file source diff --git a/docs/doc_yamls/full_configuration_template.yaml b/docs/doc_yamls/full_configuration_template.yaml deleted file mode 100644 index 6f90e77b1..000000000 --- a/docs/doc_yamls/full_configuration_template.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Full Configuration Template for NePS -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true - epochs: - lower: 5 - upper: 20 - is_fidelity: true - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget -max_cost_total: - -# Debug and Monitoring -overwrite_working_directory: true -post_run_summary: false -development_stage_id: -task_id: - -# Parallelization Setup -max_evaluations_per_run: -continue_until_max_evaluation_completed: false - -# Error Handling -objective_to_minimize_value_on_error: -cost_value_on_error: -ignore_errors: - -# Customization Options -searcher: hyperband # Internal key to select a NePS optimizer. - -# Hooks -pre_load_hooks: diff --git a/docs/doc_yamls/loading_own_optimizer.yaml b/docs/doc_yamls/loading_own_optimizer.yaml deleted file mode 100644 index 5d656372f..000000000 --- a/docs/doc_yamls/loading_own_optimizer.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Loading Optimizer Class -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - optimizer: - choices: [adam, sgd, adamw] - epochs: 50 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget -searcher: - path: path/to/your/searcher.py # Path to the class - name: CustomOptimizer # class name within the file - # Specific arguments depending on your searcher - initial_design_size: 7 diff --git a/docs/doc_yamls/loading_pipeline_space_dict.yaml b/docs/doc_yamls/loading_pipeline_space_dict.yaml deleted file mode 100644 index 53ff9ec6a..000000000 --- a/docs/doc_yamls/loading_pipeline_space_dict.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Loading pipeline space from a python dict -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - path: path/to/your/search_space.py # Path to the dict file - name: pipeline_space # Name of the dict instance - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget diff --git a/docs/doc_yamls/outsourcing_optimizer.yaml b/docs/doc_yamls/outsourcing_optimizer.yaml deleted file mode 100644 index 1ca3a764f..000000000 --- a/docs/doc_yamls/outsourcing_optimizer.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Optimizer settings from YAML configuration -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - optimizer: - choices: [adam, sgd, adamw] - epochs: 50 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget - -searcher: path/to/your/searcher_setup.yaml diff --git a/docs/doc_yamls/outsourcing_pipeline_space.yaml b/docs/doc_yamls/outsourcing_pipeline_space.yaml deleted file mode 100644 index d3f7a65f8..000000000 --- a/docs/doc_yamls/outsourcing_pipeline_space.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Pipeline space settings from separate YAML -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: path/to/your/pipeline_space.yaml - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget - diff --git a/docs/doc_yamls/pipeline_space.yaml b/docs/doc_yamls/pipeline_space.yaml deleted file mode 100644 index 8f877b98f..000000000 --- a/docs/doc_yamls/pipeline_space.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Pipeline_space including priors and fidelity -learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - prior: 1e-2 - prior_confidence: "medium" -epochs: - lower: 5 - upper: 20 - is_fidelity: true -dropout_rate: - lower: 0.1 - upper: 0.5 - prior: 0.2 - prior_confidence: "high" -optimizer: - choices: [adam, sgd, adamw] - prior: adam - # if prior confidence is not defined it gets its default 'low' -batch_size: 64 diff --git a/docs/doc_yamls/run_pipeline.py b/docs/doc_yamls/run_pipeline.py deleted file mode 100644 index 29b33e723..000000000 --- a/docs/doc_yamls/run_pipeline.py +++ /dev/null @@ -1,7 +0,0 @@ - - -def example_pipeline(learning_rate, optimizer, epochs): - model = initialize_model() - training_objective_to_minimize = train_model(model, optimizer, learning_rate, epochs) - evaluation_objective_to_minimize = evaluate_model(model) - return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize} diff --git a/docs/doc_yamls/run_pipeline_architecture.py b/docs/doc_yamls/run_pipeline_architecture.py deleted file mode 100644 index c3314489d..000000000 --- a/docs/doc_yamls/run_pipeline_architecture.py +++ /dev/null @@ -1,24 +0,0 @@ -from torch import nn - - -def example_pipeline(architecture, optimizer, learning_rate): - in_channels = 3 - base_channels = 16 - n_classes = 10 - out_channels_factor = 4 - - # E.g., in shape = (N, 3, 32, 32) => out shape = (N, 10) - model = architecture.to_pytorch() - model = nn.Sequential( - nn.Conv2d(in_channels, base_channels, 3, padding=1, bias=False), - nn.BatchNorm2d(base_channels), - model, - nn.BatchNorm2d(base_channels * out_channels_factor), - nn.ReLU(inplace=True), - nn.AdaptiveAvgPool2d(1), - nn.Flatten(), - nn.Linear(base_channels * out_channels_factor, n_classes), - ) - training_objective_to_minimize = train_model(model, optimizer, learning_rate) - evaluation_objective_to_minimize = evaluate_model(model) - return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize} diff --git a/docs/doc_yamls/run_pipeline_big_search_space.py b/docs/doc_yamls/run_pipeline_big_search_space.py deleted file mode 100644 index 346fe3bb4..000000000 --- a/docs/doc_yamls/run_pipeline_big_search_space.py +++ /dev/null @@ -1,6 +0,0 @@ - -def example_pipeline(learning_rate, optimizer, epochs, batch_size, dropout_rate): - model = initialize_model(dropout_rate) - training_objective_to_minimize = train_model(model, optimizer, learning_rate, epochs, batch_size) - evaluation_objective_to_minimize = evaluate_model(model) - return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize} diff --git a/docs/doc_yamls/run_pipeline_extended.py b/docs/doc_yamls/run_pipeline_extended.py deleted file mode 100644 index 7a57f0719..000000000 --- a/docs/doc_yamls/run_pipeline_extended.py +++ /dev/null @@ -1,6 +0,0 @@ - -def example_pipeline(learning_rate, optimizer, epochs, batch_size): - model = initialize_model() - training_objective_to_minimize = train_model(model, optimizer, learning_rate, epochs, batch_size) - evaluation_objective_to_minimize = evaluate_model(model) - return {"objective_to_minimize": evaluation_objective_to_minimize, "training_objective_to_minimize": training_objective_to_minimize} diff --git a/docs/doc_yamls/set_up_optimizer.yaml b/docs/doc_yamls/set_up_optimizer.yaml deleted file mode 100644 index 94922d78a..000000000 --- a/docs/doc_yamls/set_up_optimizer.yaml +++ /dev/null @@ -1,5 +0,0 @@ -strategy: bayesian_optimization -# Specific arguments depending on the searcher -initial_design_size: 7 -use_priors: true -sample_prior_first: false diff --git a/docs/doc_yamls/simple_example.yaml b/docs/doc_yamls/simple_example.yaml deleted file mode 100644 index 35cde1216..000000000 --- a/docs/doc_yamls/simple_example.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Basic NePS Configuration Example -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: true - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget diff --git a/docs/doc_yamls/simple_example_including_run_pipeline.yaml b/docs/doc_yamls/simple_example_including_run_pipeline.yaml deleted file mode 100644 index ce9cc163a..000000000 --- a/docs/doc_yamls/simple_example_including_run_pipeline.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Simple NePS configuration including evaluate_pipeline -evaluate_pipeline: - path: path/to/your/evaluate_pipeline.py # Path to the function file - name: example_pipeline # Function name within the file - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: true # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: true - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: path/to/results # Directory for result storage -max_evaluations_total: 20 # Budget diff --git a/docs/getting_started.md b/docs/getting_started.md index 78cad7640..7dcb2a552 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -4,103 +4,59 @@ Getting started with NePS involves a straightforward yet powerful process, cente This approach ensures flexibility and efficiency in evaluating different architecture and hyperparameter configurations for your problem. -NePS requires Python 3.8 or higher. You can install it via pip or from source. +NePS requires Python 3.10 or higher. +You can install it via `pip` or from [source](https://github.com/automl/neps/). ```bash pip install neural-pipeline-search ``` ## The 3 Main Components -1. **Execute with [`neps.run()`](./reference/neps_run.md)**: -Optimize your `run_pipeline=` over the `pipeline_space=` using this function. -For a thorough overview of the arguments and their explanations, check out the detailed documentation. - -2. **Define a [`run_pipeline=`](./reference/run_pipeline.md) Function**: -This function is essential for evaluating different configurations. -You'll implement the specific logic for your problem within this function. -For detailed instructions on initializing and effectively using `run_pipeline=`, refer to the guide. - -3. **Establish a [`pipeline_space=`](./reference/pipeline_space.md)**: -Your search space for defining parameters. -You can structure this in various formats, including dictionaries, YAML, or ConfigSpace. -The guide offers insights into defining and configuring your search space. - -By following these steps and utilizing the extensive resources provided in the guides, you can tailor NePS to meet -your specific requirements, ensuring a streamlined and effective optimization process. - -## Basic Usage -In code, the usage pattern can look like this: - +1. **Establish a [`pipeline_space=`](reference/pipeline_space.md)**: ```python -import neps -import logging - - -def run_pipeline( # (1)! - hyperparameter_a: float, - hyperparameter_b: int, - architecture_parameter: str, -) -> dict: - # insert here your own model - model = MyModel(architecture_parameter) - - # insert here your training/evaluation pipeline - validation_error, training_error = train_and_eval( - model, hyperparameter_a, hyperparameter_b - ) +pipeline_space={ + "some_parameter": (0.0, 1.0), # float + "another_parameter": (0, 10), # integer + "optimizer": ["sgd", "adam"], # categorical + "epoch": neps.Integer(lower=1, upper=100, is_fidelity=True), + "learning_rate": neps.Float(lower=1e-5, uperr=1, log=True), + "alpha": neps.Float(lower=0.1, upper=1.0, prior=0.99, prior_confidence="high") +} - return { - "loss": validation_error, # ! (2) - "info_dict": { - "training_error": training_error - # + Other metrics - }, - } +``` +2. **Define an `evaluate_pipeline()` function**: +```python +def evaluate_pipeline(some_parameter: float, + another_parameter: float, + optimizer: str, epoch: int, + learning_rate: float, alpha: float) -> float: + model = make_model(...) + loss = eval_model(model) + return loss +``` -pipeline_space = { # (3)! - "hyperparameter_b": neps.Integer(1, 42, is_fidelity=True), # ! (4) - "hyperparameter_a": neps.Float(1e-3, 1e-1, log=True) # ! (5) - "architecture_parameter": neps.Categorical(["option_a", "option_b", "option_c"]), -} +1. **Execute with [`neps.run()`](reference/neps_run.md)**: -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - neps.run( - run_pipeline=run_pipeline, - pipeline_space=pipeline_space, - root_directory="path/to/save/results", # Replace with the actual path. - max_evaluations_total=100, - searcher="hyperband" # Optional specifies the search strategy, - # otherwise NePs decides based on your data. - ) +```python +neps.run(evaluate_pipeline, pipeline_space) ``` -1. Define a function that accepts hyperparameters and computes the validation error. -2. Return a dictionary with the objective to minimize and any additional information. -3. Define a search space of the parameters of interest; ensure that the names are consistent with those defined in the run_pipeline function. -4. Use `is_fidelity=True` for a multi-fidelity approach. -5. Use `log=True` for a log-spaced hyperparameter. +--- -!!! tip - - Please visit the [full reference](./reference/neps_run.md) for a more comprehensive walkthrough of defining budgets, - optimizers, YAML configuration, parallelism, and more. +You can find a longer walk through in the [reference](reference/neps_run.md)! ## Examples Discover the features of NePS through these practical examples: -* **[Hyperparameter Optimization (HPO)](./examples/template/basic_template.md)**: +* **[Hyperparameter Optimization (HPO)](examples/template/basic.md)**: Learn the essentials of hyperparameter optimization with NePS. -* **[Architecture Search with Primitives](./examples/basic_usage/architecture.md)**: -Dive into architecture search using primitives in NePS. - -* **[Multi-Fidelity Optimization](./examples/efficiency/multi_fidelity.md)**: +* **[Multi-Fidelity Optimization](examples/efficiency/multi_fidelity.md)**: Understand how to leverage multi-fidelity optimization for efficient model tuning. -* **[Utilizing Expert Priors for Hyperparameters](./examples/template/priorband_template.md)**: +* **[Utilizing Expert Priors for Hyperparameters](examples/template/priorband.md)**: Learn how to incorporate expert priors for more efficient hyperparameter selection. -* **[Additional NePS Examples](./examples/index.md)**: +* **[Additional NePS Examples](examples/index.md)**: Explore more examples, including various use cases and advanced configurations in NePS. diff --git a/docs/index.md b/docs/index.md index ec27c7193..c46516924 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ [![PyPI version](https://img.shields.io/pypi/v/neural-pipeline-search?color=informational)](https://pypi.org/project/neural-pipeline-search/) [![Python versions](https://img.shields.io/pypi/pyversions/neural-pipeline-search)](https://pypi.org/project/neural-pipeline-search/) -[![License](https://img.shields.io/pypi/l/neural-pipeline-search?color=informational)](LICENSE) +[![License](https://img.shields.io/pypi/l/neural-pipeline-search?color=informational)](https://github.com/automl/neps/blob/master/LICENSE) [![Tests](https://github.com/automl/neps/actions/workflows/tests.yaml/badge.svg)](https://github.com/automl/neps/actions) Welcome to NePS, a powerful and flexible Python library for hyperparameter optimization (HPO) and neural architecture search (NAS) with its primary goal: **make HPO and NAS usable for deep learners in practice**. @@ -30,9 +30,9 @@ In addition to the features offered by traditional HPO and NAS libraries, NePS s Check out: * [Reference documentation](./reference/neps_run.md) for a quick overview. - * [API](./api/neps/api.md) for a more detailed reference. + * [API](api/neps/api.md) for a more detailed reference. * [Colab Tutorial](https://colab.research.google.com/drive/11IOhkmMKsIUhWbHyMYzT0v786O9TPWlH?usp=sharing) walking through NePS's main features. - * [Examples](./examples) for basic code snippets to get started. + * [Examples](examples/index.md) for basic code snippets to get started. ## Installation @@ -46,10 +46,10 @@ pip install neural-pipeline-search Using `neps` always follows the same pattern: -1. Define a `run_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations +1. Define a `evalute_pipeline` function capable of evaluating different architectural and/or hyperparameter configurations for your problem. 1. Define a search space named `pipeline_space` of those Parameters e.g. via a dictionary -1. Call `neps.run` to optimize `run_pipeline` over `pipeline_space` +1. Call `neps.run` to optimize `evalute_pipeline` over `pipeline_space` In code, the usage pattern can look like this: @@ -59,7 +59,7 @@ import logging # 1. Define a function that accepts hyperparameters and computes the validation error -def run_pipeline( +def evalute_pipeline( hyperparameter_a: float, hyperparameter_b: int, architecture_parameter: str ) -> dict: # Create your model @@ -72,7 +72,7 @@ def run_pipeline( return validation_error -# 2. Define a search space of parameters; use the same parameter names as in run_pipeline +# 2. Define a search space of parameters; use the same parameter names as in evalute_pipeline pipeline_space = dict( hyperparameter_a=neps.Float( lower=0.001, upper=0.1, log=True # The search space is sampled in log space @@ -84,7 +84,7 @@ pipeline_space = dict( # 3. Run the NePS optimization logging.basicConfig(level=logging.INFO) neps.run( - run_pipeline=run_pipeline, + evalute_pipeline=evalute_pipeline, pipeline_space=pipeline_space, root_directory="path/to/save/results", # Replace with the actual path. max_evaluations_total=100, @@ -95,20 +95,18 @@ neps.run( Discover how NePS works through these examples: -- **[Hyperparameter Optimization](./examples/basic_usage/hyperparameters.py)**: Learn the essentials of hyperparameter optimization with NePS. +- **[Hyperparameter Optimization](examples/basic_usage/hyperparameters.md)**: Learn the essentials of hyperparameter optimization with NePS. -- **[Multi-Fidelity Optimization](./examples/efficiency/multi_fidelity.py)**: Understand how to leverage multi-fidelity optimization for efficient model tuning. +- **[Multi-Fidelity Optimization](examples/efficiency/multi_fidelity.md)**: Understand how to leverage multi-fidelity optimization for efficient model tuning. -- **[Utilizing Expert Priors for Hyperparameters](./examples/efficiency/expert_priors_for_hyperparameters.py)**: Learn how to incorporate expert priors for more efficient hyperparameter selection. +- **[Utilizing Expert Priors for Hyperparameters](examples/efficiency/expert_priors_for_hyperparameters.md)**: Learn how to incorporate expert priors for more efficient hyperparameter selection. -- **[Architecture Search](./examples/basic_usage/architecture.py)**: Dive into (hierarchical) architecture search in NePS. - -- **[Additional NePS Examples](./examples/)**: Explore more examples, including various use cases and advanced configurations in NePS. +- **[Additional NePS Examples](examples/index.md)**: Explore more examples, including various use cases and advanced configurations in NePS. ## Contributing -Please see the [documentation for contributors](./dev_docs/contributing/). +Please see the [documentation for contributors](dev_docs/contributing.md). ## Citations -For pointers on citing the NePS package and papers refer to our [documentation on citations](./citations.md). +For pointers on citing the NePS package and papers refer to our [documentation on citations](citations.md). diff --git a/docs/reference/analyse.md b/docs/reference/analyse.md index ec4d16e91..0b8d6967e 100644 --- a/docs/reference/analyse.md +++ b/docs/reference/analyse.md @@ -77,7 +77,7 @@ NePS will also generate a summary CSV file for you. The `config_data.csv` contains all configuration details in CSV format, ordered by ascending `loss`. -Details include configuration hyperparameters, any returned result from the `run_pipeline` function, and metadata information. +Details include configuration hyperparameters, any returned result from the `evalute_pipeline` function, and metadata information. The `run_status.csv` provides general run details, such as the number of sampled configs, best configs, number of failed configs, best loss, etc. @@ -122,7 +122,7 @@ tblogger.log( !!! tip - The logger function is primarily designed for use within the `run_pipeline` function during the training of the neural network. + The logger function is primarily designed for use within the `evalute_pipeline` function during the training of the neural network. ??? example "Quick Reference" diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 86c23dce5..a3febcdc6 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1,16 +1,8 @@ # NePS Command Line Interface This section provides a brief overview of the commands available in the NePS CLI. -!!! note "Support of Development and Task ID" - The NePS arguments `development_stage_id` and `task_id` are only partially - supported. To retrieve results for a specific task or development stage, you must modify the `root_directory` to - point to the corresponding folder of your NePS results. For example, if you have task_id 1 and development_stage_id 4, - update your root_directory to root_directory/task_1/development_4. This can be done either by specifying the - --root-directory option in your command or by updating the root_directory in your corresponding `run_args` yaml - file. - - --- + ## **`init` Command** Generates a default `run_args` YAML configuration file, providing a template that you can customize for your experiments. @@ -40,24 +32,22 @@ Executes the optimization based on the provided configuration. This command serv - `-h, --help` (Optional): show this help message and exit - `--run-args` (Optional): Path to the YAML configuration file. -- `--run-pipeline` (Optional): Optional: Provide the path to a Python file and a function name separated by a colon, e.g., 'path/to/module.py:function_name'. If provided, it overrides the run_pipeline setting from the YAML configuration. +- `--evaluate-pipeline` (Optional): Optional: Provide the path to a Python file and a function name separated by a colon, e.g., 'path/to/module.py:function_name'. If provided, it overrides the evaluate_pipeline setting from the YAML configuration. - `--pipeline-space` (Optional): Path to the YAML file defining the search space for the optimization. This can be provided here or defined within the 'run_args' YAML file. - `--root-directory` (Optional): The directory to save progress to. This is also used to synchronize multiple calls for parallelization. -- `--overwrite-working-directory` (Optional): If set, deletes the working directory at the start of the run. This is useful, for example, when debugging a run_pipeline function. -- `--development-stage-id` (Optional): Identifier for the current development stage, used in multi-stage projects. -- `--task-id` (Optional): Identifier for the current task, useful in projects with multiple tasks. +- `--overwrite-working-directory` (Optional): If set, deletes the working directory at the start of the run. This is useful, for example, when debugging a evalute_pipeline function. - `--post-run-summary` (Optional): Provide a summary of the results after running. - `--no-post-run-summary` (Optional): Do not provide a summary of the results after running. - `--max-evaluations-total` (Optional): Total number of evaluations to run. - `--max-evaluations-per-run` (Optional): Number of evaluations a specific call should maximally do. - `--continue-until-max-evaluation-completed` (Optional): If set, only stop after max-evaluations-total have been completed. This is only relevant in the parallel setting. - `--max-cost-total` (Optional): No new evaluations will start when this cost is exceeded. Requires returning a cost - in the run_pipeline function. + in the evalute_pipeline function. - `--ignore-errors` (Optional): If set, ignore errors during the optimization process. - `--loss-value-on-error` (Optional): Loss value to assume on error. - `--cost-value-on-error` (Optional): Cost value to assume on error. -- `--searcher` (Optional): String key of searcher algorithm to use for optimization. -- `--searcher-kwargs` (Optional): Additional keyword arguments as key=value pairs for the searcher. +- `--optimizer` (Optional): String key of optimizer algorithm to use for optimization. +- `--optimizer-kwargs` (Optional): Additional keyword arguments as key=value pairs for the optimizer. **Example Usage:** @@ -266,5 +256,3 @@ neps report-config 42 success --worker-id worker_1 --loss 0.95 --duration 120 - **`--worker_id worker_1`**: Identifies the worker reporting the configuration. - **`--loss 0.95`**: The loss value obtained from the trial. - **`--duration 120`**: The duration of the evaluation in seconds. - - diff --git a/docs/reference/declarative_usage.md b/docs/reference/declarative_usage.md index be26cae4e..db5b311b6 100644 --- a/docs/reference/declarative_usage.md +++ b/docs/reference/declarative_usage.md @@ -1,268 +1,148 @@ - ## Introduction -### Configuring with YAML -Configure your experiments using a YAML file, which serves as a central reference for setting up your project. -This approach simplifies sharing, reproducing and modifying configurations. +If you prefer to use yaml for experiment configuration, +[`neps.run()`][neps.api.run] supports yaml serialized input. -!!! note "Argument Handling and Prioritization" - You can partially define and provide arguments via `run_args` (YAML file) and partially provide the arguments - directly to `neps.run`. Arguments directly provided to `neps.run` get prioritized over those defined in the YAML file. An exception to this - is for `searcher_kwargs` where a merge happens between the configurations. In this case, the directly provided arguments - are still prioritized, but the values from both the directly provided arguments and the YAML file are merged. +We make no assumption on how you'd like to structure you experimentation +and you are free to run it as you wish! +Please check [`neps.run()`][neps.api.run] for complete information on the arguments. #### Simple YAML Example -Below is a straightforward YAML configuration example for NePS covering the required arguments. +Below is a YAML configuration example for NePS covering the required arguments. +The arguments match those that you can pass to [`neps.run()`][neps.api.run]. + +In general, you can encode any [`Parameter`][neps.space.Parameter] into a YAML format. + === "config.yaml" + ```yaml - --8<-- "docs/doc_yamls/simple_example.yaml" + # Basic NePS Configuration Example + pipeline_space: + + batch_size: 64 # Constant + + optimizer: [adam, sgd, adamw] # Categorical + + alpha: [0.01, 1.0] # Uniform Float + + n_layers: [1, 10] # Uniform Integer + + learning_rate: # Log scale Float with a prior + lower: 1e-5 + upper: 1e-1 + log: true + prior: 1e-3 + prior_confidence: high + + epochs: # Integer fidelity + lower: 5 + upper: 20 + is_fidelity: true + + root_directory: path/to/results # Directory for result storage + max_evaluations_total: 20 # Budget + + optimizer: + name: hyperband # Which optimizer to use ``` === "run_neps.py" + ```python import neps + import yaml - def run_pipeline(learning_rate, optimizer, epochs): + def evaluate_pipeline(learning_rate, optimizer, epochs, batch_size): model = initialize_model() training_loss = train_model(model, optimizer, learning_rate, epochs) evaluation_loss = evaluate_model(model) - return {"loss": evaluation_loss, "training_loss": training_loss} + return {"objective_value_to_minimize": evaluation_loss, "training_loss": training_loss} if __name__ == "__main__": - neps.run(run_pipeline, run_args="path/to/your/config.yaml") + with open("path/config.yaml") as f: + settings = yaml.safe_load(f) + + neps.run(evaluate_pipeline, **settings) ``` +!!! tip "Merging multiple yaml files" + + If you would like to seperate parts of your configuration into multiple yamls, + for example, to seperate out your search spaces and optimizers, + you can use the `neps.load_yamls` function to merge them, checking for conflicts. -#### Including `run_pipeline` in `run_args` for External Referencing -In addition to setting experimental parameters via YAML, this configuration example also specifies the pipeline function -and its location, enabling more flexible project structures. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/simple_example_including_run_pipeline.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline_extended.py" - ``` -=== "run_neps.py" ```python import neps - # No need to define run_pipeline here. NePS loads it directly from the specified path. - neps.run(run_args="path/to/your/config.yaml") + + def evaluate_pipeline(...): + ... + + if __name__ == "__main__": + settings = neps.load_yamls("path/to/your/config.yaml", "path/to/your/optimizer.yaml") + neps.run(evaluate_pipeline, **settings) ``` + #### Comprehensive YAML Configuration Template This example showcases a more comprehensive YAML configuration, which includes not only the essential parameters but also advanced settings for more complex setups. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/full_configuration_template.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline_extended.py" - ``` -=== "run_neps.py" - ```python - import neps - # Executes the configuration specified in your YAML file - neps.run(run_args="path/to/your/config.yaml") - ``` -The `searcher` key used in the YAML configuration corresponds to the same keys used for selecting an optimizer directly -through `neps.run`. For a detailed list of integrated optimizers, see [here](optimizers.md#list-available-searchers) -!!! note "Note on undefined keys in `run_args` (config.yaml)" - Not all configurations are explicitly defined in this template. Any undefined key in the YAML file is mapped to - the internal default settings of NePS. This ensures that your experiments can run even if certain parameters are - omitted. - -## Different Use Cases -### Customizing NePS optimizer -Customize an internal NePS optimizer by specifying its parameters directly under the key `searcher` in the -`config.yaml` file. - -!!! note - For `searcher_kwargs` of `neps.run`, the optimizer arguments passed via the YAML file and those passed directly via - `neps.run` will be merged. In this special case, if the same argument is referenced in both places, - `searcher_kwargs` will be prioritized and set for this argument. === "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/customizing_neps_optimizer.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline.py" - ``` -=== "run_neps.py" - ```python - import neps - neps.run(run_args="path/to/your/config.yaml") - ``` - -For detailed information about the available optimizers and their parameters, please visit the [optimizer page](optimizers.md#list-available-searching-algorithms) - -### Testing Multiple Optimizer Configurations -Simplify experiments with multiple optimizer settings by outsourcing the optimizer configuration. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/outsourcing_optimizer.yaml" - ``` -=== "searcher_setup.yaml" ```yaml - --8<-- "docs/doc_yamls/set_up_optimizer.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline.py" - ``` -=== "run_neps.py" - ```python - import neps - neps.run(run_args="path/to/your/config.yaml") - ``` - -### Handling Large Search Spaces -Manage large search spaces by outsourcing the pipeline space configuration in a separate YAML file or for keeping track -of your experiments. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/outsourcing_pipeline_space.yaml" + # Full Configuration Template for NePS + evaluate_pipeline: path/to/your/evaluate_pipeline.py::example_pipeline + + pipeline_space: + learning_rate: + lower: 1e-5 + upper: 1e-1 + log: true + epochs: + lower: 5 + upper: 20 + is_fidelity: true + optimizer: + choices: [adam, sgd, adamw] + batch_size: 64 + + root_directory: path/to/results # Directory for result storage + max_evaluations_total: 20 # Budget + max_cost_total: + + # Debug and Monitoring + overwrite_working_directory: true + post_run_summary: false + + # Parallelization Setup + max_evaluations_per_run: + continue_until_max_evaluation_completed: false + + # Error Handling + objective_value_on_error: + cost_value_on_error: + ignore_errors: + + optimizer: + name: hyperband ``` -=== "pipeline_space.yaml" - ```yaml - --8<-- "docs/doc_yamls/pipeline_space.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline_big_search_space.py" - ``` === "run_neps.py" - ```python - import neps - neps.run(run_args="path/to/your/config.yaml") - ``` - - -### Using Architecture Search Spaces -Since the option for defining the search space via YAML is limited to HPO, grammar-based search spaces or architecture -search spaces must be loaded via a dictionary, which is then referenced in the `config.yaml`. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/loading_pipeline_space_dict.yaml" - ``` -=== "search_space.py" - ```python - --8<-- "docs/doc_yamls/architecture_search_space.py" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline_architecture.py" - ``` -=== "run_neps.py" ```python - import neps - neps.run(run_args="path/to/your/config.yaml") - ``` + if __name__ == "__main__": + import neps -### Integrating Custom Optimizers -For people who want to write their own optimizer class as a subclass of the base optimizer, you can load your own -custom optimizer class and define its arguments in `config.yaml`. + with open("path/config.yaml") as f: + settings = yaml.safe_load(f) -Note: You can still overwrite arguments via searcher_kwargs of `neps.run` like for the internal searchers. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/loading_own_optimizer.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline.py" - ``` -=== "run_neps.py" - ```python - import neps - neps.run(run_args="path/to/your/config.yaml") + # Note, we specified our run function in the yaml itself! + neps.run(**settings) ``` +## CLI Usage +!!! warning "CLI Usage" -### Adding Custom Hooks to Your Configuration -Define hooks in your YAML configuration to extend the functionality of your experiment. -=== "config.yaml" - ```yaml - --8<-- "docs/doc_yamls/defining_hooks.yaml" - ``` -=== "run_pipeline.py" - ```python - --8<-- "docs/doc_yamls/run_pipeline_extended.py" - ``` -=== "run_neps.py" - ```python - import neps - neps.run(run_args="path/to/your/config.yaml") - ``` - -## CLI Usage -This section provides a brief overview of the primary commands available in the NePS CLI. -For additional command options, you can directly refer to the help documentation -provided by each command using --help. - - -### **`init` Command** - -Generates a default `run_args` YAML configuration file, providing a template that you can customize for your experiments. - -**Options:** - - - `--config-path `: *Optional*. Specify the custom path for generating the configuration file. Defaults to - `run_config.yaml` in the current working directory. - - `--template [basic|complete]`: *Optional*. Choose between a basic or complete template. The basic template includes only required settings, while the complete template includes all NePS configurations. - - `--state-machine`: *Optional*. Creates a NEPS state if set, which requires an existing `run_config.yaml`. - -**Example Usage:** -```bash -neps init --config-path custom/path/config.yaml --template complete -``` - -### **`run` Command** - -Executes the optimization based on the provided configuration. This command serves as a CLI wrapper around `neps.run`, effectively mapping each CLI argument to a parameter in `neps.run`. It offers a flexible interface that allows you to override the existing settings specified in the YAML configuration file, facilitating dynamic adjustments for managing your experiments. - -**Options:** - - - `--run-args `: Path to the YAML configuration file containing the run arguments. - - `--run-pipeline `: *Optional*. Specify the path to the Python module and function to use for running the pipeline. Overrides any settings in the YAML file. - - `--pipeline-space `: Path to the YAML file defining the search space for the optimization. - - `--root-directory `: *Optional*. Directory for saving progress and synchronizing multiple processes. Defaults to the `root_directory` from `run_config.yaml` if not provided. - - `--overwrite-working-directory`: *Optional*. If set, deletes the working directory at the start of the run. - - `--development-stage-id `: *Optional*. Identifier for the current development stage, useful for multi-stage projects. - - `--task-id `: *Optional*. Identifier for the current task, useful for managing projects with multiple tasks. - - `--post-run-summary/--no-post-run-summary`: *Optional*. Provides a summary of the run after execution. Enabled by default. - - `--max-evaluations-total `: *Optional*. Specifies the total number of evaluations to run. - - `--max-evaluations-per-run `: *Optional*. Number of evaluations to run per call. - - `--continue-until-max-evaluation-completed`: *Optional*. If set, ensures the run continues until `max-evaluations-total` has been reached. - - `--max-cost-total `: *Optional*. Specifies a cost threshold. No new evaluations will start if this cost is exceeded. - - `--ignore-errors`: *Optional*. If set, errors during the optimization will be ignored. - - `--loss-value-on-error `: *Optional*. Specifies the loss value to assume in case of an error. - - `--cost-value-on-error `: *Optional*. Specifies the cost value to assume in case of an error. - - `--searcher `: Specifies the searcher algorithm for optimization. - - `--searcher-kwargs `: *Optional*. Additional keyword arguments for the searcher. - -**Example Usage:** -```bash -neps run --run-args path/to/config.yaml --max-evaluations-total 50 -``` - -### **`status` Command** -Executes the optimization based on the provided configuration. This command serves as a CLI wrapper around neps.run, -effectively mapping each CLI argument to a parameter in neps.run. This setup offers a flexible interface that allows -you to override the existing settings specified in the YAML configuration file, facilitating dynamic adjustments for -managing your experiments. - -**Example Usage:** -```bash -neps run --run-args path/to/config.yaml -``` + The CLI is still in development and may not be fully functional. diff --git a/docs/reference/run_pipeline.md b/docs/reference/evaluate_pipeline.md similarity index 75% rename from docs/reference/run_pipeline.md rename to docs/reference/evaluate_pipeline.md index ff7282d42..a0bb2a3dd 100644 --- a/docs/reference/run_pipeline.md +++ b/docs/reference/evaluate_pipeline.md @@ -2,7 +2,7 @@ ## Introduction -The `run_pipeline=` function is crucial for NePS. It encapsulates the objective function to be minimized, which could range from a regular equation to a full training and evaluation pipeline for a neural network. +The `evaluate_pipeline=` function is crucial for NePS. It encapsulates the objective function to be minimized, which could range from a regular equation to a full training and evaluation pipeline for a neural network. This function receives the configuration to be utilized from the parameters defined in the search space. Consequently, it executes the same set of instructions or equations based on the provided configuration to minimize the objective function. @@ -13,10 +13,10 @@ We will show some basic usages and some functionalites this function would requi ### 1. Single Value Assuming the `pipeline_space=` was already created (have a look at [pipeline space](./pipeline_space.md) for more details). -A `run_pipeline=` function with an objective of minimizing the loss will resemble the following: +A `evaluate_pipeline=` function with an objective of minimizing the loss will resemble the following: ```python -def run_pipeline( +def evaluate_pipeline( **config, # The hyperparameters to be used in the pipeline ): element_1 = config["element_1"] @@ -30,7 +30,7 @@ def run_pipeline( ### 2. Dictionary -In this section, we will outline the special variables that are expected to be returned when the `run_pipeline=` function returns a dictionary. +In this section, we will outline the special variables that are expected to be returned when the `evaluate_pipeline=` function returns a dictionary. #### Loss @@ -41,7 +41,7 @@ One crucial return variable is the `loss`. This metric serves as a fundamental i Loss can be any value that is to be minimized by the objective function. ```python -def run_pipeline( +def evaluate_pipeline( **config, # The hyperparameters to be used in the pipeline ): @@ -53,7 +53,7 @@ def run_pipeline( reverse_loss = -loss return { - "loss": loss, + "objective_value_to_minimize": loss, "info_dict": { "reverse_loss": reverse_loss ... @@ -63,7 +63,7 @@ def run_pipeline( #### Cost -Along with the return of the `loss`, the `run_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `max_cost_total` parameter is being utilized in the `neps.run` function. +Along with the return of the `loss`, the `evaluate_pipeline=` function would optionally need to return a `cost` in certain cases. Specifically when the `max_cost_total` parameter is being utilized in the `neps.run` function. !!! note @@ -75,7 +75,7 @@ import neps import logging -def run_pipeline( +def evaluate_pipeline( **config, # The hyperparameters to be used in the pipeline ): @@ -87,18 +87,18 @@ def run_pipeline( cost = 2 return { - "loss": loss, + "objective_value_to_minimize": loss, "cost": cost, } if __name__ == "__main__": logging.basicConfig(level=logging.INFO) neps.run( - run_pipeline=run_pipeline, + evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space, # Assuming the pipeline space is defined root_directory="results/bo", max_cost_total=10, - searcher="bayesian_optimization", + optimizer="bayesian_optimization", ) ``` @@ -106,12 +106,12 @@ Each evaluation carries a cost of 2. Hence in this example, the Bayesian optimiz ## Arguments for Convenience -NePS also provides the `pipeline_directory` and the `previous_pipeline_directory` as arguments in the `run_pipeline=` function for user convenience. +NePS also provides the `pipeline_directory` and the `previous_pipeline_directory` as arguments in the `evaluate_pipeline=` function for user convenience. -Regard an example to be run with a multi-fidelity searcher, some checkpointing would be advantageos such that one does not have to train the configuration from scratch when the configuration qualifies to higher fidelity brackets. +Regard an example to be run with a multi-fidelity optimizer, some checkpointing would be advantageos such that one does not have to train the configuration from scratch when the configuration qualifies to higher fidelity brackets. ```python -def run_pipeline( +def evaluate_pipeline( pipeline_directory, # The directory where the config is saved previous_pipeline_directory, # The directory of the immediate lower fidelity config **config, # The hyperparameters to be used in the pipeline diff --git a/docs/reference/neps_run.md b/docs/reference/neps_run.md index 1d9f3c884..074dda5dd 100644 --- a/docs/reference/neps_run.md +++ b/docs/reference/neps_run.md @@ -1,14 +1,16 @@ # Configuring and Running Optimizations -The [`neps.run()`][neps.api.run] function is the core interface for running Hyperparameter and/or architecture search using optimizers in NePS. +The [`neps.run()`][neps.api.run] function is the core interface for running Hyperparameter +and/or architecture search using optimizers in NePS. +You can find most of the features NePS provides through the API of this function. This document breaks down the core arguments that allow users to control the optimization process in NePS. -Please see the documentation of [`neps.run()`][neps.api.run] for a full list. +--- ## Required Arguments -To operate, NePS requires at minimum the following three arguments -[`neps.run(run_pipeline=..., pipeline_space=..., root_directory=...)`][neps.api.run]: +To operate, NePS requires at minimum the following two arguments +[`neps.run(evaluate_pipeline=..., pipeline_space=...)`][neps.api.run]: ```python import neps @@ -19,7 +21,7 @@ def run(learning_rate: float, epochs: int) -> float: return loss neps.run( - run_pipeline=run, # (1)! + evaluate_pipeline=run, # (1)! pipeline_space={, # (2)! "learning_rate": neps.Float(1e-3, 1e-1, log=True), "epochs": neps.Integer(10, 100) @@ -30,22 +32,24 @@ neps.run( 1. The objective function, targeted by NePS for minimization, by evaluation various configurations. It requires these configurations as input and should return either a dictionary or a sole loss value as the output. - For correct setup instructions, refer to the [run pipeline page](../reference/run_pipeline.md) 2. This defines the search space for the configurations from which the optimizer samples. It accepts either a dictionary with the configuration names as keys, a path to a YAML configuration file, or a [`configSpace.ConfigurationSpace`](https://automl.github.io/ConfigSpace/) object. For comprehensive information and examples, please refer to the detailed guide available [here](../reference/pipeline_space.md) 3. The directory path where the information about the optimization and its progress gets stored. This is also used to synchronize multiple calls to `neps.run()` for parallelization. -To learn more about the `run_pipeline` function and the `pipeline_space` configuration, please refer to the [run pipeline](../reference/run_pipeline.md) and [pipeline space](../reference/pipeline_space.md) pages. + +See the following for more: + +* What kind of [pipeline space](../reference/pipeline_space.md) can you define? +* What goes in and what goes out of [`evaluate_pipeline()`](../reference/evaluate_pipeline.md)? ## Budget, how long to run? -To define a budget, provide `max_evaluations_total=` to [`neps.run()`][neps.api.run], to specify the total number of evaluations to conduct before halting the optimization process, +To define a budget, provide `max_evaluations_total=` to [`neps.run()`][neps.api.run], +to specify the total number of evaluations to conduct before halting the optimization process, or `max_cost_total=` to specify a cost threshold for your own custom cost metric, such as time, energy, or monetary. -```python - ```python def run(learning_rate: float, epochs: int) -> float: start = time.time() @@ -53,7 +57,7 @@ def run(learning_rate: float, epochs: int) -> float: # Your code here end = time.time() duration = end - start - return {"loss": loss, "cost": duration} + return {"objective_function_to_minimize": loss, "cost": duration} neps.run( max_evaluations_total=10, # (1)! @@ -64,12 +68,12 @@ neps.run( 1. Specifies the total number of evaluations to conduct before halting the optimization process. 2. Prevents the initiation of new evaluations once this cost threshold is surpassed. This can be any kind of cost metric you like, such as time, energy, or monetary, as long as you can calculate it. - This requires adding a cost value to the output of the `run_pipeline` function, for example, return `#!python {'loss': loss, 'cost': cost}`. - For more details, please refer [here](../reference/run_pipeline.md) + This requires adding a cost value to the output of the `evaluate_pipeline` function, for example, return `#!python {'loss': loss, 'cost': cost}`. + For more details, please refer [here](../reference/evaluate_pipeline.md) ## Getting some feedback, logging -Most of NePS will not print anything to the console. -To view the progress of workers, you can do so by enabling logging through [logging.basicConfig][]. +NePS will not print anything to the console. To view the progress of workers, +you can enable logging through python's [logging.basicConfig][]. ```python import logging @@ -92,7 +96,7 @@ def run(learning_rate: float, epochs: int) -> float: # Your code here end = time.time() duration = end - start - return {"loss": loss, "cost": duration} + return {"objective_value_to_minimize": loss, "cost": duration} neps.run( # Increase the total number of trials from 10 as set previously to 50 @@ -119,26 +123,31 @@ neps.run( This will delete the folder specified by `root_directory=` and all its contents. ## Getting the results -The results of the optimization process are stored in the `root_directory=` provided to [`neps.run()`][neps.api.run]. -To obtain a summary of the optimization process, you can enable the `post_run_summary=True` argument in [`neps.run()`][neps.api.run], while will generate a summary csv after the run has finished. +The results of the optimization process are stored in the `root_directory=` +provided to [`neps.run()`][neps.api.run]. +To obtain a summary of the optimization process, you can enable the +`post_run_summary=True` argument in [`neps.run()`][neps.api.run], +while will generate a summary csv after the run has finished. === "Result Directory" The root directory after utilizing this argument will look like the following: ``` - ROOT_DIRECTORY - ├── results - │ └── config_1 - │ ├── config.yaml - │ ├── metadata.yaml - │ └── result.yaml - ├── summary_csv # Only if post_run_summary=True + root_directory + ├── configs + │ ├── config_1 + │ │ ├── config.yaml # The configuration + │ │ ├── report.yaml # The results of this run, if any + │ │ └── metadata.json # Metadata about this run, such as state and times + │ └── config_2 + │ ├── config.yaml + │ └── metadata.json + ├── summary_csv # Only if post_run_summary=True │ ├── config_data.csv │ └── run_status.csv - ├── all_losses_and_configs.txt - ├── best_loss_trajectory.txt - └── best_loss_with_config_trajectory.txt + ├── optimizer_info.yaml # The optimizer's configuration + └── optimizer_state.pkl # The optimizer's state, shared between workers ``` === "python" @@ -152,7 +161,8 @@ closer to NePS. For more information, please refer to the [analyses page](../ref ## Parallelization NePS utilizes the file-system and locks as a means of communication for implementing parallelization and resuming runs. -As a result, you can start multiple [`neps.run()`][neps.api.run] from different processes however you like and they will synchronize, **as long as they share the same `root_directory=`**. +As a result, you can start multiple [`neps.run()`][neps.api.run] from different processes however you like +and they will synchronize, **as long as they share the same `root_directory=`**. Any new workers that come online will automatically pick up work and work together to until the budget is exhausted. === "Worker script" @@ -160,7 +170,7 @@ Any new workers that come online will automatically pick up work and work togeth ```python # worker.py neps.run( - run_pipeline=..., + evaluate_pipeline=..., pipeline_space=..., root_directory="some/path", max_evaluations_total=100, @@ -189,38 +199,58 @@ Any new workers that come online will automatically pick up work and work togeth ``` ## YAML Configuration -You have the option to configure all arguments using a YAML file through [`neps.run(run_args=...)`][neps.api.run]. -For more on yaml usage, please visit the dedicated [page on usage of YAML with NePS](../reference/declarative_usage.md). +We support arguments to [`neps.run()`][neps.api.run] that have been seriliazed into a +YAML file. This means you can manage your configurations in a more human-readable format +if you prefer. -Parameters not explicitly defined within this file will receive their default values. +For more on yaml usage, please visit the dedicated +[page on usage of YAML with NePS](../reference/declarative_usage.md). -=== "Yaml Configuration" +=== "`config.yaml`" ```yaml - # path/to/your/config.yaml - run_pipeline: - path: "path/to/your/run_pipeline.py" # File path of the run_pipeline function - name: "name_of_your_run_pipeline" # Function name - pipeline_space: "path/to/your/search_space.yaml" # Path of the search space yaml file + # We allow specifying the evaluate_pipeline as a module path and function name + evaluate_pipeline: path/to/evaluate_pipeline.py:eval_func_name + + pipeline_space: + batch_size: 64 # Constant + optimizer: [adam, sgd, adamw] # Categorical + alpha: [0.01, 1.0] # Uniform Float + n_layers: [1, 10] # Uniform Integer + learning_rate: # Log scale Float with a prior + lower: 1e-5 + upper: 1e-1 + log: true + prior: 1e-3 + prior_confidence: high + epochs: # Integer fidelity + lower: 5 + upper: 20 + is_fidelity: true + root_directory: "neps_results" # Output directory for results max_evaluations_total: 100 - post_run_summary: # Defaults applied if left empty - searcher: - strategy: "bayesian_optimization" + optimizer: + name: "bayesian_optimization" initial_design_size: 5 - surrogate_model: "gp" + cost_aware: true ``` -=== "Python" +=== "`run_neps.py`" ```python - neps.run(run_args="path/to/your/config.yaml") + with open("config.yaml", "r") as file: + settings = yaml.safe_load(file) + + neps.run(**settings) ``` ## Handling Errors Things go wrong during optimization runs and it's important to consider what to do in these cases. -By default, NePS will halt the optimization process when an error but you can choose to `ignore_errors=`, providing a `loss_value_on_error=` and `cost_value_on_error=` to control what values should be reported to the optimization process. +By default, NePS will halt the optimization process when an error but you can choose to `ignore_errors=`, +providing a `loss_value_on_error=` and `cost_value_on_error=` to control what values should be +reported to the optimization process. ```python def run(learning_rate: float, epochs: int) -> float: @@ -245,44 +275,46 @@ neps.run( Any runs that error will still count towards the total `max_evaluations_total` or `max_evaluations_per_run`. -## Selecting an Optimizer -By default NePS intelligently selects the most appropriate search strategy based on your defined configurations in `pipeline_space=`, one of the arguments to [`neps.run()`][neps.api.run]. +### Re-running Failed Configurations +Sometimes things go wrong but not due to the configuration itself. +Sometimes you'd also like to change the state so that you re-evaluate that configuration. -The characteristics of your search space, as represented in the `pipeline_space=`, play a crucial role in determining which optimizer NePS will choose. -This automatic selection process ensures that the strategy aligns with the specific requirements and nuances of your search space, thereby optimizing the effectiveness of the hyperparameter and/or architecture optimization. - -You can also manually select a specific or custom optimizer that better matches your specific needs. -For more information about the available searchers and how to customize your own, refer [here](../reference/optimizers.md). - -## Managing Experiments -While tuning pipelines, it is common to run multiple experiments, perhaps varying the search space, the metric, the model or any other factors of your development. -We provide two extra arguments to help manage directories for these, `development_stage_id=` and `task_id=`. - -```python +If you need to go in there and change anything, **the entire optimization state** is editable on disk. +You can follow these steps to modify the state of things. -def run1(learning_rate: float, epochs: int) -> float: - # Only tuning learning rate - - return - -def run2(learning_rate: float, l2: float, epochs: int) -> float: - # Tuning learning rate and l2 regularization - - return - -neps.run( - ..., - task_id="l2_regularization", # (1)! - development_stage_id="003", # (2)! -) ``` +root_directory +├── configs +│ ├── .trial_cache.pkl # A cache of all trial information for optimizers +│ ├── config_1 +│ │ ├── config.yaml # The configuration +│ │ ├── report.yaml # The results of this run, if any +│ │ ├── metadata.json # Metadata about this run, such as state and times +│ └── config_2 +│ ├── config.yaml +│ └── metadata.json +├── optimizer_info.yaml +└── optimizer_state.pkl # The optimizer's state, shared between workers +``` + +1. The first thing you should do is make sure no workers are running. +2. Next, delete `optimizer_state.pkl` and `configs/.trial_cache.pkl`. This is cached information to share betwen the + workers. +3. Lastly, you can go in and modify any of the following files: -1. An identifier used when working with multiple development stages. - Instead of creating new root directories, use this identifier to save the results of an optimization run in a separate dev_id folder within the root_directory. -2. An identifier used when the optimization process involves multiple tasks. - This functions similarly to `development_stage_id=`, but it creates a folder named after the `task_id=`, providing an organized way to separate results for different tasks within the `root_directory=`. + * `config.yaml` - The configuration to be run. This was samled from your search space. + * `report.yaml` - The results of the run. This is where you can change what was reported back. + * `metadata.json` - Metadata about the run. Here you can change the `"state"` key to one + of [`State`][neps.state.trial.State] to re-run the configuration, usually you'd want to set it + to `"pending"` such that the next worker will pick it up and re-run it. +4. Once you've made your changes, you can start the workers again and they will pick up the new state + re-creating the caches as necessary. -## Others +## Selecting an Optimizer +By default NePS intelligently selects the most appropriate optimizer based on your defined configurations in `pipeline_space=`, one of the arguments to [`neps.run()`][neps.api.run]. -* `pre_load_hooks=`: A list of hook functions to be called before loading results. +The characteristics of your search space, as represented in the `pipeline_space=`, play a crucial role in determining which optimizer NePS will choose. +This automatic selection process ensures that the optimizer aligns with the specific requirements and nuances of your search space, thereby optimizing the effectiveness of the hyperparameter and/or architecture optimization. +You can also manually select a specific or custom optimizer that better matches your specific needs. +For more information about the available optimizers and how to customize your own, refer [here](../reference/optimizers.md). diff --git a/docs/reference/optimizers.md b/docs/reference/optimizers.md index 26e8a0d7c..e59116b3b 100644 --- a/docs/reference/optimizers.md +++ b/docs/reference/optimizers.md @@ -7,7 +7,7 @@ preferences and requirements. ### 1. Automatic Optimizer Selection If you prefer not to specify a particular optimizer for your AutoML task, you can simply pass `"default"` or `None` -for the neps searcher. NePS will automatically choose the best optimizer based on the characteristics of your search +for the neps optimizer. NePS will automatically choose the best optimizer based on the characteristics of your search space. This provides a hassle-free way to get started quickly. The optimizer selection is based on the following characteristics of your `pipeline_space`: @@ -17,15 +17,15 @@ The optimizer selection is based on the following characteristics of your `pipel - If it has a prior: `pibo` - If it has neither: `bayesian_optimization` -For example, running the following format, without specifying a searcher will choose an optimizer depending on +For example, running the following format, without specifying a optimizer will choose an optimizer depending on the `pipeline_space` passed. ```python neps.run( - run_pipeline=run_function, + evalute_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", max_evaluations_total=25, - # no searcher specified + # no optimizer specified ) ``` @@ -33,35 +33,33 @@ neps.run( We have also prepared some optimizers with specific hyperparameters that we believe can generalize well to most AutoML tasks and use cases. For more details on the available default optimizers and the algorithms that can be called, -please refer to the next section on [SearcherConfigs](#searcher-configurations). +please refer to the next section on [PredefinedOptimizerConfigs](#optimizer-configurations). ```python neps.run( - run_pipeline=run_function, + evalute_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", max_evaluations_total=25, - # searcher specified, along with an argument - searcher="bayesian_optimization", + # optimizer specified, along with an argument + optimizer="bayesian_optimization", initial_design_size=5, ) ``` -For more optimizers, please refer [here](#list-available-searchers) . +For more optimizers, please refer [here](#list-available-optimizers) . ### 3. Custom Optimizer Configuration via YAML For users who want more control over the optimizer's hyperparameters, you can create your own YAML configuration file. In this file, you can specify the hyperparameters for your preferred optimizer. To use this custom configuration, -provide the path to your YAML file using the `searcher` parameter when running the optimizer. +provide the path to your YAML file using the `optimizer` parameter when running the optimizer. The library will then load your custom settings and use them for optimization. Here's the format of a custom YAML (`custom_bo.yaml`) configuration using `Bayesian Optimization` as an example: ```yaml -strategy: bayesian_optimization -name: my_custom_bo # optional; otherwise, your searcher will be named after your YAML file, here 'custom_bo'. -# Specific arguments depending on the searcher +name: bayesian_optimization initial_design_size: 7 surrogate_model: gp acquisition: EI @@ -74,11 +72,11 @@ sample_prior_first: false ```python neps.run( - run_pipeline=run_function, + evalute_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", max_evaluations_total=25, - searcher="path/to/custom_bo.yaml", + optimizer="path/to/custom_bo.yaml", ) ``` @@ -91,11 +89,11 @@ precedence over those specified in the YAML configuration. ```python neps.run( - run_pipeline=run_function, + evalute_pipeline=run_function, pipeline_space=pipeline_space, root_directory="results/", max_evaluations_total=25, - searcher="path/to/custom_bo.yaml", + optimizer="path/to/custom_bo.yaml", initial_design_size=5, # overrides value in custom_bo.yaml random_interleave_prob=0.25 # overrides value in custom_bo.yaml ) @@ -103,60 +101,60 @@ neps.run( ## Note for Contributors -When designing a new optimizer, it's essential to create a YAML configuration file in the `default_searcher` folder under `neps.src.optimizers`. This YAML file should contain the default configuration settings that you believe should be used when the user chooses the searcher. +When designing a new optimizer, it's essential to create a YAML configuration file in the `optimizer_yaml_files` folder under `neps.src.optimizers`. This YAML file should contain the default configuration settings that you believe should be used when the user chooses the optimizer. -Even when many hyperparameters might be set to their default values as specified in the code, it is still considered good practice to include them in the YAML file. This is because the `SearcherConfigs` method relies on the arguments from the YAML file to display the optimizer's configuration to the user. +Even when many hyperparameters might be set to their default values as specified in the code, it is still considered good practice to include them in the YAML file. This is because the `PredefinedOptimizerConfigs` method relies on the arguments from the YAML file to display the optimizer's configuration to the user. -## Searcher Configurations +## Optimizer Configurations -The `SearcherConfigs` class provides a set of useful functions to manage and retrieve default configuration details for NePS optimizers. These functions can help you understand and interact with the available searchers and their associated algorithms and configurations. +The `PredefinedOptimizerConfigs` class provides a set of useful functions to manage and retrieve default configuration details for NePS optimizers. These functions can help you understand and interact with the available optimizers and their associated algorithms and configurations. -### Importing `SearcherConfigs` +### Importing `PredefinedOptimizerConfigs` -Before you can use the `SearcherConfigs` class to manage and retrieve default configuration details for NePS optimizers, make sure to import it into your Python script. You can do this with the following import statement: +Before you can use the `PredefinedOptimizerConfigs` class to manage and retrieve default configuration details for NePS optimizers, make sure to import it into your Python script. You can do this with the following import statement: ```python -from neps.optimizers.info import SearcherConfigs +from neps.optimizers.info import PredefinedOptimizerConfigs ``` -Once you have imported the class, you can proceed to use its functions to explore the available searchers, algorithms, and configuration details. +Once you have imported the class, you can proceed to use its functions to explore the available optimizers, algorithms, and configuration details. -### List Available Searchers +### List Available Optimizers -To list all the available searchers that can be used in NePS runs, you can use the `get_searchers` function. It provides you with a list of searcher names: +To list all the available optimizers that can be used in NePS runs, you can use the `get_optimizers` function. It provides you with a list of optimizer names: ```python -searchers = SearcherConfigs.get_searchers() -print("Available searchers:", searchers) +optimizers = PredefinedOptimizerConfigs.get_optimizers() +print("Available optimizers:", optimizers) ``` ### List Available Searching Algorithms -The `get_available_algorithms` function helps you discover the searching algorithms available within the NePS searchers: +The `get_available_algorithms` function helps you discover the searching algorithms available within the NePS optimizers: ```python -algorithms = SearcherConfigs.get_available_algorithms() +algorithms = PredefinedOptimizerConfigs.get_available_algorithms() print("Available searching algorithms:", algorithms) ``` -### Find Searchers Using a Specific Algorithm +### Find Optimizers Using a Specific Algorithm -If you want to identify which NePS searchers are using a specific searching algorithm (e.g., Bayesian Optimization, Hyperband, PriorBand...), you can use the `get_searcher_from_algorithm` function. It returns a list of searchers utilizing the specified algorithm: +If you want to identify which NePS optimizers are using a specific searching algorithm (e.g., Bayesian Optimization, Hyperband, PriorBand...), you can use the `get_optimizer_from_algorithm` function. It returns a list of optimizers utilizing the specified algorithm: ```python algorithm = "bayesian_optimization" # Replace with the desired algorithm -searchers = SearcherConfigs.get_searcher_from_algorithm(algorithm) -print(f"Searchers using {algorithm}:", searchers) +optimizers = PredefinedOptimizerConfigs.get_optimizer_from_algorithm(algorithm) +print(f"optimizers using {algorithm}:", optimizers) ``` -### Retrieve Searcher Configuration Details +### Retrieve Optimizer Configuration Details -To access the configuration details of a specific searcher, you can use the `get_searcher_kwargs` function. Provide the name of the searcher you are interested in, and it will return the searcher's configuration: +To access the configuration details of a specific optimizer, you can use the `get_optimizer_kwargs` function. Provide the name of the optimizer you are interested in, and it will return the optimizer's configuration: ```python -searcher_name = "pibo" # Replace with the desired NePS searcher name -searcher_kwargs = SearcherConfigs.get_searcher_kwargs(searcher_name) -print(f"Configuration of {searcher_name}:", searcher_kwargs) +optimizer_name = "pibo" # Replace with the desired NePS optimizer name +optimizer_kwargs = PredefinedOptimizerConfigs.get_optimizer_kwargs(optimizer_name) +print(f"Configuration of {optimizer_name}:", optimizer_kwargs) ``` -These functions empower you to explore and manage the available NePS searchers and their configurations effectively. +These functions empower you to explore and manage the available NePS optimizers and their configurations effectively. diff --git a/docs/reference/pipeline_space.md b/docs/reference/pipeline_space.md index f0780378c..e31792957 100644 --- a/docs/reference/pipeline_space.md +++ b/docs/reference/pipeline_space.md @@ -12,10 +12,10 @@ effectively incorporate various parameter types, ensuring that NePS can utilize ## Parameters NePS currently features 4 primary hyperparameter types: -* [`Categorical`][neps.search_spaces.hyperparameters.categorical.Categorical] -* [`Float`][neps.search_spaces.hyperparameters.float.Float] -* [`Integer`][neps.search_spaces.hyperparameters.integer.Integer] -* [`Constant`][neps.search_spaces.hyperparameters.constant.Constant] +* [`Categorical`][neps.space.Categorical] +* [`Float`][neps.space.Float] +* [`Integer`][neps.space.Integer] +* [`Constant`][neps.space.Constant] Using these types, you can define the parameters that NePS will optimize during the search process. The most basic way to pass these parameters is through a Python dictionary, where each key-value @@ -27,38 +27,38 @@ for optimizing a deep learning model: pipeline_space = { "learning_rate": neps.Float(0.00001, 0.1, log=True), "num_epochs": neps.Integer(3, 30, is_fidelity=True), - "optimizer": neps.Categorical(["adam", "sgd", "rmsprop"]), - "dropout_rate": neps.Constant(0.5), + "optimizer": ["adam", "sgd", "rmsprop"], # Categorical + "dropout_rate": 0.5, # Constant } -neps.run(.., pipeline_space = pipeline_space) +neps.run(.., pipeline_space=pipeline_space) ``` ??? example "Quick Parameter Reference" === "`Categorical`" - ::: neps.search_spaces.hyperparameters.categorical.Categorical + ::: neps.space.Categorical === "`Float`" - ::: neps.search_spaces.hyperparameters.float.Float + ::: neps.space.Float === "`Integer`" - ::: neps.search_spaces.hyperparameters.integer.Integer + ::: neps.space.Integer === "`Constant`" - ::: neps.search_spaces.hyperparameters.constant.Constant + ::: neps.space.Constant ## Using your knowledge, providing a Prior -When optimizing, you can provide your own knowledge using the parameters `default=`. -By indicating a `default=` we take this to be your user prior, +When optimizing, you can provide your own knowledge using the parameters `prior=`. +By indicating a `prior=` we take this to be your user prior, **your knowledge about where a good value for this parameter lies**. -You can also specify a `default_confidence=` to indicate how strongly you want NePS, +You can also specify a `prior_confidence=` to indicate how strongly you want NePS, to focus on these, one of either `"low"`, `"medium"`, or `"high"`. Currently the two major algorithms that exploit this in NePS are `PriorBand` @@ -70,21 +70,21 @@ import neps neps.run( ..., pipeline_space={ - "learning_rate": neps.Float(1e-4, 1e-1, log=True, default=1e-2, default_confidence="medium"), + "learning_rate": neps.Float(1e-4, 1e-1, log=True, prior=1e-2, prior_confidence="medium"), "num_epochs": neps.Integer(3, 30, is_fidelity=True), - "optimizer": neps.Categorical(["adam", "sgd", "rmsprop"], default="adam", default_confidence="low"), + "optimizer": neps.Categorical(["adam", "sgd", "rmsprop"], prior="adam", prior_confidence="low"), "dropout_rate": neps.Constant(0.5), } ) ``` -!!! warning "Must set `default=` for all parameters, if any" +!!! warning "Must set `prior=` for all parameters, if any" - If you specify `default=` for one parameter, you must do so for all your variables. + If you specify `prior=` for one parameter, you must do so for all your variables. This will be improved in future versions. !!! warning "Interaction with `is_fidelity`" - If you specify `is_fidelity=True` for one parameter, the `default=` and `default_confidence=` are ignored. + If you specify `is_fidelity=True` for one parameter, the `prior=` and `prior_confidence=` are ignored. This will be dissallowed in future versions. ## Defining a pipeline space using YAML diff --git a/docs/stylesheets/custom.css b/docs/stylesheets/custom.css index a7782f17a..d2ea0e9cf 100644 --- a/docs/stylesheets/custom.css +++ b/docs/stylesheets/custom.css @@ -10,7 +10,7 @@ code.highlight.language-python span.kc { :root { --md-tooltip-width: 500px; } -/* api doc attribute cards */ +/* api doc attribute cards div.doc-class > div.doc-contents > div.doc-children > div.doc-object { padding-right: 20px; padding-left: 20px; @@ -22,4 +22,4 @@ div.doc-class > div.doc-contents > div.doc-children > div.doc-object { border-color: rgba(0, 0, 0, 0.2); border-width: 1px; border-style: solid; -} +} */ diff --git a/mkdocs.yml b/mkdocs.yml index 464d6fc1b..50869ed88 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -104,8 +104,8 @@ plugins: returns_multiple_items: false show_docstring_attributes: true show_docstring_description: true - show_root_heading: true - show_root_toc_entry: true + show_root_heading: false + show_root_toc_entry: false show_object_full_path: false show_root_members_full_path: false signature_crossrefs: true @@ -136,7 +136,7 @@ nav: - Search Space: 'reference/pipeline_space.md' - Optimizers: 'reference/optimizers.md' - Declarative Usage: 'reference/declarative_usage.md' - - The Run Function: 'reference/run_pipeline.md' + - The Evaluate Function: 'reference/evaluate_pipeline.md' - CLI Usage: 'reference/cli.md' - Analysing Runs: 'reference/analyse.md' - Examples: "examples/" # auto-generated diff --git a/neps/__init__.py b/neps/__init__.py index 52ee91f73..caad18c2b 100644 --- a/neps/__init__.py +++ b/neps/__init__.py @@ -1,40 +1,18 @@ from neps.api import run from neps.plot.plot import plot from neps.plot.tensorboard_eval import tblogger -from neps.search_spaces import ( - Architecture, - ArchitectureParameter, - Categorical, - CategoricalParameter, - Constant, - ConstantParameter, - Float, - FloatParameter, - Function, - FunctionParameter, - GraphGrammar, - Integer, - IntegerParameter, -) +from neps.space import Categorical, Constant, Float, Integer, SearchSpace from neps.status.status import get_summary_dict, status +from neps.utils.files import load_and_merge_yamls as load_yamls __all__ = [ - "Architecture", - "ArchitectureParameter", "Categorical", - "CategoricalParameter", "Constant", - "ConstantParameter", "Float", - "FloatParameter", - "Function", - "FunctionParameter", - "GraphGrammar", - "GraphGrammarCell", - "GraphGrammarRepetitive", "Integer", - "IntegerParameter", + "SearchSpace", "get_summary_dict", + "load_yamls", "plot", "run", "status", diff --git a/neps/api.py b/neps/api.py index ac80e4a34..0e5b47205 100644 --- a/neps/api.py +++ b/neps/api.py @@ -1,250 +1,437 @@ """API for the neps package.""" -import inspect +from __future__ import annotations + import logging -import warnings +from collections.abc import Callable, Mapping from pathlib import Path -from typing import Callable, Iterable, Literal - -import ConfigSpace as CS -from neps.utils.run_args import Settings, Default +from typing import TYPE_CHECKING, Any, Literal -from neps.utils.common import instance_from_map +from neps.optimizers import OptimizerChoice, load_optimizer from neps.runtime import _launch_runtime -from neps.optimizers import BaseOptimizer, SearcherMapping -from neps.search_spaces.parameter import Parameter -from neps.search_spaces.search_space import ( - SearchSpace, - pipeline_space_from_configspace, - pipeline_space_from_yaml, -) +from neps.space.parsing import convert_to_space from neps.status.status import post_run_csv -from neps.utils.common import get_searcher_data, get_value -from neps.optimizers.info import SearcherConfigs +from neps.utils.common import dynamic_load_object + +if TYPE_CHECKING: + from ConfigSpace import ConfigurationSpace + + from neps.optimizers.optimizer import AskFunction + from neps.space import Parameter, SearchSpace logger = logging.getLogger(__name__) -def run( - evaluate_pipeline: Callable | None = Default(None), - root_directory: str | Path | None = Default(None), +def run( # noqa: PLR0913 + evaluate_pipeline: Callable | str, pipeline_space: ( - dict[str, Parameter] | str | Path | CS.ConfigurationSpace | None - ) = Default(None), - run_args: str | Path | None = Default(None), - overwrite_working_directory: bool = Default(False), - post_run_summary: bool = Default(True), - development_stage_id=Default(None), - task_id=Default(None), - max_evaluations_total: int | None = Default(None), - max_evaluations_per_run: int | None = Default(None), - continue_until_max_evaluation_completed: bool = Default(False), - max_cost_total: int | float | None = Default(None), - ignore_errors: bool = Default(False), - objective_to_minimize_value_on_error: None | float = Default(None), - cost_value_on_error: None | float = Default(None), - pre_load_hooks: Iterable | None = Default(None), - sample_batch_size: int | None = Default(None), - searcher: ( - Literal[ - "default", - "bayesian_optimization", - "random_search", - "hyperband", - "priorband", - "mobster", - "asha", - ] - | BaseOptimizer - | Path - ) = Default("default"), - **searcher_kwargs, + Mapping[str, dict | str | int | float | Parameter] + | SearchSpace + | ConfigurationSpace + ), + *, + root_directory: str | Path = "neps_results", + overwrite_working_directory: bool = False, + post_run_summary: bool = True, + max_evaluations_total: int | None = None, + max_evaluations_per_run: int | None = None, + continue_until_max_evaluation_completed: bool = False, + max_cost_total: int | float | None = None, + ignore_errors: bool = False, + objective_value_on_error: float | None = None, + cost_value_on_error: float | None = None, + sample_batch_size: int | None = None, + optimizer: ( + OptimizerChoice + | Mapping[str, Any] + | tuple[OptimizerChoice, Mapping[str, Any]] + | tuple[Callable[..., AskFunction], Mapping[str, Any]] + | Callable[..., AskFunction] + | Literal["auto"] + ) = "auto", ) -> None: - """Run a neural pipeline search. - - To parallelize: - To run a neural pipeline search with multiple processes or machines, - simply call run(.) multiple times (optionally on different machines). Make sure - that root_directory points to the same folder on the same filesystem, otherwise, - the multiple calls to run(.) will be independent. + """Run the optimization. + + !!! tip "Parallelization" + + To run with multiple processes or machines, execute the script that + calls `neps.run()` multiple times. They will keep in sync using + the file-sytem, requiring that `root_directory` be shared between them. + + + ```python + import neps + import logging + + logging.basicConfig(level=logging.INFO) + + def evaluate_pipeline(some_parameter: float) -> float: + validation_error = -some_parameter + return validation_error + + pipeline_space = dict(some_parameter=neps.Float(lower=0, upper=1)) + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space={ + "some_parameter": (0.0, 1.0), # float + "another_parameter": (0, 10), # integer + "optimizer": ["sgd", "adam"], # categorical + "epoch": neps.Integer( # fidelity integer + lower=1, + upper=100, + is_fidelity=True + ), + "learning_rate": neps.Float( # log spaced float + lower=1e-5, + uperr=1, + log=True + ), + "alpha": neps.Float( # float with a prior + lower=0.1, + upper=1.0, + prior=0.99, + prior_confidence="high", + ) + }, + root_directory="usage_example", + max_evaluations_total=5, + ) + ``` Args: - evaluate_pipeline: The objective function to minimize. + evaluate_pipeline: The objective function to minimize. This will be called + with a configuration from the `pipeline_space=` that you define. + + The function should return one of the following: + + * A `float`, which is the objective value to minimize. + * A `dict` which can have the following keys: + + ```python + { + "objective_to_minimize": float, # The thing to minimize (required) + "cost": float, # The cost of the evaluate_pipeline, used by some algorithms + "info_dict": dict, # Any additional information you want to store, should be YAML serializable + } + ``` + + ??? note "`str` usage for dynamic imports" + + If a string, it should be in the format `"/path/to/:function"`. + to specify the function to call. You may also directly provide + an mode to import, e.g., `"my.module.something:evaluate_pipeline"`. + pipeline_space: The search space to minimize over. - root_directory: The directory to save progress to. This is also used to - synchronize multiple calls to run(.) for parallelization. - run_args: An option for providing the optimization settings e.g. - max_evaluations_total in a YAML file. + + This most direct way to specify the search space is as follows: + + ```python + neps.run( + pipeline_space={ + "dataset": "mnist", # constant + "nlayers": (2, 10), # integer + "alpha": (0.1, 1.0), # float + "optimizer": [ # categorical + "adam", "sgd", "rmsprop" + ], + "learning_rate": neps.Float(, # log spaced float + lower=1e-5, upper=1, log=True + ), + "epochs": neps.Integer( # fidelity integer + lower=1, upper=100, is_fidelity=True + ), + "batch_size": neps.Integer( # integer with a prior + lower=32, upper=512, prior=128 + ), + + } + ) + ``` + + You can also directly instantiate any of the parameters + defined by [`Parameter`][neps.space.parameters.Parameter] + and provide them directly. + + Some important properties you can set on parameters are: + + * `prior=`: If you have a good idea about what a good setting + for a parameter may be, you can set this as the prior for + a parameter. You can specify this along with `prior_confidence` + if you would like to assign a `"low"`, `"medium"`, or `"high"` + confidence to the prior. + + + !!! note "Yaml support" + + To support spaces defined in yaml, you may also define the parameters + as dictionarys, e.g., + + ```python + neps.run( + pipeline_space={ + "dataset": "mnist", + "nlayers": {"type": "int", "lower": 2, "upper": 10}, + "alpha": {"type": "float", "lower": 0.1, "upper": 1.0}, + "optimizer": {"type": "cat", "choices": ["adam", "sgd", "rmsprop"]}, + "learning_rate": {"type": "float", "lower": 1e-5, "upper": 1, "log": True}, + "epochs": {"type": "int", "lower": 1, "upper": 100, "is_fidelity": True}, + "batch_size": {"type": "int", "lower": 32, "upper": 512, "prior": 128}, + } + ) + ``` + + !!! note "ConfigSpace support" + + You may also use a `ConfigurationSpace` object from the + `ConfigSpace` library. + + root_directory: The directory to save progress to. + overwrite_working_directory: If true, delete the working directory at the start of the run. This is, e.g., useful when debugging a evaluate_pipeline function. + post_run_summary: If True, creates a csv file after each worker is done, holding summary information about the configs and results. - development_stage_id: ID for the current development stage. Only needed if - you work with multiple development stages. - task_id: ID for the current task. Only needed if you work with multiple - tasks. + + max_evaluations_per_run: Number of evaluations this specific call should do. + max_evaluations_total: Number of evaluations after which to terminate. - max_evaluations_per_run: Number of evaluations the specific call to run(.) should - maximally do. - continue_until_max_evaluation_completed: If true, only stop after - max_evaluations_total have been completed. This is only relevant in the - parallel setting. + This is shared between all workers operating in the same `root_directory`. + + continue_until_max_evaluation_completed: + If true, only stop after max_evaluations_total have been completed. + This is only relevant in the parallel setting. + max_cost_total: No new evaluations will start when this cost is exceeded. Requires returning a cost in the evaluate_pipeline function, e.g., `return dict(loss=loss, cost=cost)`. ignore_errors: Ignore hyperparameter settings that threw an error and do not raise an error. Error configs still count towards max_evaluations_total. - objective_to_minimize_value_on_error: Setting this and cost_value_on_error to any float will + objective_value_on_error: Setting this and cost_value_on_error to any float will supress any error and will use given objective_to_minimize value instead. default: None - cost_value_on_error: Setting this and objective_to_minimize_value_on_error to any float will + cost_value_on_error: Setting this and objective_value_on_error to any float will supress any error and will use given cost value instead. default: None - pre_load_hooks: List of functions that will be called before load_results(). - sample_batch_size: The number of samples to ask for in a single call to the - optimizer. - searcher: Which optimizer to use. Can be a string identifier, an - instance of BaseOptimizer, or a Path to a custom optimizer. - **searcher_kwargs: Will be passed to the searcher. This is usually only needed by - neps develolpers. - - Raises: - ValueError: If deprecated argument working_directory is used. - ValueError: If root_directory is None. - - - Example: - >>> import neps - - >>> def evaluate_pipeline(some_parameter: float): - >>> validation_error = -some_parameter - >>> return validation_error - - >>> pipeline_space = dict(some_parameter=neps.Float(lower=0, upper=1)) - - >>> logging.basicConfig(level=logging.INFO) - >>> neps.run( - >>> evaluate_pipeline=evaluate_pipeline, - >>> pipeline_space=pipeline_space, - >>> root_directory="usage_example", - >>> max_evaluations_total=5, - >>> ) - """ - if "working_directory" in searcher_kwargs: - raise ValueError( - "The argument 'working_directory' is deprecated, please use 'root_directory' " - "instead" - ) - if "budget" in searcher_kwargs: - warnings.warn( - "The argument: 'budget' is deprecated. In the neps.run call, please, use " - "'max_cost_total' instead. In future versions using `budget` will fail.", - DeprecationWarning, - stacklevel=2, - ) - max_cost_total = searcher_kwargs["budget"] - del searcher_kwargs["budget"] - - settings = Settings(locals(), run_args) - # TODO: check_essentials, - - # DO NOT use any neps arguments directly; instead, access them via the Settings class. - if settings.pre_load_hooks is None: - settings.pre_load_hooks = [] - - logger.info(f"Starting neps.run using root directory {settings.root_directory}") - - # Used to create the yaml holding information about the searcher. - # Also important for testing and debugging the api. - searcher_info = { - "searcher_name": "", - "searcher_alg": "", - "searcher_selection": "", - "neps_decision_tree": True, - "searcher_args": {}, - } - - # special case if you load your own optimizer via run_args - if inspect.isclass(settings.searcher): - if issubclass(settings.searcher, BaseOptimizer): - search_space = SearchSpace(**settings.pipeline_space) - # aligns with the behavior of the internal neps searcher which also overwrites - # its arguments by using searcher_kwargs - # TODO habe hier searcher kwargs gedroppt, sprich das merging muss davor statt - # finden - searcher_info["searcher_args"] = settings.searcher_kwargs - settings.searcher = settings.searcher( - search_space, **settings.searcher_kwargs - ) - else: - # Raise an error if searcher is not a subclass of BaseOptimizer - raise TypeError( - "The provided searcher must be a class that inherits from BaseOptimizer." + sample_batch_size: + The number of samples to ask for in a single call to the optimizer. + + ??? tip "When to use this?" + + This is only useful in scenarios where you have many workers + available, and the optimizers sample time prevents full + worker utilization, as can happen with Bayesian optimizers. + + In this case, the currently active worker will first + check if there are any new configurations to evaluate, + and if not, generate `sample_batch_size` new configurations + that the proceeding workers will then pick up and evaluate. + + We advise to only use this if: + + * You are using a `#!python "ifbo"` or `#!python "bayesian_optimization"`. + * You have a fast to evaluate `evaluate_pipeline` + * You have a significant amount of workers available, relative to the + time it takes to evaluate a single configuration. + + ??? warning "Downsides of batching" + + The primary downside of batched optimization is that + the next `sample_batch_size` configurations will not + be able to take into account the results of any new + evaluations, even if they were to come in relatively + quickly. + + optimizer: Which optimizer to use. + + Not sure which to use? Leave this at `"auto"` and neps will + choose the optimizer based on the search space given. + + ??? note "Available optimizers" + + --- + + * `#!python "bayesian_optimization"`, + + ::: neps.optimizers.algorithms.bayesian_optimization + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "ifbo"` + + ::: neps.optimizers.algorithms.ifbo + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "successive_halving"`: + + ::: neps.optimizers.algorithms.successive_halving + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "hyperband"`: + + ::: neps.optimizers.algorithms.hyperband + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "priorband"`: + + ::: neps.optimizers.algorithms.priorband + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "asha"`: + + ::: neps.optimizers.algorithms.asha + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "async_hb"`: + + ::: neps.optimizers.algorithms.async_hb + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "random_search"`: + + ::: neps.optimizers.algorithms.random_search + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + * `#!python "grid_search"`: + + ::: neps.optimizers.algorithms.grid_search + options: + show_root_heading: false + show_signature: false + show_source: false + + --- + + + With any optimizer choice, you also may provide some additional parameters to the optimizers. + We do not recommend this unless you are familiar with the optimizer you are using. You + may also specify an optimizer as a dictionary for supporting reading in serialized yaml + formats: + + ```python + neps.run( + ..., + optimzier={ + "name": "priorband", + "sample_prior_first": True, + } ) + ``` + + ??? tip "Own optimzier" + + Lastly, you may also provide your own optimizer which must satisfy + the [`AskFunction`][neps.optimizers.optimizer.AskFunction] signature. + + ```python + class MyOpt: + + def __init__(self, space: SearchSpace): + ... + + def __call__( + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + # Sample a new configuration. + # + # Args: + # trials: All of the trials that are known about. + # budget_info: information about the budget constraints. + # + # Returns: + # The sampled configuration(s) + + + neps.run( + ..., + optimizer=MyOpt, + ) + ``` - if isinstance(settings.searcher, BaseOptimizer): - searcher_instance = settings.searcher - searcher_info["searcher_name"] = "baseoptimizer" - searcher_info["searcher_alg"] = settings.searcher.whoami() - searcher_info["searcher_selection"] = "user-instantiation" - searcher_info["neps_decision_tree"] = False - else: - ( - searcher_instance, - searcher_info, - ) = _run_args( - searcher_info=searcher_info, - pipeline_space=settings.pipeline_space, - max_cost_total=settings.max_cost_total, - ignore_errors=settings.ignore_errors, - objective_to_minimize_value_on_error=settings.objective_to_minimize_value_on_error, - cost_value_on_error=settings.cost_value_on_error, - searcher=settings.searcher, - **settings.searcher_kwargs, - ) + This is mainly meant for internal development but allows you to use the NePS + runtime to run your optimizer. - # Check to verify if the target directory contains history of another optimizer state - # This check is performed only when the `searcher` is built during the run - if not isinstance(settings.searcher, (BaseOptimizer, str, dict, Path)): - raise ValueError( - f"Unrecognized `searcher` of type {type(settings.searcher)}. Not str or " - f"BaseOptimizer." - ) - elif isinstance(settings.searcher, BaseOptimizer): - # This check is not strict when a user-defined neps.optimizer is provided - logger.warning( - "An instantiated optimizer is provided. The safety checks of NePS will be " - "skipped. Accurate continuation of runs can no longer be guaranteed!" - ) + """ # noqa: E501 + logger.info(f"Starting neps.run using root directory {root_directory}") + space = convert_to_space(pipeline_space) + _optimizer_ask, _optimizer_info = load_optimizer(optimizer=optimizer, space=space) - if settings.task_id is not None: - settings.root_directory = Path(settings.root_directory) / ( - f"task_" f"{settings.task_id}" - ) - if settings.development_stage_id is not None: - settings.root_directory = ( - Path(settings.root_directory) / f"dev_{settings.development_stage_id}" + _eval: Callable + if isinstance(evaluate_pipeline, str): + module, funcname = evaluate_pipeline.rsplit(":", 1) + eval_pipeline = dynamic_load_object(module, funcname) + if not callable(eval_pipeline): + raise ValueError( + f"'{funcname}' in module '{module}' is not a callable function." + ) + _eval = eval_pipeline + elif callable(evaluate_pipeline): + _eval = evaluate_pipeline + else: + raise ValueError( + "evaluate_pipeline must be a callable or a string in the format" + "'module:function'." ) _launch_runtime( - evaluation_fn=settings.evaluate_pipeline, - optimizer=searcher_instance, - optimizer_info=searcher_info, - max_cost_total=settings.max_cost_total, - optimization_dir=Path(settings.root_directory), - max_evaluations_total=settings.max_evaluations_total, - max_evaluations_for_worker=settings.max_evaluations_per_run, - continue_until_max_evaluation_completed=settings.continue_until_max_evaluation_completed, - objective_to_minimize_value_on_error=settings.objective_to_minimize_value_on_error, - cost_value_on_error=settings.cost_value_on_error, - ignore_errors=settings.ignore_errors, - overwrite_optimization_dir=settings.overwrite_working_directory, - pre_load_hooks=settings.pre_load_hooks, - sample_batch_size=settings.sample_batch_size, + evaluation_fn=_eval, # type: ignore + optimizer=_optimizer_ask, + optimizer_info=_optimizer_info, + max_cost_total=max_cost_total, + optimization_dir=Path(root_directory), + max_evaluations_total=max_evaluations_total, + max_evaluations_for_worker=max_evaluations_per_run, + continue_until_max_evaluation_completed=continue_until_max_evaluation_completed, + objective_value_on_error=objective_value_on_error, + cost_value_on_error=cost_value_on_error, + ignore_errors=ignore_errors, + overwrite_optimization_dir=overwrite_working_directory, + sample_batch_size=sample_batch_size, ) - if settings.post_run_summary: - assert settings.root_directory is not None - config_data_path, run_data_path = post_run_csv(settings.root_directory) + if post_run_summary: + config_data_path, run_data_path = post_run_csv(root_directory) logger.info( "The post run summary has been created, which is a csv file with the " "output of all data in the run." @@ -259,173 +446,4 @@ def run( ) -def _run_args( - searcher_info: dict, - pipeline_space: ( - dict[str, Parameter | CS.ConfigurationSpace] - | str - | Path - | CS.ConfigurationSpace - | None - ) = None, - max_cost_total: int | float | None = None, - ignore_errors: bool = False, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - searcher: ( - Literal[ - "default", - "bayesian_optimization", - "random_search", - "hyperband", - "priorband", - "mobster", - "asha", - ] - | BaseOptimizer - | dict - ) = "default", - **searcher_kwargs, -) -> tuple[BaseOptimizer, dict]: - try: - # Raising an issue if pipeline_space is None - if pipeline_space is None: - raise ValueError( - "The choice of searcher requires a pipeline space to be provided" - ) - # Support pipeline space as ConfigurationSpace definition - if isinstance(pipeline_space, CS.ConfigurationSpace): - pipeline_space = pipeline_space_from_configspace(pipeline_space) - # Support pipeline space as YAML file - elif isinstance(pipeline_space, (str, Path)): - pipeline_space = pipeline_space_from_yaml(pipeline_space) - - # Support pipeline space as mix of ConfigurationSpace and neps parameters - new_pipeline_space: dict[str, Parameter] = dict() - for key, value in pipeline_space.items(): - if isinstance(value, CS.ConfigurationSpace): - config_space_parameters = pipeline_space_from_configspace(value) - new_pipeline_space = {**new_pipeline_space, **config_space_parameters} - else: - new_pipeline_space[key] = value - pipeline_space = new_pipeline_space - - # Transform to neps internal representation of the pipeline space - pipeline_space = SearchSpace(**pipeline_space) - except TypeError as e: - message = f"The pipeline_space has invalid type: {type(pipeline_space)}" - raise TypeError(message) from e - - # Load the information of the optimizer - if ( - isinstance(searcher, (str, Path)) - and searcher not in SearcherConfigs.get_searchers() - and searcher != "default" - ): - # The users have their own custom searcher provided via yaml. - logging.info("Preparing to run user created searcher") - - searcher_config, file_name = get_searcher_data( - searcher, loading_custom_searcher=True - ) - # name defined via key or the filename of the yaml - searcher_name = searcher_config.pop("name", file_name) - searcher_info["searcher_selection"] = "user-yaml" - searcher_info["neps_decision_tree"] = False - elif isinstance(searcher, dict): - custom_config = searcher - default_config, searcher_name = get_searcher_data(searcher["strategy"]) - searcher_config = {**default_config, **custom_config} - if "name" not in searcher_config: - searcher_name = "custom_" + searcher_name - else: - searcher_name = searcher_config.pop("name") - searcher_info["searcher_selection"] = "user-run_args-yaml" - searcher_info["neps_decision_tree"] = False - else: - if searcher in ["default", None]: - # NePS decides the searcher according to the pipeline space. - if pipeline_space.has_prior: - searcher = "priorband" if len(pipeline_space.fidelities) > 0 else "pibo" - else: - searcher = ( - "hyperband" - if len(pipeline_space.fidelities) > 0 - else "bayesian_optimization" - ) - searcher_info["searcher_selection"] = "neps-default" - else: - # Users choose one of NePS searchers. - searcher_info["neps_decision_tree"] = False - searcher_info["searcher_selection"] = "neps-default" - # Fetching the searcher data, throws an error when the searcher is not found - searcher_config, searcher_name = get_searcher_data(searcher) - - # Check for deprecated 'algorithm' argument - if "algorithm" in searcher_config: - warnings.warn( - "The 'algorithm' argument is deprecated and will be removed in " - "future versions. Please use 'strategy' instead.", - DeprecationWarning, - ) - # Map the old 'algorithm' argument to 'strategy' - searcher_config["strategy"] = searcher_config.pop("algorithm") - - if "strategy" in searcher_config: - searcher_alg = searcher_config.pop("strategy") - else: - raise KeyError(f"Missing key strategy in searcher config:{searcher_config}") - - logger.info(f"Running {searcher_name} as the searcher") - logger.info(f"Strategy: {searcher_alg}") - - # Used to create the yaml holding information about the searcher. - # Also important for testing and debugging the api. - searcher_info["searcher_name"] = searcher_name - searcher_info["searcher_alg"] = searcher_alg - - # Updating searcher arguments from searcher_kwargs - for key, value in searcher_kwargs.items(): - if not searcher_info["neps_decision_tree"]: - if key not in searcher_config or searcher_config[key] != value: - searcher_config[key] = value - logger.info( - f"Updating the current searcher argument '{key}'" - f" with the value '{get_value(value)}'" - ) - else: - logger.info( - f"The searcher argument '{key}' has the same" - f" value '{get_value(value)}' as default." - ) - else: - # No searcher argument updates when NePS decides the searcher. - logger.info(35 * "=" + "WARNING" + 35 * "=") - logger.info("CHANGINE ARGUMENTS ONLY WORKS WHEN SEARCHER IS DEFINED") - logger.info( - f"The searcher argument '{key}' will not change to '{value}'" - f" because NePS chose the searcher" - ) - - searcher_info["searcher_args"] = get_value(searcher_config) - - searcher_config.update( - { - "objective_to_minimize_value_on_error": objective_to_minimize_value_on_error, - "cost_value_on_error": cost_value_on_error, - "ignore_errors": ignore_errors, - } - ) - - searcher_instance = instance_from_map( - SearcherMapping, searcher_alg, "searcher", as_class=True - )( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, # TODO: use max_cost_total everywhere - **searcher_config, - ) - - return ( - searcher_instance, - searcher_info, - ) +__all__ = ["run"] diff --git a/neps/optimizers/bayesian_optimization/__init__.py b/neps/cli/__init__.py similarity index 100% rename from neps/optimizers/bayesian_optimization/__init__.py rename to neps/cli/__init__.py diff --git a/neps/optimizers/__init__.py b/neps/optimizers/__init__.py index b3ea4f3a0..db3d9f254 100644 --- a/neps/optimizers/__init__.py +++ b/neps/optimizers/__init__.py @@ -1,42 +1,86 @@ +from __future__ import annotations + from collections.abc import Callable, Mapping -from functools import partial - -from neps.optimizers.base_optimizer import BaseOptimizer -from neps.optimizers.bayesian_optimization.optimizer import BayesianOptimization -from neps.optimizers.grid_search.optimizer import GridSearch -from neps.optimizers.multi_fidelity import ( - IFBO, - MOBSTER, - AsynchronousSuccessiveHalving, - AsynchronousSuccessiveHalvingWithPriors, - Hyperband, - HyperbandCustomDefault, - SuccessiveHalving, - SuccessiveHalvingWithPriors, -) -from neps.optimizers.multi_fidelity_prior import ( - PriorBand, - PriorBandAsha, - PriorBandAshaHB, +from typing import TYPE_CHECKING, Any, Literal + +from neps.optimizers.algorithms import ( + OptimizerChoice, + PredefinedOptimizers, + determine_optimizer_automatically, ) -from neps.optimizers.random_search.optimizer import RandomSearch - -# TODO: Rename Searcher to Optimizer... -SearcherMapping: Mapping[str, Callable[..., BaseOptimizer]] = { - "bayesian_optimization": partial(BayesianOptimization, use_priors=False), - "pibo": partial(BayesianOptimization, use_priors=True), - "random_search": RandomSearch, - "grid_search": GridSearch, - "successive_halving": SuccessiveHalving, - "successive_halving_prior": SuccessiveHalvingWithPriors, - "asha": AsynchronousSuccessiveHalving, - "hyperband": Hyperband, - "asha_prior": AsynchronousSuccessiveHalvingWithPriors, - "hyperband_custom_default": HyperbandCustomDefault, - "priorband": PriorBand, - "priorband_bo": partial(PriorBand, model_based=True), - "priorband_asha": PriorBandAsha, - "priorband_asha_hyperband": PriorBandAshaHB, - "mobster": MOBSTER, - "ifbo": IFBO, -} +from neps.optimizers.optimizer import AskFunction # noqa: TC001 +from neps.utils.common import extract_keyword_defaults + +if TYPE_CHECKING: + from neps.space import SearchSpace + + +def _load_optimizer_from_string( + optimizer: OptimizerChoice | Literal["auto"], + space: SearchSpace, + *, + optimizer_kwargs: Mapping[str, Any] | None = None, +) -> tuple[AskFunction, dict[str, Any]]: + if optimizer == "auto": + _optimizer = determine_optimizer_automatically(space) + else: + _optimizer = optimizer + + optimizer_build = PredefinedOptimizers.get(_optimizer) + if optimizer_build is None: + raise ValueError( + f"Unrecognized `optimizer` of type {type(optimizer)}." + f" {optimizer}. Available optimizers are:" + f" {PredefinedOptimizers.keys()}" + ) + + info = extract_keyword_defaults(optimizer_build) + info["name"] = _optimizer + + optimizer_kwargs = optimizer_kwargs or {} + opt = optimizer_build(space, **optimizer_kwargs) + return opt, info + + +def load_optimizer( + optimizer: ( + OptimizerChoice + | Mapping[str, Any] + | tuple[OptimizerChoice | Callable[..., AskFunction], Mapping[str, Any]] + | Callable[..., AskFunction] + | Literal["auto"] + ), + space: SearchSpace, +) -> tuple[AskFunction, dict[str, Any]]: + match optimizer: + # Predefined string + case str(): + return _load_optimizer_from_string(optimizer, space) + + # class/builder + case _ if callable(optimizer): + info = extract_keyword_defaults(optimizer) + _optimizer = optimizer(space) + info["name"] = optimizer.__name__ + return _optimizer, info + + # Predefined string with kwargs + case (opt, kwargs) if isinstance(opt, str): + return _load_optimizer_from_string(opt, space, optimizer_kwargs=kwargs) # type: ignore + + # class/builder with kwargs + case (opt, kwargs): + info = extract_keyword_defaults(opt) # type: ignore + info["name"] = opt.__name__ # type: ignore + _optimizer = opt(space, **kwargs) # type: ignore + return _optimizer, info + + # Mapping with a name + case {"name": name, **_kwargs}: + return _load_optimizer_from_string(name, space, optimizer_kwargs=_kwargs) # type: ignore + + case _: + raise ValueError( + f"Unrecognized `optimizer` of type {type(optimizer)}." + f" {optimizer}. Must either be a string or a callable." + ) diff --git a/neps/optimizers/acquisition/__init__.py b/neps/optimizers/acquisition/__init__.py new file mode 100644 index 000000000..0d2d27efa --- /dev/null +++ b/neps/optimizers/acquisition/__init__.py @@ -0,0 +1,5 @@ +from neps.optimizers.acquisition.cost_cooling import cost_cooled_acq +from neps.optimizers.acquisition.pibo import pibo_acquisition +from neps.optimizers.acquisition.weighted_acquisition import WeightedAcquisition + +__all__ = ["WeightedAcquisition", "cost_cooled_acq", "pibo_acquisition"] diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py b/neps/optimizers/acquisition/cost_cooling.py similarity index 89% rename from neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py rename to neps/optimizers/acquisition/cost_cooling.py index 46fe1309e..8faf30aa8 100644 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/cost_cooling.py +++ b/neps/optimizers/acquisition/cost_cooling.py @@ -5,9 +5,7 @@ import torch from botorch.acquisition.logei import partial -from neps.optimizers.bayesian_optimization.acquisition_functions.weighted_acquisition import ( # noqa: E501 - WeightedAcquisition, -) +from neps.optimizers.acquisition.weighted_acquisition import WeightedAcquisition if TYPE_CHECKING: from botorch.acquisition import AcquisitionFunction @@ -31,7 +29,7 @@ def apply_cost_cooling( # -- x = acq / cost^alpha # -- log(x) = log(acq) - alpha * log(cost) w = alpha * cost.log() - return acq_values - w + return acq_values - w # type: ignore # https://github.com/pytorch/botorch/discussions/2194 w = cost.pow(alpha) diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/pibo.py b/neps/optimizers/acquisition/pibo.py similarity index 84% rename from neps/optimizers/bayesian_optimization/acquisition_functions/pibo.py rename to neps/optimizers/acquisition/pibo.py index 3cba54e56..a87c18710 100644 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/pibo.py +++ b/neps/optimizers/acquisition/pibo.py @@ -18,17 +18,14 @@ from botorch.acquisition.logei import partial -from neps.optimizers.bayesian_optimization.acquisition_functions.weighted_acquisition import ( # noqa: E501 - WeightedAcquisition, -) +from neps.optimizers.acquisition.weighted_acquisition import WeightedAcquisition if TYPE_CHECKING: from botorch.acquisition.acquisition import AcquisitionFunction from torch import Tensor - from neps.sampling.priors import Prior - from neps.search_spaces.domain import Domain - from neps.search_spaces.encoding import ConfigEncoder + from neps.sampling import Prior + from neps.space import ConfigEncoder, Domain def apply_pibo_acquisition_weight( diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/weighted_acquisition.py b/neps/optimizers/acquisition/weighted_acquisition.py similarity index 97% rename from neps/optimizers/bayesian_optimization/acquisition_functions/weighted_acquisition.py rename to neps/optimizers/acquisition/weighted_acquisition.py index fd23d3319..8d5ab795d 100644 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/weighted_acquisition.py +++ b/neps/optimizers/acquisition/weighted_acquisition.py @@ -126,7 +126,7 @@ def __init__( self._log = acq._log # Taken from PiBO implementation in botorch (PriorGuidedAcquisitionFunction). - @concatenate_pending_points + @concatenate_pending_points # type: ignore @t_batch_mode_transform() # type: ignore def forward(self, X: Tensor) -> Tensor: """Evaluate a weighted acquisition function on the candidate set X. @@ -145,9 +145,9 @@ def forward(self, X: Tensor) -> Tensor: weighted_acq_values = self.apply_weight(acq_values, X, self.acq) q_reduced_acq = self.acq._q_reduction(weighted_acq_values) sample_reduced_acq = self.acq._sample_reduction(q_reduced_acq) - return sample_reduced_acq.squeeze(-1) + return sample_reduced_acq.squeeze(-1) # type: ignore # shape: batch x q-candidates acq_values = self.acq(X).unsqueeze(-1) weighted_acq_values = self.apply_weight(acq_values, X, self.acq) - return weighted_acq_values.squeeze(-1) + return weighted_acq_values.squeeze(-1) # type: ignore diff --git a/neps/optimizers/algorithms.py b/neps/optimizers/algorithms.py new file mode 100644 index 000000000..e11a9f84c --- /dev/null +++ b/neps/optimizers/algorithms.py @@ -0,0 +1,856 @@ +"""The selection of optimization algorithms available in NePS. + +This module conveniently starts with 'a' to be at the top and +is where most of the code documentation for optimizers can be found. + +Below you will find some functions with some sane defaults documenting +the parameters available. You can pass these functoins to `neps.run()` +if you like, otherwise you may also refer to them by their string name. +""" +# NOTE: If updating this file with new optimizers, please be aware that +# the documentation here is what is shown in the `neps.run()` documentation. +# Heres a checklist: +# 1. Add you function and document it +# 2. Add it to the `OptimizerChoice` at the bottom of this file. +# 3. Add a section to `neps.run()` + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING, Concatenate, Literal, TypeAlias + +import torch + +from neps.optimizers.bayesian_optimization import BayesianOptimization +from neps.optimizers.bracket_optimizer import BracketOptimizer +from neps.optimizers.grid_search import GridSearch +from neps.optimizers.ifbo import IFBO +from neps.optimizers.models.ftpfn import FTPFNSurrogate +from neps.optimizers.optimizer import AskFunction # noqa: TC001 +from neps.optimizers.priorband import PriorBandArgs +from neps.optimizers.random_search import RandomSearch +from neps.sampling import Prior, Sampler, Uniform +from neps.space.encoding import CategoricalToUnitNorm, ConfigEncoder + +if TYPE_CHECKING: + import pandas as pd + + from neps.optimizers.utils.brackets import Bracket + from neps.space import SearchSpace + + +def _bo( + pipeline_space: SearchSpace, + *, + initial_design_size: int | Literal["ndim"] = "ndim", + use_priors: bool, + cost_aware: bool | Literal["log"], + sample_prior_first: bool, + device: torch.device | str | None, +) -> BayesianOptimization: + """Initialise the BO loop. + + Args: + pipeline_space: Space in which to search + initial_design_size: Number of samples used before using the surrogate model. + If "ndim", it will use the number of parameters in the search space. + use_priors: Whether to use priors set on the hyperparameters during search. + cost_aware: Whether to consider reported "cost" from configurations in decision + making. If True, the optimizer will weigh potential candidates by how much + they cost, incentivising the optimizer to explore cheap, good performing + configurations. This amount is modified over time. If "log", the cost + will be log-transformed before being used. + + !!! warning + + If using `cost`, cost must be provided in the reports of the trials. + + sample_prior_first: Whether to sample the default configuration first. + device: Device to use for the optimization. + + Raises: + ValueError: if initial_design_size < 1 + """ + if any(pipeline_space.fidelities): + raise ValueError( + "Fidelities are not supported for BayesianOptimization." + " Please consider setting the fidelity to a constant value." + f" Got: {pipeline_space.fidelities}" + ) + + match initial_design_size: + case "ndim": + n_initial_design_size = len(pipeline_space.numerical) + len( + pipeline_space.categoricals + ) + case int(): + if initial_design_size < 1: + raise ValueError("initial_design_size should be greater than 0") + + n_initial_design_size = initial_design_size + case _: + raise ValueError( + "initial_design_size should be either 'ndim' or a positive integer" + ) + + match device: + case str(): + device = torch.device(device) + case None: + device = torch.get_default_device() + case torch.device(): + pass + case _: + raise ValueError("device should be a string, torch.device or None") + + return BayesianOptimization( + pipeline_space=pipeline_space, + encoder=ConfigEncoder.from_space(space=pipeline_space), + n_initial_design=n_initial_design_size, + cost_aware=cost_aware, + prior=Prior.from_space(pipeline_space) if use_priors is True else None, + sample_prior_first=sample_prior_first, + device=device, + ) + + +def _bracket_optimizer( # noqa: C901 + pipeline_space: SearchSpace, + *, + bracket_type: Literal["successive_halving", "hyperband", "asha", "async_hb"], + eta: int, + sampler: Literal["uniform", "prior", "priorband"] | PriorBandArgs | Sampler, + sample_prior_first: bool | Literal["highest_fidelity"], + # NOTE: This is the only argument to get a default, since it + # is not required for hyperband style algorithms, only single bracket + # style ones. + early_stopping_rate: int | None = None, +) -> BracketOptimizer: + """Initialise a bracket optimizer. + + Args: + pipeline_space: Space in which to search + bracket_type: The type of bracket to use. Can be one of: + + * "successive_halving": Successive Halving + * "hyperband": HyperBand + * "asha": ASHA + * "async_hb": Async + + eta: The reduction factor used for building brackets + early_stopping_rate: Determines the number of rungs in a bracket + Choosing 0 creates maximal rungs given the fidelity bounds. + + !!! warning + + This is only used for Successive Halving and Asha. If set + to not `None`, then the bracket type must be one of those. + + sampler: The type of sampling procedure to use: + + * If "uniform", samples uniformly from the space when it needs to sample + * If "prior", samples from the prior distribution built from the prior + and prior_confidence values in the pipeline space. + * If "priorband", samples with weights according to the PriorBand + algorithm. See: https://arxiv.org/abs/2306.12370 + + * If a `PriorBandArgs` object, samples with weights according to the + PriorBand algorithm with the given parameters. + * If a `Sampler` object, samples from the space using the sampler. + + sample_prior_first: Whether to sample the prior configuration first. + """ + assert pipeline_space.fidelity is not None + fidelity_name, fidelity = pipeline_space.fidelity + + if len(pipeline_space.fidelities) != 1: + raise ValueError( + "Only one fidelity should be defined in the pipeline space." + f"\nGot: {pipeline_space.fidelities}" + ) + + if sample_prior_first not in (True, False, "highest_fidelity"): + raise ValueError( + "sample_prior_first should be either True, False or 'highest_fidelity'" + ) + + from neps.optimizers.utils import brackets + + # Determine the strategy for creating brackets for sampling + create_brackets: Callable[[pd.DataFrame], Sequence[Bracket] | Bracket] + match bracket_type: + case "successive_halving": + assert early_stopping_rate is not None + rung_to_fidelity, rung_sizes = brackets.calculate_sh_rungs( + bounds=(fidelity.lower, fidelity.upper), + eta=eta, + early_stopping_rate=early_stopping_rate, + ) + create_brackets = partial( + brackets.Sync.create_repeating, rung_sizes=rung_sizes + ) + case "hyperband": + assert early_stopping_rate is None + rung_to_fidelity, bracket_layouts = brackets.calculate_hb_bracket_layouts( + bounds=(fidelity.lower, fidelity.upper), + eta=eta, + ) + create_brackets = partial( + brackets.Hyperband.create_repeating, + bracket_layouts=bracket_layouts, + ) + case "asha": + assert early_stopping_rate is not None + rung_to_fidelity, _rung_sizes = brackets.calculate_sh_rungs( + bounds=(fidelity.lower, fidelity.upper), + eta=eta, + early_stopping_rate=early_stopping_rate, + ) + create_brackets = partial( + brackets.Async.create, rungs=list(rung_to_fidelity), eta=eta + ) + case "async_hb": + assert early_stopping_rate is None + rung_to_fidelity, bracket_layouts = brackets.calculate_hb_bracket_layouts( + bounds=(fidelity.lower, fidelity.upper), + eta=eta, + ) + # We don't care about the capacity of each bracket, we need the rung layout + bracket_rungs = [list(bracket.keys()) for bracket in bracket_layouts] + create_brackets = partial( + brackets.AsyncHyperband.create, + bracket_rungs=bracket_rungs, + eta=eta, + ) + case _: + raise ValueError(f"Unknown bracket type: {bracket_type}") + + encoder = ConfigEncoder.from_space(pipeline_space, include_fidelity=False) + + _sampler: Sampler | PriorBandArgs + match sampler: + case "uniform": + _sampler = Sampler.uniform(ndim=encoder.ndim) + case "prior": + _sampler = Prior.from_config(pipeline_space.prior, space=pipeline_space) + case "priorband": + _sampler = PriorBandArgs(mutation_rate=0.5, mutation_std=0.25) + case PriorBandArgs() | Sampler(): + _sampler = sampler + case _: + raise ValueError(f"Unknown sampler: {sampler}") + + return BracketOptimizer( + pipeline_space=pipeline_space, + encoder=encoder, + eta=eta, + rung_to_fid=rung_to_fidelity, + fid_min=fidelity.lower, + fid_max=fidelity.upper, + fid_name=fidelity_name, + sampler=_sampler, + sample_prior_first=sample_prior_first, + create_brackets=create_brackets, + ) + + +def determine_optimizer_automatically(space: SearchSpace) -> str: + if len(space.prior) > 0: + return "priorband" if len(space.fidelities) > 0 else "pibo" + + return "hyperband" if len(space.fidelities) > 0 else "bayesian_optimization" + + +def random_search( + pipeline_space: SearchSpace, + *, + use_priors: bool = False, + ignore_fidelity: bool = True, +) -> RandomSearch: + """A simple random search algorithm that samples configurations uniformly at random. + + You may also `use_priors=` to sample from a distribution centered around your defined + priors. + + Args: + pipeline_space: The search space to sample from. + use_priors: Whether to use priors when sampling. + ignore_fidelity: Whether to ignore fidelity when sampling. + In this case, the max fidelity is always used. + """ + encoder = ConfigEncoder.from_space( + pipeline_space, include_fidelity=not ignore_fidelity + ) + + sampler: Sampler + if use_priors: + sampler = Prior.from_space(pipeline_space, include_fidelity=not ignore_fidelity) + else: + sampler = Uniform(ndim=encoder.ndim) + + return RandomSearch( + pipeline_space=pipeline_space, + encoder=encoder, + sampler=sampler, + ignore_fidelity=ignore_fidelity, + ) + + +def grid_search(pipeline_space: SearchSpace) -> GridSearch: + """A simple grid search algorithm which discretizes the search + space and evaluates all possible configurations. + + Args: + pipeline_space: The search space to sample from. + """ + from neps.optimizers.utils.grid import make_grid + + return GridSearch( + pipeline_space=pipeline_space, + configs_list=make_grid(pipeline_space), + ) + + +def ifbo( + pipeline_space: SearchSpace, + *, + step_size: int | float = 1, + use_priors: bool = False, + sample_prior_first: bool = False, + initial_design_size: int | Literal["ndim"] = "ndim", + device: torch.device | str | None = None, + surrogate_path: str | Path | None = None, + surrogate_version: str = "0.0.1", +) -> IFBO: + """A transformer that has been trained to predict loss curves of deep-learing + models, used to guide the optimization procedure and select configurations which + are most promising to evaluate. + + !!! tip "When to use this?" + + Use this when you think that early signal in your loss curve could be used + to distinguish which configurations are likely to achieve a good performance. + + This algorithm will take many small steps in evaluating your configuration + so we also advise that saving and loading your model checkpoint should + be relatively fast. + + This algorithm requires a _fidelity_ parameter, such as `epochs`, to be present. + Each time we evaluate a configuration, we will only evaluate it for a single + epoch, before returning back to the ifbo algorithm to select the next configuration. + + ??? tip "Fidelities?" + + A fidelity parameter lets you control how many resources to invest in + a single evaluation. For example, a common one for deep-learing is + `epochs`. We can evaluate a model for just a single epoch, (fidelity step) + to gain more information about the model's performance and decide what + to do next. + + * **Paper**: https://openreview.net/forum?id=VyoY3Wh9Wd + * **Github**: https://github.com/automl/ifBO/tree/main + + Args: + pipeline_space: Space in which to search + step_size: The size of the step to take in the fidelity domain. + sample_prior_first: Whether to sample the default configuration first + initial_design_size: Number of configs to sample before starting optimization + + If `None`, the number of configs will be equal to the number of dimensions. + + device: Device to use for the model + surrogate_path: Path to the surrogate model to use + surrogate_version: Version of the surrogate model to use + """ + from neps.optimizers.ifbo import _adjust_space_to_match_stepsize + + # TODO: I'm not sure how this might effect tables, whose lowest fidelity + # might be below to possibly increased lower bound. + space, fid_bins = _adjust_space_to_match_stepsize(pipeline_space, step_size) + assert space.fidelity is not None + fidelity_name, fidelity = space.fidelity + + match initial_design_size: + case "ndim": + _initial_design_size = len(space.numerical) + len(space.categoricals) + case _: + _initial_design_size = initial_design_size + + match device: + case str(): + device = torch.device(device) + case None: + device = torch.get_default_device() + case torch.device(): + pass + case _: + raise ValueError("device should be a string, torch.device or None") + + return IFBO( + pipeline_space=pipeline_space, + n_fidelity_bins=fid_bins, + device=device, + sample_prior_first=sample_prior_first, + n_initial_design=_initial_design_size, + fid_domain=fidelity.domain, + fidelity_name=fidelity_name, + prior=(Prior.from_space(space, include_fidelity=False) if use_priors else None), + ftpfn=FTPFNSurrogate( + target_path=Path(surrogate_path) if surrogate_path is not None else None, + version=surrogate_version, + device=device, + ), + encoder=ConfigEncoder.from_space( + space=space, + # FTPFN doesn't support categoricals and we were recomended + # to just evenly distribute in the unit norm + custom_transformers={ + cat_name: CategoricalToUnitNorm(choices=cat.choices) + for cat_name, cat in space.categoricals.items() + }, + ), + ) + + +def successive_halving( + space: SearchSpace, + *, + sampler: Literal["uniform", "prior"] = "uniform", + eta: int = 3, + early_stopping_rate: int = 0, + sample_prior_first: bool | Literal["highest_fidelity"] = False, +) -> BracketOptimizer: + """ + A bandit-based optimization algorithm that uses a _fidelity_ parameter + to gradually invest resources into more promising configurations. + + ??? tip "Fidelities?" + + A fidelity parameter lets you control how many resources to invest in + a single evaluation. For example, a common one for deep-learing is + `epochs`. By evaluating a model for just a few epochs, we can quickly + get a sense if the model is promising or not. Only those that perform + well get _promoted_ and evaluated at a higher epoch. + + !!! tip "When to use this?" + + When you think that the rank of N configurations at a lower fidelity correlates + very well with the rank if you were to evaluate those configurations at higher + fidelities. + + It does this by creating a competition between N configurations and + racing them in a _bracket_ against each other. + This _bracket_ has a series of incrementing _rungs_, where lower rungs + indicate less resources invested. The amount of resources is related + to your fidelity parameter, with the highest rung relating to the + maximum of your fidelity parameter. + + Those that perform well get _promoted_ and evaluated with more resources. + + ``` + # A bracket indicating the rungs and configurations. + # Those which performed best get promoted through the rungs. + + | | fidelity | c1 | c2 | c3 | c4 | c5 | ... | cN | + | Rung 0 | (3 epochs) | o | o | o | o | o | ... | o | + | Rung 1 | (9 epochs) | o | | o | o | | ... | o | + | Rung 2 | (27 epochs) | o | | | | | ... | | + ``` + + By default, new configurations are sampled using a _uniform_ distribution, + however you can also specify to prefer sampling from around a distribution you + think is more promising by setting the `prior` and the `prior_confidence` + of parameters of your search space. + + You can choose between these by setting `#!python sampler="uniform"` + or `#!python sampler="prior"`. + + Args: + space: The search space to sample from. + eta: The reduction factor used for building brackets + early_stopping_rate: Determines the number of rungs in a bracket + Choosing 0 creates maximal rungs given the fidelity bounds. + sampler: The type of sampling procedure to use: + + * If `#!python "uniform"`, samples uniformly from the space when + it needs to sample. + * If `#!python "prior"`, samples from the prior + distribution built from the `prior` and `prior_confidence` + values in the search space. + + sample_prior_first: Whether to sample the prior configuration first, + and if so, should it be at the highest fidelity level. + """ + return _bracket_optimizer( + pipeline_space=space, + bracket_type="successive_halving", + eta=eta, + early_stopping_rate=early_stopping_rate, + sampler=sampler, + sample_prior_first=sample_prior_first, + ) + + +def hyperband( + space: SearchSpace, + *, + eta: int = 3, + sampler: Literal["uniform", "prior"] = "uniform", + sample_prior_first: bool | Literal["highest_fidelity"] = False, +) -> BracketOptimizer: + """Another bandit-based optimization algorithm that uses a _fidelity_ parameter, + very similar to [`successive_halving`][neps.optimizers.algorithms.successive_halving], + but hedges a bit more on the safe side, just incase your _fidelity_ parameters + isn't as well correlated as you'd like. + + !!! tip "When to use this?" + + Use this when you think lower fidelity evaluations of your configurations carries + some signal about their ranking at higher fidelities, but not enough to be certain + + Hyperband is like Successive Halving but it instead of always having the same bracket + layout, it runs different brackets with different rungs. + + This helps hedge against scenarios where rankings at the lowest fidelity do + not correlate well with the upper fidelity. + + + ``` + # Hyperband runs different successive halving brackets + + | Bracket 1 | | Bracket 2 | | Bracket 3 | + | Rung 0 | ... | | (skipped) | | (skipped) | + | Rung 1 | ... | | Rung 1 | ... | | (skipped) | + | Rung 2 | ... | | Rung 2 | ... | | Rung 2 | ... | + ``` + + For more information, see the + [`successive_halving`][neps.optimizers.algorithms.successive_halving] documentation, + as this algorithm could be considered an extension of it. + + Args: + space: The search space to sample from. + eta: The reduction factor used for building brackets + sampler: The type of sampling procedure to use: + + * If `#!python "uniform"`, samples uniformly from the space when + it needs to sample. + * If `#!python "prior"`, samples from the prior + distribution built from the `prior` and `prior_confidence` + values in the search space. + + sample_prior_first: Whether to sample the prior configuration first, + and if so, should it be at the highest fidelity level. + """ + return _bracket_optimizer( + pipeline_space=space, + bracket_type="hyperband", + eta=eta, + sampler=sampler, + sample_prior_first=sample_prior_first, + ) + + +def asha( + space: SearchSpace, + *, + eta: int = 3, + early_stopping_rate: int = 0, + sampler: Literal["uniform", "prior"] = "uniform", + sample_prior_first: bool | Literal["highest_fidelity"] = False, +) -> BracketOptimizer: + """A bandit-based optimization algorithm that uses a _fidelity_ parameter, + the _asynchronous_ version of + [`successive_halving`][neps.optimizers.algorithms.successive_halving]. + one that scales better to many parallel workers. + + !!! tip "When to use this?" + + Use this when you think lower fidelity evaluations of your configurations carries + a strong signal about their ranking at higher fidelities, and you have many + workers available to evaluate configurations in parallel. + + It does this by maintaining one big bracket, i.e. one + big on-going competition, with a promotion rule based on the sizes of each rung. + + ``` + # ASHA maintains one big bracket with an exponentially decreasing amount of + # configurations promoted, relative to those in the rung below. + + | | fidelity | c1 | c2 | c3 | c4 | c5 | ... + | Rung 0 | (3 epochs) | o | o | o | o | o | ... + | Rung 1 | (9 epochs) | o | | o | o | | ... + | Rung 2 | (27 epochs) | o | | | o | | ... + ``` + + For more information, see the + [`successive_halving`][neps.optimizers.algorithms.successive_halving] documentation, + as this algorithm could be considered an extension of it. + + Args: + space: The search space to sample from. + eta: The reduction factor used for building brackets + sampler: The type of sampling procedure to use: + + * If `#!python "uniform"`, samples uniformly from the space when + it needs to sample. + * If `#!python "prior"`, samples from the prior + distribution built from the `prior` and `prior_confidence` + values in the search space. + + sample_prior_first: Whether to sample the prior configuration first, + and if so, should it be at the highest fidelity. + """ + return _bracket_optimizer( + pipeline_space=space, + bracket_type="asha", + eta=eta, + early_stopping_rate=early_stopping_rate, + sampler=sampler, + sample_prior_first=sample_prior_first, + ) + + +def async_hb( + space: SearchSpace, + *, + eta: int = 3, + sampler: Literal["uniform", "prior"] = "uniform", + sample_prior_first: bool = False, +) -> BracketOptimizer: + """An _asynchronous_ version of [`hyperband`][neps.optimizers.algorithms.hyperband], + where the brackets are run asynchronously, and the promotion rule is based on the + number of evaluations each configuration has had. + + !!! tip "When to use this?" + + Use this when you think lower fidelity evaluations of your configurations carries + some signal about their ranking at higher fidelities, but not confidently, and + you have many workers available to evaluate configurations in parallel. + + ``` + # Async HB runs different "asha" brackets, which are unbounded in the number + # of configurations that can be in each. The bracket chosen at each iteration + # is a sampling function based on the resources invested in each bracket. + + | Bracket 1 | | Bracket 2 | | Bracket 3 | + | Rung 0 | ... | (skipped) | | (skipped) | + | Rung 1 | ... | Rung 1 | ... | (skipped) | + | Rung 2 | ... | Rung 2 | ... | Rung 2 | ... + ``` + + For more information, see the + [`hyperband`][neps.optimizers.algorithms.hyperband] documentation, + [`successive_halving`][neps.optimizers.algorithms.successive_halving] documentation, + and the [`asha`][neps.optimizers.algorithms.asha] documentation, as this algorithm + takes elements from each. + + Args: + space: The search space to sample from. + eta: The reduction factor used for building brackets + sampler: The type of sampling procedure to use: + + * If `#!python "uniform"`, samples uniformly from the space when + it needs to sample. + * If `#!python "prior"`, samples from the prior + distribution built from the `prior` and `prior_confidence` + values in the search space. + + sample_prior_first: Whether to sample the prior configuration first. + """ + return _bracket_optimizer( + pipeline_space=space, + bracket_type="async_hb", + eta=eta, + sampler=sampler, + sample_prior_first=sample_prior_first, + ) + + +def priorband( + space: SearchSpace, + *, + eta: int = 3, + sample_prior_first: bool | Literal["highest_fidelity"] = False, + base: Literal["successive_halving", "hyperband", "asha", "async_hb"] = "hyperband", +) -> BracketOptimizer: + """Priorband is also a bandit-based optimization algorithm that uses a _fidelity_, + providing a general purpose sampling extension to other algorithms. It makes better + use of the prior information you provide in the search space along with the fact + that you can afford to explore and take more risk at lower fidelities. + + !!! tip "When to use this?" + + Use this when you have a good idea of what good parameters look like and + can specify them through the `prior` and `prior_confidence` parameters in + the search space. + + As `priorband` is flexible, you may choose between the existing tradeoffs + the other algorithms provide through the use of `base=`. + + Priorband works by adjusting the sampling procedure to sample from one of + the following three distributions: + + * 1) a uniform distribution + * 2) a prior distribution + * 3) a distribution around the best found configuration so far. + + By weighing the likelihood of good configurations having been sampled + from each of these distribution, we can score them against each other to aid + selection. We further use the fact that we can afford to explore and take more + risk at lower fidelities, which is factored into the sampling procedure. + + See: https://openreview.net/forum?id=uoiwugtpCH¬eId=xECpK2WH6k + + Args: + space: The search space to sample from. + eta: The reduction factor used for building brackets + sample_prior_first: Whether to sample the prior configuration first. + base: The base algorithm to use for the bracketing. + """ + return _bracket_optimizer( + pipeline_space=space, + bracket_type=base, + eta=eta, + sampler="priorband", + sample_prior_first=sample_prior_first, + ) + + +def bayesian_optimization( + space: SearchSpace, + *, + initial_design_size: int | Literal["ndim"] = "ndim", + cost_aware: bool | Literal["log"] = False, + device: torch.device | str | None = None, +) -> BayesianOptimization: + """Models the relation between hyperparameters in your `pipeline_space` + and the results of `evaluate_pipeline` using bayesian optimization. + This acts as a cheap _surrogate model_ of you `evaluate_pipeline` function + that can be used for optimization. + + !!! tip "When to use this?" + + Bayesion optimization is a good general purpose choice, especially + if the size of your search space is not too large. It is also the best + option to use if you do not have or want to use a _fidelity_ parameter. + + Note that acquiring the next configuration to evaluate with bayesian + optimization can become prohibitvely expensive as the number of + configurations evaluated increases. + + If there is some numeric cost associated with evaluating a configuration, + you can provide this as a `cost` when returning the results from your + `evaluate_pipeline` function. By specifying `#!python cost_aware=True`, + the optimizer will attempt to balance getting the best result while + minimizing the cost. + + If you have _priors_, we recommend looking at + [`pibo`][neps.optimizers.algorithms.pibo]. + + Args: + space: The search space to sample from. + initial_design_size: Number of samples used before using the surrogate model. + If "ndim", it will use the number of parameters in the search space. + cost_aware: Whether to consider reported "cost" from configurations in decision + making. If True, the optimizer will weigh potential candidates by how much + they cost, incentivising the optimizer to explore cheap, good performing + configurations. This amount is modified over time. If "log", the cost + will be log-transformed before being used. + + !!! warning + + If using `cost`, cost must be provided in the reports of the trials. + + device: Device to use for the optimization. + """ + return _bo( + pipeline_space=space, + initial_design_size=initial_design_size, + cost_aware=cost_aware, + device=device, + use_priors=False, + sample_prior_first=False, + ) + + +def pibo( + space: SearchSpace, + *, + initial_design_size: int | Literal["ndim"] = "ndim", + cost_aware: bool | Literal["log"] = False, + device: torch.device | str | None = None, + sample_prior_first: bool = False, +) -> BayesianOptimization: + """A modification of + [`bayesian_optimization`][neps.optimizers.algorithms.bayesian_optimization] + that also incorporates the use of priors in the search space. + + !!! tip "When to use this?" + + Use this if you'd like to use bayesian optimization while also having + a good idea of what good parameters look like and can specify them + through the `prior` and `prior_confidence` parameters in the search space. + + Note that this incurs the same tradeoffs that bayesian optimization + has. + + Args: + space: The search space to sample from. + initial_design_size: Number of samples used before using the surrogate model. + If "ndim", it will use the number of parameters in the search space. + cost_aware: Whether to consider reported "cost" from configurations in decision + making. If True, the optimizer will weigh potential candidates by how much + they cost, incentivising the optimizer to explore cheap, good performing + configurations. This amount is modified over time. If "log", the cost + will be log-transformed before being used. + + !!! warning + + If using `cost`, cost must be provided in the reports of the trials. + + device: Device to use for the optimization. + """ + return _bo( + pipeline_space=space, + initial_design_size=initial_design_size, + cost_aware=cost_aware, + device=device, + use_priors=True, + sample_prior_first=sample_prior_first, + ) + + +PredefinedOptimizers: Mapping[ + str, + Callable[Concatenate[SearchSpace, ...], AskFunction], +] = { + f.__name__: f + for f in ( + bayesian_optimization, + pibo, + random_search, + grid_search, + ifbo, + successive_halving, + hyperband, + asha, + async_hb, + priorband, + ) +} + +OptimizerChoice: TypeAlias = Literal[ + "bayesian_optimization", + "pibo", + "successive_halving", + "hyperband", + "asha", + "async_hb", + "priorband", + "random_search", + "grid_search", + "ifbo", +] diff --git a/neps/optimizers/base_optimizer.py b/neps/optimizers/base_optimizer.py deleted file mode 100644 index 06adef711..000000000 --- a/neps/optimizers/base_optimizer.py +++ /dev/null @@ -1,173 +0,0 @@ -from __future__ import annotations - -import logging -from abc import abstractmethod -from collections.abc import Mapping -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any - -from neps.state.trial import Report, Trial - -if TYPE_CHECKING: - from neps.search_spaces.search_space import SearchSpace - from neps.state.optimizer import BudgetInfo - from neps.utils.types import ERROR, ResultDict - - -def _get_objective_to_minimize( - result: ERROR | ResultDict | float, - objective_to_minimize_value_on_error: float | None = None, - *, - ignore_errors: bool = False, -) -> ERROR | float: - if result == "error": - if ignore_errors: - return "error" - - if objective_to_minimize_value_on_error is not None: - return objective_to_minimize_value_on_error - - raise ValueError( - "An error happened during the execution of your evaluate_pipeline function." - " You have three options: 1. If the error is expected and corresponds to" - " an objective_to_minimize value in your application (e.g., 0% accuracy)," - " you can set objective_to_minimize_value_on_error to some float. 2. If " - " sometimes your pipeline crashes randomly, you can set ignore_errors=True." - " 3. Fix your error." - ) - - if isinstance(result, dict): - return float(result["objective_to_minimize"]) - - assert isinstance(result, float) - return float(result) - - -def _get_cost( - result: ERROR | ResultDict | float, - cost_value_on_error: float | None = None, - *, - ignore_errors: bool = False, -) -> float | Any: - if result == "error": - if ignore_errors: - return "error" - - if cost_value_on_error is None: - raise ValueError( - "An error happened during the execution of your evaluate_pipeline" - " function. You have three options: 1. If the error is expected and" - " corresponds to a cost value in your application, you can set" - " cost_value_on_error to some float. 2. If sometimes your pipeline" - " crashes randomly, you can set ignore_errors=True. 3. Fix your error." - ) - - return cost_value_on_error - - if isinstance(result, Mapping): - return float(result["cost"]) - - return float(result) - - -@dataclass -class SampledConfig: - id: str - config: Mapping[str, Any] - previous_config_id: str | None = None - - -class BaseOptimizer: - """Base sampler class. Implements all the low-level work.""" - - # TODO: Remove a lot of these init params - # Ideally we just make this a `Protocol`, i.e. an interface - # and it has no functionality - def __init__( - self, - *, - pipeline_space: SearchSpace, - patience: int = 50, - logger: logging.Logger | None = None, - max_cost_total: int | float | None = None, - objective_to_minimize_value_on_error: float | None = None, - cost_value_on_error: float | None = None, - learning_curve_on_error: float | list[float] | None = None, - ignore_errors: bool = False, - ) -> None: - if patience < 1: - raise ValueError("Patience should be at least 1") - - self.max_cost_total = max_cost_total - self.pipeline_space = pipeline_space - self.patience = patience - self.logger = logger or logging.getLogger("neps") - self.objective_to_minimize_value_on_error = objective_to_minimize_value_on_error - self.cost_value_on_error = cost_value_on_error - self.learning_curve_on_error = learning_curve_on_error - self.ignore_errors = ignore_errors - - @abstractmethod - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None, - n: int | None = None, - ) -> SampledConfig | list[SampledConfig]: - """Sample a new configuration. - - Args: - trials: All of the trials that are known about. - budget_info: information about the budget constraints. - - Returns: - The sampled configuration(s) - """ - ... - - def get_objective_to_minimize( - self, result: ERROR | ResultDict | float | Report - ) -> float | ERROR: - """Calls result.utils.get_objective_to_minimize() and passes the error handling - through. Please use self.get_objective_to_minimize() instead of - get_objective_to_minimize() in all optimizer classes. - """ - # TODO(eddiebergman): This is a forward change for whenever we can have optimizers - # use `Trial` and `Report`, they already take care of this and save having to do - # this `_get_objective_to_minimize` at every call. We can also then just use - # `None` instead of the string `"error"` - if isinstance(result, Report): - return ( - result.objective_to_minimize - if result.objective_to_minimize is not None - else "error" - ) - - return _get_objective_to_minimize( - result, - objective_to_minimize_value_on_error=self.objective_to_minimize_value_on_error, - ignore_errors=self.ignore_errors, - ) - - def get_cost(self, result: ERROR | ResultDict | float | Report) -> float | ERROR: - """Calls result.utils.get_cost() and passes the error handling through. - Please use self.get_cost() instead of get_cost() in all optimizer classes. - """ - # TODO(eddiebergman): This is a forward change for whenever we can have optimizers - # use `Trial` and `Report`, they already take care of this and save having to do - # this `_get_objective_to_minimize` at every call - if isinstance(result, Report): - return ( - result.objective_to_minimize - if result.objective_to_minimize is not None - else "error" - ) - - return _get_cost( - result, - cost_value_on_error=self.cost_value_on_error, - ignore_errors=self.ignore_errors, - ) - - def whoami(self) -> str: - return type(self).__name__ diff --git a/neps/optimizers/bayesian_optimization/optimizer.py b/neps/optimizers/bayesian_optimization.py similarity index 57% rename from neps/optimizers/bayesian_optimization/optimizer.py rename to neps/optimizers/bayesian_optimization.py index 471fd0706..1a1969edf 100644 --- a/neps/optimizers/bayesian_optimization/optimizer.py +++ b/neps/optimizers/bayesian_optimization.py @@ -2,25 +2,24 @@ import math from collections.abc import Mapping -from typing import TYPE_CHECKING, Any -from typing_extensions import override +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal import torch from botorch.acquisition import LinearMCObjective from botorch.acquisition.logei import qLogNoisyExpectedImprovement -from neps.optimizers.base_optimizer import BaseOptimizer, SampledConfig -from neps.optimizers.bayesian_optimization.models.gp import ( +from neps.optimizers.models.gp import ( encode_trials_for_gp, fit_and_acquire_from_gp, make_default_single_obj_gp, ) -from neps.optimizers.initial_design import make_initial_design -from neps.sampling import Prior -from neps.search_spaces.encoding import ConfigEncoder +from neps.optimizers.optimizer import SampledConfig +from neps.optimizers.utils.initial_design import make_initial_design if TYPE_CHECKING: - from neps.search_spaces import SearchSpace + from neps.sampling import Prior + from neps.space import ConfigEncoder, SearchSpace from neps.state import BudgetInfo, Trial @@ -54,89 +53,32 @@ def _pibo_exp_term( return math.exp(-n_bo_samples / ndims) -class BayesianOptimization(BaseOptimizer): - """Implements the basic BO loop.""" +@dataclass +class BayesianOptimization: + """Uses `botorch` as an engine for doing bayesian optimiziation.""" - def __init__( - self, - pipeline_space: SearchSpace, - *, - initial_design_size: int | None = None, - use_priors: bool = False, - use_cost: bool = False, - cost_on_log_scale: bool = True, - sample_prior_first: bool = False, - device: torch.device | None = None, - encoder: ConfigEncoder | None = None, - seed: int | None = None, - max_cost_total: Any | None = None, # TODO: remove - surrogate_model: Any | None = None, # TODO: remove - objective_to_minimize_value_on_error: Any | None = None, # TODO: remove - cost_value_on_error: Any | None = None, # TODO: remove - ignore_errors: Any | None = None, # TODO: remove - ): - """Initialise the BO loop. - - Args: - pipeline_space: Space in which to search - initial_design_size: Number of samples used before using the surrogate model. - If None, it will use the number of parameters in the search space. - use_priors: Whether to use priors set on the hyperparameters during search. - use_cost: Whether to consider reported "cost" from configurations in decision - making. If True, the optimizer will weigh potential candidates by how much - they cost, incentivising the optimizer to explore cheap, good performing - configurations. This amount is modified over time - - !!! warning - - If using `cost`, cost must be provided in the reports of the trials. - - cost_on_log_scale: Whether to use the log of the cost when using cost. - sample_prior_first: Whether to sample the default configuration first. - seed: Seed to use for the random number generator of samplers. - device: Device to use for the optimization. - encoder: Encoder to use for encoding the configurations. If None, it will - will use the default encoder. - - Raises: - ValueError: if initial_design_size < 1 - ValueError: if no kernel is provided - """ - if seed is not None: - raise NotImplementedError( - "Seed is not implemented yet for BayesianOptimization" - ) - if any(pipeline_space.graphs): - raise NotImplementedError("Only supports flat search spaces for now!") - if any(pipeline_space.fidelities): - raise ValueError( - "Fidelities are not supported for BayesianOptimization." - " Please consider setting the fidelity to a constant value." - f" Got: {pipeline_space.fidelities}" - ) + pipeline_space: SearchSpace + """The search space to use.""" - super().__init__(pipeline_space=pipeline_space) + encoder: ConfigEncoder + """The encoder to use for encoding and decoding configurations.""" - self.encoder = encoder or ConfigEncoder.from_space( - space=pipeline_space, - include_constants_when_decoding=True, - ) - self.prior = Prior.from_space(pipeline_space) if use_priors is True else None - self.use_cost = use_cost - self.use_priors = use_priors - self.cost_on_log_scale = cost_on_log_scale - self.device = device - self.sample_prior_first = sample_prior_first - - if initial_design_size is not None: - self.n_initial_design = initial_design_size - else: - self.n_initial_design = len(pipeline_space.numerical) + len( - pipeline_space.categoricals - ) + prior: Prior | None + """The prior to use for sampling configurations and inferring their likelihood.""" + + sample_prior_first: bool + """Whether to sample the prior configuration first.""" - @override - def ask( + cost_aware: bool | Literal["log"] + """Whether to consider the cost of configurations in decision making.""" + + n_initial_design: int + """The number of initial design samples to use before fitting the GP.""" + + device: torch.device | None + """The device to use for the optimization.""" + + def __call__( self, trials: Mapping[str, Trial], budget_info: BudgetInfo | None = None, @@ -189,7 +131,11 @@ def ask( ) cost_percent = None - if self.use_cost: + if self.cost_aware: + # TODO: Interaction with `"log"` cost aware + if self.cost_aware == "log": + raise NotImplementedError("Log cost aware not implemented yet.") + if budget_info is None: raise ValueError( "Must provide a 'cost' to configurations if using cost" @@ -204,9 +150,7 @@ def ask( pibo_exp_term = None prior = None if self.prior: - pibo_exp_term = _pibo_exp_term( - n_sampled, encoder.ncols, self.n_initial_design - ) + pibo_exp_term = _pibo_exp_term(n_sampled, encoder.ndim, self.n_initial_design) # If the exp term is insignificant, skip prior acq. weighting prior = None if pibo_exp_term < 1e-4 else self.prior @@ -228,9 +172,9 @@ def ask( prior=prior, n_candidates_required=_n, pibo_exp_term=pibo_exp_term, - costs=data.cost if self.use_cost else None, + costs=data.cost if self.cost_aware is not False else None, cost_percentage_used=cost_percent, - costs_on_log_scale=self.cost_on_log_scale, + costs_on_log_scale=self.cost_aware == "log", ) configs = encoder.decode(candidates) diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py b/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py deleted file mode 100644 index 31cb5b2b0..000000000 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -from collections.abc import Callable -from functools import partial - -from neps.optimizers.bayesian_optimization.acquisition_functions.base_acquisition import ( - BaseAcquisition, -) -from neps.optimizers.bayesian_optimization.acquisition_functions.ei import ( - ComprehensiveExpectedImprovement, -) -from neps.optimizers.bayesian_optimization.acquisition_functions.ucb import ( - UpperConfidenceBound, -) - -AcquisitionMapping: dict[str, Callable] = { - "EI": partial( - ComprehensiveExpectedImprovement, - in_fill="best", - augmented_ei=False, - ), - "LogEI": partial( - ComprehensiveExpectedImprovement, - in_fill="best", - augmented_ei=False, - log_ei=True, - ), - ## Uses the augmented EI heuristic and changed the in-fill criterion to the best test - ## location with the highest *posterior mean*, which are preferred when the - ## optimisation is noisy. - "AEI": partial( - ComprehensiveExpectedImprovement, - in_fill="posterior", - augmented_ei=True, - ), - "UCB": partial( - UpperConfidenceBound, - maximize=False, - ), -} - -__all__ = [ - "AcquisitionMapping", - "BaseAcquisition", - "ComprehensiveExpectedImprovement", - "UpperConfidenceBound", -] diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py b/neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py deleted file mode 100644 index 17a1a974f..000000000 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/base_acquisition.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import Iterable -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - import numpy as np - import torch - - -class BaseAcquisition(ABC): - def __init__(self): - self.surrogate_model: Any | None = None - - @abstractmethod - def eval( - self, - x: Iterable, - *, - asscalar: bool = False, - ) -> np.ndarray | torch.Tensor | float: - """Evaluate the acquisition function at point x2.""" - raise NotImplementedError - - def __call__(self, *args: Any, **kwargs: Any) -> np.ndarray | torch.Tensor | float: - return self.eval(*args, **kwargs) - - def set_state(self, surrogate_model: Any, **kwargs: Any) -> None: - self.surrogate_model = surrogate_model diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/ei.py b/neps/optimizers/bayesian_optimization/acquisition_functions/ei.py deleted file mode 100644 index b8ee5f752..000000000 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/ei.py +++ /dev/null @@ -1,128 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import TYPE_CHECKING, Any - -import torch -from torch.distributions import Normal - -from .base_acquisition import BaseAcquisition - -if TYPE_CHECKING: - import numpy as np - - from neps.search_spaces import SearchSpace - - -class ComprehensiveExpectedImprovement(BaseAcquisition): - def __init__( - self, - *, - augmented_ei: bool = False, - xi: float = 0.0, - in_fill: str = "best", - log_ei: bool = False, - optimize_on_max_fidelity: bool = True, - ): - """This is the graph BO version of the expected improvement - key differences are: - - 1. The input x2 is a networkx graph instead of a vectorial input - - 2. The search space (a collection of x1_graphs) is discrete, so there is no - gradient-based optimisation. Instead, we compute the EI at all candidate points - and empirically select the best position during optimisation - - Args: - augmented_ei: Using the Augmented EI heuristic modification to the standard - expected improvement algorithm according to Huang (2006). - xi: manual exploration-exploitation trade-off parameter. - in_fill: the criterion to be used for in-fill for the determination of mu_star - 'best' means the empirical best observation so far (but could be - susceptible to noise), 'posterior' means the best *posterior GP mean* - encountered so far, and is recommended for optimization of more noisy - functions. Defaults to "best". - log_ei: log-EI if true otherwise usual EI. - """ - super().__init__() - - if in_fill not in ["best", "posterior"]: - raise ValueError(f"Invalid value for in_fill ({in_fill})") - self.augmented_ei = augmented_ei - self.xi = xi - self.in_fill = in_fill - self.log_ei = log_ei - self.incumbent: float | None = None - self.optimize_on_max_fidelity = optimize_on_max_fidelity - - def eval( - self, - x: Sequence[SearchSpace], - *, - asscalar: bool = False, - ) -> np.ndarray | torch.Tensor | float: - """Return the negative expected improvement at the query point x2.""" - assert self.incumbent is not None, "EI function not fitted on model" - assert self.surrogate_model is not None - - space = x[0] - if len(space.fidelities) > 0 and self.optimize_on_max_fidelity: - assert len(space.fidelities) == 1 - fid_name, fid = next(iter(space.fidelities.items())) - _x = [space.from_dict({**e._values, fid_name: fid.upper}) for e in x] - else: - _x = list(x) - - mu, cov = self.surrogate_model.predict(_x) - - std = torch.sqrt(torch.diag(cov)) - mu_star = self.incumbent - - gauss = Normal(torch.zeros(1, device=mu.device), torch.ones(1, device=mu.device)) - # > u = (mu - mu_star - self.xi) / std - # > ei = std * updf + (mu - mu_star - self.xi) * ucdf - if self.log_ei: - # we expect that f_min is in log-space - f_min = mu_star - self.xi - v = (f_min - mu) / std - ei = torch.exp(f_min) * gauss.cdf(v) - torch.exp( - 0.5 * torch.diag(cov) + mu - ) * gauss.cdf(v - std) - else: - u = (mu_star - mu - self.xi) / std - try: - ucdf = gauss.cdf(u) - except ValueError as e: - print(f"u: {u}") # noqa: T201 - print(f"mu_star: {mu_star}") # noqa: T201 - print(f"mu: {mu}") # noqa: T201 - print(f"std: {std}") # noqa: T201 - print(f"diag: {cov.diag()}") # noqa: T201 - raise e - updf = torch.exp(gauss.log_prob(u)) - ei = std * updf + (mu_star - mu - self.xi) * ucdf - if self.augmented_ei: - sigma_n = self.surrogate_model.likelihood - ei *= 1.0 - torch.sqrt(torch.tensor(sigma_n, device=mu.device)) / torch.sqrt( - sigma_n + torch.diag(cov) - ) - if isinstance(_x, list) and asscalar: - return ei.detach().numpy() - - if asscalar: - ei = ei.detach().numpy().item() - - return ei - - def set_state(self, surrogate_model: Any, **kwargs: Any) -> None: - super().set_state(surrogate_model, **kwargs) - assert self.surrogate_model is not None - - # Compute incumbent - if self.in_fill == "best": - self.incumbent = float(torch.min(self.surrogate_model.y_)) - else: - x = self.surrogate_model.x - mu_train, _ = self.surrogate_model.predict(x) - incumbent_idx = torch.argmin(mu_train) - self.incumbent = self.surrogate_model.y_[incumbent_idx] diff --git a/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py b/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py deleted file mode 100644 index 52587a7a8..000000000 --- a/neps/optimizers/bayesian_optimization/acquisition_functions/ucb.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Iterable -from typing import Any - -import numpy as np -import torch - -from neps.optimizers.bayesian_optimization.acquisition_functions.base_acquisition import ( - BaseAcquisition, -) - -logger = logging.getLogger(__name__) - - -class UpperConfidenceBound(BaseAcquisition): - def __init__(self, *, beta: float = 1.0, maximize: bool = False): - """Upper Confidence Bound (UCB) acquisition function. - - Args: - beta: Controls the balance between exploration and exploitation. - maximize: If True, maximize the given model, else minimize. - DEFAULT=False, assumes minimzation. - """ - super().__init__() - self.beta = beta # can be updated as part of the state for dynamism or a schedule - self.maximize = maximize - - # to be initialized as part of the state - self.surrogate_model = None - - def set_state(self, surrogate_model: Any, **kwargs: Any) -> None: - super().set_state(surrogate_model) - self.surrogate_model = surrogate_model - if "beta" in kwargs: - if not isinstance(kwargs["beta"], list | np.array): - self.beta = kwargs["beta"] - else: - logger.warning("Beta is a list, not updating beta value!") - - def eval( - self, - x: Iterable, - *, - asscalar: bool = False, - ) -> np.ndarray | torch.Tensor | float: - assert self.surrogate_model is not None, "Surrogate model is not set." - try: - mu, cov = self.surrogate_model.predict(x) - std = torch.sqrt(torch.diag(cov)) - except ValueError as e: - raise e - sign = 1 if self.maximize else -1 # LCB is performed if minimize=True - ucb_scores = mu + sign * np.sqrt(self.beta) * std - # if LCB, minimize acquisition, or maximize -acquisition - return ucb_scores.detach().numpy() * sign diff --git a/neps/optimizers/bayesian_optimization/models/__init__.py b/neps/optimizers/bayesian_optimization/models/__init__.py deleted file mode 100755 index 034049a33..000000000 --- a/neps/optimizers/bayesian_optimization/models/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from neps.optimizers.bayesian_optimization.models.ftpfn import FTPFNSurrogate -from neps.optimizers.bayesian_optimization.models.gp import make_default_single_obj_gp - -__all__ = ["FTPFNSurrogate", "make_default_single_obj_gp"] diff --git a/neps/optimizers/bracket_optimizer.py b/neps/optimizers/bracket_optimizer.py new file mode 100644 index 000000000..7c5af28ce --- /dev/null +++ b/neps/optimizers/bracket_optimizer.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import logging +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +import numpy as np +import pandas as pd + +from neps.optimizers.optimizer import SampledConfig +from neps.optimizers.priorband import PriorBandArgs, sample_with_priorband +from neps.optimizers.utils.brackets import PromoteAction, SampleAction +from neps.sampling.samplers import Sampler + +if TYPE_CHECKING: + from neps.optimizers.utils.brackets import Bracket + from neps.space import SearchSpace + from neps.space.encoding import ConfigEncoder + from neps.state.optimizer import BudgetInfo + from neps.state.trial import Trial + + +logger = logging.getLogger(__name__) + + +def trials_to_table(trials: Mapping[str, Trial]) -> pd.DataFrame: + id_index = np.empty(len(trials), dtype=int) + rungs_index = np.empty(len(trials), dtype=int) + perfs = np.empty(len(trials), dtype=np.float64) + configs = np.empty(len(trials), dtype=object) + + for i, (trial_id, trial) in enumerate(trials.items()): + config_id_str, rung_str = trial_id.split("_") + _id, _rung = int(config_id_str), int(rung_str) + + if trial.report is None: + perf = np.nan # Pending + elif trial.report.objective_to_minimize is None: + perf = np.inf # Error? Either way, we wont promote it + else: + perf = trial.report.objective_to_minimize + + id_index[i] = _id + rungs_index[i] = _rung + perfs[i] = perf + configs[i] = trial.config + + id_index = pd.MultiIndex.from_arrays([id_index, rungs_index], names=["id", "rung"]) + df = pd.DataFrame(data={"config": configs, "perf": perfs}, index=id_index) + return df.sort_index(ascending=True) + + +@dataclass +class BracketOptimizer: + """Implements an optimizer over brackets. + + This is the main class behind algorithms like `"priorband"`, + `"successive_halving"`, `"asha"`, `"hyperband"`, etc. + """ + + pipeline_space: SearchSpace + """The pipeline space to optimize over.""" + + encoder: ConfigEncoder + """The encoder to use for the pipeline space.""" + + sample_prior_first: bool | Literal["highest_fidelity"] + """Whether or not to sample the prior first. + + If set to `"highest_fidelity"`, the prior will be sampled at the highest fidelity, + otherwise at the lowest fidelity. + """ + + eta: int + """The eta parameter for the algorithm.""" + + rung_to_fid: Mapping[int, int | float] + """The mapping from rung to fidelity value.""" + + create_brackets: Callable[[pd.DataFrame], Sequence[Bracket] | Bracket] + """A function that creates the brackets from the table of trials.""" + + sampler: Sampler | PriorBandArgs + """The sampler used to generate new trials.""" + + fid_min: int | float + """The minimum fidelity value.""" + + fid_max: int | float + """The maximum fidelity value.""" + + fid_name: str + """The name of the fidelity in the space.""" + + def __call__( # noqa: PLR0912, C901 + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + assert n is None, "TODO" + space = self.pipeline_space + + # If we have no trials, we either go with the prior or just a sampled config + if len(trials) == 0: + match self.sample_prior_first: + case "highest_fidelity": + config = {**space.prior, self.fid_name: self.fid_max} + rung = max(self.rung_to_fid) + return SampledConfig(id=f"0_{rung}", config=config) + case True: + config = {**space.prior, self.fid_name: self.fid_min} + rung = min(self.rung_to_fid) + return SampledConfig(id=f"0_{rung}", config=config) + case False: + pass + + # We have to special case this as we don't want it ending up in a bracket + if self.sample_prior_first == "highest_fidelity": + table = trials_to_table(trials=trials)[1:] + assert isinstance(table, pd.DataFrame) + else: + table = trials_to_table(trials=trials) + + if len(table) == 0: + nxt_id = 0 + else: + nxt_id = int(table.index.get_level_values("id").max()) + 1 # type: ignore + + # Get and execute the next action from our brackets that are not pending or done + brackets = self.create_brackets(table) + if not isinstance(brackets, Sequence): + brackets = [brackets] + + next_action = next( + ( + action + for bracket in brackets + if (action := bracket.next()) not in ("done", "pending") + ), + None, + ) + + if next_action is None: + raise RuntimeError( + f"{self.__class__.__name__} never got a 'sample' or 'pending' action!" + ) + + match next_action: + # The bracket would like us to promote a configuration + case PromoteAction(config=config, id=config_id, new_rung=new_rung): + config = {**config, self.fid_name: self.rung_to_fid[new_rung]} + return SampledConfig( + id=f"{config_id}_{new_rung}", + config=config, + previous_config_id=f"{config_id}_{new_rung - 1}", + ) + + # The bracket would like us to sample a new configuration for a rung + case SampleAction(rung=rung): + match self.sampler: + case Sampler(): + config = self.sampler.sample_config(to=self.encoder) + config = {**config, self.fid_name: self.rung_to_fid[rung]} + return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + case PriorBandArgs(): + config = sample_with_priorband( + table=table, + space=space, + rung_to_sample_for=rung, + fid_bounds=(self.fid_min, self.fid_max), + encoder=self.encoder, + inc_mutation_rate=self.sampler.mutation_rate, + inc_mutation_std=self.sampler.mutation_std, + eta=self.eta, + seed=None, # TODO + ) + config = {**config, self.fid_name: self.rung_to_fid[rung]} + return SampledConfig(id=f"{nxt_id}_{rung}", config=config) + case _: + raise RuntimeError(f"Unknown sampler: {self.sampler}") + case _: + raise RuntimeError(f"Unknown bracket action: {next_action}") diff --git a/neps/optimizers/default_searchers/asha.yaml b/neps/optimizers/default_searchers/asha.yaml deleted file mode 100644 index 5a4fcc82b..000000000 --- a/neps/optimizers/default_searchers/asha.yaml +++ /dev/null @@ -1,13 +0,0 @@ -strategy: asha -# Arguments that can be modified by the user -eta: 3 -early_stopping_rate: 0 -initial_design_type: max_budget -use_priors: false -random_interleave_prob: 0.0 -sample_prior_first: false -sample_prior_at_target: false - -# Arguments that can not be modified by the user -# sampling_policy: RandomUniformPolicy -# promotion_policy: AsyncPromotionPolicy diff --git a/neps/optimizers/default_searchers/asha_prior.yaml b/neps/optimizers/default_searchers/asha_prior.yaml deleted file mode 100644 index 4122c7972..000000000 --- a/neps/optimizers/default_searchers/asha_prior.yaml +++ /dev/null @@ -1,13 +0,0 @@ -strategy: asha_prior -# Arguments that can be modified by the user -eta: 3 -early_stopping_rate: 0 -initial_design_type: max_budget -prior_confidence: medium # or {"low", "high"} -random_interleave_prob: 0.0 -sample_prior_first: false -sample_prior_at_target: false - -# Arguments that can not be modified by the user -# sampling_policy: FixedPriorPolicy -# promotion_policy: AsyncPromotionPolicy diff --git a/neps/optimizers/default_searchers/bayesian_optimization.yaml b/neps/optimizers/default_searchers/bayesian_optimization.yaml deleted file mode 100644 index 49e9fbae6..000000000 --- a/neps/optimizers/default_searchers/bayesian_optimization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -strategy: bayesian_optimization -# Arguments that can be modified by the user -initial_design_size: null # Defaults to depending on number or hyperparameters -use_cost: false # Whether to factor in cost when selecting new configurations -use_priors: false # Whether to use user set priors in optimization -sample_prior_first: false # Whether to sample the default configuration first -device: null # Device to load the gaussian process model on with torch diff --git a/neps/optimizers/default_searchers/hyperband.yaml b/neps/optimizers/default_searchers/hyperband.yaml deleted file mode 100644 index 77bfd5a88..000000000 --- a/neps/optimizers/default_searchers/hyperband.yaml +++ /dev/null @@ -1,12 +0,0 @@ -strategy: hyperband -# Arguments that can be modified by the user -eta: 3 -initial_design_type: max_budget -use_priors: false -random_interleave_prob: 0.0 -sample_prior_first: false -sample_prior_at_target: false - -# Arguments that can not be modified by the user -# sampling_policy: RandomUniformPolicy -# promotion_policy: SyncPromotionPolicy diff --git a/neps/optimizers/default_searchers/ifbo.yaml b/neps/optimizers/default_searchers/ifbo.yaml deleted file mode 100644 index 3e9ecb2ba..000000000 --- a/neps/optimizers/default_searchers/ifbo.yaml +++ /dev/null @@ -1,11 +0,0 @@ -strategy: ifbo -surrogate_model_args: - version: "0.0.1" - target_path: null # Defaults to current_working_directory/.model -step_size: 1 # Step size to use for partial evaluations -use_priors: false # Whether to use priors set through `prior` and `prior_confidence` -sample_prior_first: false # Whether to sample the default configuration first -sample_prior_at_target: false # Whether to evaluate the default at the maximum fidelity or not -initial_design_size: "ndim" # How many initial samples to try before using the model -n_acquisition_new_configs: 1_000 # Number samples of new configs to include during acqusition -device: null # Device to load the model on with torch diff --git a/neps/optimizers/default_searchers/mobster.yaml b/neps/optimizers/default_searchers/mobster.yaml deleted file mode 100644 index d1f0ed0a8..000000000 --- a/neps/optimizers/default_searchers/mobster.yaml +++ /dev/null @@ -1,24 +0,0 @@ -strategy: mobster -# Arguments that can be modified by the user -eta: 3 -initial_design_type: max_budget -use_priors: false -random_interleave_prob: 0.0 -sample_prior_first: false -sample_prior_at_target: false - -# arguments for model -surrogate_model: gp -acquisition: EI # or {"LogEI", "AEI"} -log_prior_weighted: false - -# Arguments that can not be modified by the user -# sampling_policy: RandomUniformPolicy -# promotion_policy: AsyncPromotionPolicy -# model_policy: ModelPolicy - -# Other arguments -# surrogate_model_args: None # type: dict -# domain_se_kernel: None # type: str -# graph_kernels: None # type: list -# hp_kernels: None # type: list diff --git a/neps/optimizers/default_searchers/pibo.yaml b/neps/optimizers/default_searchers/pibo.yaml deleted file mode 100644 index eb44b8b2b..000000000 --- a/neps/optimizers/default_searchers/pibo.yaml +++ /dev/null @@ -1,7 +0,0 @@ -strategy: pibo -# Arguments that can be modified by the user -initial_design_size: null # Defaults to depending on number or hyperparameters -use_cost: false # Whether to factor in cost when selecting new configurations -use_priors: true # Whether to use user set priors in optimization -sample_prior_first: true # Whether to sample the default configuration first -device: null # Device to load the gaussian process model on with torch diff --git a/neps/optimizers/default_searchers/priorband.yaml b/neps/optimizers/default_searchers/priorband.yaml deleted file mode 100644 index 3bb2dcc55..000000000 --- a/neps/optimizers/default_searchers/priorband.yaml +++ /dev/null @@ -1,20 +0,0 @@ -strategy: priorband -# Arguments that can be modified by the user -eta: 3 -initial_design_type: max_budget -prior_confidence: medium # or {"low", "high"} -random_interleave_prob: 0.0 -sample_prior_first: true -sample_prior_at_target: false -prior_weight_type: geometric -inc_sample_type: mutation -inc_mutation_rate: 0.5 -inc_mutation_std: 0.25 -inc_style: dynamic - -# arguments for model -model_based: false # crucial argument to set to allow model-search - -# Arguments that can not be modified by the user -# sampling_policy: EnsemblePolicy -# promotion_policy: SyncPromotionPolicy diff --git a/neps/optimizers/default_searchers/priorband_bo.yaml b/neps/optimizers/default_searchers/priorband_bo.yaml deleted file mode 100644 index 49083df25..000000000 --- a/neps/optimizers/default_searchers/priorband_bo.yaml +++ /dev/null @@ -1,32 +0,0 @@ -strategy: priorband -# Arguments that can be modified by the user -eta: 3 -initial_design_type: max_budget -prior_confidence: medium # or {"low", "high"} -random_interleave_prob: 0.0 -sample_prior_first: true -sample_prior_at_target: false -prior_weight_type: geometric -inc_sample_type: mutation -inc_mutation_rate: 0.5 -inc_mutation_std: 0.25 -inc_style: dynamic - -# arguments for model -model_based: true # crucial argument to set to allow model-search -modelling_type: joint -initial_design_size: 10 -surrogate_model: gp -acquisition: EI # or {"LogEI", "AEI"} -log_prior_weighted: false - -# Arguments that can not be modified by the user -# sampling_policy: EnsemblePolicy -# promotion_policy: SyncPromotionPolicy -# model_policy: ModelPolicy - - # Other arguments - # surrogate_model_args: None # type: dict - # domain_se_kernel: None # type: str - # graph_kernels: None # type: list - # hp_kernels: None # type: list diff --git a/neps/optimizers/default_searchers/random_search.yaml b/neps/optimizers/default_searchers/random_search.yaml deleted file mode 100644 index e7e8879cc..000000000 --- a/neps/optimizers/default_searchers/random_search.yaml +++ /dev/null @@ -1,4 +0,0 @@ -strategy: random_search -# Arguments that can be modified by the user -use_priors: false -ignore_fidelity: true diff --git a/neps/optimizers/default_searchers/successive_halving.yaml b/neps/optimizers/default_searchers/successive_halving.yaml deleted file mode 100644 index 038e56efe..000000000 --- a/neps/optimizers/default_searchers/successive_halving.yaml +++ /dev/null @@ -1,13 +0,0 @@ -strategy: successive_halving -# Arguments that can be modified by the user -eta: 3 -early_stopping_rate: 0 -initial_design_type: max_budget -use_priors: false -random_interleave_prob: 0.0 -sample_prior_first: false -sample_prior_at_target: false - -# Arguments that can not be modified by the user -# sampling_policy: RandomUniformPolicy -# promotion_policy: SyncPromotionPolicy diff --git a/neps/optimizers/default_searchers/successive_halving_prior.yaml b/neps/optimizers/default_searchers/successive_halving_prior.yaml deleted file mode 100644 index 2b198f7b4..000000000 --- a/neps/optimizers/default_searchers/successive_halving_prior.yaml +++ /dev/null @@ -1,13 +0,0 @@ -strategy: successive_halving_prior -# Arguments that can be modified by the user -eta: 3 -early_stopping_rate: 0 -initial_design_type: max_budget -prior_confidence: medium # or {"low", "high"} -random_interleave_prob: 0.0 -sample_prior_first: false -sample_prior_at_target: false - -# Arguments that can not be modified by the user -# sampling_policy: FixedPriorPolicy -# promotion_policy: SyncPromotionPolicy diff --git a/neps/optimizers/grid_search.py b/neps/optimizers/grid_search.py new file mode 100644 index 000000000..74ccfadd0 --- /dev/null +++ b/neps/optimizers/grid_search.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import random +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from neps.optimizers.optimizer import SampledConfig + +if TYPE_CHECKING: + from neps.space import SearchSpace + from neps.state import BudgetInfo, Trial + + +@dataclass +class GridSearch: + """Evaluates a fixed list of configurations in order.""" + + pipeline_space: SearchSpace + """The search space from which the configurations are derived.""" + + configs_list: list[dict[str, Any]] + """The list of configurations to evaluate.""" + + def __call__( + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + assert n is None, "TODO" + _num_previous_configs = len(trials) + if _num_previous_configs > len(self.configs_list) - 1: + raise ValueError("Grid search exhausted!") + + rng = random.Random() + configs = rng.sample(self.configs_list, len(self.configs_list)) + + config = configs[_num_previous_configs] + config_id = str(_num_previous_configs) + return SampledConfig(config=config, id=config_id, previous_config_id=None) diff --git a/neps/optimizers/grid_search/optimizer.py b/neps/optimizers/grid_search/optimizer.py deleted file mode 100644 index 9bef62baa..000000000 --- a/neps/optimizers/grid_search/optimizer.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import random -from collections.abc import Mapping -from itertools import product -from typing import TYPE_CHECKING, Any -from typing_extensions import override - -import torch - -from neps.optimizers.base_optimizer import BaseOptimizer, SampledConfig -from neps.search_spaces import Categorical, Constant, Float, Integer -from neps.search_spaces.architecture.graph_grammar import GraphParameter -from neps.search_spaces.domain import UNIT_FLOAT_DOMAIN - -if TYPE_CHECKING: - from neps.search_spaces.search_space import SearchSpace - from neps.state.optimizer import BudgetInfo - from neps.state.trial import Trial - - -def _make_grid( - space: SearchSpace, - *, - size_per_numerical_hp: int = 10, -) -> list[dict[str, Any]]: - """Get a grid of configurations from the search space. - - For [`Numerical`][neps.search_spaces.Numerical] hyperparameters, - the parameter `size_per_numerical_hp=` is used to determine a grid. If there are - any duplicates, e.g. for an - [`Integer`][neps.search_spaces.Integer], then we will - remove duplicates. - - For [`Categorical`][neps.search_spaces.Categorical] - hyperparameters, we include all the choices in the grid. - - For [`Constant`][neps.search_spaces.Constant] hyperparameters, - we include the constant value in the grid. - - !!! note "TODO" - - Does not support graph parameters currently. - - !!! note "TODO" - - Include default hyperparameters in the grid. - If all HPs have a `default` then add a single configuration. - If only partial HPs have defaults then add all combinations of defaults, but - only to the end of the list of configs. - - Args: - size_per_numerical_hp: The size of the grid for each numerical hyperparameter. - - Returns: - A list of configurations from the search space. - """ - param_ranges: dict[str, list[Any]] = {} - for name, hp in space.hyperparameters.items(): - match hp: - # NOTE(eddiebergman): This is a temporary fix to avoid graphs - # If this is resolved, please update the docstring! - case GraphParameter(): - raise ValueError("Trying to create a grid for graphs!") - case Categorical(): - param_ranges[name] = list(hp.choices) - case Constant(): - param_ranges[name] = [hp.value] - case Integer() | Float(): - if hp.is_fidelity: - param_ranges[name] = [hp.upper] - continue - - if hp.domain.cardinality is None: - steps = size_per_numerical_hp - else: - steps = min(size_per_numerical_hp, hp.domain.cardinality) - - xs = torch.linspace(0, 1, steps=steps) - numeric_values = hp.domain.cast(xs, frm=UNIT_FLOAT_DOMAIN) - uniq_values = torch.unique(numeric_values).tolist() - param_ranges[name] = uniq_values - case _: - raise NotImplementedError(f"Unknown Parameter type: {type(hp)}\n{hp}") - - values = product(*param_ranges.values()) - keys = list(space.hyperparameters.keys()) - - return [dict(zip(keys, p, strict=False)) for p in values] - - -class GridSearch(BaseOptimizer): - def __init__(self, pipeline_space: SearchSpace, seed: int | None = None): - super().__init__(pipeline_space=pipeline_space) - self.configs_list = _make_grid(pipeline_space) - self.seed = seed - - @override - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None, - n: int | None = None, - ) -> SampledConfig: - assert n is None, "TODO" - _num_previous_configs = len(trials) - if _num_previous_configs > len(self.configs_list) - 1: - raise ValueError("Grid search exhausted!") - - rng = random.Random(self.seed) - configs = rng.sample(self.configs_list, len(self.configs_list)) - - config = configs[_num_previous_configs] - config_id = str(_num_previous_configs) - return SampledConfig(config=config, id=config_id, previous_config_id=None) diff --git a/neps/optimizers/ifbo.py b/neps/optimizers/ifbo.py new file mode 100755 index 000000000..ed4e46751 --- /dev/null +++ b/neps/optimizers/ifbo.py @@ -0,0 +1,266 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import numpy as np +import torch + +from neps.optimizers.models.ftpfn import ( + FTPFNSurrogate, + acquire_next_from_ftpfn, + decode_ftpfn_data, + encode_ftpfn, +) +from neps.optimizers.optimizer import SampledConfig +from neps.optimizers.utils.initial_design import make_initial_design +from neps.sampling import Prior, Sampler +from neps.space import ConfigEncoder, Domain, Float, Integer, SearchSpace + +if TYPE_CHECKING: + from neps.state import BudgetInfo, Trial + +# NOTE: Ifbo was trained using 32 bit +FTPFN_DTYPE = torch.float32 + + +def _adjust_space_to_match_stepsize( + space: SearchSpace, + step_size: int | float, +) -> tuple[SearchSpace, int]: + """Adjust the pipeline space to be evenly divisible by the step size. + + This is done by incrementing the lower bound of the fidelity domain to the + that enables this. + + Args: + space: The pipeline space to adjust + step_size: The size of the step to take in the fidelity domain. + + Returns: + The adjusted pipeline space and the number of bins it can be divided into + """ + assert space.fidelity is not None + fidelity_name, fidelity = space.fidelity + + if fidelity.log: + raise NotImplementedError("Log fidelity not yet supported") + + # Can't use mod since it's quite innacurate for floats + # Use the fact that we can always write x = n*k + r + # where k = stepsize and x = (fid_upper - fid_lower) + + x = fidelity.upper - fidelity.lower + + # > x = n*k + r + # > n = x // k + n = int(x // step_size) + + if n <= 0: + raise ValueError( + f"Step size ({step_size}) is too large for the fidelity domain {fidelity}." + "Considering lowering this parameter to ifBO." + ) + + # > r = x - n*k + r = x - n * step_size + new_lower = fidelity.lower + r + + new_fid: Float | Integer + match fidelity: + case Float(): + new_fid = Float( + lower=float(new_lower), + upper=float(fidelity.upper), + log=fidelity.log, + prior=fidelity.prior, + is_fidelity=True, + prior_confidence=fidelity.prior_confidence, + ) + case Integer(): + new_fid = Integer( + lower=int(new_lower), + upper=int(fidelity.upper), + log=fidelity.log, + prior=fidelity.prior, + is_fidelity=True, + prior_confidence=fidelity.prior_confidence, + ) + case _: + raise ValueError(f"Unsupported fidelity type: {type(fidelity)}") + new_space = SearchSpace({**space.parameters, fidelity_name: new_fid}) + return new_space, n + + +@dataclass +class IFBO: + """The ifBO optimizer. + + * Paper: https://openreview.net/forum?id=VyoY3Wh9Wd + * Github: https://github.com/automl/ifBO/tree/main + """ + + pipeline_space: SearchSpace + """The search space for the pipeline.""" + + encoder: ConfigEncoder + """The encoder to use for the pipeline space.""" + + sample_prior_first: bool + """Whether to sample the prior first.""" + + prior: Prior | None + """The prior to use for sampling the pipeline space.""" + + n_initial_design: int + """The number of initial designs to sample.""" + + device: torch.device | None + """The device to use for the optimizer.""" + + ftpfn: FTPFNSurrogate + """The FTPFN surrogate to use.""" + + fid_domain: Domain + """The domain of the fidelity parameter.""" + + fidelity_name: str + """The name of the fidelity parameter.""" + + n_fidelity_bins: int + """The number of bins to divide the fidelity domain into. + + Each one will be treated as an individual fidelity level. + """ + + def __call__( + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None = None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + assert n is None, "TODO" + ids = [int(config_id.split("_", maxsplit=1)[0]) for config_id in trials] + new_id = max(ids) + 1 if len(ids) > 0 else 0 + + min_fid = self.fid_domain.lower + max_fid = self.fid_domain.upper + + # The FTPFN surrogate takes in a budget in the range [0, 1] + # We also need to be able to map these to discrete integers + # Hence we use the two domains below to do so. + + # Domain in which we should pass budgets to ifbo model + budget_domain = Domain.floating(lower=1 / max_fid, upper=1) + + # Domain from which we assign an index to each budget + budget_index_domain = Domain.indices(self.n_fidelity_bins) + + # If we havn't passed the intial design phase + if new_id < self.n_initial_design: + init_design = make_initial_design( + space=self.pipeline_space, + encoder=self.encoder, + sample_prior_first=self.sample_prior_first, + sampler="sobol" if self.prior is None else self.prior, + seed=None, # TODO: + sample_fidelity="min", + sample_size=self.n_initial_design, + ) + config = init_design[new_id] + config[self.fidelity_name] = min_fid + return SampledConfig(id=f"{new_id}_0", config=config) + + X, y = encode_ftpfn( + trials=trials, + space=self.pipeline_space, + encoder=self.encoder, + budget_domain=budget_domain, + device=self.device, + pending_value=torch.nan, + ) + + # Fantasize if needed + pending_mask = torch.isnan(y) + if pending_mask.any(): + not_pending_mask = ~pending_mask + not_pending_X = X[not_pending_mask] + y[pending_mask] = self.ftpfn.get_mean_performance( + train_x=not_pending_X, + train_y=y[not_pending_mask], + test_x=X[pending_mask], + ) + else: + not_pending_X = X + + # NOTE: Can't really abstract this, requires knowledge that: + # 1. The encoding is such that the objective_to_minimize is 1 - + # objective_to_minimize + # 2. The budget is the second column + # 3. The budget is encoded between 1/max_fid and 1 + rng = np.random.RandomState(len(trials)) + # Cast the a random budget index into the ftpfn budget domain + horizon_increment = budget_domain.cast_one( + rng.randint(*budget_index_domain.bounds) + 1, + frm=budget_index_domain, + ) + f_best = y.max().item() + threshold = f_best + (10 ** rng.uniform(-4, -1)) * (1 - f_best) + + def _mfpi_random(samples: torch.Tensor) -> torch.Tensor: + # HACK: Because we are modifying the samples inplace, we do, + # and then undo the addition + original_budget_column = samples[..., 1].clone() + samples[..., 1].add_(horizon_increment).clamp_max_(budget_domain.upper) + + scores = self.ftpfn.get_pi(X, y, samples, y_best=threshold) + + samples[..., 1] = original_budget_column + return scores + + # Do acquisition on ftpfn + # TODO: Parametrize some of this + sample_dims = self.encoder.ndim + best_row = acquire_next_from_ftpfn( + ftpfn=self.ftpfn, + # How to encode + encoder=self.encoder, + budget_domain=budget_domain, + # Acquisition function + acq_function=_mfpi_random, + # Which acquisition samples to consider for continuation + continuation_samples=not_pending_X, + # How to generate some initial samples + initial_samplers=[ + (Sampler.sobol(ndim=sample_dims), 512), + (Sampler.uniform(ndim=sample_dims), 512), + (Sampler.borders(ndim=sample_dims), 256), + ], + seed=None, # TODO: Seeding + # A next step local sampling around best point found by initial_samplers + local_search_sample_size=256, + local_search_confidence=0.95, + ) + _id, fid, config = decode_ftpfn_data( + best_row, + self.encoder, + budget_domain=budget_domain, + fidelity_domain=self.fid_domain, + )[0] + + if _id is None: + config[self.fidelity_name] = fid + return SampledConfig(id=f"{new_id}_0", config=config) + + # Convert fidelity to budget index, bump by 1 and convert back + budget_ix = budget_index_domain.cast_one(fid, frm=self.fid_domain) + next_ix = budget_ix + 1 + next_fid = self.fid_domain.cast_one(next_ix, frm=budget_index_domain) + + config[self.fidelity_name] = next_fid + return SampledConfig( + id=f"{_id}_{next_ix}", + config=config, + previous_config_id=f"{_id}_{budget_ix}", + ) diff --git a/neps/optimizers/info.py b/neps/optimizers/info.py deleted file mode 100644 index b2494c20c..000000000 --- a/neps/optimizers/info.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import yaml - -HERE = Path(__file__).parent.resolve() - - -class SearcherConfigs: - """This class provides methods to access default configuration details - for NePS optimizers. - """ - - @staticmethod - def _get_searchers_folder_path() -> Path: - """Helper method to get the folder path for default searchers. - - Returns: - str: The absolute path to the default searchers folder. - """ - return HERE / "default_searchers" - - @staticmethod - def get_searchers() -> list[str]: - """List all the searcher names that can be used in neps run. - - Returns: - list[str]: A list of searcher names. - """ - folder_path = SearcherConfigs._get_searchers_folder_path() - searchers = [] - - for file in folder_path.iterdir(): - if file.suffix == ".yaml": - searchers.append(file.stem) - - return searchers - - @staticmethod - def get_available_algorithms() -> list[str]: - """List all available algorithms used by NePS searchers. - - Returns: - list[str]: A list of algorithm names. - """ - folder_path = SearcherConfigs._get_searchers_folder_path() - prev_algorithms = set() - - for file in folder_path.iterdir(): - if file.suffix == ".yaml": - with file.open("r") as f: - searcher_config = yaml.safe_load(f) - algorithm = searcher_config.get("strategy") - if algorithm: - prev_algorithms.add(algorithm) - - return list(prev_algorithms) - - @staticmethod - def get_searcher_from_algorithm(algorithm: str) -> list[str]: - """Get all NePS searchers that use a specific searching algorithm. - - Args: - algorithm (str): The name of the algorithm needed for the search. - - Returns: - list[str]: A list of searcher names using the specified algorithm. - """ - folder_path = SearcherConfigs._get_searchers_folder_path() - searchers = [] - - for file in folder_path.iterdir(): - if file.suffix == ".yaml": - with file.open("r") as f: - searcher_config = yaml.safe_load(f) - if searcher_config.get("strategy") == algorithm: - searchers.append(file.stem) - - return searchers - - @staticmethod - def get_searcher_kwargs(searcher: str) -> str: - """Get the kwargs and algorithm setup for a specific searcher. - - Args: - searcher (str): The name of the searcher to check the details of. - - Returns: - str: The raw content of the searcher's configuration - """ - folder_path = SearcherConfigs._get_searchers_folder_path() - - for file in folder_path.iterdir(): - if file.suffix == ".yaml" and file.stem.startswith(searcher): - return file.read_text() - - raise FileNotFoundError( - f"Searcher {searcher} not found in default searchers folder." - ) diff --git a/neps/optimizers/models/__init__.py b/neps/optimizers/models/__init__.py new file mode 100755 index 000000000..a634e7366 --- /dev/null +++ b/neps/optimizers/models/__init__.py @@ -0,0 +1,4 @@ +from neps.optimizers.models.ftpfn import FTPFNSurrogate +from neps.optimizers.models.gp import make_default_single_obj_gp + +__all__ = ["FTPFNSurrogate", "make_default_single_obj_gp"] diff --git a/neps/optimizers/bayesian_optimization/models/ftpfn.py b/neps/optimizers/models/ftpfn.py similarity index 95% rename from neps/optimizers/bayesian_optimization/models/ftpfn.py rename to neps/optimizers/models/ftpfn.py index 2990b095b..4b36e1d79 100644 --- a/neps/optimizers/bayesian_optimization/models/ftpfn.py +++ b/neps/optimizers/models/ftpfn.py @@ -7,13 +7,10 @@ import torch from ifbo import FTPFN -from neps.sampling.priors import Prior +from neps.sampling import Prior, Sampler if TYPE_CHECKING: - from neps.sampling.samplers import Sampler - from neps.search_spaces.domain import Domain - from neps.search_spaces.encoding import ConfigEncoder - from neps.search_spaces.search_space import SearchSpace + from neps.space import ConfigEncoder, Domain, SearchSpace from neps.state.trial import Trial @@ -145,8 +142,9 @@ def encode_ftpfn( # Select all trials which have something we can actually use for modelling # The absence of a report signifies pending selected = dict(trials.items()) - assert space.fidelity_name is not None assert space.fidelity is not None + fidelity_name, fidelity = space.fidelity + assert 0 <= error_value <= 1 train_configs = encoder.encode( [t.config for t in selected.values()], device=device, dtype=dtype @@ -160,12 +158,12 @@ def encode_ftpfn( ids = ids + 1 train_fidelities = torch.tensor( - [t.config[space.fidelity_name] for t in selected.values()], + [t.config[fidelity_name] for t in selected.values()], device=device, dtype=dtype, ) train_max_cost_total = budget_domain.cast( - train_fidelities, frm=space.fidelity.domain, dtype=dtype + train_fidelities, frm=fidelity.domain, dtype=dtype ) # TODO: Document that it's on the user to ensure these are already all bounded @@ -269,7 +267,7 @@ def acquire_next_from_ftpfn( # ... update best if needed sample_best_ix = acq_scores.argmax() - sample_best_score = acq_scores[sample_best_ix] + sample_best_score = acq_scores[sample_best_ix].item() sample_best_row = X_test[sample_best_ix].clone().detach() if sample_best_score > best_score: best_score = sample_best_score @@ -293,7 +291,7 @@ def acquire_next_from_ftpfn( acq_scores = acq_function(X_test) local_best_ix = acq_scores.argmax() - local_best_score = acq_scores[local_best_ix].clone().detach() + local_best_score = acq_scores[local_best_ix].clone().detach().item() if local_best_score > best_score: best_score = local_best_score best_row = X_test[local_best_ix].clone().detach() @@ -334,7 +332,7 @@ def __init__( def _get_logits( self, train_x: torch.Tensor, train_y: torch.Tensor, test_x: torch.Tensor ) -> torch.Tensor: - return self.ftpfn.model( + return self.ftpfn.model( # type: ignore _cast_tensor_shapes(train_x), _cast_tensor_shapes(train_y), _cast_tensor_shapes(test_x), @@ -348,7 +346,7 @@ def get_mean_performance( test_x: torch.Tensor, ) -> torch.Tensor: logits = self._get_logits(train_x, train_y, test_x).squeeze() - return self.ftpfn.model.criterion.mean(logits) + return self.ftpfn.model.criterion.mean(logits) # type: ignore @torch.no_grad() # type: ignore def get_pi( @@ -359,7 +357,7 @@ def get_pi( y_best: torch.Tensor | float, ) -> torch.Tensor: logits = self._get_logits(train_x, train_y, test_x) - return self.ftpfn.model.criterion.pi(logits.squeeze(), best_f=y_best) + return self.ftpfn.model.criterion.pi(logits.squeeze(), best_f=y_best) # type: ignore @torch.no_grad() # type: ignore def get_ei( @@ -370,7 +368,7 @@ def get_ei( y_best: torch.Tensor | float, ) -> torch.Tensor: logits = self._get_logits(train_x, train_y, test_x) - return self.ftpfn.model.criterion.ei(logits.squeeze(), best_f=y_best) + return self.ftpfn.model.criterion.ei(logits.squeeze(), best_f=y_best) # type: ignore @torch.no_grad() # type: ignore def get_lcb( @@ -381,7 +379,7 @@ def get_lcb( beta: float = (1 - 0.682) / 2, ) -> torch.Tensor: logits = self._get_logits(train_x, train_y, test_x) - return self.ftpfn.model.criterion.ucb( + return self.ftpfn.model.criterion.ucb( # type: ignore logits=logits, best_f=None, rest_prob=beta, @@ -398,7 +396,7 @@ def get_ucb( beta: float = (1 - 0.682) / 2, ) -> torch.Tensor: logits = self._get_logits(train_x, train_y, test_x) - return self.ftpfn.model.criterion.ucb( + return self.ftpfn.model.criterion.ucb( # type: ignore logits=logits, best_f=None, rest_prob=beta, diff --git a/neps/optimizers/bayesian_optimization/models/gp.py b/neps/optimizers/models/gp.py similarity index 94% rename from neps/optimizers/bayesian_optimization/models/gp.py rename to neps/optimizers/models/gp.py index 2210e44b4..7e2fbf47f 100644 --- a/neps/optimizers/bayesian_optimization/models/gp.py +++ b/neps/optimizers/models/gp.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from functools import reduce from itertools import product -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any import gpytorch.constraints import torch @@ -20,27 +20,19 @@ from gpytorch import ExactMarginalLogLikelihood from gpytorch.kernels import ScaleKernel -from neps.optimizers.bayesian_optimization.acquisition_functions.cost_cooling import ( - cost_cooled_acq, -) -from neps.optimizers.bayesian_optimization.acquisition_functions.pibo import ( - pibo_acquisition, -) -from neps.search_spaces.encoding import CategoricalToIntegerTransformer, ConfigEncoder +from neps.optimizers.acquisition import cost_cooled_acq, pibo_acquisition +from neps.space.encoding import CategoricalToIntegerTransformer, ConfigEncoder if TYPE_CHECKING: from botorch.acquisition import AcquisitionFunction from neps.sampling.priors import Prior - from neps.search_spaces.search_space import SearchSpace + from neps.space import SearchSpace from neps.state.trial import Trial logger = logging.getLogger(__name__) -T = TypeVar("T") - - @dataclass class GPEncodedData: """Tensor data of finished configurations.""" @@ -158,7 +150,7 @@ def optimize_acq( # Cap out at 4096 when len(bounds) >= 8 n_intial_start_points = min(64 * len(bounds) ** 2, 4096) - return optimize_acqf( + return optimize_acqf( # type: ignore acq_function=acq_fn, bounds=bounds, q=n_candidates_required, @@ -207,7 +199,7 @@ def optimize_acq( # TODO: we should deterministically shuffle the fixed_categoricals # as the underlying function does not. - return optimize_acqf_mixed( + return optimize_acqf_mixed( # type: ignore acq_function=acq_fn, bounds=bounds, num_restarts=min(num_restarts // n_combos, 2), @@ -225,16 +217,24 @@ def encode_trials_for_gp( encoder: ConfigEncoder | None = None, device: torch.device | None = None, ) -> tuple[GPEncodedData, ConfigEncoder]: + """Encode the trials for use in a GP. + + Args: + trials: The trials to encode. + space: The search space. + encoder: The encoder to use. If `None`, one will be created. + device: The device to use. + + Returns: + The encoded data and the encoder + """ train_configs: list[Mapping[str, Any]] = [] train_losses: list[float] = [] train_costs: list[float] = [] pending_configs: list[Mapping[str, Any]] = [] if encoder is None: - encoder = ConfigEncoder.from_space( - space=space, - include_constants_when_decoding=True, - ) + encoder = ConfigEncoder.from_space(space=space) for trial in trials.values(): if trial.report is None: @@ -299,9 +299,9 @@ def fit_and_acquire_from_gp( Please see the following for: * Making a GP to pass in: - [`make_default_single_obj_gp`][neps.optimizers.bayesian_optimization.models.gp.make_default_single_obj_gp] + [`make_default_single_obj_gp()`][neps.optimizers.models.gp.make_default_single_obj_gp] * Encoding configurations: - [`encode_trails_for_gp`][neps.optimizers.bayesian_optimization.models.gp.encode_trails_for_gp] + [`encode_trials_for_gp()`][neps.optimizers.models.gp.encode_trials_for_gp] Args: gp: The GP model to use. diff --git a/neps/optimizers/multi_fidelity/__init__.py b/neps/optimizers/multi_fidelity/__init__.py deleted file mode 100644 index 02e29dc96..000000000 --- a/neps/optimizers/multi_fidelity/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -from neps.optimizers.multi_fidelity.hyperband import ( - MOBSTER, - AsynchronousHyperband, - AsynchronousHyperbandWithPriors, - Hyperband, - HyperbandCustomDefault, - HyperbandWithPriors, -) -from neps.optimizers.multi_fidelity.ifbo import IFBO -from neps.optimizers.multi_fidelity.successive_halving import ( - AsynchronousSuccessiveHalving, - AsynchronousSuccessiveHalvingWithPriors, - SuccessiveHalving, - SuccessiveHalvingWithPriors, -) - -__all__ = [ - "IFBO", - "MOBSTER", - "AsynchronousHyperband", - "AsynchronousHyperbandWithPriors", - "AsynchronousSuccessiveHalving", - "AsynchronousSuccessiveHalvingWithPriors", - "Hyperband", - "HyperbandCustomDefault", - "HyperbandWithPriors", - "SuccessiveHalving", - "SuccessiveHalvingWithPriors", -] diff --git a/neps/optimizers/multi_fidelity/hyperband.py b/neps/optimizers/multi_fidelity/hyperband.py deleted file mode 100644 index ce78dd200..000000000 --- a/neps/optimizers/multi_fidelity/hyperband.py +++ /dev/null @@ -1,564 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from collections.abc import Mapping -from copy import deepcopy -from typing import TYPE_CHECKING, Any, Literal -from typing_extensions import override - -import numpy as np -import pandas as pd - -from neps.optimizers.base_optimizer import SampledConfig -from neps.optimizers.multi_fidelity.mf_bo import MFBOBase -from neps.optimizers.multi_fidelity.promotion_policy import ( - AsyncPromotionPolicy, - SyncPromotionPolicy, -) -from neps.optimizers.multi_fidelity.sampling_policy import ( - EnsemblePolicy, - FixedPriorPolicy, - ModelPolicy, - RandomUniformPolicy, -) -from neps.optimizers.multi_fidelity.successive_halving import ( - AsynchronousSuccessiveHalving, - SuccessiveHalving, - SuccessiveHalvingBase, -) -from neps.sampling.priors import Prior - -if TYPE_CHECKING: - from neps.optimizers.bayesian_optimization.acquisition_functions import ( - BaseAcquisition, - ) - from neps.search_spaces.search_space import SearchSpace - from neps.state.optimizer import BudgetInfo - from neps.state.trial import Trial - from neps.utils.types import ConfigResult, RawConfig - - -class HyperbandBase(SuccessiveHalvingBase): - """Implements a Hyperband procedure with a sampling and promotion policy.""" - - early_stopping_rate = 0 - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - use_priors: bool = False, - sampling_policy: Any = RandomUniformPolicy, - promotion_policy: Any = SyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] | None = None, - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - args = { - "pipeline_space": pipeline_space, - "max_cost_total": max_cost_total, - "eta": eta, - "early_stopping_rate": self.early_stopping_rate, # HB subsumes this from SH - "initial_design_type": initial_design_type, - "use_priors": use_priors, - "sampling_policy": sampling_policy, - "promotion_policy": promotion_policy, - "objective_to_minimize_value_on_error": objective_to_minimize_value_on_error, - "cost_value_on_error": cost_value_on_error, - "ignore_errors": ignore_errors, - "prior_confidence": prior_confidence, - "random_interleave_prob": random_interleave_prob, - "sample_prior_first": sample_prior_first, - "sample_prior_at_target": sample_prior_at_target, - } - super().__init__(**args) - # stores the flattened sequence of SH brackets to loop over - the HB heuristic - # for (n,r) pairing, i.e., (num. configs, fidelity) - self.full_rung_trace = [] - self.sh_brackets: dict[int, SuccessiveHalvingBase] = {} - for s in range(self.max_rung + 1): - args.update({"early_stopping_rate": s}) - self.sh_brackets[s] = SuccessiveHalving(**args) - # `full_rung_trace` contains the index of SH bracket to run sequentially - self.full_rung_trace.extend([s] * len(self.sh_brackets[s].full_rung_trace)) - # book-keeping variables - self.current_sh_bracket: int = 0 - - def _update_sh_bracket_state(self) -> None: - # `load_results()` for each of the SH bracket objects are not called as they are - # not part of the main Hyperband loop. For correct promotions and sharing of - # optimization history, the promotion handler of the current SH bracket needs the - # optimization state. Calling `load_results()` is an option but leads to - # redundant data processing. - # `clean_active_brackets` takes care of setting rung information and promotion - # for the current SH bracket in HB - # TODO: can we avoid copying full observation history - bracket = self.sh_brackets[self.current_sh_bracket] - bracket.observed_configs = self.observed_configs.copy() - - def clear_old_brackets(self) -> None: - """Enforces reset at each new bracket.""" - # unlike synchronous SH, the state is not reset at each rung and a configuration - # is promoted if the rung has eta configs if it is the top performing - # base class allows for retaining the whole optimization state - return - - def _handle_promotions(self) -> None: - self.promotion_policy.set_state( - max_rung=self.max_rung, - members=self.rung_members, - performances=self.rung_members_performance, - **self.promotion_policy_kwargs, - ) - # promotions are handled by the individual SH brackets which are explicitly - # called in the _update_sh_bracket_state() function - # overloaded function disables the need for retrieving promotions for HB overall - - @override - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None, - n: int | None = None, - ) -> SampledConfig: - assert n is None, "TODO" - completed: dict[str, ConfigResult] = { - trial_id: trial.into_config_result(self.pipeline_space.from_dict) - for trial_id, trial in trials.items() - if trial.report is not None - } - pending: dict[str, SearchSpace] = { - trial_id: self.pipeline_space.from_dict(trial.config) - for trial_id, trial in trials.items() - if trial.report is None - } - - self.rung_histories = { - rung: {"config": [], "perf": []} - for rung in range(self.min_rung, self.max_rung + 1) - } - - self.observed_configs = pd.DataFrame([], columns=("config", "rung", "perf")) - - # previous optimization run exists and needs to be loaded - self._load_previous_observations(completed) - - # account for pending evaluations - self._handle_pending_evaluations(pending) - - # process optimization state and bucket observations per rung - self._get_rungs_state() - - # filter/reset old SH brackets - self.clear_old_brackets() - - # identifying promotion list per rung - self._handle_promotions() - - # fit any model/surrogates - self._fit_models() - - # important for the global HB to run the right SH - self._update_sh_bracket_state() - - config, _id, previous_id = self.get_config_and_ids() - return SampledConfig(id=_id, config=config, previous_config_id=previous_id) - - @abstractmethod - def get_config_and_ids(self) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - raise NotImplementedError - - -class Hyperband(HyperbandBase): - def clear_old_brackets(self) -> None: - """Enforces reset at each new bracket. - - The _get_rungs_state() function creates the `rung_promotions` dict mapping which - is used by the promotion policies to determine the next step: promotion/sample. - To simulate reset of rungs like in vanilla HB, the algorithm is viewed as a - series of SH brackets, where the SH brackets comprising HB is repeated. This is - done by iterating over the closed loop of possible SH brackets (self.sh_brackets). - The oldest, active, incomplete SH bracket is searched for to choose the next - evaluation. If either all brackets are over or waiting, a new SH bracket, - corresponding to the SH bracket under HB as registered by `current_SH_bracket`. - """ - n_sh_brackets = len(self.sh_brackets) - # iterates over the different SH brackets - self.current_sh_bracket = 0 # indexing from range(0, n_sh_brackets) - start = 0 - _min_rung = self.sh_brackets[self.current_sh_bracket].min_rung - end = self.sh_brackets[self.current_sh_bracket].config_map[_min_rung] - - if self.sample_prior_first and self.sample_prior_at_target: - start += 1 - end += 1 - - # stores the base rung size for each SH bracket in HB - base_rung_sizes = [] # sorted(self.config_map.values(), reverse=True) - for bracket in self.sh_brackets.values(): - base_rung_sizes.append(sorted(bracket.config_map.values(), reverse=True)[0]) - while end <= len(self.observed_configs): - # subsetting only this SH bracket from the history - sh_bracket = self.sh_brackets[self.current_sh_bracket] - sh_bracket.clean_rung_information() - # for the SH bracket in start-end, calculate total SH budget used, from the - # correct SH bracket object to make the right budget calculations - - assert isinstance(sh_bracket, SuccessiveHalving) - bracket_budget_used = sh_bracket._calc_budget_used_in_bracket( - deepcopy(self.observed_configs.rung.values[start:end]) - ) - # if budget used is less than the total SH budget then still an active bracket - current_bracket_full_budget = sum(sh_bracket.full_rung_trace) - if bracket_budget_used < current_bracket_full_budget: - # updating rung information of the current bracket - - sh_bracket._get_rungs_state(self.observed_configs.iloc[start:end]) - # extra call to use the updated rung member info to find promotions - # SyncPromotion signals a wait if a rung is full but with - # incomplete/pending evaluations, signals to starts a new SH bracket - sh_bracket._handle_promotions() - promotion_count = 0 - for _, promotions in sh_bracket.rung_promotions.items(): - promotion_count += len(promotions) - # if no promotion candidates are returned, then the current bracket - # is active and waiting - if promotion_count: - # returns the oldest active bracket if a promotion found which is the - # current SH bracket at this scope - return - # if no promotions, ensure an empty state explicitly to disable bracket - sh_bracket.clean_rung_information() - start = end - # updating pointer to the next SH bracket in HB - self.current_sh_bracket = (self.current_sh_bracket + 1) % n_sh_brackets - end = start + base_rung_sizes[self.current_sh_bracket] - # reaches here if all old brackets are either waiting or finished - - # updates rung info with the latest active, incomplete bracket - sh_bracket = self.sh_brackets[self.current_sh_bracket] - - sh_bracket._get_rungs_state(self.observed_configs.iloc[start:end]) - sh_bracket._handle_promotions() - # self._handle_promotion() need not be called as it is called by load_results() - - def get_config_and_ids(self) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - config, config_id, previous_config_id = self.sh_brackets[ - self.current_sh_bracket - ].get_config_and_ids() - return config, config_id, previous_config_id - - -class HyperbandWithPriors(Hyperband): - """Implements a Hyperband procedure with a sampling and promotion policy.""" - - use_priors = True - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = FixedPriorPolicy, - promotion_policy: Any = SyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - initial_design_type=initial_design_type, - use_priors=self.use_priors, # key change to the base HB class - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - - -class HyperbandCustomDefault(HyperbandWithPriors): - """If prior specified, does 50% times priors and 50% random search like vanilla-HB.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = EnsemblePolicy, - promotion_policy: Any = SyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - initial_design_type=initial_design_type, - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - self.sampling_args = { - "inc": None, - "weights": { - "prior": 0.5, - "inc": 0, - "random": 0.5, - }, - } - for _, sh in self.sh_brackets.items(): - sh.sampling_args = self.sampling_args - - -class AsynchronousHyperband(HyperbandBase): - """Implements ASHA but as Hyperband. - - Implements the Promotion variant of ASHA as used in Mobster. - """ - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - use_priors: bool = False, - sampling_policy: Any = RandomUniformPolicy, - promotion_policy: Any = AsyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] | None = None, - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - args = { - "pipeline_space": pipeline_space, - "max_cost_total": max_cost_total, - "eta": eta, - "initial_design_type": initial_design_type, - "use_priors": use_priors, - "sampling_policy": sampling_policy, - "promotion_policy": promotion_policy, - "objective_to_minimize_value_on_error": objective_to_minimize_value_on_error, - "cost_value_on_error": cost_value_on_error, - "ignore_errors": ignore_errors, - "prior_confidence": prior_confidence, - "random_interleave_prob": random_interleave_prob, - "sample_prior_first": sample_prior_first, - "sample_prior_at_target": sample_prior_at_target, - } - super().__init__(**args) - # overwrite parent class SH brackets with Async SH brackets - self.sh_brackets: dict[int, SuccessiveHalvingBase] = {} - for s in range(self.max_rung + 1): - args.update({"early_stopping_rate": s}) - # key difference from vanilla HB where it runs synchronous SH brackets - self.sh_brackets[s] = AsynchronousSuccessiveHalving(**args) - - def _update_sh_bracket_state(self) -> None: - # `load_results()` for each of the SH bracket objects are not called as they are - # not part of the main Hyperband loop. For correct promotions and sharing of - # optimization history, the promotion handler of the SH brackets need the - # optimization state. Calling `load_results()` is an option but leads to - # redundant data processing. - for _, bracket in self.sh_brackets.items(): - bracket.promotion_policy.set_state( - max_rung=self.max_rung, - members=self.rung_members, - performances=self.rung_members_performance, - config_map=bracket.config_map, - ) - bracket.rung_promotions = bracket.promotion_policy.retrieve_promotions() - bracket.observed_configs = self.observed_configs.copy() - - def _get_bracket_to_run(self) -> int: - """Samples the ASHA bracket to run. - - The selected bracket always samples at its minimum rung. Thus, selecting a bracket - effectively selects the rung that a new sample will be evaluated at. - """ - # Sampling distribution derived from Appendix A (https://arxiv.org/abs/2003.10865) - # Adapting the distribution based on the current optimization state - # s \in [0, max_rung] and to with the denominator's constraint, we have K > s - 1 - # and thus K \in [1, ..., max_rung, ...] - # Since in this version, we see the full SH rung, we fix the K to max_rung - K = self.max_rung - bracket_probs = [ - self.eta ** (K - s) * (K + 1) / (K - s + 1) for s in range(self.max_rung + 1) - ] - bracket_probs = np.array(bracket_probs) / sum(bracket_probs) - return int(np.random.choice(range(self.max_rung + 1), p=bracket_probs)) - - def get_config_and_ids(self) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - # the rung to sample at - bracket_to_run = self._get_bracket_to_run() - config, config_id, previous_config_id = self.sh_brackets[ - bracket_to_run - ].get_config_and_ids() - return config, config_id, previous_config_id - - -class AsynchronousHyperbandWithPriors(AsynchronousHyperband): - """Implements ASHA but as Hyperband.""" - - use_priors = True - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = FixedPriorPolicy, - promotion_policy: Any = AsyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - initial_design_type=initial_design_type, - use_priors=self.use_priors, # key change to the base Async HB class - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - - -class MOBSTER(MFBOBase, AsynchronousHyperband): - model_based = True - modelling_type = "rung" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - use_priors: bool = False, - sampling_policy: Any = RandomUniformPolicy, - promotion_policy: Any = AsyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] | None = None, - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - # new arguments for model - model_policy: Any = ModelPolicy, - surrogate_model: str | Any = "gp", # TODO: Remove - domain_se_kernel: str | None = None, # TODO: Remove - hp_kernels: list | None = None, # TODO: Remove - surrogate_model_args: dict | None = None, # TODO: Remove - acquisition: str | BaseAcquisition = "EI", # TODO: Remove - log_prior_weighted: bool = False, # TODO: Remove - acquisition_sampler: str = "random", # TODO: Remove - ): - hb_args = { - "pipeline_space": pipeline_space, - "max_cost_total": max_cost_total, - "eta": eta, - "initial_design_type": initial_design_type, - "use_priors": use_priors, - "sampling_policy": sampling_policy, - "promotion_policy": promotion_policy, - "objective_to_minimize_value_on_error": objective_to_minimize_value_on_error, - "cost_value_on_error": cost_value_on_error, - "ignore_errors": ignore_errors, - "prior_confidence": prior_confidence, - "random_interleave_prob": random_interleave_prob, - "sample_prior_first": sample_prior_first, - "sample_prior_at_target": sample_prior_at_target, - } - super().__init__(**hb_args) - - self.pipeline_space.has_prior = self.use_priors - - # counting non-fidelity dimensions in search space - ndims = sum( - 1 - for _, hp in self.pipeline_space.hyperparameters.items() - if not hp.is_fidelity - ) - n_min = ndims + 1 - self.init_size = n_min + 1 # in BOHB: init_design >= N_min + 2 - - if self.use_priors: - prior = Prior.from_space(self.pipeline_space, include_fidelity=False) - else: - prior = None - - self.model_policy = model_policy(pipeline_space=pipeline_space, prior=prior) - - for _, sh in self.sh_brackets.items(): - sh.model_policy = self.model_policy # type: ignore - sh.sample_new_config = self.sample_new_config # type: ignore - - -# TODO: TrulyAsyncHyperband diff --git a/neps/optimizers/multi_fidelity/ifbo.py b/neps/optimizers/multi_fidelity/ifbo.py deleted file mode 100755 index ae7dfc411..000000000 --- a/neps/optimizers/multi_fidelity/ifbo.py +++ /dev/null @@ -1,293 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Literal -from typing_extensions import override - -import numpy as np -import torch - -from neps.optimizers.base_optimizer import BaseOptimizer, SampledConfig -from neps.optimizers.bayesian_optimization.models.ftpfn import ( - FTPFNSurrogate, - acquire_next_from_ftpfn, - decode_ftpfn_data, - encode_ftpfn, -) -from neps.optimizers.initial_design import make_initial_design -from neps.sampling.priors import Prior -from neps.sampling.samplers import Sampler -from neps.search_spaces.domain import Domain -from neps.search_spaces.encoding import CategoricalToUnitNorm, ConfigEncoder -from neps.search_spaces.search_space import Float, Integer, SearchSpace - -if TYPE_CHECKING: - from neps.state.optimizer import BudgetInfo - from neps.state.trial import Trial - -# NOTE: Ifbo was trained using 32 bit -FTPFN_DTYPE = torch.float32 - - -def _adjust_pipeline_space_to_match_stepsize( - pipeline_space: SearchSpace, - step_size: int | float, -) -> tuple[SearchSpace, int]: - """Adjust the pipeline space to be evenly divisible by the step size. - - This is done by incrementing the lower bound of the fidelity domain to the - that enables this. - - Args: - pipeline_space: The pipeline space to adjust - step_size: The size of the step to take in the fidelity domain. - - Returns: - The adjusted pipeline space and the number of bins it can be divided into - """ - fidelity = pipeline_space.fidelity - fidelity_name = pipeline_space.fidelity_name - assert fidelity_name is not None - assert isinstance(fidelity, Float | Integer) - if fidelity.log: - raise NotImplementedError("Log fidelity not yet supported") - - # Can't use mod since it's quite innacurate for floats - # Use the fact that we can always write x = n*k + r - # where k = stepsize and x = (fid_upper - fid_lower) - # > x = n*k + r - # > n = x // k - # > r = x - n*k - x = fidelity.upper - fidelity.lower - n = int(x // step_size) - - if n <= 0: - raise ValueError( - f"Step size ({step_size}) is too large for the fidelity domain {fidelity}." - "Considering lowering this parameter to ifBO." - ) - - r = x - n * step_size - new_lower = fidelity.lower + r - new_fid = fidelity.__class__( - lower=new_lower, - upper=fidelity.upper, - log=fidelity.log, - prior=fidelity.prior, - is_fidelity=True, - prior_confidence=fidelity.prior_confidence_choice, - ) - return ( - SearchSpace(**{**pipeline_space.hyperparameters, fidelity_name: new_fid}), - n, - ) - - -class IFBO(BaseOptimizer): - """Base class for MF-BO algorithms that use DyHPO-like acquisition and budgeting.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - step_size: int | float = 1, - use_priors: bool = False, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - surrogate_model_args: dict | None = None, - initial_design_size: int | Literal["ndim"] = "ndim", - n_acquisition_new_configs: int = 1_000, - device: torch.device | None = None, - max_cost_total: int | float | None = None, # TODO: Remove - objective_to_minimize_value_on_error: float | None = None, # TODO: Remove - cost_value_on_error: float | None = None, # TODO: Remove - ignore_errors: bool = False, # TODO: Remove - ): - """Initialise. - - Args: - pipeline_space: Space in which to search - step_size: The size of the step to take in the fidelity domain. - sampling_policy: The type of sampling procedure to use - promotion_policy: The type of promotion procedure to use - sample_prior_first: Whether to sample the default configuration first - initial_design_size: Number of configs to sample before starting optimization - - If None, the number of configs will be equal to the number of dimensions. - - device: Device to use for the model - """ - # TODO: I'm not sure how this might effect tables, whose lowest fidelity - # might be below to possibly increased lower bound. - space, fid_bins = _adjust_pipeline_space_to_match_stepsize( - pipeline_space, step_size - ) - assert space.fidelity is not None - assert isinstance(space.fidelity_name, str) - - super().__init__(pipeline_space=space) - self.step_size = step_size - self.use_priors = use_priors - self.sample_prior_first = sample_prior_first - self.sample_prior_at_target = sample_prior_at_target - self.device = device - self.n_initial_design: int | Literal["ndim"] = initial_design_size - self.n_acquisition_new_configs = n_acquisition_new_configs - self.surrogate_model_args = surrogate_model_args or {} - - self._min_budget: int | float = space.fidelity.lower - self._max_budget: int | float = space.fidelity.upper - self._fidelity_name: str = space.fidelity_name - self._initial_design: list[dict[str, Any]] | None = None - - self._prior: Prior | None - if use_priors: - self._prior = Prior.from_space(space, include_fidelity=False) - else: - self._prior = None - - self._config_encoder: ConfigEncoder = ConfigEncoder.from_space( - space=space, - include_constants_when_decoding=True, - # FTPFN doesn't support categoricals and we were recomended - # to just evenly distribute in the unit norm - custom_transformers={ - cat_name: CategoricalToUnitNorm(choices=cat.choices) - for cat_name, cat in space.categoricals.items() - }, - ) - - # Domain of fidelity values, i.e. what is given in the configs that we - # give to the user to evaluate at. - self._fid_domain = space.fidelity.domain - - # Domain in which we should pass budgets to ifbo model - self._budget_domain = Domain.floating(1 / self._max_budget, 1) - - # Domain from which we assign an index to each budget - self._budget_ix_domain = Domain.indices(fid_bins) - - @override - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None = None, - n: int | None = None, - ) -> SampledConfig: - assert n is None, "TODO" - ids = [int(config_id.split("_", maxsplit=1)[0]) for config_id in trials] - new_id = max(ids) + 1 if len(ids) > 0 else 0 - - # If we havn't passed the intial design phase - if self._initial_design is None: - self._initial_design = make_initial_design( - space=self.pipeline_space, - encoder=self._config_encoder, - sample_prior_first=self.sample_prior_first, - sampler="sobol" if self._prior is None else self._prior, - seed=None, # TODO: - sample_fidelity="min", - sample_size=self.n_initial_design, - ) - - if new_id < len(self._initial_design): - config = self._initial_design[new_id] - config[self._fidelity_name] = self._min_budget - return SampledConfig(id=f"{new_id}_0", config=config) - - # Otherwise, we proceed to surrogate phase - ftpfn = FTPFNSurrogate( - target_path=self.surrogate_model_args.get("target_path", None), - version=self.surrogate_model_args.get("version", "0.0.1"), - device=self.device, - ) - X, y = encode_ftpfn( - trials=trials, - space=self.pipeline_space, - encoder=self._config_encoder, - budget_domain=self._budget_domain, - device=self.device, - pending_value=torch.nan, - ) - - # Fantasize if needed - pending_mask = torch.isnan(y) - if pending_mask.any(): - not_pending_mask = ~pending_mask - not_pending_X = X[not_pending_mask] - y[pending_mask] = ftpfn.get_mean_performance( - train_x=not_pending_X, - train_y=y[not_pending_mask], - test_x=X[pending_mask], - ) - else: - not_pending_X = X - - # NOTE: Can't really abstract this, requires knowledge that: - # 1. The encoding is such that the objective_to_minimize is 1 - - # objective_to_minimize - # 2. The budget is the second column - # 3. The budget is encoded between 1/max_fid and 1 - rng = np.random.RandomState(len(trials)) - # Cast the a random budget index into the ftpfn budget domain - horizon_increment = self._budget_domain.cast_one( - rng.randint(*self._budget_ix_domain.bounds) + 1, - frm=self._budget_ix_domain, - ) - f_best = y.max().item() - threshold = f_best + (10 ** rng.uniform(-4, -1)) * (1 - f_best) - - def _mfpi_random(samples: torch.Tensor) -> torch.Tensor: - # HACK: Because we are modifying the samples inplace, we do, - # and then undo the addition - original_budget_column = samples[..., 1].clone() - samples[..., 1].add_(horizon_increment).clamp_max_(self._budget_domain.upper) - - scores = ftpfn.get_pi(X, y, samples, y_best=threshold) - - samples[..., 1] = original_budget_column - return scores - - # Do acquisition on ftpfn - sample_dims = self._config_encoder.ncols - best_row = acquire_next_from_ftpfn( - ftpfn=ftpfn, - # How to encode - encoder=self._config_encoder, - budget_domain=self._budget_domain, - # Acquisition function - acq_function=_mfpi_random, - # Which acquisition samples to consider for continuation - continuation_samples=not_pending_X, - # How to generate some initial samples - initial_samplers=[ - (Sampler.sobol(ndim=sample_dims), 512), - (Sampler.uniform(ndim=sample_dims), 512), - (Sampler.borders(ndim=sample_dims), 256), - ], - seed=None, # TODO: Seeding - # A next step local sampling around best point found by initial_samplers - local_search_sample_size=256, - local_search_confidence=0.95, - ) - _id, fid, config = decode_ftpfn_data( - best_row, - self._config_encoder, - budget_domain=self._budget_domain, - fidelity_domain=self._fid_domain, - )[0] - - if _id is None: - config[self._fidelity_name] = fid - return SampledConfig(id=f"{new_id}_0", config=config) - # Convert fidelity to budget index, bump by 1 and convert back - budget_ix = self._budget_ix_domain.cast_one(fid, frm=self._fid_domain) - next_ix = budget_ix + 1 - next_fid = self._fid_domain.cast_one(next_ix, frm=self._budget_ix_domain) - - config[self._fidelity_name] = next_fid - return SampledConfig( - id=f"{_id}_{next_ix}", - config=config, - previous_config_id=f"{_id}_{budget_ix}", - ) diff --git a/neps/optimizers/multi_fidelity/mf_bo.py b/neps/optimizers/multi_fidelity/mf_bo.py deleted file mode 100755 index f4355585e..000000000 --- a/neps/optimizers/multi_fidelity/mf_bo.py +++ /dev/null @@ -1,211 +0,0 @@ -from __future__ import annotations - -import logging -from copy import deepcopy -from typing import TYPE_CHECKING, Any, Literal - -from neps.search_spaces.functions import sample_one_old - - -def update_fidelity(config: SearchSpace, fidelity: int | float) -> SearchSpace: - assert config.fidelity is not None - config.fidelity.set_value(fidelity) - # TODO: Place holder until we can get rid of passing around search spaces - # as configurations - assert config.fidelity_name is not None - config._values[config.fidelity_name] = fidelity - return config - - -if TYPE_CHECKING: - import pandas as pd - - from neps.search_spaces import SearchSpace - -logger = logging.getLogger(__name__) - - -class MFBOBase: - """Designed to work with model-based search on SH-based multi-fidelity algorithms. - - Requires certain strict assumptions about fidelities and rung maps. - """ - - # TODO: Make pure function... - model_based: bool - pipeline_space: SearchSpace - observed_configs: pd.DataFrame - rung_map: dict - max_budget: float - modelling_type: Literal["rung", "joint"] - rung_histories: dict - min_rung: int - max_rung: int - model_policy: Any - sampling_args: dict - sampling_policy: Any - patience: int - use_priors: bool - init_size: int - - def _fit_models(self) -> None: - """Performs necessary procedures to build and use models.""" - if not self.model_based: - # do nothing here if the algorithm has model-based search disabled - return - - if self.is_init_phase(): - return - - if self.pipeline_space.has_prior: - # PriorBand + BO - valid_perf_mask = self.observed_configs["perf"].notna() - rungs = self.observed_configs.loc[valid_perf_mask, "rung"] - total_resources = sum(self.rung_map[r] for r in rungs) - decay_t = total_resources / self.max_budget - else: - # Mobster - decay_t = None - - # extract pending configurations - # doing this separately as `rung_histories` do not record pending configs - pending_df = self.observed_configs[self.observed_configs.perf.isna()] - if self.modelling_type == "rung": - # collect only the finished configurations at the highest active `rung` - # for training the surrogate and considering only those pending - # evaluations at `rung` for fantasization - # important to set the fidelity value of the training data configuration - # such that the fidelity variable is rendered ineffective in the model - - rung = self._active_rung() - # inside here rung should not be None - if rung is None: - raise ValueError( - "Returned rung is None. Should not be so when not init phase." - ) - logger.info(f"Building model at rung {rung}") - # collecting finished evaluations at `rung` - train_df = self.observed_configs.loc[ - self.rung_histories[rung]["config"] - ].copy() - - # setting the fidelity value and performance to match the rung history - # a promoted configuration may have a different fidelity than the - # rung history recorded - fidelities = [self.rung_map[rung]] * len(train_df) - train_x = deepcopy(train_df.config.values.tolist()) - # update fidelity - train_x = list(map(update_fidelity, train_x, fidelities)) - train_y = deepcopy(self.rung_histories[rung]["perf"]) - # extract only the pending configurations that are at `rung` - pending_df = pending_df[pending_df.rung == rung] - pending_x = deepcopy(pending_df["config"].values.tolist()) - # update fidelity - fidelities = [self.rung_map[rung]] * len(pending_x) - pending_x = list(map(update_fidelity, pending_x, fidelities)) - - elif self.modelling_type == "joint": - # collect ALL configurations ever recorded for training the surrogate - # and considering all pending evaluations for fantasization - # the fidelity for all these configurations should be set for each of the - # rungs they were evaluated at in the entire optimization history - - # NOTE: pandas considers mutable objects inside dataframes as antipattern - train_x = [] - train_y = [] - pending_x = [] - for rung in range(self.min_rung, self.max_rung + 1): - _ids = self.rung_histories[rung]["config"] - _x = deepcopy(self.observed_configs.loc[_ids].config.values.tolist()) - # update fidelity - fidelity = [self.rung_map[rung]] * len(_x) - _x = list(map(update_fidelity, _x, fidelity)) - _y = deepcopy(self.rung_histories[rung]["perf"]) - train_x.extend(_x) - train_y.extend(_y) - # setting the fidelity value of the pending configs appropriately - get_fidelity = lambda _rung: self.rung_map[_rung] - fidelities = list(map(get_fidelity, pending_df.rung.values)) - _pending_x = list( - map(update_fidelity, pending_df.config.values.tolist(), fidelities) - ) - pending_x.extend(_pending_x) - else: - raise ValueError("Choice of modelling_type not in {{'rung', 'joint'}}") - # the `model_policy` class should define a function to train the surrogates - # and set the acquisition states - self.model_policy.update_model(train_x, train_y, pending_x, decay_t=decay_t) - - def _active_rung(self) -> int | None: - """The highest rung that can fit a model, `None` if no rung is eligible.""" - rung = self.max_rung - while rung >= self.min_rung: - if len(self.rung_histories[rung]["config"]) >= self.init_size: - return rung - rung -= 1 - return None - - def is_init_phase(self) -> bool: - """Returns True is in the warmstart phase and False under model-based search.""" - if self.modelling_type == "rung": - # build a model per rung or per fidelity - # in this case, the initial design checks if `init_size` number of - # configurations have finished at a rung or not and the highest such rung is - # chosen for model building at teh current iteration - if self._active_rung() is None: - return True - elif self.modelling_type == "joint": - # builds a model across all fidelities with the fidelity as a dimension - # in this case, calculate the total number of function evaluations spent - # and in vanilla BO fashion use that to compare with the initital design size - valid_perf_mask = self.observed_configs["perf"].notna() - rungs = self.observed_configs.loc[valid_perf_mask, "rung"] - total_resources = sum(self.rung_map[r] for r in rungs) - resources = total_resources / self.max_budget - if resources < self.init_size: - return True - else: - raise ValueError("Choice of modelling_type not in {{'rung', 'joint'}}") - return False - - def sample_new_config( - self, - rung: int | None = None, - **kwargs: Any, - ) -> SearchSpace: - """Samples configuration from policies or random.""" - if self.model_based and not self.is_init_phase(): - incumbent = None - if self.modelling_type == "rung": - # `rung` should not be None when not in init phase - active_max_rung = self._active_rung() - fidelity = None - active_max_fidelity = self.rung_map[active_max_rung] - elif self.modelling_type == "joint": - fidelity = self.rung_map[rung] - active_max_fidelity = None - # IMPORTANT step for correct 2-step acquisition - incumbent = min(self.rung_histories[rung]["perf"]) - else: - raise ValueError("Choice of modelling_type not in 'rung', 'joint'") - assert ( - (fidelity is None and active_max_fidelity is not None) - or (active_max_fidelity is None and fidelity is not None) - or (active_max_fidelity is not None and fidelity is not None) - ), "Either condition needs to be not None!" - config = self.model_policy.sample( - active_max_fidelity=active_max_fidelity, - fidelity=fidelity, - incumbent=incumbent, - **self.sampling_args, - ) - elif self.sampling_policy is not None: - config = self.sampling_policy.sample(**self.sampling_args) - else: - config = sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=self.use_priors, - ignore_fidelity=True, - ) - return config diff --git a/neps/optimizers/multi_fidelity/promotion_policy.py b/neps/optimizers/multi_fidelity/promotion_policy.py deleted file mode 100644 index b94cbf0e6..000000000 --- a/neps/optimizers/multi_fidelity/promotion_policy.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any - -import numpy as np - - -class PromotionPolicy(ABC): - """Base class for implementing a sampling straregy for SH and its subclasses.""" - - def __init__(self, eta: int): - self.rung_members: dict = {} - self.rung_members_performance: dict = {} - self.rung_promotions: dict = {} - self.eta: int = eta - self.max_rung: int | None = None - - def set_state( - self, - *, - max_rung: int, - members: dict, - performances: dict, - **kwargs: Any, - ) -> None: - self.max_rung = max_rung - self.rung_members = members - self.rung_members_performance = performances - - @abstractmethod - def retrieve_promotions(self) -> dict: - raise NotImplementedError - - -class SyncPromotionPolicy(PromotionPolicy): - """Implements a synchronous promotion from lower to higher fidelity. - - Promotes only when all predefined number of config slots are full. - """ - - def __init__(self, eta: int, **kwargs: Any): - super().__init__(eta, **kwargs) - self.config_map: dict | None = None - self.rung_promotions: dict | None = None - - def set_state( - self, - *, - max_rung: int, - members: dict, - performances: dict, - config_map: dict, - **kwargs: Any, - ) -> None: - super().set_state(max_rung=max_rung, members=members, performances=performances) - self.config_map = config_map - - def retrieve_promotions(self) -> dict: - """Returns the top 1/eta configurations per rung if enough configurations seen.""" - assert self.config_map is not None - - self.rung_promotions = {rung: [] for rung in self.config_map} - total_rung_evals = 0 - for rung in sorted(self.config_map.keys(), reverse=True): - total_rung_evals += len(self.rung_members[rung]) - if ( - total_rung_evals >= self.config_map[rung] - and np.isnan(self.rung_members_performance[rung]).sum() - ): - # if rung is full but incomplete evaluations, pause on promotions, wait - return self.rung_promotions - if rung == self.max_rung: - # cease promotions for the highest rung (configs at max budget) - continue - if ( - total_rung_evals >= self.config_map[rung] - and np.isnan(self.rung_members_performance[rung]).sum() == 0 - ): - # if rung is full and no incomplete evaluations, find promotions - top_k = (self.config_map[rung] // self.eta) - ( - self.config_map[rung] - len(self.rung_members[rung]) - ) - selected_idx = np.argsort(self.rung_members_performance[rung])[:top_k] - self.rung_promotions[rung] = self.rung_members[rung][selected_idx] - return self.rung_promotions - - -class AsyncPromotionPolicy(PromotionPolicy): - """Implements an asynchronous promotion from lower to higher fidelity. - - Promotes whenever a higher fidelity has at least eta configurations. - """ - - def __init__(self, eta: int, **kwargs: Any): - super().__init__(eta, **kwargs) - - def retrieve_promotions(self) -> dict: - """Returns the top 1/eta configurations per rung if enough configurations seen.""" - assert self.max_rung is not None - for rung in range(self.max_rung + 1): - if rung == self.max_rung: - # cease promotions for the highest rung (configs at max budget) - continue - # if less than eta configurations seen, no promotions occur as top_k=0 - top_k = len(self.rung_members_performance[rung]) // self.eta - _ordered_idx = np.argsort(self.rung_members_performance[rung]) - self.rung_promotions[rung] = np.array(self.rung_members[rung])[_ordered_idx][ - :top_k - ].tolist() - return self.rung_promotions diff --git a/neps/optimizers/multi_fidelity/sampling_policy.py b/neps/optimizers/multi_fidelity/sampling_policy.py deleted file mode 100644 index dd510c1c6..000000000 --- a/neps/optimizers/multi_fidelity/sampling_policy.py +++ /dev/null @@ -1,430 +0,0 @@ -from __future__ import annotations - -import logging -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, Literal - -import numpy as np -import pandas as pd -import torch -from botorch.acquisition import ( - AcquisitionFunction, - LinearMCObjective, - qLogNoisyExpectedImprovement, -) -from botorch.fit import fit_gpytorch_mll -from gpytorch import ExactMarginalLogLikelihood - -from neps.optimizers.bayesian_optimization.acquisition_functions.pibo import ( - pibo_acquisition, -) -from neps.optimizers.bayesian_optimization.models.gp import make_default_single_obj_gp -from neps.sampling.priors import Prior -from neps.sampling.samplers import Sampler -from neps.search_spaces.encoding import ConfigEncoder -from neps.search_spaces.functions import sample_one_old - -if TYPE_CHECKING: - from botorch.acquisition.analytic import SingleTaskGP - - from neps.search_spaces.search_space import SearchSpace - -TOLERANCE = 1e-2 # 1% -SAMPLE_THRESHOLD = 1000 # num samples to be rejected for increasing hypersphere radius -DELTA_THRESHOLD = 1e-2 # 1% -TOP_EI_SAMPLE_COUNT = 10 - -logger = logging.getLogger(__name__) - - -def update_fidelity(config: SearchSpace, fidelity: int | float) -> SearchSpace: - assert config.fidelity is not None - config.fidelity.set_value(fidelity) - return config - - -class SamplingPolicy(ABC): - """Base class for implementing a sampling strategy for SH and its subclasses.""" - - def __init__(self, pipeline_space: SearchSpace, patience: int = 100): - self.pipeline_space = pipeline_space - self.patience = patience - - @abstractmethod - def sample(self, *args: Any, **kwargs: Any) -> SearchSpace: ... - - -class RandomUniformPolicy(SamplingPolicy): - """A random policy for sampling configuration, i.e. the default for SH / hyperband. - - Args: - SamplingPolicy ([type]): [description] - """ - - def __init__(self, pipeline_space: SearchSpace): - super().__init__(pipeline_space=pipeline_space) - - def sample(self, *args: Any, **kwargs: Any) -> SearchSpace: - return sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=False, - ignore_fidelity=True, - ) - - -class FixedPriorPolicy(SamplingPolicy): - """A random policy for sampling configuration, i.e. the default for SH but samples - a fixed fraction from the prior. - """ - - def __init__(self, pipeline_space: SearchSpace, fraction_from_prior: float = 1): - super().__init__(pipeline_space=pipeline_space) - assert 0 <= fraction_from_prior <= 1 - self.fraction_from_prior = fraction_from_prior - - def sample(self, *args: Any, **kwargs: Any) -> SearchSpace: - """Samples from the prior with a certain probabiliyu. - - Returns: - SearchSpace: [description] - """ - user_priors = False - if np.random.uniform() < self.fraction_from_prior: - user_priors = True - - return sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=user_priors, - ignore_fidelity=True, - ) - - -class EnsemblePolicy(SamplingPolicy): - """Ensemble of sampling policies including sampling randomly, from prior & incumbent. - - Args: - SamplingPolicy ([type]): [description] - """ - - def __init__( - self, - pipeline_space: SearchSpace, - inc_type: Literal[ - "hypersphere", "gaussian", "crossover", "mutation" - ] = "mutation", - ): - """Samples a policy as per its weights and performs the selected sampling. - - Args: - pipeline_space: Space in which to search - inc_type: str - if "hypersphere", uniformly samples from around the incumbent within its - distance from the nearest neighbour in history - if "gaussian", samples from a gaussian around the incumbent - if "crossover", generates a config by crossover between a random sample - and the incumbent - if "mutation", generates a config by perturbing each hyperparameter with - 50% (mutation_rate=0.5) probability of selecting each hyperparmeter - for perturbation, sampling a deviation N(value, mutation_std=0.5)) - """ - super().__init__(pipeline_space=pipeline_space) - self.inc_type = inc_type - # setting all probabilities uniformly - self.policy_map = {"random": 0.33, "prior": 0.34, "inc": 0.33} - - def sample_neighbour( - self, - incumbent: SearchSpace, - distance: float, - tolerance: float = TOLERANCE, - ) -> SearchSpace: - """Samples a config from around the `incumbent` within radius as `distance`.""" - # TODO: how does tolerance affect optimization on landscapes of different scale - sample_counter = 0 - from neps.optimizers.multi_fidelity_prior.utils import ( - compute_config_dist, - ) - - while True: - config = sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=False, - ignore_fidelity=False, - ) - # computing distance from incumbent - d = compute_config_dist(config, incumbent) - # checking if sample is within the hypersphere around the incumbent - if d < max(distance, tolerance): - # accept sample - break - sample_counter += 1 - if sample_counter > SAMPLE_THRESHOLD: - # reset counter for next increased radius for hypersphere - sample_counter = 0 - # if no sample falls within the radius, increase the threshold radius 1% - distance += distance * DELTA_THRESHOLD - # end of while - return config - - def sample( # noqa: PLR0912, C901, PLR0915 - self, - inc: SearchSpace | None = None, - weights: dict[str, float] | None = None, - *args: Any, - **kwargs: Any, - ) -> SearchSpace: - """Samples from the prior with a certain probability. - - Returns: - SearchSpace: [description] - """ - from neps.optimizers.multi_fidelity_prior.utils import ( - custom_crossover, - local_mutation, - ) - - if weights is not None: - for key, value in sorted(weights.items()): - self.policy_map[key] = value - else: - logger.info(f"Using default policy weights: {self.policy_map}") - prob_weights = [v for _, v in sorted(self.policy_map.items())] - policy_idx = np.random.choice(range(len(prob_weights)), p=prob_weights) - policy = sorted(self.policy_map.keys())[policy_idx] - - logger.info(f"Sampling from {policy} with weights (i, p, r)={prob_weights}") - - if policy == "prior": - config = sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=True, - ignore_fidelity=True, - ) - elif policy == "inc": - if ( - hasattr(self.pipeline_space, "has_prior") - and self.pipeline_space.has_prior - ): - user_priors = True - else: - user_priors = False - - if inc is None: - inc = self.pipeline_space.from_dict(self.pipeline_space.prior_config) - logger.warning( - "No incumbent config found, using default as the incumbent." - ) - - if self.inc_type == "hypersphere": - distance = kwargs["distance"] - config = self.sample_neighbour(inc, distance) - elif self.inc_type == "gaussian": - # TODO: These could be lifted higher, ideall we pass - # down the encoder we want, where we want it. Also passing - # around a `Prior` should be the evidence that we want to use - # a prior, not whether the searchspace has a flag active or not. - encoder = ConfigEncoder.from_space(inc) - sampler = ( - Prior.from_space(inc) - if user_priors - else Sampler.uniform(ndim=encoder.ncols) - ) - - config_tensor = sampler.sample(1, to=encoder.domains) - config_dict = encoder.decode(config_tensor)[0] - _fids = {fid_name: fid.value for fid_name, fid in inc.fidelities.items()} - - config = inc.from_dict({**config_dict, **_fids}) - - elif self.inc_type == "crossover": - # choosing the configuration for crossover with incumbent - # the weight distributed across prior adnd inc - _w_priors = 1 - self.policy_map["random"] - # re-calculate normalized score ratio for prior-inc - w_prior = np.clip(self.policy_map["prior"] / _w_priors, a_min=0, a_max=1) - w_inc = np.clip(self.policy_map["inc"] / _w_priors, a_min=0, a_max=1) - # calculating difference of prior and inc score - score_diff = np.abs(w_prior - w_inc) - # using the difference in score as the weight of what to sample when - # if the score difference is small, crossover between incumbent and prior - # if the score difference is large, crossover between incumbent and random - probs = [1 - score_diff, score_diff] # the order is [prior, random] - if ( - hasattr(self.pipeline_space, "has_prior") - and not self.pipeline_space.has_prior - ): - user_priors = False - else: - user_priors = np.random.choice([True, False], p=probs) - logger.info( - f"Crossing over with user_priors={user_priors} with p={probs}" - ) - # sampling a configuration either randomly or from a prior - _config = sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=user_priors, - ignore_fidelity=True, - ) - # injecting hyperparameters from the sampled config into the incumbent - # TODO: ideally lower crossover prob overtime - config = custom_crossover(inc, _config, crossover_prob=0.5) - elif self.inc_type == "mutation": - if "inc_mutation_rate" in kwargs: - config = local_mutation( - inc, - mutation_rate=kwargs["inc_mutation_rate"], - std=kwargs["inc_mutation_std"], - ) - else: - config = local_mutation(inc) - else: - raise ValueError( - f"{self.inc_type} is not in " - f"{{'mutation', 'crossover', 'hypersphere', 'gaussian'}}" - ) - else: - config = sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=False, - ignore_fidelity=True, - ) - return config - - -class ModelPolicy(SamplingPolicy): - """A policy for sampling configuration, i.e. the default for SH / hyperband. - - Args: - SamplingPolicy ([type]): [description] - """ - - def __init__( - self, - *, - pipeline_space: SearchSpace, - prior: Prior | None = None, - use_cost: bool = False, - device: torch.device | None = None, - ): - if use_cost: - raise NotImplementedError("Cost is not implemented yet.") - - super().__init__(pipeline_space=pipeline_space) - self.device = device - self.prior = prior - self._encoder = ConfigEncoder.from_space( - pipeline_space, - include_constants_when_decoding=True, - ) - self._model: SingleTaskGP | None = None - self._acq: AcquisitionFunction | None = None - - def update_model( - self, - train_x: list[SearchSpace], - train_y: list[float], - pending_x: list[SearchSpace], - decay_t: float | None = None, - ) -> None: - x_train = self._encoder.encode([config._values for config in train_x]) - x_pending = self._encoder.encode([config._values for config in pending_x]) - y_train = torch.tensor(train_y, dtype=torch.float64, device=self.device) - - # TODO: Most of this just copies BO and the duplication can be replaced - # once we don't have the two stage `update_model()` and `sample()` - y_model = make_default_single_obj_gp(x_train, y_train, encoder=self._encoder) - - fit_gpytorch_mll( - ExactMarginalLogLikelihood(likelihood=y_model.likelihood, model=y_model), - ) - acq = qLogNoisyExpectedImprovement( - y_model, - X_baseline=x_train, - X_pending=x_pending, - # Unfortunatly, there's no option to indicate that we minimize - # the AcqFunction so we need to do some kind of transformation. - # https://github.com/pytorch/botorch/issues/2316#issuecomment-2085964607 - objective=LinearMCObjective(weights=torch.tensor([-1.0])), - ) - - # If we have a prior, wrap the above acquisitionm with a prior weighting - if self.prior is not None: - assert decay_t is not None - # TODO: Ideally we have something based on budget and dimensions, not an - # arbitrary term. This 10 is extracted from the old DecayingWeightedPrior - pibo_exp_term = 10 / decay_t - significant_lower_bound = 1e-4 # No significant impact beyond this point - if pibo_exp_term < significant_lower_bound: - acq = pibo_acquisition( - acq, - prior=self.prior, - prior_exponent=pibo_exp_term, - x_domain=self._encoder.domains, - ) - - self._y_model = y_model - self._acq = acq - - # TODO: rework with MFBO - def sample( - self, - active_max_fidelity: int | None = None, - fidelity: int | None = None, - **kwargs: Any, - ) -> SearchSpace: - """Performs the equivalent of optimizing the acquisition function. - - Performs 2 strategies as per the arguments passed: - * If fidelity is not None, triggers the case when the surrogate has been - trained jointly with the fidelity dimension, i.e., all observations ever - recorded. In this case, the EI for random samples is evaluated at the - `fidelity` where the new sample will be evaluated. The top-10 are selected, - and the EI for them is evaluated at the target/mmax fidelity. - * If active_max_fidelity is not None, triggers the case when a surrogate is - trained per fidelity. In this case, all samples have their fidelity - variable set to the same value. This value is same as that of the fidelity - value of the configs in the training data. - """ - # sampling random configurations - samples = [ - sample_one_old(self.pipeline_space, user_priors=False, ignore_fidelity=True) - for _ in range(SAMPLE_THRESHOLD) - ] - - if fidelity is not None: - # w/o setting this flag, the AF eval will set all fidelities to max - self.acquisition.optimize_on_max_fidelity = False - _inc_copy = self.acquisition.incumbent - # TODO: better design required, for example, not import torch - # right now this case handles the 2-step acquisition in `sample` - if "incumbent" in kwargs: - # sets the incumbent to the best score at the required fidelity for - # correct computation of EI scores - self.acquisition.incumbent = torch.tensor(kwargs["incumbent"]) - # updating the fidelity of the sampled configurations - samples = list(map(update_fidelity, samples, [fidelity] * len(samples))) - # computing EI at the given `fidelity` - eis = self.acquisition.eval(x=samples, asscalar=True) - # extracting the 10 highest scores - _ids = np.argsort(eis)[-TOP_EI_SAMPLE_COUNT:] - samples = pd.Series(samples).iloc[_ids].values.tolist() - # setting the fidelity to the maximum fidelity - self.acquisition.optimize_on_max_fidelity = True - self.acquisition.incumbent = _inc_copy - - if active_max_fidelity is not None: - # w/o setting this flag, the AF eval will set all fidelities to max - self.acquisition.optimize_on_max_fidelity = False - fidelity = active_max_fidelity - samples = list(map(update_fidelity, samples, [fidelity] * len(samples))) - - # computes the EI for all `samples` - eis = self.acquisition.eval(x=samples, asscalar=True) - # extracting the highest scored sample - return samples[np.argmax(eis)] diff --git a/neps/optimizers/multi_fidelity/successive_halving.py b/neps/optimizers/multi_fidelity/successive_halving.py deleted file mode 100644 index 08d8c8d1f..000000000 --- a/neps/optimizers/multi_fidelity/successive_halving.py +++ /dev/null @@ -1,677 +0,0 @@ -from __future__ import annotations - -import logging -import random -from collections.abc import Mapping -from copy import deepcopy -from typing import TYPE_CHECKING, Any, Literal -from typing_extensions import override - -import numpy as np -import pandas as pd - -from neps.optimizers.base_optimizer import BaseOptimizer, SampledConfig -from neps.optimizers.multi_fidelity.promotion_policy import ( - AsyncPromotionPolicy, - SyncPromotionPolicy, -) -from neps.optimizers.multi_fidelity.sampling_policy import ( - FixedPriorPolicy, - RandomUniformPolicy, -) -from neps.search_spaces import ( - Categorical, - Constant, - Float, - Integer, - SearchSpace, -) -from neps.search_spaces.functions import sample_one_old - -if TYPE_CHECKING: - from neps.state.optimizer import BudgetInfo - from neps.state.trial import Trial - from neps.utils.types import ConfigResult, RawConfig - -logger = logging.getLogger(__name__) - -CUSTOM_FLOAT_CONFIDENCE_SCORES = dict(Float.DEFAULT_CONFIDENCE_SCORES) -CUSTOM_FLOAT_CONFIDENCE_SCORES.update({"ultra": 0.05}) - -CUSTOM_CATEGORICAL_CONFIDENCE_SCORES = dict(Categorical.PRIOR_CONFIDENCE_SCORES) -CUSTOM_CATEGORICAL_CONFIDENCE_SCORES.update({"ultra": 8}) - - -class SuccessiveHalvingBase(BaseOptimizer): - """Implements a SuccessiveHalving procedure with a sampling and promotion policy.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int | None = None, - eta: int = 3, - early_stopping_rate: int = 0, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - use_priors: bool = False, - sampling_policy: Any = RandomUniformPolicy, - promotion_policy: Any = SyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] | None = None, - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - """Initialise an SH bracket. - - Args: - pipeline_space: Space in which to search - max_cost_total: Maximum budget - eta: The reduction factor used by SH - early_stopping_rate: Determines the number of rungs in an SH bracket - Choosing 0 creates maximal rungs given the fidelity bounds - initial_design_type: Type of initial design to switch to BO - Legacy parameter from NePS BO design. Could be used to extend to MF-BO. - use_priors: Allows random samples to be generated from a default - Samples generated from a Gaussian centered around the default value - sampling_policy: The type of sampling procedure to use - promotion_policy: The type of promotion procedure to use - objective_to_minimize_value_on_error: Setting this and cost_value_on_error to - any float will supress any error during bayesian optimization and will - use given objective_to_minimize value instead. default: None - cost_value_on_error: Setting this and objective_to_minimize_value_on_error to - any float will supress any error during bayesian optimization and will - use given cost value instead. default: None - prior_confidence: The range of confidence to have on the prior - The higher the confidence, the smaller is the standard deviation of the - prior distribution centered around the default - random_interleave_prob: Chooses the fraction of samples from random vs prior - sample_prior_first: Whether to sample the prior configuration first - sample_prior_at_target: Whether to evaluate the prior configuration at - the target fidelity or max budget - """ - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - ) - if random_interleave_prob < 0 or random_interleave_prob > 1: - raise ValueError("random_interleave_prob should be in [0.0, 1.0]") - self.random_interleave_prob = random_interleave_prob - self.sample_prior_first = sample_prior_first - self.sample_prior_at_target = sample_prior_at_target - - assert self.pipeline_space.fidelity is not None, "Fidelity parameter not set." - self.min_budget = self.pipeline_space.fidelity.lower - self.max_budget = self.pipeline_space.fidelity.upper - self.eta = eta - # SH implicitly sets early_stopping_rate to 0 - # the parameter is exposed to allow HB to call SH with different stopping rates - self.early_stopping_rate = early_stopping_rate - self.sampling_policy = sampling_policy(pipeline_space=self.pipeline_space) - self.promotion_policy = promotion_policy(self.eta) - - # `max_budget_init` checks for the number of configurations that have been - # evaluated at the target budget - self.initial_design_type = initial_design_type - self.use_priors = use_priors - - # check to ensure no rung ID is negative - # equivalent to s_max in https://arxiv.org/pdf/1603.06560.pdf - self.stopping_rate_limit = np.floor( - np.log(self.max_budget / self.min_budget) / np.log(self.eta) - ).astype(int) - assert self.early_stopping_rate <= self.stopping_rate_limit - - # maps rungs to a fidelity value for an SH bracket with `early_stopping_rate` - self.rung_map = self._get_rung_map(self.early_stopping_rate) - self.config_map = self._get_config_map(self.early_stopping_rate) - - self.min_rung = min(list(self.rung_map.keys())) - self.max_rung = max(list(self.rung_map.keys())) - - # placeholder args for varying promotion and sampling policies - self.promotion_policy_kwargs: dict = {} - self.promotion_policy_kwargs.update({"config_map": self.config_map}) - self.sampling_args: dict = {} - - self.fidelities = list(self.rung_map.values()) - # stores the observations made and the corresponding fidelity explored - # crucial data structure used for determining promotion candidates - self.observed_configs = pd.DataFrame([], columns=("config", "rung", "perf")) - # stores which configs occupy each rung at any time - self.rung_members: dict = {} # stores config IDs per rung - self.rung_members_performance: dict = {} # performances recorded per rung - self.rung_promotions: dict = {} # records a promotable config per rung - - # setup SH state counter - self.full_rung_trace = SuccessiveHalving._get_rung_trace( - self.rung_map, self.config_map - ) - - ############################# - # Setting prior confidences # - ############################# - # the std. dev or peakiness of distribution - self.prior_confidence = prior_confidence - self._enhance_priors() - self.rung_histories: dict[ - int, dict[Literal["config", "perf"], list[int | float]] - ] = {} - - @classmethod - def _get_rung_trace(cls, rung_map: dict, config_map: dict) -> list[int]: - """Lists the rung IDs in sequence of the flattened SH tree.""" - rung_trace = [] - for rung in sorted(rung_map.keys()): - rung_trace.extend([rung] * config_map[rung]) - return rung_trace - - def _get_rung_map(self, s: int = 0) -> dict: - """Maps rungs (0,1,...,k) to a fidelity value based on fidelity bounds, eta, s.""" - assert s <= self.stopping_rate_limit - new_min_budget = self.min_budget * (self.eta**s) - nrungs = ( - np.floor(np.log(self.max_budget / new_min_budget) / np.log(self.eta)).astype( - int - ) - + 1 - ) - _max_budget = self.max_budget - rung_map = {} - for i in reversed(range(nrungs)): - rung_map[i + s] = ( - int(_max_budget) - if isinstance(self.pipeline_space.fidelity, Integer) - else _max_budget - ) - _max_budget /= self.eta - return rung_map - - def _get_config_map(self, s: int = 0) -> dict: - """Maps rungs (0,1,...,k) to the number of configs for each fidelity.""" - assert s <= self.stopping_rate_limit - new_min_budget = self.min_budget * (self.eta**s) - nrungs = ( - np.floor(np.log(self.max_budget / new_min_budget) / np.log(self.eta)).astype( - int - ) - + 1 - ) - s_max = self.stopping_rate_limit + 1 - _s = self.stopping_rate_limit - s - # L2 from Alg 1 in https://arxiv.org/pdf/1603.06560.pdf - _n_config = np.floor(s_max / (_s + 1)) * self.eta**_s - config_map = {} - for i in range(nrungs): - config_map[i + s] = int(_n_config) - _n_config //= self.eta - return config_map - - @classmethod - def _get_config_id_split(cls, config_id: str) -> tuple[str, str]: - # assumes config IDs of the format `[unique config int ID]_[int rung ID]` - _config, _rung = config_id.split("_") - return _config, _rung - - def _load_previous_observations( - self, - previous_results: dict[str, ConfigResult], - ) -> None: - for config_id, config_val in previous_results.items(): - _config, _rung = self._get_config_id_split(config_id) - perf = self.get_objective_to_minimize(config_val.result) - if int(_config) in self.observed_configs.index: - # config already recorded in dataframe - rung_recorded = self.observed_configs.at[int(_config), "rung"] - if rung_recorded < int(_rung): - # config recorded for a lower rung but higher rung eval available - self.observed_configs.at[int(_config), "config"] = config_val.config - self.observed_configs.at[int(_config), "rung"] = int(_rung) - self.observed_configs.at[int(_config), "perf"] = perf - else: - _df = pd.DataFrame( - [[config_val.config, int(_rung), perf]], - columns=self.observed_configs.columns, - index=pd.Series(int(_config)), # key for config_id - ) - if self.observed_configs.empty: - self.observed_configs = _df - else: - self.observed_configs = pd.concat( - (self.observed_configs, _df) - ).sort_index() - # for efficiency, redefining the function to have the - # `rung_histories` assignment inside the for loop - # rung histories are collected only for `previous` and not `pending` configs - self.rung_histories[int(_rung)]["config"].append(int(_config)) - self.rung_histories[int(_rung)]["perf"].append(perf) - - def _handle_pending_evaluations( - self, pending_evaluations: dict[str, SearchSpace] - ) -> None: - # iterates over all pending evaluations and updates the list of observed - # configs with the rung and performance as None - for config_id, config in pending_evaluations.items(): - _config, _rung = self._get_config_id_split(config_id) - if int(_config) not in self.observed_configs.index: - _df = pd.DataFrame( - [[config, int(_rung), np.nan]], - columns=self.observed_configs.columns, - index=pd.Series(int(_config)), # key for config_id - ) - self.observed_configs = pd.concat( - (self.observed_configs, _df) - ).sort_index() - else: - self.observed_configs.at[int(_config), "rung"] = int(_rung) - self.observed_configs.at[int(_config), "perf"] = np.nan - - def clean_rung_information(self) -> None: - self.rung_members = {k: [] for k in self.rung_map} - self.rung_members_performance = {k: [] for k in self.rung_map} - self.rung_promotions = {k: [] for k in self.rung_map} - - def _get_rungs_state(self, observed_configs: pd.DataFrame | None = None) -> None: - """Collects info on configs at a rung and their performance there.""" - # to account for incomplete evaluations from being promoted --> working on a copy - observed_configs = ( - self.observed_configs.copy().dropna(inplace=False) - if observed_configs is None - else observed_configs - ) - # remove the default from being part of a Successive-Halving bracket - if ( - self.sample_prior_first - and self.sample_prior_at_target - and 0 in observed_configs.index.values - ): - observed_configs = observed_configs.drop(index=0) - # iterates over the list of explored configs and buckets them to respective - # rungs depending on the highest fidelity it was evaluated at - self.clean_rung_information() - for _rung in observed_configs.rung.unique(): - idxs = observed_configs.rung == _rung - self.rung_members[_rung] = observed_configs.index[idxs].values - self.rung_members_performance[_rung] = observed_configs.perf[idxs].values - - def _handle_promotions(self) -> None: - self.promotion_policy.set_state( - max_rung=self.max_rung, - members=self.rung_members, - performances=self.rung_members_performance, - **self.promotion_policy_kwargs, - ) - self.rung_promotions = self.promotion_policy.retrieve_promotions() - - def clear_old_brackets(self) -> None: - return - - def _fit_models(self) -> None: - # define any model or surrogate training and acquisition function state setting - # if adding model-based search to the basic multi-fidelity algorithm - return - - @override - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None, - n: int | None = None, - ) -> SampledConfig | list[SampledConfig]: - """This is basically the fit method.""" - assert n is None, "TODO" - completed: dict[str, ConfigResult] = { - trial_id: trial.into_config_result(self.pipeline_space.from_dict) - for trial_id, trial in trials.items() - if trial.report is not None - } - pending: dict[str, SearchSpace] = { - trial_id: self.pipeline_space.from_dict(trial.config) - for trial_id, trial in trials.items() - if trial.report is None - } - - self.rung_histories = { - rung: {"config": [], "perf": []} - for rung in range(self.min_rung, self.max_rung + 1) - } - - self.observed_configs = pd.DataFrame([], columns=("config", "rung", "perf")) - - # previous optimization run exists and needs to be loaded - self._load_previous_observations(completed) - - # account for pending evaluations - self._handle_pending_evaluations(pending) - - # process optimization state and bucket observations per rung - self._get_rungs_state() - - # filter/reset old SH brackets - self.clear_old_brackets() - - # identifying promotion list per rung - self._handle_promotions() - - # fit any model/surrogates - self._fit_models() - - config, _id, previous_id = self.get_config_and_ids() - return SampledConfig(id=_id, config=config, previous_config_id=previous_id) - - def is_init_phase(self) -> bool: - return True - - def sample_new_config( - self, - rung: int | None = None, - **kwargs: Any, - ) -> SearchSpace: - # Samples configuration from policy or random - if self.sampling_policy is None: - return sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=self.use_priors, - ignore_fidelity=True, - ) - - return self.sampling_policy.sample(**self.sampling_args) - - def _generate_new_config_id(self) -> int: - if len(self.observed_configs) == 0: - return 0 - - _max = self.observed_configs.index.max() - return int(_max) + 1 # type: ignore - - def is_promotable(self) -> int | None: - """Returns an int if a rung can be promoted, else a None.""" - rung_to_promote = None - # # iterates starting from the highest fidelity promotable to the lowest fidelity - for rung in reversed(range(self.min_rung, self.max_rung)): - if len(self.rung_promotions[rung]) > 0: - rung_to_promote = rung - # stop checking when a promotable config found - # no need to search at lower fidelities - break - return rung_to_promote - - def get_config_and_ids(self) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - fidelity_name = self.pipeline_space.fidelity_name - assert fidelity_name is not None - - rung_to_promote = self.is_promotable() - if rung_to_promote is not None: - # promotes the first recorded promotable config in the argsort-ed rung - row = self.observed_configs.iloc[self.rung_promotions[rung_to_promote][0]] - config = row["config"].clone() - rung = rung_to_promote + 1 - # assigning the fidelity to evaluate the config at - - config_values = config._values - config_values[fidelity_name] = self.rung_map[rung] - - # updating config IDs - previous_config_id = f"{row.name}_{rung_to_promote}" - config_id = f"{row.name}_{rung}" - else: - rung_id = self.min_rung - # using random instead of np.random to be consistent with NePS BO - rng = random.Random(None) # TODO: Seeding - if ( - self.use_priors - and self.sample_prior_first - and len(self.observed_configs) == 0 - ): - if self.sample_prior_at_target: - # sets the default config to be evaluated at the target fidelity - rung_id = self.max_rung - logger.info("Next config will be evaluated at target fidelity.") - logger.info("Sampling the default configuration...") - config = self.pipeline_space.from_dict(self.pipeline_space.prior_config) - elif rng.random() < self.random_interleave_prob: - config = sample_one_old( - self.pipeline_space, - patience=self.patience, - user_priors=False, # sample uniformly random - ignore_fidelity=True, - ) - else: - config = self.sample_new_config(rung=rung_id) - - fidelity_value = self.rung_map[rung_id] - config_values = config._values - config_values[fidelity_name] = fidelity_value - - previous_config_id = None - config_id = f"{self._generate_new_config_id()}_{rung_id}" - - return config_values, config_id, previous_config_id - - def _enhance_priors(self, confidence_score: dict[str, float] | None = None) -> None: - """Only applicable when priors are given along with a confidence. - - Args: - confidence_score: dict - The confidence scores for the types. - Example: {"categorical": 5.2, "numeric": 0.15} - """ - if not self.use_priors or self.prior_confidence is None: - return - - for k, v in self.pipeline_space.items(): - if v.is_fidelity or isinstance(v, Constant): - continue - if isinstance(v, Float | Integer): - if confidence_score is None: - confidence = CUSTOM_FLOAT_CONFIDENCE_SCORES[self.prior_confidence] - else: - confidence = confidence_score["numeric"] - self.pipeline_space[k].prior_confidence_score = confidence - elif isinstance(v, Categorical): - if confidence_score is None: - confidence = CUSTOM_CATEGORICAL_CONFIDENCE_SCORES[ - self.prior_confidence - ] - else: - confidence = confidence_score["categorical"] - self.pipeline_space[k].prior_confidence_score = confidence - - -class SuccessiveHalving(SuccessiveHalvingBase): - def _calc_budget_used_in_bracket(self, config_history: list[int]) -> int: - max_cost_total = 0 - for rung in self.config_map: - count = sum(config_history == rung) - # `range(min_rung, rung+1)` counts the black-box cost of promotions since - # SH budgets assume each promotion involves evaluation from scratch - max_cost_total += count * sum(np.arange(self.min_rung, rung + 1)) - return max_cost_total - - def clear_old_brackets(self) -> None: - """Enforces reset at each new bracket. - - The _get_rungs_state() function creates the `rung_promotions` dict mapping which - is used by the promotion policies to determine the next step: promotion/sample. - The key to simulating reset of rungs like in vanilla SH is by subsetting only the - relevant part of the observation history that corresponds to one SH bracket. - Under a parallel run, multiple SH brackets can be spawned. The oldest, active, - incomplete SH bracket is searched for to choose the next evaluation. If either - all brackets are over or waiting, a new SH bracket is spawned. - There are no waiting or blocking calls. - """ - # indexes to mark separate brackets - start = 0 - end = self.config_map[self.min_rung] # length of lowest rung in a bracket - if self.sample_prior_at_target and self.sample_prior_first: - start += 1 - end += 1 - # iterates over the different SH brackets which span start-end by index - while end <= len(self.observed_configs): - # for the SH bracket in start-end, calculate total SH budget used - - # TODO(eddiebergman): Not idea what the type is of the stuff in the deepcopy - # but should work on removing the deepcopy - bracket_budget_used = self._calc_budget_used_in_bracket( - deepcopy(self.observed_configs.rung.values[start:end]) - ) - # if budget used is less than a SH bracket budget then still an active bracket - if bracket_budget_used < sum(self.full_rung_trace): - # subsetting only this SH bracket from the history - self._get_rungs_state(self.observed_configs.iloc[start:end]) - # extra call to use the updated rung member info to find promotions - # SyncPromotion signals a wait if a rung is full but with - # incomplete/pending evaluations, and signals to starts a new SH bracket - self._handle_promotions() - promotion_count = 0 - for _, promotions in self.rung_promotions.items(): - promotion_count += len(promotions) - # if no promotion candidates are returned, then the current bracket - # is active and waiting - if promotion_count: - # returns the oldest active bracket if a promotion found - return - # else move to next SH bracket recorded by an offset (= lowest rung length) - start = end - end = start + self.config_map[self.min_rung] - - # updates rung info with the latest active, incomplete bracket - self._get_rungs_state(self.observed_configs.iloc[start:end]) - # _handle_promotion() need not be called as it is called by load_results() - return - - -class SuccessiveHalvingWithPriors(SuccessiveHalving): - """Implements a SuccessiveHalving procedure with a sampling and promotion policy.""" - - use_priors = True - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - early_stopping_rate: int = 0, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = FixedPriorPolicy, - promotion_policy: Any = SyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", # medium = 0.25 - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - early_stopping_rate=early_stopping_rate, - initial_design_type=initial_design_type, - use_priors=self.use_priors, - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - - -class AsynchronousSuccessiveHalving(SuccessiveHalvingBase): - """Implements ASHA with a sampling and asynchronous promotion policy.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - early_stopping_rate: int = 0, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - use_priors: bool = False, - sampling_policy: Any = RandomUniformPolicy, - promotion_policy: Any = AsyncPromotionPolicy, # key difference from SH - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] | None = None, - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - early_stopping_rate=early_stopping_rate, - initial_design_type=initial_design_type, - use_priors=use_priors, - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - - -class AsynchronousSuccessiveHalvingWithPriors(AsynchronousSuccessiveHalving): - """Implements ASHA with a sampling and asynchronous promotion policy.""" - - use_priors = True - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - early_stopping_rate: int = 0, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = FixedPriorPolicy, - promotion_policy: Any = AsyncPromotionPolicy, # key difference from SH - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = False, - sample_prior_at_target: bool = False, - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - early_stopping_rate=early_stopping_rate, - initial_design_type=initial_design_type, - use_priors=self.use_priors, - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) diff --git a/neps/optimizers/multi_fidelity/utils.py b/neps/optimizers/multi_fidelity/utils.py deleted file mode 100644 index bbc6557f5..000000000 --- a/neps/optimizers/multi_fidelity/utils.py +++ /dev/null @@ -1,251 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from copy import deepcopy -from typing import Any - -import numpy as np -import pandas as pd - - -class MFObservedData: - """(Under development). - - This module is used to unify the data access across different Multi-Fidelity - optimizers. It stores column names and index names. Possible optimizations - and extensions of the observed data should be handled by this class. - - So far this is just a draft class containing the DataFrame and some properties. - """ - - default_config_idx = "config_id" - default_budget_idx = "budget_id" - default_config_col = "config" - default_perf_col = "perf" - default_lc_col = "learning_curves" - # TODO: deepcopy all the mutable outputs from the dataframe - - def __init__( - self, - columns: list[str] | None = None, - index_names: list[str] | None = None, - ): - if columns is None: - columns = [self.default_config_col, self.default_perf_col] - if index_names is None: - index_names = [self.default_config_idx, self.default_budget_idx] - - self.config_col = columns[0] - self.perf_col = columns[1] - - if len(columns) > 2: - self.lc_col_name = columns[2] - else: - self.lc_col_name = self.default_lc_col - - if len(index_names) == 1: - index_names += ["budget_id"] - - self.config_idx = index_names[0] - self.budget_idx = index_names[1] - self.index_names = index_names - - index = pd.MultiIndex.from_tuples([], names=index_names) - - self.df = pd.DataFrame([], columns=columns, index=index) - - @property - def pending_condition(self) -> pd.Series: - return self.df[self.perf_col].isna() - - @property - def error_condition(self) -> pd.Series: - return self.df[self.perf_col] == "error" - - @property - def seen_config_ids(self) -> list: - return self.df.index.levels[0].to_list() - - @property - def seen_budget_levels(self) -> list: - # Considers pending and error budgets as seen - return self.df.index.levels[1].to_list() - - @property - def pending_runs_index(self) -> pd.Index | pd.MultiIndex: - return self.df.loc[self.pending_condition].index - - @property - def completed_runs(self) -> pd.DataFrame: - return self.df[~(self.pending_condition | self.error_condition)] - - @property - def completed_runs_index(self) -> pd.Index | pd.MultiIndex: - return self.completed_runs.index - - def next_config_id(self) -> int: - if len(self.seen_config_ids): - return max(self.seen_config_ids) + 1 - return 0 - - def add_data( - self, - data: list[Any] | list[list[Any]], - index: tuple[int, ...] | Sequence[tuple[int, ...]] | Sequence[int] | int, - *, - error: bool = False, - ) -> None: - """Add data only if none of the indices are already existing in the DataFrame.""" - # TODO: If index is only config_id extend it - if not isinstance(index, list): - index_list = [index] - data_list = [data] - else: - index_list = index - data_list = data - - if not self.df.index.isin(index_list).any(): - index = pd.MultiIndex.from_tuples(index_list, names=self.index_names) - _df = pd.DataFrame(data_list, columns=self.df.columns, index=index) - self.df = _df.copy() if self.df.empty else pd.concat((self.df, _df)) - elif error: - raise ValueError( - f"Data with at least one of the given indices already " - f"exists: {self.df[self.df.index.isin(index_list)]}\n" - f"Given indices: {index_list}" - ) - - def update_data( - self, - data_dict: dict[str, list[Any]], - index: tuple[int, ...] | Sequence[tuple[int, ...]] | Sequence[int] | int, - *, - error: bool = False, - ) -> None: - """Update data if all the indices already exist in the DataFrame.""" - index_list = [index] if not isinstance(index, list) else index - if self.df.index.isin(index_list).sum() == len(index_list): - column_names, data = zip(*data_dict.items(), strict=False) - data = list(zip(*data, strict=False)) - self.df.loc[index_list, list(column_names)] = data - - elif error: - raise ValueError( - f"Data with at least one of the given indices doesn't " - f"exist.\n Existing indices: {self.df.index}\n" - f"Given indices: {index_list}" - ) - - def get_learning_curves(self) -> pd.DataFrame: - return self.df.pivot_table( - index=self.df.index.names[0], - columns=self.df.index.names[1], - values=self.perf_col, - ) - - def all_configs_list(self) -> list[Any]: - return self.df.loc[:, self.config_col].sort_index().values.tolist() - - def get_best_learning_curve_id(self, *, maximize: bool = False) -> int: - """Returns a single configuration id of the best observed performance. - - Note: this will always return the single best lowest ID - if two configurations has the same performance - """ - learning_curves = self.get_learning_curves() - if maximize: - return learning_curves.max(axis=1).idxmax() - return learning_curves.min(axis=1).idxmin() - - def get_best_seen_performance(self, *, maximize: bool = False) -> float: - learning_curves = self.get_learning_curves() - if maximize: - return learning_curves.max(axis=1).max() - return learning_curves.min(axis=1).min() - - def add_budget_column(self) -> pd.DataFrame: - combined_df = self.df.reset_index(level=1) - return combined_df.set_index(keys=[self.budget_idx], drop=False, append=True) - - def reduce_to_max_seen_budgets(self) -> pd.DataFrame: - self.df = self.df.sort_index() - combined_df = self.add_budget_column() - return combined_df.groupby(level=0).last() - - def get_partial_configs_at_max_seen(self) -> pd.Series: - return self.reduce_to_max_seen_budgets()[self.config_col] - - def extract_learning_curve( - self, config_id: int, budget_id: int | None = None - ) -> list[float]: - if budget_id is None: - # budget_id only None when predicting - # extract full observed learning curve for prediction pipeline - budget_id = ( - max(self.df.loc[config_id].index.get_level_values("budget_id").values) + 1 - ) - - # For the first epoch we have no learning curve available - if budget_id == 0: - return [] - # reduce budget_id to discount the current validation objective_to_minimize - # both during training and prediction phase - budget_id = max(0, budget_id - 1) - if self.lc_col_name in self.df.columns: - lc = self.df.loc[(config_id, budget_id), self.lc_col_name] - else: - lcs = self.get_learning_curves() - lc = lcs.loc[config_id, :budget_id].values.flatten().tolist() - return deepcopy(lc) - - def get_best_performance_per_config(self, *, maximize: bool = False) -> pd.Series: - """Returns the best score recorded per config across fidelities seen.""" - op = np.max if maximize else np.min - return ( - self.df.sort_values( - "budget_id", ascending=False - ) # sorts with largest budget first - .groupby("config_id") # retains only config_id - .first() # retrieves the largest budget seen for each config_id - .learning_curves.apply( # extracts all values seen till largest budget - op - ) # finds the minimum over per-config learning curve - ) - - def get_max_observed_fidelity_level_per_config(self) -> pd.Series: - """Returns the highest fidelity level recorded per config seen.""" - max_z_observed = { - _id: self.df.loc[_id, :].index.sort_values()[-1] - for _id in self.df.index.get_level_values("config_id").sort_values() - } - return pd.Series(max_z_observed) - - -if __name__ == "__main__": - # TODO: Either delete these or convert them to tests (karibbov) - """ - Here are a few examples of how to manage data with this class: - """ - data = MFObservedData(["config", "perf"], index_names=["config_id", "budget_id"]) - - # When adding multiple indices data should be list of rows(lists) and the - # index should be list of tuples - data.add_data( - [["conf1", 0.5], ["conf2", 0.7], ["conf1", 0.6], ["conf2", 0.4]], - index=[(0, 0), (1, 1), (0, 3), (1, 0)], - ) - data.add_data( - [["conf1", 0.5], ["conf2", 0.10], ["conf1", 0.11]], - index=[(0, 2), (1, 2), (0, 1)], - ) - - # When updating multiple indices at a time both the values in the data dictionary - # and the indices should be lists - data.update_data({"perf": [1.8, 1.5]}, index=[(1, 1), (0, 0)]) - - data = MFObservedData(["config", "perf"], index_names=["config_id", "budget_id"]) - - # when adding a single row second level list is not necessary - data.add_data(["conf1", 0.5], index=(0, 0)) - - data.update_data({"perf": [1.8], "budget_col": [5]}, index=(0, 0)) diff --git a/neps/optimizers/multi_fidelity_prior/__init__.py b/neps/optimizers/multi_fidelity_prior/__init__.py deleted file mode 100644 index f272be75b..000000000 --- a/neps/optimizers/multi_fidelity_prior/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from neps.optimizers.multi_fidelity_prior.async_priorband import ( - PriorBandAsha, - PriorBandAshaHB, -) -from neps.optimizers.multi_fidelity_prior.priorband import PriorBand - -__all__ = [ - "PriorBand", - "PriorBandAsha", - "PriorBandAshaHB", -] diff --git a/neps/optimizers/multi_fidelity_prior/async_priorband.py b/neps/optimizers/multi_fidelity_prior/async_priorband.py deleted file mode 100644 index c664eeb07..000000000 --- a/neps/optimizers/multi_fidelity_prior/async_priorband.py +++ /dev/null @@ -1,320 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Literal -from typing_extensions import override - -import numpy as np -import pandas as pd - -from neps.optimizers.base_optimizer import SampledConfig -from neps.optimizers.multi_fidelity.mf_bo import MFBOBase -from neps.optimizers.multi_fidelity.promotion_policy import AsyncPromotionPolicy -from neps.optimizers.multi_fidelity.sampling_policy import EnsemblePolicy, ModelPolicy -from neps.optimizers.multi_fidelity.successive_halving import ( - AsynchronousSuccessiveHalvingWithPriors, -) -from neps.optimizers.multi_fidelity_prior.priorband import PriorBandBase -from neps.sampling.priors import Prior - -if TYPE_CHECKING: - from neps.optimizers.bayesian_optimization.acquisition_functions import ( - BaseAcquisition, - ) - from neps.search_spaces.search_space import SearchSpace - from neps.state.optimizer import BudgetInfo - from neps.state.trial import Trial - from neps.utils.types import ConfigResult, RawConfig - - -class PriorBandAsha(MFBOBase, PriorBandBase, AsynchronousSuccessiveHalvingWithPriors): - """Implements a PriorBand on top of ASHA.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - early_stopping_rate: int = 0, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = EnsemblePolicy, # key difference to ASHA - promotion_policy: Any = AsyncPromotionPolicy, # key difference from SH - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = True, - sample_prior_at_target: bool = True, - prior_weight_type: Literal["geometric", "linear", "50-50"] = "geometric", - inc_sample_type: Literal[ - "crossover", "gaussian", "hypersphere", "mutation" - ] = "mutation", - inc_mutation_rate: float = 0.5, - inc_mutation_std: float = 0.25, - inc_style: Literal["dynamic", "constant", "decay"] = "dynamic", - # arguments for model - model_based: bool = False, # crucial argument to set to allow model-search - modelling_type: Literal["joint", "rung"] = "joint", - initial_design_size: int | None = None, - model_policy: Any = ModelPolicy, - # TODO: Remove these when fixing model policy - surrogate_model: str | Any = "gp", - domain_se_kernel: str | None = None, - hp_kernels: list | None = None, - surrogate_model_args: dict | None = None, - acquisition: str | BaseAcquisition = "EI", - log_prior_weighted: bool = False, - acquisition_sampler: str = "random", - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - early_stopping_rate=early_stopping_rate, - initial_design_type=initial_design_type, - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - self.prior_weight_type = prior_weight_type - self.inc_sample_type = inc_sample_type - self.inc_mutation_rate = inc_mutation_rate - self.inc_mutation_std = inc_mutation_std - self.sampling_policy = sampling_policy( - pipeline_space=pipeline_space, inc_type=self.inc_sample_type - ) - # determines the kind of trade-off between incumbent and prior weightage - self.inc_style = inc_style # used by PriorBandBase - self.sampling_args = { - "inc": None, - "weights": { - "prior": 1, # begin with only prior sampling - "inc": 0, - "random": 0, - }, - } - - self.model_based = model_based - self.modelling_type = modelling_type - self.initial_design_size = initial_design_size - # counting non-fidelity dimensions in search space - ndims = sum( - 1 - for _, hp in self.pipeline_space.hyperparameters.items() - if not hp.is_fidelity - ) - n_min = ndims + 1 - self.init_size = n_min + 1 # in BOHB: init_design >= N_dim + 2 - if self.modelling_type == "joint" and self.initial_design_size is not None: - self.init_size = self.initial_design_size - - prior_dist = Prior.from_space(self.pipeline_space) - self.model_policy = model_policy(pipeline_space=pipeline_space, prior=prior_dist) - - def get_config_and_ids( - self, - ) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - rung_to_promote = self.is_promotable() - rung = rung_to_promote + 1 if rung_to_promote is not None else self.min_rung - self._set_sampling_weights_and_inc(rung=rung) - # performs standard ASHA but sampling happens as per the EnsemblePolicy - return super().get_config_and_ids() - - -class PriorBandAshaHB(PriorBandAsha): - """Implements a PriorBand on top of ASHA-HB (Mobster).""" - - early_stopping_rate: int = 0 - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = EnsemblePolicy, # key difference to ASHA - promotion_policy: Any = AsyncPromotionPolicy, # key difference from PB - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = True, - sample_prior_at_target: bool = True, - prior_weight_type: Literal["geometric", "linear", "50-50"] = "geometric", - inc_sample_type: Literal[ - "crossover", "gaussian", "hypersphere", "mutation" - ] = "mutation", - inc_mutation_rate: float = 0.5, - inc_mutation_std: float = 0.25, - inc_style: Literal["dynamic", "constant", "decay"] = "dynamic", - # arguments for model - model_based: bool = False, # crucial argument to set to allow model-search - modelling_type: Literal["joint", "rung"] = "joint", - initial_design_size: int | None = None, - model_policy: Any = ModelPolicy, - # TODO: Remove these when fixing model policy - surrogate_model: str | Any = "gp", - domain_se_kernel: str | None = None, - hp_kernels: list | None = None, - surrogate_model_args: dict | None = None, - acquisition: str | BaseAcquisition = "EI", - log_prior_weighted: bool = False, - acquisition_sampler: str = "random", - ): - # collecting arguments required by ASHA - args: dict[str, Any] = { - "pipeline_space": pipeline_space, - "max_cost_total": max_cost_total, - "eta": eta, - "early_stopping_rate": self.early_stopping_rate, - "initial_design_type": initial_design_type, - "sampling_policy": sampling_policy, - "promotion_policy": promotion_policy, - "objective_to_minimize_value_on_error": objective_to_minimize_value_on_error, - "cost_value_on_error": cost_value_on_error, - "ignore_errors": ignore_errors, - "prior_confidence": prior_confidence, - "random_interleave_prob": random_interleave_prob, - "sample_prior_first": sample_prior_first, - "sample_prior_at_target": sample_prior_at_target, - } - super().__init__( - **args, - prior_weight_type=prior_weight_type, - inc_sample_type=inc_sample_type, - inc_mutation_rate=inc_mutation_rate, - inc_mutation_std=inc_mutation_std, - inc_style=inc_style, - model_based=model_based, - modelling_type=modelling_type, - initial_design_size=initial_design_size, - model_policy=model_policy, - ) - - # Creating the ASHA (SH) brackets that Hyperband iterates over - self.sh_brackets = {} - for s in range(self.max_rung + 1): - args.update({"early_stopping_rate": s}) - # key difference from vanilla HB where it runs synchronous SH brackets - self.sh_brackets[s] = AsynchronousSuccessiveHalvingWithPriors(**args) - self.sh_brackets[s].sampling_policy = self.sampling_policy - self.sh_brackets[s].sampling_args = self.sampling_args - self.sh_brackets[s].model_policy = self.model_policy # type: ignore - self.sh_brackets[s].sample_new_config = self.sample_new_config # type: ignore - - def _update_sh_bracket_state(self) -> None: - # `load_results()` for each of the SH bracket objects are not called as they are - # not part of the main Hyperband loop. For correct promotions and sharing of - # optimization history, the promotion handler of the SH brackets need the - # optimization state. Calling `load_results()` is an option but leads to - # redundant data processing. - for _, bracket in self.sh_brackets.items(): - bracket.promotion_policy.set_state( - max_rung=self.max_rung, - members=self.rung_members, - performances=self.rung_members_performance, - config_map=bracket.config_map, - ) - bracket.rung_promotions = bracket.promotion_policy.retrieve_promotions() - bracket.observed_configs = self.observed_configs.copy() - bracket.rung_histories = self.rung_histories - - @override - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None, - n: int | None = None, - ) -> SampledConfig: - """This is basically the fit method.""" - assert n is None, "TODO" - completed: dict[str, ConfigResult] = { - trial_id: trial.into_config_result(self.pipeline_space.from_dict) - for trial_id, trial in trials.items() - if trial.report is not None - } - pending: dict[str, SearchSpace] = { - trial_id: self.pipeline_space.from_dict(trial.config) - for trial_id, trial in trials.items() - if trial.report is None - } - - self.rung_histories = { - rung: {"config": [], "perf": []} - for rung in range(self.min_rung, self.max_rung + 1) - } - - self.observed_configs = pd.DataFrame([], columns=("config", "rung", "perf")) - - # previous optimization run exists and needs to be loaded - self._load_previous_observations(completed) - - # account for pending evaluations - self._handle_pending_evaluations(pending) - - # process optimization state and bucket observations per rung - self._get_rungs_state() - - # filter/reset old SH brackets - self.clear_old_brackets() - - # identifying promotion list per rung - self._handle_promotions() - - # fit any model/surrogates - self._fit_models() - - # important for the global HB to run the right SH - self._update_sh_bracket_state() - - config, _id, previous_id = self.get_config_and_ids() - return SampledConfig(id=_id, config=config, previous_config_id=previous_id) - - def _get_bracket_to_run(self) -> int: - """Samples the ASHA bracket to run. - - The selected bracket always samples at its minimum rung. Thus, selecting a bracket - effectively selects the rung that a new sample will be evaluated at. - """ - # Sampling distribution derived from Appendix A (https://arxiv.org/abs/2003.10865) - # Adapting the distribution based on the current optimization state - # s \in [0, max_rung] and to with the denominator's constraint, we have K > s - 1 - # and thus K \in [1, ..., max_rung, ...] - # Since in this version, we see the full SH rung, we fix the K to max_rung - K = self.max_rung - bracket_probs = [ - self.eta ** (K - s) * (K + 1) / (K - s + 1) for s in range(self.max_rung + 1) - ] - bracket_probs = np.array(bracket_probs) / sum(bracket_probs) - return int(np.random.choice(range(self.max_rung + 1), p=bracket_probs)) - - def get_config_and_ids(self) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - # the rung to sample at - bracket_to_run = self._get_bracket_to_run() - - self._set_sampling_weights_and_inc(rung=bracket_to_run) - self.sh_brackets[bracket_to_run].sampling_args = self.sampling_args - config, config_id, previous_config_id = self.sh_brackets[ - bracket_to_run - ].get_config_and_ids() - return config, config_id, previous_config_id diff --git a/neps/optimizers/multi_fidelity_prior/priorband.py b/neps/optimizers/multi_fidelity_prior/priorband.py deleted file mode 100644 index bcbd8c9c4..000000000 --- a/neps/optimizers/multi_fidelity_prior/priorband.py +++ /dev/null @@ -1,407 +0,0 @@ -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any, Literal - -import numpy as np - -from neps.optimizers.multi_fidelity.hyperband import HyperbandCustomDefault -from neps.optimizers.multi_fidelity.mf_bo import MFBOBase -from neps.optimizers.multi_fidelity.promotion_policy import SyncPromotionPolicy -from neps.optimizers.multi_fidelity.sampling_policy import EnsemblePolicy, ModelPolicy -from neps.optimizers.multi_fidelity.successive_halving import SuccessiveHalvingBase -from neps.optimizers.multi_fidelity_prior.utils import ( - compute_config_dist, - compute_scores, - get_prior_weight_for_decay, -) -from neps.sampling.priors import Prior -from neps.search_spaces.search_space import SearchSpace - -if TYPE_CHECKING: - import pandas as pd - - from neps.optimizers.bayesian_optimization.acquisition_functions import ( - BaseAcquisition, - ) - from neps.utils.types import RawConfig - -logger = logging.getLogger(__name__) - - -# TODO: We should just make these functions... -class PriorBandBase: - """Class that defines essential properties needed by PriorBand. - - Designed to work with the topmost parent class as SuccessiveHalvingBase. - """ - - # TODO: Dependant variables which should just be made into functions - observed_configs: pd.DataFrame - eta: int - pipeline_space: SearchSpace - inc_sample_type: Literal["hypersphere", "mutation", "crossover", "gaussian"] - inc_mutation_rate: float - inc_mutation_std: float - rung_histories: dict[int, dict[Literal["config", "perf"], list[int | float]]] - max_rung: int - min_rung: int - rung_map: dict - prior_weight_type: Literal["geometric", "linear", "50-50"] - sampling_args: dict[str, Any] - inc_style: Literal["dynamic", "decay", "constant"] - min_budget: int | float - max_budget: int | float - - def find_all_distances_from_incumbent(self, incumbent: SearchSpace) -> list[float]: - """Finds the distance to the nearest neighbour.""" - dist = lambda x: compute_config_dist(incumbent, x) - # computing distance of incumbent from all seen points in history - distances = [dist(config) for config in self.observed_configs.config] - # ensuring the distances exclude 0 or the distance from itself - return [d for d in distances if d > 0] - - def find_1nn_distance_from_incumbent(self, incumbent: SearchSpace) -> float: - """Finds the distance to the nearest neighbour.""" - distances = self.find_all_distances_from_incumbent(incumbent) - return min(distances) - - def find_incumbent(self, rung: int | None = None) -> SearchSpace: - """Find the best performing configuration seen so far.""" - rungs = self.observed_configs.rung.values - idxs = self.observed_configs.index.values - while rung is not None: - # enters this scope is `rung` argument passed and not left empty or None - if rung not in rungs: - logger.warning(f"{rung} not in {np.unique(idxs)}") # type: ignore - # filtering by rung based on argument passed - idxs = self.observed_configs.rung.values == rung - # checking width of current rung - if len(idxs) < self.eta: - logger.warn( - f"Selecting incumbent from a rung with width less than {self.eta}" - ) - # extracting the incumbent configuration - if len(idxs): - # finding the config with the lowest recorded performance - _perfs = self.observed_configs.loc[idxs].perf.values - inc_idx = np.nanargmin([np.nan if t is None else t for t in _perfs]) - inc = self.observed_configs.loc[idxs].iloc[inc_idx].config - assert isinstance(inc, SearchSpace) - else: - # THIS block should not ever execute, but for runtime anomalies, if no - # incumbent can be extracted, the prior is treated as the incumbent - inc = self.pipeline_space.from_dict(self.pipeline_space.prior_config) - logger.warning( - "Treating the prior as the incumbent. " - "Please check if this should not happen." - ) - return inc - - def _set_sampling_weights_and_inc(self, rung: int) -> dict: - sampling_args = self.calc_sampling_args(rung) - if not self.is_activate_inc(): - sampling_args["prior"] += sampling_args["inc"] - sampling_args["inc"] = 0 - inc = None - - self.sampling_args = {"inc": inc, "weights": sampling_args} - else: - inc = self.find_incumbent() - - self.sampling_args = {"inc": inc, "weights": sampling_args} - if self.inc_sample_type == "hypersphere": - min_dist = self.find_1nn_distance_from_incumbent(inc) - self.sampling_args.update({"distance": min_dist}) - elif self.inc_sample_type == "mutation": - self.sampling_args.update( - { - "inc_mutation_rate": self.inc_mutation_rate, - "inc_mutation_std": self.inc_mutation_std, - } - ) - return self.sampling_args - - def is_activate_inc(self) -> bool: - """Function to check optimization state to allow/disallow incumbent sampling. - - This function checks if the total resources used for the finished evaluations - sums to the budget of one full SH bracket. - """ - activate_inc = False - - # calculate total resource cost required for the first SH bracket in HB - sh_brackets = getattr(self, "sh_brackets", None) - if sh_brackets is not None and len(sh_brackets) > 1: - # for HB or AsyncHB which invokes multiple SH brackets - bracket = sh_brackets[self.min_rung] - else: - # for SH or ASHA which do not invoke multiple SH brackets - bracket = self - - assert isinstance(bracket, SuccessiveHalvingBase) - - # calculating the total resources spent in the first SH bracket, taking into - # account the continuations, that is, the resources spent on a promoted config is - # not fidelity[rung] but (fidelity[rung] - fidelity[rung - 1]) - continuation_resources = bracket.rung_map[bracket.min_rung] - resources = bracket.config_map[bracket.min_rung] * continuation_resources - for r in range(1, len(bracket.rung_map)): - rung = sorted(bracket.rung_map.keys(), reverse=False)[r] - continuation_resources = bracket.rung_map[rung] - bracket.rung_map[rung - 1] - resources += bracket.config_map[rung] * continuation_resources - - # find resources spent so far for all finished evaluations - valid_perf_mask = self.observed_configs["perf"].notna() - rungs = self.observed_configs.loc[valid_perf_mask, "rung"] - resources_used = sum(self.rung_map[r] for r in rungs) - - if resources_used >= resources and len( - self.rung_histories[self.max_rung]["config"] - ): - # activate incumbent-based sampling if a total resources is at least - # equivalent to one SH bracket resource usage, and additionally, for the - # asynchronous case with large number of workers, the check enforces that - # at least one configuration has been evaluated at the highest fidelity - activate_inc = True - return activate_inc - - def calc_sampling_args(self, rung: int) -> dict: - """Sets the weights for each of the sampling techniques.""" - if self.prior_weight_type == "geometric": - _w_random = 1 - # scales weight of prior by eta raised to the current rung level - # at the base rung thus w_prior = w_random - # at the max rung r, w_prior = eta^r * w_random - _w_prior = (self.eta**rung) * _w_random - elif self.prior_weight_type == "linear": - _w_random = 1 - w_prior_min_rung = 1 * _w_random - w_prior_max_rung = self.eta * _w_random - num_rungs = len(self.rung_map) - # linearly increasing prior weight such that - # at base rung, w_prior = w_random - # at max rung, w_prior = self.eta * w_random - _w_prior = np.linspace( - start=w_prior_min_rung, - stop=w_prior_max_rung, - endpoint=True, - num=num_rungs, - )[rung] - elif self.prior_weight_type == "50-50": - _w_random = 1 - _w_prior = 1 - else: - raise ValueError(f"{self.prior_weight_type} not in {{'linear', 'geometric'}}") - - # normalizing weights of random and prior sampling - w_prior = _w_prior / (_w_prior + _w_random) - w_random = _w_random / (_w_prior + _w_random) - # calculating ratio of prior and incumbent weights - _w_prior, _w_inc = self.prior_to_incumbent_ratio() - # scaling back such that w_random + w_prior + w_inc = 1 - w_inc = _w_inc * w_prior - w_prior = _w_prior * w_prior - - return { - "prior": w_prior, - "inc": w_inc, - "random": w_random, - } - - def prior_to_incumbent_ratio(self) -> tuple[float, float]: - """Calculates the normalized weight distribution between prior and incumbent. - - Sum of the weights should be 1. - """ - if self.inc_style == "constant": - return self._prior_to_incumbent_ratio_constant() - if self.inc_style == "decay": - valid_perf_mask = self.observed_configs["perf"].notna() - rungs = self.observed_configs.loc[valid_perf_mask, "rung"] - resources = sum(self.rung_map[r] for r in rungs) - return self._prior_to_incumbent_ratio_decay( - resources, self.eta, self.min_budget, self.max_budget - ) - if self.inc_style == "dynamic": - return self._prior_to_incumbent_ratio_dynamic(self.max_rung) - raise ValueError(f"Invalid option {self.inc_style}") - - def _prior_to_incumbent_ratio_decay( - self, resources: float, eta: int, min_budget: int | float, max_budget: int | float - ) -> tuple[float, float]: - """Decays the prior weightage and increases the incumbent weightage.""" - w_prior = get_prior_weight_for_decay(resources, eta, min_budget, max_budget) - w_inc = 1 - w_prior - return w_prior, w_inc - - def _prior_to_incumbent_ratio_constant(self) -> tuple[float, float]: - """Fixes the weightage of incumbent sampling to 1/eta of prior sampling.""" - # fixing weight of incumbent to 1/eta of prior - _w_prior = self.eta - _w_inc = 1 - w_prior = _w_prior / (_w_prior + _w_inc) - w_inc = _w_inc / (_w_prior + _w_inc) - return w_prior, w_inc - - def _prior_to_incumbent_ratio_dynamic(self, rung: int) -> tuple[float, float]: - """Dynamically determines the ratio of weights for prior and incumbent sampling. - - Finds the highest rung with eta configurations recorded. Picks the top-1/eta - configs from this rung. Each config is then ranked by performance and scored by - the Gaussian centered around the prior configuration and the Gaussian centered - around the current incumbent. This scores each of the top-eta configs with the - likelihood of being sampled by the prior or the incumbent. A weighted sum is - performed on these scores based on their ranks. The ratio of the scores is used - as the weights for prior and incumbent sampling. These weighs are calculated - before every sampling operation. - """ - # requires at least eta completed configurations to begin computing scores - if len(self.rung_histories[rung]["config"]) >= self.eta: - # retrieve the prior - prior = self.pipeline_space.from_dict(self.pipeline_space.prior_config) - # retrieve the global incumbent - inc = self.find_incumbent() - # subsetting the top 1/eta configs from the rung - top_n = max(len(self.rung_histories[rung]["perf"]) // self.eta, self.eta) - # ranking by performance - config_idxs = np.argsort(self.rung_histories[rung]["perf"])[:top_n] - # find the top-eta configurations in the rung - top_configs = np.array(self.rung_histories[rung]["config"])[config_idxs] - top_config_scores = np.array( - [ - # `compute_scores` returns a tuple of scores resp. by prior and inc - compute_scores( - self.observed_configs.loc[config_id].config, prior, inc - ) - for config_id in top_configs - ] - ) - # adding positional weights to the score, with the best config weighed most - weights = np.flip(np.arange(1, top_config_scores.shape[0] + 1)).reshape(-1, 1) - # calculating weighted sum of scores - weighted_top_config_scores = np.sum(top_config_scores * weights, axis=0) - prior_score, inc_score = weighted_top_config_scores - # normalizing scores to be weighted ratios - w_prior = prior_score / sum(weighted_top_config_scores) - w_inc = inc_score / sum(weighted_top_config_scores) - elif rung == self.min_rung: - # setting `w_inc = eta * w_prior` as default till score calculation begins - w_prior = self.eta / (1 + self.eta) - w_inc = 1 / (1 + self.eta) - else: - # if rung > min.rung then the lower rung could already have enough - # configurations and thus can be recursively queried till the base rung - return self._prior_to_incumbent_ratio_dynamic(rung - 1) - return w_prior, w_inc - - -# order of inheritance (method resolution order) extremely essential for correct behaviour -class PriorBand(MFBOBase, HyperbandCustomDefault, PriorBandBase): - """PriorBand optimizer for multi-fidelity optimization.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - max_cost_total: int, - eta: int = 3, - initial_design_type: Literal["max_budget", "unique_configs"] = "max_budget", - sampling_policy: Any = EnsemblePolicy, - promotion_policy: Any = SyncPromotionPolicy, - objective_to_minimize_value_on_error: None | float = None, - cost_value_on_error: None | float = None, - ignore_errors: bool = False, - prior_confidence: Literal["low", "medium", "high"] = "medium", - random_interleave_prob: float = 0.0, - sample_prior_first: bool = True, - sample_prior_at_target: bool = True, - prior_weight_type: Literal["geometric", "linear", "50-50"] = "geometric", - inc_sample_type: Literal[ - "hypersphere", "mutation", "crossover", "gaussian" - ] = "mutation", - inc_mutation_rate: float = 0.5, - inc_mutation_std: float = 0.25, - inc_style: Literal["dynamic", "decay", "constant"] = "dynamic", - # arguments for model - model_based: bool = False, # crucial argument to set to allow model-search - modelling_type: Literal["joint", "rung"] = "joint", - initial_design_size: int | None = None, - model_policy: Any = ModelPolicy, - # TODO: Remove these when fixing ModelPolicy - surrogate_model: str | Any = "gp", - surrogate_model_args: dict | None = None, # TODO: Remove - acquisition: str | BaseAcquisition = "EI", # TODO: Remove - log_prior_weighted: bool = False, # TODO: Remove - acquisition_sampler: str = "random", # TODO: Remove - ): - super().__init__( - pipeline_space=pipeline_space, - max_cost_total=max_cost_total, - eta=eta, - initial_design_type=initial_design_type, - sampling_policy=sampling_policy, - promotion_policy=promotion_policy, - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, - cost_value_on_error=cost_value_on_error, - ignore_errors=ignore_errors, - prior_confidence=prior_confidence, - random_interleave_prob=random_interleave_prob, - sample_prior_first=sample_prior_first, - sample_prior_at_target=sample_prior_at_target, - ) - self.prior_weight_type = prior_weight_type - self.inc_sample_type = inc_sample_type - self.inc_mutation_rate = inc_mutation_rate - self.inc_mutation_std = inc_mutation_std - self.sampling_policy = sampling_policy( - pipeline_space=pipeline_space, inc_type=self.inc_sample_type - ) - # determines the kind of trade-off between incumbent and prior weightage - self.inc_style = inc_style # used by PriorBandBase - self.sampling_args: dict[str, Any] = { - "inc": None, - "weights": { - "prior": 1, # begin with only prior sampling - "inc": 0, - "random": 0, - }, - } - - self.model_based = model_based - self.modelling_type = modelling_type - self.initial_design_size = initial_design_size - # counting non-fidelity dimensions in search space - ndims = sum( - 1 - for _, hp in self.pipeline_space.hyperparameters.items() - if not hp.is_fidelity - ) - n_min = ndims + 1 - self.init_size = n_min + 1 # in BOHB: init_design >= N_min + 2 - if self.modelling_type == "joint" and self.initial_design_size is not None: - self.init_size = self.initial_design_size - - # TODO: We also create a prior later inside of `compute_scores()`, - # in which we should really just pass in the prior dist as it does not move - # around in the space. - prior_dist = Prior.from_space(self.pipeline_space) - self.model_policy = model_policy(pipeline_space=pipeline_space, prior=prior_dist) - - for _, sh in self.sh_brackets.items(): - sh.sampling_policy = self.sampling_policy - sh.sampling_args = self.sampling_args - sh.model_policy = self.model_policy # type: ignore - sh.sample_new_config = self.sample_new_config # type: ignore - - def get_config_and_ids(self) -> tuple[RawConfig, str, str | None]: - """...and this is the method that decides which point to query. - - Returns: - [type]: [description] - """ - self._set_sampling_weights_and_inc(rung=self.current_sh_bracket) - - for _, sh in self.sh_brackets.items(): - sh.sampling_args = self.sampling_args - return super().get_config_and_ids() diff --git a/neps/optimizers/multi_fidelity_prior/utils.py b/neps/optimizers/multi_fidelity_prior/utils.py deleted file mode 100644 index c8a8c7c78..000000000 --- a/neps/optimizers/multi_fidelity_prior/utils.py +++ /dev/null @@ -1,179 +0,0 @@ -from __future__ import annotations - -from typing import Any - -import numpy as np -import torch - -from neps.sampling.priors import Prior -from neps.search_spaces import ( - Categorical, - Constant, - GraphParameter, - Float, - Integer, - SearchSpace, -) -from neps.search_spaces.encoding import ConfigEncoder -from neps.search_spaces.functions import sample_one_old, pairwise_dist - - -def update_fidelity(config: SearchSpace, fidelity: int | float) -> SearchSpace: - assert config.fidelity is not None - config.fidelity.set_value(fidelity) - return config - - -# TODO(eddiebergman): This would be much faster -# if done in a vectorized manner... -def local_mutation( - config: SearchSpace, - std: float = 0.25, - mutation_rate: float = 0.5, - patience: int = 50, -) -> SearchSpace: - """Performs a local search by mutating randomly chosen hyperparameters.""" - # Used to check uniqueness later. - # TODO: Seeding - space = config - parameters_to_keep = {} - parameters_to_mutate = {} - - for name, parameter in space.hyperparameters.items(): - if ( - parameter.is_fidelity - or isinstance(parameter, Constant) - or np.random.uniform() > mutation_rate - ): - parameters_to_keep[name] = parameter.value - else: - parameters_to_mutate[name] = parameter - - if len(parameters_to_mutate) == 0: - return space.from_dict(parameters_to_keep) - - new_config: dict[str, Any] = {} - - for hp_name, hp in parameters_to_mutate.items(): - match hp: - case Categorical(): - assert hp._value_index is not None - perm: list[int] = torch.randperm(len(hp.choices)).tolist() - ix = perm[0] if perm[0] != hp._value_index else perm[1] - new_config[hp_name] = hp.choices[ix] - case GraphParameter(): - new_config[hp_name] = hp.mutate(mutation_strategy="bananas") - case Integer() | Float(): - prior = Prior.from_parameters( - {hp_name: hp}, - confidence_values={hp_name: (1 - std)}, - ) - - for _ in range(patience): - sample = prior.sample(1, to=hp.domain).item() - if sample != hp.value: - new_config[hp_name] = hp.value - break - else: - raise ValueError( - f"Exhausted patience trying to mutate parameter '{hp_name}'" - f" with value {hp.value}" - ) - case _: - raise NotImplementedError(f"Unknown hp type for {hp_name}: {type(hp)}") - - return space.from_dict(new_config) - - -def custom_crossover( - config1: SearchSpace, - config2: SearchSpace, - crossover_prob: float = 0.5, - patience: int = 50, -) -> SearchSpace: - """Performs a crossover of config2 into config1. - - Returns a configuration where each HP in config1 has `crossover_prob`% chance of - getting config2's value of the corresponding HP. By default, crossover rate is 50%. - """ - _existing = config1._values - - for _ in range(patience): - child_config = {} - for key, hyperparameter in config1.items(): - if not hyperparameter.is_fidelity and np.random.random() < crossover_prob: - child_config[key] = config2[key].value - else: - child_config[key] = hyperparameter.value - - if _existing != child_config: - return config1.from_dict(child_config) - - # fail safe check to handle edge cases where config1=config2 or - # config1 extremely local to config2 such that crossover fails to - # generate new config in a discrete (sub-)space - return sample_one_old( - config1, - patience=patience, - user_priors=False, - ignore_fidelity=True, - ) - - -def compute_config_dist(config1: SearchSpace, config2: SearchSpace) -> float: - """Computes distance between two configurations. - - Divides the search space into continuous and categorical subspaces. - Normalizes all the continuous values while gives numerical encoding to categories. - Distance returned is the sum of the Euclidean distance of the continous subspace and - the Hamming distance of the categorical subspace. - """ - encoder = ConfigEncoder.from_parameters({**config1.numerical, **config1.categoricals}) - configs = encoder.encode([config1._values, config2._values]) - dist = pairwise_dist(configs, encoder, square_form=False) - return float(dist.item()) - - -def compute_scores( - config: SearchSpace, - prior: SearchSpace, - inc: SearchSpace, - *, - include_fidelity: bool = False, -) -> tuple[float, float]: - """Scores the config by a Gaussian around the prior and the incumbent.""" - # TODO: This could lifted up and just done in the class itself - # in a vectorized form. - encoder = ConfigEncoder.from_space(config, include_fidelity=include_fidelity) - encoded_config = encoder.encode([config._values]) - - prior_dist = Prior.from_space( - prior, - center_values=prior._values, - include_fidelity=include_fidelity, - ) - inc_dist = Prior.from_space( - inc, - center_values=inc._values, - include_fidelity=include_fidelity, - ) - - prior_score = prior_dist.pdf(encoded_config, frm=encoder).item() - inc_score = inc_dist.pdf(encoded_config, frm=encoder).item() - return prior_score, inc_score - - -def get_prior_weight_for_decay( - resources_used: float, eta: int, min_budget: int | float, max_budget: int | float -) -> float: - r"""Creates a step function schedule for the prior weight decay. - - The prior weight ratio is decayed every time the total resources used is - equivalent to the cost of one successive halving bracket within the HB schedule. - This is approximately eta \times max_budget resources for one evaluation. - """ - # decay factor for the prior - decay = 2 - unit_resources = eta * max_budget - idx = resources_used // unit_resources - return 1 / decay**idx diff --git a/neps/optimizers/optimizer.py b/neps/optimizers/optimizer.py new file mode 100644 index 000000000..1eef2d7ee --- /dev/null +++ b/neps/optimizers/optimizer.py @@ -0,0 +1,66 @@ +"""Optimizer interface. + +By implementing the [`AskFunction`][neps.optimizers.optimizer.AskFunction] protocol, +you can inject your own optimizer into the neps runtime. + +```python +class MyOpt: + + def __init__(self, space: SearchSpace, ...): ... + + def __call__( + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: ... + +neps.run(..., optimizer=MyOpt) + +# Or with optimizer hyperparameters +neps.run(..., optimizer=(MyOpt, {"a": 1, "b": 2})) +``` +""" + +from __future__ import annotations + +from abc import abstractmethod +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from neps.state.optimizer import BudgetInfo + from neps.state.trial import Trial + + +@dataclass +class SampledConfig: + id: str + config: Mapping[str, Any] + previous_config_id: str | None = None + + +class AskFunction(Protocol): + """Interface to implement the ask of optimizer.""" + + @abstractmethod + def __call__( + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + """Sample a new configuration. + + Args: + trials: All of the trials that are known about. + budget_info: information about the budget constraints. + n: The number of configurations to sample. If you do not support + sampling multiple configurations at once, you should raise + a `ValueError`. + + Returns: + The sampled configuration(s) + """ + ... diff --git a/neps/optimizers/priorband.py b/neps/optimizers/priorband.py new file mode 100644 index 000000000..06551151e --- /dev/null +++ b/neps/optimizers/priorband.py @@ -0,0 +1,212 @@ +"""Implements functionallity for the priorband sampling strategy.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar + +import numpy as np +import torch + +from neps.optimizers.utils import brackets +from neps.sampling import Prior, Sampler +from neps.space import ConfigEncoder, SearchSpace + +if TYPE_CHECKING: + import pandas as pd + + +@dataclass +class PriorBandArgs: + """Arguments for the PriorBand sampler. + + Args: + mutation_rate: The mutation rate for the PriorBand algorithm when sampling + from the incumbent. + mutation_std: The standard deviation for the mutation rate when sampling + from the incumbent. + """ + + name: ClassVar = "priorband" + + mutation_rate: float + mutation_std: float + + +def mutate_config( + config: dict[str, Any], + space: SearchSpace, + *, + mutation_rate: float = 0.5, + std: float = 0.25, + include_fidelity: bool = False, + seed: torch.Generator | None = None, +) -> dict[str, Any]: + if seed is not None: + raise NotImplementedError("Seed is not implemented yet.") + + parameters = {**space.numerical, **space.categoricals} + + # Assign a confidence of 0 to our current categoricals to ensure they dont get sampled + confidence_values = { + key: 0 if hp.domain.is_categorical else (1 - std) + for key, hp in parameters.items() + } + + # This prior places a guassian on the numericals and places a 0 probability on the + # current value of the categoricals. + mutate_prior = Prior.from_config( + config, + space=space, + confidence_values=confidence_values, + include_fidelity=include_fidelity, + ) + config_encoder = ConfigEncoder.from_space(space, include_fidelity=include_fidelity) + + mutant: dict[str, Any] = mutate_prior.sample_config(to=config_encoder) + mutatant_selection = torch.rand(len(config), generator=seed) < mutation_rate + + return { + key: mutant[key] if select_mutant else config[key] + for key, select_mutant in zip(mutant.keys(), mutatant_selection, strict=False) + } + + +def sample_with_priorband( + *, + table: pd.DataFrame, + rung_to_sample_for: int, + # Search Space + space: SearchSpace, + encoder: ConfigEncoder, + # Inc sampling params + inc_mutation_rate: float, + inc_mutation_std: float, + # SH parameters to calculate the rungs + eta: int, + early_stopping_rate: int = 0, + fid_bounds: tuple[int, int] | tuple[float, float], + # Extra + seed: torch.Generator | None = None, +) -> dict[str, Any]: + """Samples a configuration using the PriorBand algorithm. + + Args: + table: The table of all the trials that have been run. + rung_to_sample_for: The rung to sample for. + space: The search space to sample from. + encoder: The encoder to use for the search space. + inc_mutation_rate: The mutation rate for the incumbent. + inc_mutation_std: The standard deviation for the incumbent mutation rate. + eta: The eta parameter for the Successive Halving algorithm. + early_stopping_rate: The early stopping rate for the Successive Halving algorithm. + fid_bounds: The bounds for the fidelity parameter. + seed: The seed to use for the random number generator. + + Returns: + The sampled configuration. + """ + rung_to_fid, rung_sizes = brackets.calculate_sh_rungs( + bounds=fid_bounds, + eta=eta, + early_stopping_rate=early_stopping_rate, + ) + max_rung = max(rung_sizes) + prior_dist = Prior.from_config(space.prior, space=space) + + # Below we will follow the "geomtric" spacing + w_random = 1 / (1 + eta**rung_to_sample_for) + w_prior = 1 - w_random + + completed: pd.DataFrame = table[table["perf"].notna()] # type: ignore + + # To see if we activate incumbent sampling, we check: + # 1) We have at least one fully complete run + # 2) We have spent at least one full SH bracket worth of fidelity + # 3) There is at least one rung with eta evaluations to get the top 1/eta configs of + completed_rungs = completed.index.get_level_values("rung") + one_complete_run_at_max_rung = (completed_rungs == max_rung).any() + + # For SH bracket cost, we include the fact we can continue runs, + # i.e. resources for rung 2 discounts the cost of evaluating to rung 1, + # only counting the difference in fidelity cost between rung 2 and rung 1. + cost_per_rung = {i: rung_to_fid[i] - rung_to_fid.get(i - 1, 0) for i in rung_to_fid} + + cost_of_one_sh_bracket = sum(rung_sizes[r] * cost_per_rung[r] for r in rung_sizes) + current_cost_used = sum(r * cost_per_rung[r] for r in completed_rungs) + spent_one_sh_bracket_worth_of_fidelity = current_cost_used >= cost_of_one_sh_bracket + + # Check that there is at least rung with `eta` evaluations + rung_counts = completed.groupby("rung").size() + any_rung_with_eta_evals = (rung_counts == eta).any() + + # If the conditions are not met, we sample from the prior or randomly depending on + # the geometrically distributed prior and uniform weights + if ( + one_complete_run_at_max_rung is False + or spent_one_sh_bracket_worth_of_fidelity is False + or any_rung_with_eta_evals is False + ): + policy = np.random.choice(["prior", "random"], p=[w_prior, w_random]) + match policy: + case "prior": + config = prior_dist.sample_config(to=encoder) + case "random": + _sampler = Sampler.uniform(ndim=encoder.ndim) + config = _sampler.sample_config(to=encoder) + + return config + + # Otherwise, we now further split the `prior` weight into `(prior, inc)` + + # 1. Select the top `1//eta` percent of configs at the highest rung that supports it + rungs_with_at_least_eta = rung_counts[rung_counts >= eta].index # type: ignore + rung_table: pd.DataFrame = completed[ # type: ignore + completed.index.get_level_values("rung") == rungs_with_at_least_eta.max() + ] + + K = len(rung_table) // eta + top_k_configs = rung_table.nsmallest(K, columns=["perf"])["config"].tolist() + + # 2. Get the global incumbent, and build a prior distribution around it + inc = completed.loc[completed["perf"].idxmin()]["config"] + inc_dist = Prior.from_config(inc, space=space) + + # 3. Calculate a ratio score of how likely each of the top K configs are under + # the prior and inc distribution, weighing them by their position in the top K + weights = torch.arange(K, 0, -1) + top_k_pdf_inc = inc_dist.pdf_configs(top_k_configs, frm=encoder) + top_k_pdf_prior = prior_dist.pdf_configs(top_k_configs, frm=encoder) + + unnormalized_inc_score = (weights * top_k_pdf_inc).sum() + unnormalized_prior_score = (weights * top_k_pdf_prior).sum() + total_score = unnormalized_inc_score + unnormalized_prior_score + + inc_ratio = float(unnormalized_inc_score / total_score) + prior_ratio = float(unnormalized_prior_score / total_score) + + # 4. And finally, we distribute the original w_prior according to this ratio + w_inc = w_prior * inc_ratio + w_prior = w_prior * prior_ratio + assert np.isclose(w_prior + w_inc + w_random, 1.0) + + # Now we use these weights to choose which sampling distribution to sample from + policy = np.random.choice(["prior", "inc", "random"], p=[w_prior, w_inc, w_random]) + match policy: + case "prior": + return prior_dist.sample_config(to=encoder) + case "random": + _sampler = Sampler.uniform(ndim=encoder.ndim) + return _sampler.sample_config(to=encoder) + case "inc": + assert inc is not None + return mutate_config( + inc, + space=space, + mutation_rate=inc_mutation_rate, + std=inc_mutation_std, + include_fidelity=False, + seed=seed, + ) + + raise RuntimeError(f"Unknown policy: {policy}") diff --git a/neps/optimizers/random_search.py b/neps/optimizers/random_search.py new file mode 100644 index 000000000..1c08e58d8 --- /dev/null +++ b/neps/optimizers/random_search.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from neps.optimizers.optimizer import SampledConfig + +if TYPE_CHECKING: + from neps.sampling import Sampler + from neps.space import ConfigEncoder, SearchSpace + from neps.state import BudgetInfo, Trial + + +@dataclass +class RandomSearch: + """A simple random search optimizer.""" + + pipeline_space: SearchSpace + ignore_fidelity: bool + encoder: ConfigEncoder + sampler: Sampler + + def __call__( + self, + trials: Mapping[str, Trial], + budget_info: BudgetInfo | None, + n: int | None = None, + ) -> SampledConfig | list[SampledConfig]: + n_trials = len(trials) + _n = 1 if n is None else n + configs = self.sampler.sample(_n, to=self.encoder.domains) + config_dicts = self.encoder.decode(configs) + if n == 1: + config = config_dicts[0] + config_id = str(n_trials + 1) + return SampledConfig(config=config, id=config_id, previous_config_id=None) + + return [ + SampledConfig( + config=config, + id=str(n_trials + i + 1), + previous_config_id=None, + ) + for i, config in enumerate(config_dicts) + ] diff --git a/neps/optimizers/random_search/optimizer.py b/neps/optimizers/random_search/optimizer.py deleted file mode 100644 index a5df59ad1..000000000 --- a/neps/optimizers/random_search/optimizer.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Random search optimizer.""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any -from typing_extensions import override - -from neps.optimizers.base_optimizer import BaseOptimizer, SampledConfig -from neps.sampling.priors import UniformPrior -from neps.search_spaces.encoding import ConfigEncoder - -if TYPE_CHECKING: - from neps.search_spaces.search_space import SearchSpace - from neps.state.optimizer import BudgetInfo - from neps.state.trial import Trial - - -class RandomSearch(BaseOptimizer): - """A simple random search optimizer.""" - - def __init__( - self, - *, - pipeline_space: SearchSpace, - use_priors: bool = False, - ignore_fidelity: bool = True, - seed: int | None = None, - **kwargs: Any, # TODO: Remove - ): - """Initialize the random search optimizer. - - Args: - pipeline_space: The search space to sample from. - use_priors: Whether to use priors when sampling. - ignore_fidelity: Whether to ignore fidelity when sampling. - In this case, the max fidelity is always used. - seed: The seed for the random number generator. - """ - super().__init__(pipeline_space=pipeline_space) - self.use_priors = use_priors - self.ignore_fidelity = ignore_fidelity - if seed is not None: - raise NotImplementedError("Seed is not implemented yet for RandomSearch") - - self.seed = seed - self.encoder = ConfigEncoder.from_space( - pipeline_space, - include_fidelity=False, - include_constants_when_decoding=True, - ) - self.sampler = UniformPrior(ndim=self.encoder.ncols) - - @override - def ask( - self, - trials: Mapping[str, Trial], - budget_info: BudgetInfo | None, - n: int | None = None, - ) -> SampledConfig | list[SampledConfig]: - n_trials = len(trials) - _n = 1 if n is None else n - configs = self.sampler.sample(_n, to=self.encoder.domains) - config_dicts = self.encoder.decode(configs) - if n == 1: - config = config_dicts[0] - config_id = str(n_trials + 1) - return SampledConfig(config=config, id=config_id, previous_config_id=None) - - return [ - SampledConfig( - config=config, - id=str(n_trials + i + 1), - previous_config_id=None, - ) - for i, config in enumerate(config_dicts) - ] diff --git a/neps/optimizers/grid_search/__init__.py b/neps/optimizers/utils/__init__.py similarity index 100% rename from neps/optimizers/grid_search/__init__.py rename to neps/optimizers/utils/__init__.py diff --git a/neps/optimizers/utils/brackets.py b/neps/optimizers/utils/brackets.py new file mode 100644 index 000000000..d134f28ac --- /dev/null +++ b/neps/optimizers/utils/brackets.py @@ -0,0 +1,566 @@ +from __future__ import annotations + +import logging +from collections.abc import Hashable, Sequence, Sized +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Literal, TypeAlias + +import numpy as np +from more_itertools import all_unique, pairwise + +if TYPE_CHECKING: + import pandas as pd + from pandas import Index + + +logger = logging.getLogger(__name__) + + +@dataclass +class PromoteAction: + config: dict[str, Any] + id: int + new_rung: int + + +@dataclass +class SampleAction: + rung: int + + +BracketAction: TypeAlias = PromoteAction | SampleAction | Literal["pending", "done"] + + +def calculate_sh_rungs( + bounds: tuple[int, int] | tuple[float, float], + eta: int, + early_stopping_rate: int, +) -> tuple[dict[int, int | float], dict[int, int]]: + bmin, bmax = bounds + budget_type = int if isinstance(bmin, int) else float + esr = early_stopping_rate + stop_rate_limit = int(np.floor(np.log(bmax / bmin) / np.log(eta))) + assert esr <= stop_rate_limit + + nrungs = int(np.floor(np.log(bmax / (bmin * (eta**esr))) / np.log(eta)) + 1) + rung_to_fidelity = { + esr + j: budget_type(bmax / (eta**i)) + for i, j in enumerate(reversed(range(nrungs))) + } + + # L2 from Alg 1 in https://arxiv.org/pdf/1603.06560.pdf + s_max = stop_rate_limit + 1 + _s = stop_rate_limit - esr + _n_config = int(np.floor(s_max / (_s + 1)) * eta**_s) + rung_sizes = {i + esr: _n_config // (eta**i) for i in range(nrungs)} + return rung_to_fidelity, rung_sizes + + +def calculate_hb_bracket_layouts( + bounds: tuple[int, int] | tuple[float, float], + eta: int, +) -> tuple[dict[int, int | float], list[dict[int, int]]]: + bmin, bmax = bounds + budget_type = int if isinstance(bmin, int) else float + stop_rate_limit = int(np.floor(np.log(bmax / bmin) / np.log(eta))) + + nrungs = int(np.floor(np.log(bmax / bmin) / np.log(eta))) + 1 + rung_to_fidelity = { + j: budget_type(bmax / (eta**i)) for i, j in enumerate(reversed(range(nrungs))) + } + + # L2 from Alg 1 in https://arxiv.org/pdf/1603.06560.pdf + bracket_layouts: list[dict[int, int]] = [] + s_max = stop_rate_limit + 1 + for esr in range(nrungs): + _s = stop_rate_limit - esr + _n_config = int(np.floor(s_max / (_s + 1)) * eta**_s) + + sh_rungs = int(np.floor(np.log(bmax / (bmin * (eta**esr))) / np.log(eta)) + 1) + rung_sizes = {i + esr: _n_config // (eta**i) for i in range(sh_rungs)} + bracket_layouts.append(rung_sizes) + + return rung_to_fidelity, bracket_layouts + + +def async_hb_sample_bracket_to_run(max_rung: int, eta: int) -> int: + # Sampling distribution derived from Appendix A (https://arxiv.org/abs/2003.10865) + # Adapting the distribution based on the current optimization state + # s \in [0, max_rung] and to with the denominator's constraint, we have K > s - 1 + # and thus K \in [1, ..., max_rung, ...] + # Since in this version, we see the full SH rung, we fix the K to max_rung + K = max_rung + bracket_probs = [eta ** (K - s) * (K + 1) / (K - s + 1) for s in range(max_rung + 1)] + bracket_probs = np.array(bracket_probs) / sum(bracket_probs) + return int(np.random.choice(range(max_rung + 1), p=bracket_probs)) + + +@dataclass +class Rung(Sized): + """A rung in a bracket""" + + value: int + """The value of a rung, used to determine order between rungs.""" + + table: pd.DataFrame + """The slice of the table that constitutes this rung.""" + + capacity: int | None + """The capacity of the rung, if any.""" + + def __len__(self) -> int: + return len(self.table) + + @property + def config_ids(self) -> list[int]: + return self.table.index.get_level_values("id").unique().tolist() # type: ignore + + def has_pending(self) -> bool: + return bool(self.table["perf"].isna().any()) + + def has_capacity(self) -> bool: + return self.capacity is None or len(self.table) < self.capacity + + def best_to_promote( + self, *, exclude: Sequence[Hashable] + ) -> tuple[int, dict[str, Any], float] | None: + if exclude: + contenders = self.table.drop(exclude) + if contenders.empty: + return None + else: + contenders = self.table + + best_ix, _best_rung = contenders["perf"].idxmin() # type: ignore + row = self.table.loc[(best_ix, _best_rung)] + config = dict(row["config"]) + perf = row["perf"] + return best_ix, config, perf + + def top_k(self, k: int) -> pd.DataFrame: + return self.table.nsmallest(k, "perf") + + +@dataclass +class Sync: + """A bracket that holds a collection of rungs with a capacity constraint.""" + + rungs: list[Rung] + """A list of unique rungs, ordered from lowest to highest. The must have + a capacity set. + """ + + def __post_init__(self) -> None: + if not all_unique(rung.value for rung in self.rungs): + raise ValueError(f"Got rungs with duplicate values\n{self.rungs}") + + if any(rung.value < 0 for rung in self.rungs): + raise ValueError(f"Got rung with negative value\n{self.rungs}") + + if any(rung.capacity is None or rung.capacity < 1 for rung in self.rungs): + raise ValueError( + "All rungs must have a capacity set greater than 1" + f"\nrungs: {len(self.rungs)}" + ) + + _sorted = sorted(self.rungs, key=lambda rung: rung.value) + + if any( + lower.capacity < upper.capacity # type: ignore + for lower, upper in pairwise(_sorted) + ): + raise ValueError(f"Rungs must have a non-increasing capacity, got {_sorted}") + + self.rungs = _sorted + + def next(self) -> BracketAction: + bottom_rung = self.rungs[0] + + # If the bottom rung has capacity, we need to sample for it. + if bottom_rung.has_capacity(): + return SampleAction(bottom_rung.value) + + if not any(rung.has_capacity() for rung in self.rungs): + return "done" + + lower, upper = next((l, u) for l, u in pairwise(self.rungs) if u.has_capacity()) + + if lower.has_pending(): + return "pending" # We need to wait before promoting + + promote_config = lower.best_to_promote(exclude=upper.config_ids) + + # If we have no promotable config, somehow the upper rung has more + # capacity then lower. We check for this in the `__post_init__` + if promote_config is None: + raise RuntimeError( + "This is a bug, either this bracket should have signified to have" + " nothing promotable or pending" + ) + + _id, config, _perf = promote_config + return PromoteAction(config, _id, upper.value) + + @classmethod + def create_repeating( + cls, table: pd.DataFrame, *, rung_sizes: dict[int, int] + ) -> list[Sync]: + """Create a list of brackets from the table. + + The table should have a multi-index of (id, rung) where rung is the + fidelity level of the configuration. + + This method will always ensure there is at least one bracket, with at least one + empty slot. For example, if each bracket houses a maximum of 9 configurations, + and there are 27 total unique configurations in the table, these will be split + into 3 brackets with 9 configurations + 1 additional bracket with 0 in it + configurations. + + ``` + # Unrealistic example showing the format of the table + (id, rung) -> config, perf + -------------------------- + 0 0 | {"hp": 0, ...}, 0.1 + 1 | {"hp": 0, ...}, 0.2 + 1 0 | {"hp": 1, ...}, 0.1 + 2 1 | {"hp": 2, ...}, 0.3 + 2 | {"hp": 2, ...}, 0.4 + 3 2 | {"hp": 3, ...}, 0.4 + ``` + + Args: + table: The table of configurations to split into brackets. + rung_sizes: A mapping of rung to the capacity of that rung. + + Returns: + Brackets which have each subselected the table with the corresponding rung + sizes. + """ + uniq_ids = table.index.get_level_values("id").unique() + + # Split the ids into N brackets of size K. + # K is the number of configurations in the lowest rung, i.e. number of config ids + K = rung_sizes[min(rung_sizes)] + + # Here we don't do `((len(uniq_ids) - 1) // K) + 1` because we want to ensure + # the extra bracket, + # i.e. if K = 9 and for varying len(uniq_ids): + # N = (26 // 9) + 1 = 3 + # N = (27 // 9) + 1 = 4 + # N = (28 // 9) + 1 = 4 + N = max(((len(uniq_ids) - 1) // K) + 1, 1) + + bracket_id_slices: list[Index] = [uniq_ids[i * K : (i + 1) * K] for i in range(N)] + bracket_datas = [table.loc[bracket_ids] for bracket_ids in bracket_id_slices] + + # [bracket] -> {rung: table} + data_for_bracket_by_rung = [ + dict(iter(d.groupby(level="rung", sort=False))) for d in bracket_datas + ] + + # Used if there is nothing for one of the rungs + empty_slice = table.loc[[]] + + return [ + Sync( + rungs=[ + Rung(rung, data_by_rung.get(rung, empty_slice), capacity) + for rung, capacity in rung_sizes.items() + ], + ) + for data_by_rung in data_for_bracket_by_rung + ] + + +@dataclass +class Async: + """A bracket that holds a collection of rungs with no capacity constraints.""" + + rungs: list[Rung] + """A list of rungs, ordered from lowest to highest.""" + + eta: int + """The eta parameter used for deciding when to promote. + + When any of the top_k configs in a rung can be promoted and have not been + promoted yet, they will be. + + Here `k = len(rung) // eta`. + """ + + def __post_init__(self) -> None: + self.rungs = sorted(self.rungs, key=lambda rung: rung.value) + if any(rung.capacity is not None for rung in self.rungs): + raise ValueError( + "AsyncBracket was given a rung that has a capacity, however" + " a rung in an async bracket should not have a capacity set." + f"\nrungs: {self.rungs}" + ) + + def next(self) -> BracketAction: + # Starting from the highest rung going down, check if any configs to promote + for lower, upper in reversed(list(pairwise(self.rungs))): + k = len(lower) // self.eta + if k == 0: + continue # Not enough configs to promote yet + + best_k = lower.top_k(k) + candidates = best_k.drop( + upper.config_ids, + axis="index", + level="id", + errors="ignore", + ) + if candidates.empty: + continue # No configs that aren't already promoted + + promotable = candidates.iloc[0] + _id, _rung = promotable.name + config = dict(promotable["config"]) + return PromoteAction(config, _id, upper.value) + + # We couldn't find any promotions, sample at the lowest rung + return SampleAction(self.rungs[0].value) + + @classmethod + def create( + cls, + table: pd.DataFrame, + *, + rungs: list[int], + eta: int, + ) -> Async: + return cls( + rungs=[ + Rung( + rung, + capacity=None, + table=table.loc[table.index.get_level_values("rung") == rung], + ) + for rung in rungs + ], + eta=eta, + ) + + +@dataclass +class Hyperband: + sh_brackets: list[Sync] + + _min_rung: int = field(init=False, repr=False) + _max_rung: int = field(init=False, repr=False) + + def __post_init__(self) -> None: + if not self.sh_brackets: + raise ValueError("HyperbandBrackets must have at least one SH bracket") + + # Sort the brackets by those which contain the lowest rung values first + self.sh_brackets = sorted( + self.sh_brackets, key=lambda sh_bracket: sh_bracket.rungs[0].value + ) + self._min_rung = min(bracket.rungs[0].value for bracket in self.sh_brackets) + self._max_rung = max(bracket.rungs[-1].value for bracket in self.sh_brackets) + + @classmethod + def create_repeating( + cls, + table: pd.DataFrame, + *, + bracket_layouts: list[dict[int, int]], + ) -> list[Hyperband]: + """Create a list of brackets from the table. + + The table should have a multi-index of (id, rung) where rung is the + fidelity level of the configuration. + + This method will always ensure there is at least one hyperband set of brackets, + with at least one empty slot. For example, if each hyperband set of brackets + houses a maximum of 9 configurations, and there are 27 total unique configurations + in the table, these will be split into 3 hyperband brackets with 9 configurations + + 1 additional hyperband bracket with 0 in it configurations. + + ``` + # Unrealistic example showing the format of the table + (id, rung) -> config, perf + -------------------------- + 0 0 | {"hp": 0, ...}, 0.1 + 1 | {"hp": 0, ...}, 0.2 + 1 0 | {"hp": 1, ...}, 0.1 + 2 1 | {"hp": 2, ...}, 0.3 + 2 | {"hp": 2, ...}, 0.4 + 3 2 | {"hp": 3, ...}, 0.4 + ``` + + Args: + table: The table of configurations to split into brackets. + bracket_layouts: A mapping of rung to the capacity of that rung. + + Returns: + HyperbandBrackets which have each subselected the table with the + corresponding rung sizes. + """ + all_ids = table.index.get_level_values("id").unique() + + # Split the ids into N hyperband brackets of size K. + # K is sum of number of configurations in the lowest rung of each SH bracket + # + # For example: + # > bracket_layouts = [ + # > {0: 81, 1: 27, 2: 9, 3: 3, 4: 1}, + # > {1: 27, 2: 9, 3: 3, 4: 1}, + # > {2: 9, 3: 3, 4: 1}, + # > ... + # > ] + # + # Corresponds to: + # bracket1 - [rung_0: 81, rung_1: 27, rung_2: 9, rung_3: 3, rung_4: 1] + # bracket2 - [rung_1: 27, rung_2: 9, rung_3: 3, rung_4: 1] + # bracket3 - [rung_2: 9, rung_3: 3, rung_4: 1] + # ... + # > K = 81 + 27 + 9 + ... + # + bottom_rung_sizes = [sh[min(sh.keys())] for sh in bracket_layouts] + K = sum(bottom_rung_sizes) + N = max(len(all_ids) // K + 1, 1) + + hb_id_slices: list[Index] = [all_ids[i * K : (i + 1) * K] for i in range(N)] + + # Used if there is nothing for one of the rungs + empty_slice = table.loc[[]] + + # Now for each of our HB brackets, we need to split them into the SH brackets + hb_brackets: list[list[Sync]] = [] + + offsets = np.cumsum([0, *bottom_rung_sizes]) + for hb_ids in hb_id_slices: + # Split the ids into each of the respective brackets, e.g. [81, 27, 9, ...] + ids_for_each_bracket = [hb_ids[s:e] for s, e in pairwise(offsets)] + + # Select the data for each of the configs allocated to these sh_brackets + data_for_each_bracket = [table.loc[_ids] for _ids in ids_for_each_bracket] + + # Create the bracket + sh_brackets: list[Sync] = [] + for data_for_bracket, layout in zip( + data_for_each_bracket, + bracket_layouts, + strict=True, + ): + rung_data = dict(iter(data_for_bracket.groupby(level="rung", sort=False))) + bracket = Sync( + rungs=[ + Rung( + value=rung, + capacity=capacity, + table=rung_data.get(rung, empty_slice), + ) + for rung, capacity in layout.items() + ] + ) + sh_brackets.append(bracket) + + hb_brackets.append(sh_brackets) + + return [cls(sh_brackets=sh_brackets) for sh_brackets in hb_brackets] + + def next(self) -> BracketAction: + # We check what each SH bracket wants to do + statuses = [sh_bracket.next() for sh_bracket in self.sh_brackets] + + # We define a priority function to sort and decide what to return: + # + # 1. "promote"/"new": tie break by rung value + # 1.1 Tie break by rung value if needed + # 1.2 Further tie break by index (bracket with lowest rung goes first) + # (1.2 is handled implicitly by the sorted order of the brackets + # 2. "pending": If there are no promotions or new samples, we say HB is pending + # 3. "done": If everything is done, then we are done. + def priority(x: BracketAction) -> tuple[int, int]: + match x: + case PromoteAction(new_rung=new_rung): + return 0, new_rung + case SampleAction(sample_at_rung): + return 0, sample_at_rung + case "pending": + return 1, 0 + case "done": + return 2, 0 + case _: + raise RuntimeError("This is a bug!") + + sorted_priorities = sorted(statuses, key=priority) # type: ignore + return sorted_priorities[0] + + +@dataclass +class AsyncHyperband: + asha_brackets: list[Async] + """A list of ASHA brackets, ordered from lowest to highest according to the lowest + rung value in each bracket.""" + + eta: int + """The eta parameter used for deciding when to promote.""" + + _min_rung: int = field(init=False, repr=False) + _max_rung: int = field(init=False, repr=False) + + def __post_init__(self) -> None: + if not self.asha_brackets: + raise ValueError("HyperbandBrackets must have at least one ASHA bracket") + + # Sort the brackets by those which contain the lowest rung values first + self.asha_brackets = sorted( + self.asha_brackets, key=lambda bracket: bracket.rungs[0].value + ) + self._min_rung = min(bracket.rungs[0].value for bracket in self.asha_brackets) + self._max_rung = max(bracket.rungs[-1].value for bracket in self.asha_brackets) + + @classmethod + def create( + cls, + table: pd.DataFrame, + *, + bracket_rungs: list[list[int]], + eta: int, + ) -> AsyncHyperband: + """Create an AsyncHyperbandBrackets from the table. + + The table should have a multi-index of (id, rung) where rung is the + fidelity level of the configuration. + ``` + # Unrealistic example showing the format of the table + (id, rung) -> config, perf + -------------------------- + 0 0 | {"hp": 0, ...}, 0.1 + 1 | {"hp": 0, ...}, 0.2 + 1 0 | {"hp": 1, ...}, 0.1 + 2 1 | {"hp": 2, ...}, 0.3 + 2 | {"hp": 2, ...}, 0.4 + 3 2 | {"hp": 3, ...}, 0.4 + ``` + + Args: + table: The table of configurations to split into brackets. + bracket_rungs: A list of rungs for each bracket. Each element of the list + is a list for that given bracket. + + Returns: + The AsyncHyperbandBrackets which have each subselected the table with the + corresponding rung sizes. + """ + return AsyncHyperband( + asha_brackets=[ + Async.create(table=table, rungs=layout, eta=eta) + for layout in bracket_rungs + ], + eta=eta, + ) + + def next(self) -> BracketAction: + # Each ASHA bracket always has an action, sample which to take + bracket_ix = async_hb_sample_bracket_to_run(self._max_rung, self.eta) + bracket = self.asha_brackets[bracket_ix] + return bracket.next() + + +Bracket: TypeAlias = Sync | Async | Hyperband | AsyncHyperband diff --git a/neps/optimizers/utils/grid.py b/neps/optimizers/utils/grid.py new file mode 100644 index 000000000..1c3072cf0 --- /dev/null +++ b/neps/optimizers/utils/grid.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from itertools import product +from typing import Any + +import torch + +from neps.space import Categorical, Constant, Domain, Float, Integer, SearchSpace + + +def make_grid( + space: SearchSpace, + *, + size_per_numerical_hp: int = 10, +) -> list[dict[str, Any]]: + """Get a grid of configurations from the search space. + + For [`Float`][neps.space.Float] and [`Integer`][neps.space.Integer] + the parameter `size_per_numerical_hp=` is used to determine a grid. + + For [`Categorical`][neps.space.Categorical] + hyperparameters, we include all the choices in the grid. + + For [`Constant`][neps.space.Constant] hyperparameters, + we include the constant value in the grid. + + Args: + size_per_numerical_hp: The size of the grid for each numerical hyperparameter. + + Returns: + A list of configurations from the search space. + """ + param_ranges: dict[str, list[Any]] = {} + for name, hp in space.parameters.items(): + match hp: + case Categorical(): + param_ranges[name] = list(hp.choices) + case Constant(): + param_ranges[name] = [hp.value] + case Integer() | Float(): + if hp.is_fidelity: + param_ranges[name] = [hp.upper] + continue + + if hp.domain.cardinality is None: + steps = size_per_numerical_hp + else: + steps = min(size_per_numerical_hp, hp.domain.cardinality) + + xs = torch.linspace(0, 1, steps=steps) + numeric_values = hp.domain.cast(xs, frm=Domain.unit_float()) + uniq_values = torch.unique(numeric_values).tolist() + param_ranges[name] = uniq_values + case _: + raise NotImplementedError(f"Unknown Parameter type: {type(hp)}\n{hp}") + + values = product(*param_ranges.values()) + keys = list(space.parameters.keys()) + + return [dict(zip(keys, p, strict=False)) for p in values] diff --git a/neps/optimizers/initial_design.py b/neps/optimizers/utils/initial_design.py similarity index 92% rename from neps/optimizers/initial_design.py rename to neps/optimizers/utils/initial_design.py index 6de3e5fb5..0cd77c921 100644 --- a/neps/optimizers/initial_design.py +++ b/neps/optimizers/utils/initial_design.py @@ -1,16 +1,13 @@ from __future__ import annotations +import random from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Literal +from typing import Any, Literal import torch -from neps.sampling import Sampler -from neps.sampling.priors import Prior - -if TYPE_CHECKING: - from neps.search_spaces.encoding import ConfigEncoder - from neps.search_spaces.search_space import SearchSpace +from neps.sampling import Prior, Sampler +from neps.space import ConfigEncoder, Domain, SearchSpace def make_initial_design( # noqa: PLR0912, C901 @@ -72,7 +69,8 @@ def make_initial_design( # noqa: PLR0912, C901 fids = lambda: _fids case True: fids = lambda: { - name: hp.sample_value() for name, hp in space.fidelities.items() + name: hp.domain.cast_one(random.random(), frm=Domain.unit_float()) + for name, hp in space.fidelities.items() } case int() | float(): if len(space.fidelities) != 1: @@ -100,7 +98,7 @@ def make_initial_design( # noqa: PLR0912, C901 ) if sample_prior_first: - configs.append({**space.prior_config, **fids()}) + configs.append({**space.prior, **fids()}) ndims = len(space.numerical) + len(space.categoricals) if sample_size == "ndim": diff --git a/neps/plot/plot.py b/neps/plot/plot.py index c36f18424..ded80e49f 100644 --- a/neps/plot/plot.py +++ b/neps/plot/plot.py @@ -60,8 +60,7 @@ def plot( # noqa: C901, PLR0913 algorithms = ["neps"] logger.info( - f"Processing {len(benchmarks)} benchmark(s) " - f"and {len(algorithms)} algorithm(s)..." + f"Processing {len(benchmarks)} benchmark(s) and {len(algorithms)} algorithm(s)..." ) ncols = 1 if len(benchmarks) == 1 else 2 diff --git a/neps/plot/plot3D.py b/neps/plot/plot3D.py index e0d835988..fa0b49e03 100644 --- a/neps/plot/plot3D.py +++ b/neps/plot/plot3D.py @@ -35,17 +35,17 @@ class Plotter3D: def __post_init__(self) -> None: if self.run_path is not None: - assert ( - Path(self.run_path).absolute().is_dir() - ), f"Path {self.run_path} is not a directory" + assert Path(self.run_path).absolute().is_dir(), ( + f"Path {self.run_path} is not a directory" + ) self.data_path = ( Path(self.run_path).absolute() / "summary_csv" / "config_data.csv" ) assert self.data_path.exists(), f"File {self.data_path} does not exist" - self.df = pd.read_csv( + self.df = pd.read_csv( # type: ignore self.data_path, index_col=0, - float_precision="round_trip", + float_precision="round_trip", # type: ignore ) # Assigned at prep_df stage @@ -55,23 +55,23 @@ def __post_init__(self) -> None: @staticmethod def get_x(df: pd.DataFrame) -> np.ndarray: """Get the x-axis values for the plot.""" - return df["epochID"].to_numpy() + return df["epochID"].to_numpy() # type: ignore @staticmethod def get_y(df: pd.DataFrame) -> np.ndarray: """Get the y-axis values for the plot.""" y_ = df["configID"].to_numpy() - return np.ones_like(y_) * y_[0] + return np.ones_like(y_) * y_[0] # type: ignore @staticmethod def get_z(df: pd.DataFrame) -> np.ndarray: """Get the z-axis values for the plot.""" - return df["result.objective_to_minimize"].to_numpy() + return df["result.objective_to_minimize"].to_numpy() # type: ignore @staticmethod def get_color(df: pd.DataFrame) -> np.ndarray: """Get the color values for the plot.""" - return df.index.to_numpy() + return df.index.to_numpy() # type: ignore def prep_df(self, df: pd.DataFrame | None = None) -> pd.DataFrame: """Prepare the dataframe for plotting.""" @@ -197,7 +197,7 @@ def plot3D( # noqa: N802, PLR0915 ax3D.axes.set_xlim3d(left=self.epochs_range[0], right=self.epochs_range[1]) # type: ignore ax3D.axes.set_ylim3d(bottom=0, top=data_groups.ngroups) # type: ignore - ax3D.axes.set_zlim3d( + ax3D.axes.set_zlim3d( # type: ignore bottom=self.objective_to_minimize_range[0], top=self.objective_to_minimize_range[1], ) # type: ignore diff --git a/neps/plot/plotting.py b/neps/plot/plotting.py index e0cc2eae8..612fad9d8 100644 --- a/neps/plot/plotting.py +++ b/neps/plot/plotting.py @@ -195,7 +195,8 @@ def _set_legend( ) for legend_item in legend.legend_handles: - legend_item.set_linewidth(2.0) + if legend_item is not None: + legend_item.set_linewidth(2.0) # type: ignore def _save_fig( diff --git a/neps/runtime.py b/neps/runtime.py index c4c8bff70..92d3b824b 100644 --- a/neps/runtime.py +++ b/neps/runtime.py @@ -7,7 +7,7 @@ import os import shutil import time -from collections.abc import Callable, Iterable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path @@ -120,8 +120,11 @@ def _set_global_trial(trial: Trial) -> Iterator[None]: ) _CURRENTLY_RUNNING_TRIAL_IN_PROCESS = trial yield + + # This is mostly for `tblogger` for _key, callback in _TRIAL_END_CALLBACKS.items(): callback(trial) + _CURRENTLY_RUNNING_TRIAL_IN_PROCESS = None @@ -149,9 +152,6 @@ class DefaultWorker(Generic[Loc]): worker_id: str """The id of the worker.""" - _pre_sample_hooks: list[Callable[[BaseOptimizer], BaseOptimizer]] | None = None - """Hooks to run before sampling a new trial.""" - worker_cumulative_eval_count: int = 0 """The number of evaluations done by this worker.""" @@ -171,7 +171,6 @@ def new( optimizer: BaseOptimizer, settings: WorkerSettings, evaluation_fn: Callable[..., float | Mapping[str, Any]], - _pre_sample_hooks: list[Callable[[BaseOptimizer], BaseOptimizer]] | None = None, worker_id: str | None = None, ) -> DefaultWorker: """Create a new worker.""" @@ -181,7 +180,6 @@ def new( settings=settings, evaluation_fn=evaluation_fn, worker_id=worker_id if worker_id is not None else _default_worker_name(), - _pre_sample_hooks=_pre_sample_hooks, ) def _check_worker_local_settings( @@ -620,14 +618,13 @@ def _launch_runtime( # noqa: PLR0913 optimization_dir: Path, max_cost_total: float | None, ignore_errors: bool = False, - objective_to_minimize_value_on_error: float | None, + objective_value_on_error: float | None, cost_value_on_error: float | None, continue_until_max_evaluation_completed: bool, overwrite_optimization_dir: bool, max_evaluations_total: int | None, max_evaluations_for_worker: int | None, sample_batch_size: int | None, - pre_load_hooks: Iterable[Callable[[BaseOptimizer], BaseOptimizer]] | None, ) -> None: if overwrite_optimization_dir and optimization_dir.exists(): logger.info( @@ -677,7 +674,7 @@ def _launch_runtime( # noqa: PLR0913 ), batch_size=sample_batch_size, default_report_values=DefaultReportValues( - objective_to_minimize_value_on_error=objective_to_minimize_value_on_error, + objective_value_on_error=objective_value_on_error, cost_value_on_error=cost_value_on_error, cost_if_not_provided=None, # TODO: User can't specify yet learning_curve_on_error=None, # TODO: User can't specify yet @@ -722,6 +719,5 @@ def _launch_runtime( # noqa: PLR0913 optimizer=optimizer, evaluation_fn=evaluation_fn, settings=settings, - _pre_sample_hooks=list(pre_load_hooks) if pre_load_hooks is not None else None, ) worker.run() diff --git a/neps/sampling/__init__.py b/neps/sampling/__init__.py index 032290d6c..68a5a813b 100644 --- a/neps/sampling/__init__.py +++ b/neps/sampling/__init__.py @@ -1,4 +1,12 @@ -from neps.sampling.priors import CenteredPrior, Prior, UniformPrior -from neps.sampling.samplers import Sampler, Sobol +from neps.sampling.priors import CenteredPrior, Prior, Uniform +from neps.sampling.samplers import BorderSampler, Sampler, Sobol, WeightedSampler -__all__ = ["CenteredPrior", "Prior", "Sampler", "Sobol", "UniformPrior"] +__all__ = [ + "BorderSampler", + "CenteredPrior", + "Prior", + "Sampler", + "Sobol", + "Uniform", + "WeightedSampler", +] diff --git a/neps/sampling/distributions.py b/neps/sampling/distributions.py index e9c93b0aa..371979b29 100644 --- a/neps/sampling/distributions.py +++ b/neps/sampling/distributions.py @@ -13,7 +13,7 @@ from torch.distributions import Distribution, Uniform, constraints from torch.distributions.utils import broadcast_all -from neps.search_spaces.domain import Domain +from neps.space import Domain if TYPE_CHECKING: from torch.distributions.constraints import Constraint @@ -40,7 +40,7 @@ class TruncatedStandardNormal(Distribution): "a": constraints.real, "b": constraints.real, } # type: ignore - has_rsample: ClassVar[bool] = True + has_rsample: ClassVar[bool] = True # type: ignore eps: ClassVar[float] = 1e-6 def __init__( @@ -111,41 +111,41 @@ def mean(self) -> torch.Tensor: @property @override # type: ignore def variance(self) -> torch.Tensor: - return self._variance + return self._variance # type: ignore @override # type: ignore def entropy(self) -> torch.Tensor: - return self._entropy + return self._entropy # type: ignore @staticmethod def _little_phi(x: torch.Tensor) -> torch.Tensor: - return (-(x**2) * 0.5).exp() * CONST_INV_SQRT_2PI + return (-(x**2) * 0.5).exp() * CONST_INV_SQRT_2PI # type: ignore def _big_phi(self, x: torch.Tensor) -> torch.Tensor: phi = 0.5 * (1 + (x * CONST_INV_SQRT_2).erf()) - return phi.clamp(self.eps, 1 - self.eps) + return phi.clamp(self.eps, 1 - self.eps) # type: ignore @staticmethod def _inv_big_phi(x: torch.Tensor) -> torch.Tensor: - return CONST_SQRT_2 * (2 * x - 1).erfinv() + return CONST_SQRT_2 * (2 * x - 1).erfinv() # type: ignore @override # type: ignore def cdf(self, value: torch.Tensor) -> torch.Tensor: if self._validate_args: self._validate_sample(value) - return ((self._big_phi(value) - self._big_phi_a) / self._Z).clamp(0, 1) + return ((self._big_phi(value) - self._big_phi_a) / self._Z).clamp(0, 1) # type: ignore @override # type: ignore def icdf(self, value: torch.Tensor) -> torch.Tensor: y = self._big_phi_a + value * self._Z y = y.clamp(self.eps, 1 - self.eps) - return self._inv_big_phi(y) + return self._inv_big_phi(y) # type: ignore @override # type: ignore def log_prob(self, value: torch.Tensor) -> torch.Tensor: if self._validate_args: self._validate_sample(value) - return CONST_LOG_INV_SQRT_2PI - self._log_Z - (value**2) * 0.5 + return CONST_LOG_INV_SQRT_2PI - self._log_Z - (value**2) * 0.5 # type: ignore @override # type: ignore def rsample(self, sample_shape: torch.Size | None = None) -> torch.Tensor: @@ -200,14 +200,14 @@ def __init__( self._entropy += self._log_scale def _to_std_rv(self, value: torch.Tensor) -> torch.Tensor: - return (value - self.loc) / self.scale + return (value - self.loc) / self.scale # type: ignore def _from_std_rv(self, value: torch.Tensor) -> torch.Tensor: - return value * self.scale + self.loc + return value * self.scale + self.loc # type: ignore @override def cdf(self, value: torch.Tensor) -> torch.Tensor: - return super().cdf(self._to_std_rv(value)) + return super().cdf(self._to_std_rv(value)) # type: ignore @override def icdf(self, value: torch.Tensor) -> torch.Tensor: @@ -221,12 +221,12 @@ def icdf(self, value: torch.Tensor) -> torch.Tensor: [sample_clip, self._non_std_b.detach().expand_as(sample)], 0 ).min(0)[0] sample.data.copy_(sample_clip) - return sample + return sample # type: ignore @override def log_prob(self, value: torch.Tensor) -> torch.Tensor: value = self._to_std_rv(value) - return super().log_prob(value) - self._log_scale + return super().log_prob(value) - self._log_scale # type: ignore class UniformWithUpperBound(Uniform): diff --git a/neps/sampling/priors.py b/neps/sampling/priors.py index be43cb01e..c2cfdea9b 100644 --- a/neps/sampling/priors.py +++ b/neps/sampling/priors.py @@ -4,7 +4,7 @@ variables, i.e. each column of a tensor is assumed to be independent and can be acted on independently. -See the class doc description of [`Prior`][neps.priors.Prior] for more details. +See the class doc description of [`Prior`][neps.sampling.Prior] for more details. """ from __future__ import annotations @@ -23,20 +23,16 @@ TruncatedNormal, ) from neps.sampling.samplers import Sampler -from neps.search_spaces import Categorical -from neps.search_spaces.domain import UNIT_FLOAT_DOMAIN, Domain -from neps.search_spaces.encoding import ConfigEncoder +from neps.space import Categorical, ConfigEncoder, Domain, Float, Integer, SearchSpace if TYPE_CHECKING: from torch.distributions import Distribution - from neps.search_spaces import Float, Integer, SearchSpace - class Prior(Sampler): """A protocol for priors over search spaces. - Extends from the [`Sampler`][neps.samplers.Sampler] protocol. + Extends from the [`Sampler`][neps.sampling.Sampler] protocol. At it's core, the two methods that need to be implemented are `log_pdf` and `sample`. The `log_pdf` method should return the @@ -45,13 +41,13 @@ class Prior(Sampler): All values given to the `log_pdf` and the ones returned from the `sample` method are assumed to be in the value domain of the prior, - i.e. the [`.domains`][neps.priors.Prior] attribute. + i.e. the [`.domains`][neps.sampling.Prior] attribute. !!! warning The domain in which samples are actually drawn from not necessarily need to match that of the value domain. For example, the - [`UniformPrior`][neps.priors.UniformPrior] class uses a unit uniform + [`Uniform`][neps.sampling.Uniform] class uses a unit uniform distribution to sample from the unit interval before converting samples to the value domain. @@ -60,7 +56,7 @@ class Prior(Sampler): For example, consider a value domain `[0, 1e9]`. You might expect the `pdf` to be `1e-9` (1 / 1e9) for any given value inside the domain. - However, since the `UniformPrior` samples from the unit interval, the `pdf` will + However, since the `Uniform` samples from the unit interval, the `pdf` will actually be `1` (1 / 1) for any value inside the domain. """ @@ -101,18 +97,25 @@ def pdf( ) -> torch.Tensor: """Compute the pdf of values in `x` under a prior. - See [`log_pdf()`][neps.priors.Prior.log_pdf] for details on shapes. + See [`log_pdf()`][neps.sampling.Prior.log_pdf] for details on shapes. """ return torch.exp(self.log_pdf(x, frm=frm)) + def pdf_configs(self, x: list[dict[str, Any]], *, frm: ConfigEncoder) -> torch.Tensor: + """Compute the pdf of values in `x` under a prior. + + See [`log_pdf()`][neps.sampling.Prior.log_pdf] for details on shapes. + """ + return self.pdf(frm.encode(x), frm=frm) + @classmethod - def uniform(cls, ncols: int) -> UniformPrior: + def uniform(cls, ncols: int) -> Uniform: """Create a uniform prior for a given list of domains. Args: ncols: The number of columns in the tensor to sample. """ - return UniformPrior(ndim=ncols) + return Uniform(ndim=ncols) @classmethod def from_parameters( @@ -122,15 +125,9 @@ def from_parameters( center_values: Mapping[str, Any] | None = None, confidence_values: Mapping[str, float] | None = None, ) -> CenteredPrior: - """Please refer to [`from_space()`][neps.priors.Prior.from_space] + """Please refer to [`from_space()`][neps.sampling.Prior.from_space] for more details. """ - # TODO: This needs to be moved to the search space class, however - # to not break the current prior based APIs used elsewhere, we can - # just manually create this here. - # We use confidence here where `0` means no confidence and `1` means - # absolute confidence. This gets translated in to std's and weights - # accordingly in a `CenteredPrior` _mapping = {"low": 0.25, "medium": 0.5, "high": 0.75} center_values = center_values or {} @@ -145,10 +142,7 @@ def from_parameters( centers.append(None) continue - confidence_score = confidence_values.get( - name, - _mapping[hp.prior_confidence_choice], - ) + confidence_score = confidence_values.get(name, _mapping[hp.prior_confidence]) center = hp.choices.index(default) if isinstance(hp, Categorical) else default centers.append((center, confidence_score)) @@ -195,9 +189,6 @@ def from_domains_and_centers( The values contained in centers should be contained within the domain. All confidence levels should be within the `[0, 1]` range. - confidence: The confidence level for the center. Entries containing `None` - should match with `centers` that are `None`. If not, this is considered an - error. device: Device to place the tensors on for distributions. Returns: @@ -215,7 +206,21 @@ def from_domains_and_centers( # the distributions to all be unit uniform as it can speed up sampling when # consistentaly the same. This still works for categoricals if center_conf is None: - distributions.append(UNIT_UNIFORM_DIST) + if domain.is_categorical: + # Uniform categorical + n_cats = domain.cardinality + assert n_cats is not None + dist = TorchDistributionWithDomain( + distribution=torch.distributions.Categorical( + probs=torch.ones(n_cats, device=device) / n_cats, + validate_args=False, + ), + domain=domain, + ) + distributions.append(dist) + else: + distributions.append(UNIT_UNIFORM_DIST) + continue center, conf = center_conf @@ -262,7 +267,7 @@ def from_domains_and_centers( device=device, validate_args=False, ), - domain=UNIT_FLOAT_DOMAIN, + domain=Domain.unit_float(), ) distributions.append(dist) @@ -310,6 +315,34 @@ def from_space( confidence_values=confidence_values, ) + @classmethod + def from_config( + cls, + config: dict[str, Any], + *, + space: SearchSpace, + confidence_values: Mapping[str, float] | None = None, + include_fidelity: bool = False, + ) -> Prior: + """Create a prior from a configuration. + + Args: + config: The configuration to create a prior from. + space: The search space to create the prior from. + confidence_values: Any confidence values to override by what's set in the + `space`. + include_fidelity: Whether to include the fidelity of the search space. + + Returns: + The prior distribution + """ + return Prior.from_space( + space, + center_values=config, + confidence_values=confidence_values, + include_fidelity=include_fidelity, + ) + @dataclass class CenteredPrior(Prior): @@ -322,7 +355,11 @@ class CenteredPrior(Prior): not have a center and confidence level, i.e. no prior information. You can create this class more easily using - [`Prior.make_centered()`][neps.priors.Prior.make_centered]. + [`Prior.from_space()`][neps.sampling.Prior.from_space] to use the prior config + of the search space. + + If you need to create a prior around a specific configuration, you can also + use the [`Prior.from_config()`][neps.sampling.Prior.from_config] method. """ distributions: list[TorchDistributionWithDomain] @@ -443,7 +480,7 @@ def sample( @dataclass -class UniformPrior(Prior): +class Uniform(Prior): """A prior that is uniform over a given domain. Uses a UnitUniform under the hood before converting to the value domain. @@ -492,4 +529,4 @@ def sample( else: samples = torch.rand(_n, device=device) - return Domain.translate(samples, frm=UNIT_FLOAT_DOMAIN, to=to, dtype=dtype) + return Domain.translate(samples, frm=Domain.unit_float(), to=to, dtype=dtype) diff --git a/neps/sampling/samplers.py b/neps/sampling/samplers.py index 43558eff6..c6f70b6af 100644 --- a/neps/sampling/samplers.py +++ b/neps/sampling/samplers.py @@ -1,26 +1,25 @@ """Samplers for generating points in a search space. -These are similar to [`Prior`][neps.priors.Prior] objects, but they +These are similar to [`Prior`][neps.sampling.Prior] objects, but they do not necessarily have an easily definable pdf. """ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from functools import reduce -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from typing_extensions import override import torch from more_itertools import all_equal -from neps.search_spaces.domain import UNIT_FLOAT_DOMAIN, Domain -from neps.search_spaces.encoding import ConfigEncoder +from neps.space import ConfigEncoder, Domain if TYPE_CHECKING: - from neps.sampling.priors import UniformPrior + from neps.sampling.priors import Uniform class Sampler(ABC): @@ -46,7 +45,7 @@ def sample( Args: n: The number of points to sample. If a torch.Size, an additional dimension - will be added with [`.ncols`][neps.samplers.Sampler.ncols]. + will be added with [`.ncols`][neps.sampling.Sampler.ncols]. For example, if `n = 5`, the output will be `(5, ncols)`. If `n = (5, 3)`, the output will be `(5, 3, ncols)`. to: If a single domain, `.ncols` columns will be produced form that one @@ -60,22 +59,40 @@ def sample( A tensor of (n, ndim) points sampled cast to the given domain. """ - def sample_one( + def sample_config( self, + to: ConfigEncoder, *, - to: Domain | list[Domain] | ConfigEncoder, seed: torch.Generator | None = None, - device: torch.device | None = None, - dtype: torch.dtype | None = None, - ) -> torch.Tensor: - """Sample a single point and convert it to the given domain. + include: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + """See [`sample_configs()`][neps.sampling.Sampler.sample_configs].""" + return self.sample_configs(1, to, seed=seed, include=include)[0] + + def sample_configs( + self, + n: int, + to: ConfigEncoder, + *, + seed: torch.Generator | None = None, + include: Mapping[str, Any] | None = None, + ) -> list[dict[str, Any]]: + """Sample configurations directly into a search space. - The configuration will be a single dimensional tensor of shape - `(ncols,)`. + Args: + n: The number of configurations to sample. + to: The encoding to sample into. + seed: The seed generator. + include: Additional values to include in the configuration. - Please see [`sample`][neps.samplers.Sampler.sample] for more details. + Returns: + A list of configurations. """ - return self.sample(1, to=to, seed=seed, device=device, dtype=dtype).squeeze(0) + tensors = self.sample(n, to=to, seed=seed) + configs = to.decode(tensors) + if include is None: + return configs + return [{**config, **include} for config in configs] @classmethod def sobol(cls, ndim: int, *, scramble: bool = True) -> Sobol: @@ -91,7 +108,7 @@ def sobol(cls, ndim: int, *, scramble: bool = True) -> Sobol: return Sobol(ndim=ndim, scramble=scramble) @classmethod - def uniform(cls, ndim: int) -> UniformPrior: + def uniform(cls, ndim: int) -> Uniform: """Create a uniform sampler. Args: @@ -100,9 +117,9 @@ def uniform(cls, ndim: int) -> UniformPrior: Returns: A uniform sampler. """ - from neps.sampling.priors import UniformPrior + from neps.sampling.priors import Uniform - return UniformPrior(ndim=ndim) + return Uniform(ndim=ndim) @classmethod def borders(cls, ndim: int) -> BorderSampler: @@ -177,7 +194,7 @@ def sample( if isinstance(n, torch.Size): x = x.view(*n, self.ncols) - return Domain.translate(x, frm=UNIT_FLOAT_DOMAIN, to=to) + return Domain.translate(x, frm=Domain.unit_float(), to=to) @dataclass @@ -187,7 +204,7 @@ class WeightedSampler(Sampler): samplers: Sequence[Sampler] """The samplers to sample from.""" - weights: torch.Tensor + weights: Sequence[float] """The weights for each sampler.""" sampler_probabilities: torch.Tensor = field(init=False, repr=False) @@ -199,20 +216,21 @@ def __post_init__(self) -> None: f"At least two samplers must be given. Got {len(self.samplers)}" ) - if self.weights.ndim != 1: - raise ValueError("Weights must be a 1D tensor.") + probs = torch.as_tensor(self.weights, dtype=torch.float64) + if probs.ndim != 1: + raise ValueError("Weights must be a 1D.") - if len(self.samplers) != len(self.weights): + if len(self.samplers) != len(probs): raise ValueError("The number of samplers and weights must be the same.") ncols = [sampler.ncols for sampler in self.samplers] if not all_equal(ncols): raise ValueError( - "All samplers must have the same number of columns." f" Got {ncols}." + f"All samplers must have the same number of columns. Got {ncols}." ) self._ncols = ncols[0] - self.sampler_probabilities = self.weights / self.weights.sum() + self.sampler_probabilities = probs / probs.sum() @property @override @@ -309,7 +327,7 @@ def ncols(self) -> int: @property def n_possible(self) -> int: """The amount of possible border configurations.""" - return 2**self.ndim + return int(2**self.ndim) @override def sample( @@ -351,4 +369,4 @@ def sample( configs = configs.unsqueeze(1).bitwise_and(bit_masks).ne(0).to(dtype) # Reshape to the output shape including ncols dimension configs = configs.view(output_shape) - return Domain.translate(configs, frm=UNIT_FLOAT_DOMAIN, to=to) + return Domain.translate(configs, frm=Domain.unit_float(), to=to) diff --git a/neps/search_spaces/__init__.py b/neps/search_spaces/__init__.py deleted file mode 100644 index b726b8ae1..000000000 --- a/neps/search_spaces/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from neps.search_spaces.architecture.api import ( - Architecture, - ArchitectureParameter, - Function, - FunctionParameter, -) -from neps.search_spaces.architecture.graph_grammar import ( - CoreGraphGrammar, - GraphGrammar, - GraphParameter, -) -from neps.search_spaces.hyperparameters import ( - Categorical, - CategoricalParameter, - Constant, - ConstantParameter, - Float, - FloatParameter, - Integer, - IntegerParameter, - Numerical, - NumericalParameter, -) -from neps.search_spaces.parameter import Parameter, ParameterWithPrior -from neps.search_spaces.search_space import SearchSpace - -__all__ = [ - "Architecture", - "ArchitectureParameter", - "Categorical", - "CategoricalParameter", - "Constant", - "ConstantParameter", - "CoreGraphGrammar", - "Float", - "FloatParameter", - "Function", - "FunctionParameter", - "GraphGrammar", - "GraphParameter", - "Integer", - "IntegerParameter", - "Numerical", - "NumericalParameter", - "Parameter", - "ParameterWithPrior", - "SearchSpace", -] diff --git a/neps/search_spaces/architecture/__init__.py b/neps/search_spaces/architecture/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/neps/search_spaces/architecture/api.py b/neps/search_spaces/architecture/api.py deleted file mode 100644 index 9521bd7aa..000000000 --- a/neps/search_spaces/architecture/api.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import annotations - -import inspect -from typing import TYPE_CHECKING, Callable - -import networkx as nx - -from .cfg import Grammar -from .cfg_variants.constrained_cfg import ConstrainedGrammar -from .graph_grammar import GraphGrammar - -if TYPE_CHECKING: - from torch import nn - - -def _dict_structure_to_str( - structure: dict, primitives: dict, repetitive_mapping: dict | None = None -) -> str: - def _save_replace(string: str, __old: str, __new: str): - while string.count(__old) > 0: - string = string.replace(__old, __new) - return string - - grammar = "" - for nonterminal, productions in structure.items(): - grammar += nonterminal + " -> " + " | ".join(productions) + "\n" - grammar = grammar.replace("(", " ") - grammar = grammar.replace(")", "") - grammar = grammar.replace(",", "") - for primitive in primitives: - grammar = _save_replace(grammar, f" {primitive} ", f' "{primitive}" ') - grammar = _save_replace(grammar, f" {primitive}\n", f' "{primitive}"\n') - if repetitive_mapping is not None: - for placeholder in repetitive_mapping: - grammar = _save_replace(grammar, f" {placeholder} ", f' "{placeholder}" ') - grammar = _save_replace(grammar, f" {placeholder}\n", f' "{placeholder}"\n') - return grammar - - -def _build(graph, set_recursive_attribute): - in_node = next(n for n in graph.nodes if graph.in_degree(n) == 0) - for n in nx.topological_sort(graph): - for pred in graph.predecessors(n): - e = (pred, n) - op_name = graph.edges[e]["op_name"] - if pred == in_node: - predecessor_values = None - else: - pred_pred = next(iter(graph.predecessors(pred))) - predecessor_values = graph.edges[(pred_pred, pred)] - graph.edges[e].update(set_recursive_attribute(op_name, predecessor_values)) - - -def Architecture(**kwargs): - """Factory function.""" - if "structure" not in kwargs: - raise ValueError("Factory function requires structure") - if not isinstance(kwargs["structure"], list) or len(kwargs["structure"]) == 1: - base = GraphGrammar - - class _FunctionParameter(base): - def __init__( - self, - structure: Grammar - | list[Grammar] - | ConstrainedGrammar - | list[ConstrainedGrammar] - | str - | list[str] - | dict - | list[dict], - primitives: dict, - # TODO: Follow this rabbit hole for `constraint_kwargs`, - # it can all be deleted my friend - constraint_kwargs: dict | None = None, - name: str = "ArchitectureParameter", - set_recursive_attribute: Callable | None = None, - **kwargs, - ): - local_vars = locals() - self.input_kwargs = { - args: local_vars[args] - for args in inspect.getfullargspec(self.__init__).args # type: ignore[misc] - if args != "self" - } - self.input_kwargs.update(**kwargs) - - if isinstance(structure, list): - structures = [ - _dict_structure_to_str( - st, - primitives, - repetitive_mapping=kwargs.get( - "terminal_to_sublanguage_map", None - ), - ) - if isinstance(st, dict) - else st - for st in structure - ] - _structures = [] - for st in structures: - if isinstance(st, str): - if constraint_kwargs is None: - _st = Grammar.fromstring(st) - else: - _st = ConstrainedGrammar.fromstring(st) - _st.set_constraints(**constraint_kwargs) - _structures.append(_st) # type: ignore[has-type] - structures = _structures - - super().__init__( - grammars=structures, - terminal_to_op_names=primitives, - edge_attr=False, - **kwargs, - ) - else: - if isinstance(structure, dict): - structure = _dict_structure_to_str(structure, primitives) - - if isinstance(structure, str): - if constraint_kwargs is None: - structure = Grammar.fromstring(structure) - else: - structure = ConstrainedGrammar.fromstring(structure) - structure.set_constraints(**constraint_kwargs) # type: ignore[union-attr] - - super().__init__( - grammar=structure, # type: ignore[arg-type] - terminal_to_op_names=primitives, - edge_attr=False, - **kwargs, - ) - - self._set_recursive_attribute = set_recursive_attribute - self.name: str = name - - def to_pytorch(self) -> nn.Module: - self.clear_graph() - if len(self.nodes()) == 0: - composed_function = self.compose_functions() - # part below is required since PyTorch has no standard functional API - self.graph_to_self(composed_function) - self.prune_graph() - - if self._set_recursive_attribute: - m = _build(self, self._set_recursive_attribute) - - if m is not None: - return m - - self.compile() - self.update_op_names() - return super().to_pytorch() # create PyTorch model - - def create_new_instance_from_id(self, identifier: str): - g = Architecture(**self.input_kwargs) # type: ignore[arg-type] - g.load_from(identifier) - return g - - return _FunctionParameter(**kwargs) - - -def ArchitectureParameter(**kwargs): - """Deprecated: Use `Architecture` instead of `ArchitectureParameter`. - - This function remains for backward compatibility and will raise a deprecation - warning if used. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.ArchitectureParameter' is deprecated and will be removed in" - " future releases. Please use 'neps.Architecture' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - - return Architecture(**kwargs) - - -Function = Architecture - - -def FunctionParameter(**kwargs): - """Deprecated: Use `Function` instead of `FunctionParameter`. - - This function remains for backward compatibility and will raise a deprecation - warning if used. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.FunctionParameter' is deprecated and will be removed in" - " future releases. Please use 'neps.Function' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - - return Function(**kwargs) diff --git a/neps/search_spaces/architecture/cfg.py b/neps/search_spaces/architecture/cfg.py deleted file mode 100644 index 392f56353..000000000 --- a/neps/search_spaces/architecture/cfg.py +++ /dev/null @@ -1,320 +0,0 @@ -from __future__ import annotations - -import math -from typing import Hashable - -import numpy as np -from nltk import CFG, Production -from nltk.grammar import Nonterminal - - -class Grammar(CFG): - """Extended context free grammar (CFG) class from the NLTK python package - We have provided functionality to sample from the CFG. - We have included generation capability within the class (before it was an external function) - Also allow sampling to return whole trees (not just the string of terminals). - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # store some extra quantities needed later - non_unique_nonterminals = [str(prod.lhs()) for prod in self.productions()] - self.nonterminals = list(set(non_unique_nonterminals)) - self.terminals = list( - {str(individual) for prod in self.productions() for individual in prod.rhs()} - - set(self.nonterminals) - ) - # collect nonterminals that are worth swapping when doing genetic operations (i.e not those with a single production that leads to a terminal) - self.swappable_nonterminals = list( - {i for i in non_unique_nonterminals if non_unique_nonterminals.count(i) > 1} - ) - - self._prior = None - - if len(set(self.terminals).intersection(set(self.nonterminals))) > 0: - raise Exception( - f"Same terminal and nonterminal symbol: {set(self.terminals).intersection(set(self.nonterminals))}!" - ) - for nt in self.nonterminals: - if len(self.productions(Nonterminal(nt))) == 0: - raise Exception(f"There is no production for nonterminal {nt}") - - @property - def prior(self): - return self._prior - - @prior.setter - def prior(self, value: dict): - def _check_prior(value: dict): - for nonterminal in self.nonterminals: - if nonterminal not in value: - raise Exception( - f"Nonterminal {nonterminal} not defined in prior distribution!" - ) - if len(value[nonterminal]) != len( - self.productions(lhs=Nonterminal(nonterminal)) - ): - raise Exception( - f"Not all RHS of nonterminal {nonterminal} have a probability!" - ) - if not math.isclose(sum(value[nonterminal]), 1.0): - raise Exception( - f"Prior for {nonterminal} is no probablility distribution (=1)!" - ) - - if value is not None: - _check_prior(value) - self._prior = value - - def sampler( - self, - n=1, - start_symbol: str | None = None, - user_priors: bool = False, - ): - # sample n sequences from the CFG - # cfactor: the factor to downweight productions (cfactor=1 returns to naive sampling strategy) - # smaller cfactor provides smaller sequences (on average) - - # Note that a simple recursive traversal of the grammar (setting convergent=False) where we choose - # productions at random, often hits Python's max recursion depth as the longer a sequnce gets, the - # less likely it is to terminate. Therefore, we set the default sampler (setting convergent=True) to - # downweight frequent productions when traversing the grammar. - # see https://eli.thegreenplace.net/2010/01/28/generating-random-sentences-from-a-context-free-236grammar - start_symbol = self.start() if start_symbol is None else Nonterminal(start_symbol) - - return [ - f"{self._sampler(symbol=start_symbol, user_priors=user_priors)})" - for _ in range(n) - ] - - def _sampler( - self, - symbol=None, - user_priors: bool = False, - *, - _cache: dict[Hashable, str] | None = None, - ): - # simple sampler where each production is sampled uniformly from all possible productions - # Tree choses if return tree or list of terminals - # recursive implementation - if _cache is None: - _cache = {} - - # init the sequence - tree = "(" + str(symbol) - # collect possible productions from the starting symbol - productions = list(self.productions(lhs=symbol)) - # sample - if len(productions) == 0: - raise Exception(f"Nonterminal {symbol} has no productions!") - if user_priors and self._prior is not None: - production = np.random.choice(productions, p=self._prior[str(symbol)]) - else: - production = np.random.choice(productions) - - for sym in production.rhs(): - if isinstance(sym, str): - ## if terminal then add string to sequence - tree = tree + " " + sym - else: - cached = _cache.get(sym) - if cached is None: - cached = self._sampler(sym, user_priors=user_priors, _cache=_cache) - _cache[sym] = cached - - tree = tree + " " + cached + ")" - - return tree - - def compute_prior(self, string_tree: str, log: bool = True) -> float: - prior_prob = 1.0 if not log else 0.0 - - symbols = self.nonterminals + self.terminals - q_production_rules: list[tuple[list, int]] = [] - non_terminal_productions: dict[str, list[Production]] = { - sym: self.productions(lhs=Nonterminal(sym)) for sym in self.nonterminals - } - - _symbols_by_size = sorted(symbols, key=len, reverse=True) - _longest = len(_symbols_by_size[0]) - - i = 0 - _tree_len = len(string_tree) - while i < _tree_len: - char = string_tree[i] - if char in " \t\n": - i += 1 - continue - - if char == "(": - if i == 0: - i += 1 - continue - - # special case: "(" is (part of) a terminal - if string_tree[i - 1 : i + 2] != " ( ": - i += 1 - continue - - if char == ")" and string_tree[i - 1] != " ": - # closing symbol of production - production = q_production_rules.pop()[0][0] - lhs_production = production.lhs() - - idx = self.productions(lhs=lhs_production).index(production) - if log: - prior_prob += np.log(self.prior[(lhs_production)][idx] + 1e-15) - else: - prior_prob *= self.prior[str(lhs_production)][idx] - i += 1 - continue - - _s = string_tree[i : i + _longest] - for sym in _symbols_by_size: - if _s.startswith(sym): - break - else: - raise RuntimeError( - f"Terminal or nonterminal at position {i} does not exist" - ) - - i += len(sym) - 1 - - if sym in self.terminals: - _productions, _count = q_production_rules[-1] - new_productions = [ - production - for production in _productions - if production.rhs()[_count] == sym - ] - q_production_rules[-1] = (new_productions, _count + 1) - elif sym in self.nonterminals: - if len(q_production_rules) > 0: - _productions, _count = q_production_rules[-1] - new_productions = [ - production - for production in _productions - if str(production.rhs()[_count]) == sym - ] - q_production_rules[-1] = (new_productions, _count + 1) - - q_production_rules.append((non_terminal_productions[sym], 0)) - else: - raise Exception(f"Unknown symbol {sym}") - i += 1 - - if len(q_production_rules) > 0: - raise Exception(f"Error in prior computation for {string_tree}") - - return prior_prob - - def mutate( - self, parent: str, subtree_index: int, subtree_node: str, patience: int = 50 - ) -> str: - """Grammar-based mutation, i.e., we sample a new subtree from a nonterminal - node in the parse tree. - - Args: - parent (str): parent of the mutation. - subtree_index (int): index pointing to the node that is root of the subtree. - subtree_node (str): nonterminal symbol of the node. - patience (int, optional): Number of tries. Defaults to 50. - - Returns: - str: mutated child from parent. - """ - # chop out subtree - pre, _, post = self.remove_subtree(parent, subtree_index) - _patience = patience - while _patience > 0: - # only sample subtree -> avoids full sampling of large parse trees - new_subtree = self.sampler(1, start_symbol=subtree_node)[0] - child = pre + new_subtree + post - if parent != child: # ensure that parent is really mutated - break - _patience -= 1 - - return child.strip() - - def rand_subtree(self, tree: str) -> tuple[str, int]: - """Helper function to choose a random subtree in a given parse tree. - Runs a single pass through the tree (stored as string) to look for - the location of swappable nonterminal symbols. - - Args: - tree (str): parse tree. - - Returns: - Tuple[str, int]: return the parent node of the subtree and its index. - """ - split_tree = tree.split(" ") - swappable_indices = [ - i - for i in range(len(split_tree)) - if split_tree[i][1:] in self.swappable_nonterminals - ] - r = np.random.randint(1, len(swappable_indices)) - chosen_non_terminal = split_tree[swappable_indices[r]][1:] - chosen_non_terminal_index = swappable_indices[r] - return chosen_non_terminal, chosen_non_terminal_index - - @staticmethod - def rand_subtree_fixed_head( - tree: str, head_node: str, swappable_indices: list | None = None - ) -> int: - # helper function to choose a random subtree from a given tree with a specific head node - # if no such subtree then return False, otherwise return the index of the subtree - - # single pass through tree (stored as string) to look for the location of swappable_non_terminmals - if swappable_indices is None: - split_tree = tree.split(" ") - swappable_indices = [ - i for i in range(len(split_tree)) if split_tree[i][1:] == head_node - ] - if not isinstance(swappable_indices, list): - raise TypeError("Expected list for swappable indices!") - if len(swappable_indices) == 0: - # no such subtree - return False - else: - # randomly choose one of these non-terminals - r = ( - np.random.randint(1, len(swappable_indices)) - if len(swappable_indices) > 1 - else 0 - ) - return swappable_indices[r] - - @staticmethod - def remove_subtree(tree: str, index: int) -> tuple[str, str, str]: - """Helper functioon to remove a subtree from a parse tree - given its index. - E.g. '(S (S (T 2)) (ADD +) (T 1))' - becomes '(S (S (T 2)) ', '(T 1))' after removing (ADD +). - - Args: - tree (str): parse tree - index (int): index of the subtree root node - - Returns: - Tuple[str, str, str]: part before the subtree, subtree, part past subtree - """ - split_tree = tree.split(" ") - pre_subtree = " ".join(split_tree[:index]) + " " - # get chars to the right of split - right = " ".join(split_tree[index + 1 :]) - # remove chosen subtree - # single pass to find the bracket matching the start of the split - counter, current_index = 1, 0 - for char in right: - if char == "(": - counter += 1 - elif char == ")": - counter -= 1 - if counter == 0: - break - current_index += 1 - post_subtree = right[current_index + 1 :] - removed = "".join(split_tree[index]) + " " + right[: current_index + 1] - return (pre_subtree, removed, post_subtree) diff --git a/neps/search_spaces/architecture/cfg_variants/__init__.py b/neps/search_spaces/architecture/cfg_variants/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/neps/search_spaces/architecture/cfg_variants/constrained_cfg.py b/neps/search_spaces/architecture/cfg_variants/constrained_cfg.py deleted file mode 100644 index aa1e05eac..000000000 --- a/neps/search_spaces/architecture/cfg_variants/constrained_cfg.py +++ /dev/null @@ -1,496 +0,0 @@ -from __future__ import annotations - -import math -from collections import deque -from functools import partial -from queue import LifoQueue - -import numpy as np -from nltk.grammar import Nonterminal - -from neps.search_spaces.architecture.cfg import Grammar - - -class Constraint: - def __init__(self, current_derivation: str | None = None) -> None: - self.current_derivation = current_derivation - - @staticmethod - def initialize_constraints(topology: str) -> Constraint: - raise NotImplementedError - - def get_not_allowed_productions(self, productions: str) -> list[bool] | bool: - raise NotImplementedError - - def update_context(self, new_part: str) -> None: - raise NotImplementedError - - def mutate_not_allowed_productions( - self, nonterminal: str, before: str, after: str, possible_productions: list - ) -> list: - raise NotImplementedError - - -class ConstrainedGrammar(Grammar): - def set_constraints(self, constraints: dict, none_operation: str | None = None): - self.constraints = constraints - self.none_operation = none_operation - self.constraint_is_class = isinstance(self.constraints, Constraint) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.constraints = None - self.none_operation = None - self.constraint_is_class: bool = False - - self._prior: dict | None = None - - @property - def prior(self): - return self._prior - - @prior.setter - def prior(self, value: dict): - def _check_prior(value: dict): - for nonterminal in self.nonterminals: - if nonterminal not in value: - raise Exception( - f"Nonterminal {nonterminal} not defined in prior distribution!" - ) - if len(value[nonterminal]) != len( - self.productions(lhs=Nonterminal(nonterminal)) - ): - raise Exception( - f"Not all RHS of nonterminal {nonterminal} have a probability!" - ) - if not math.isclose(sum(value[nonterminal]), 1.0): - raise Exception( - f"Prior for {nonterminal} is no probablility distribution (=1)!" - ) - - if value is not None: - _check_prior(value) - self._prior = value - - def sampler( # type: ignore[override] - self, - n=1, - start_symbol: str | None = None, - not_allowed_productions=None, - user_priors: bool = False, - ): - start_symbol = self.start() if start_symbol is None else Nonterminal(start_symbol) - - return [ - self._constrained_sampler( - symbol=start_symbol, - not_allowed_productions=not_allowed_productions, - user_priors=user_priors, - ) - + ")" - for _ in range(n) - ] - - def _get_not_allowed_productions( - self, potential_productions, any_production_allowed: bool - ): - return list( - filter( - lambda prod: not any_production_allowed - and len(prod.rhs()) == 1 - and prod.rhs()[0] == self.none_operation, - potential_productions, - ) - ) - - def _constrained_sampler( - self, - symbol=None, - not_allowed_productions=None, - current_derivation=None, - user_priors: bool = False, - ): - # simple sampler where each production is sampled uniformly from all possible productions - # Tree choses if return tree or list of terminals - # recursive implementation - - # init the sequence - tree = "(" + str(symbol) - # collect possible productions from the starting symbol - productions = list(self.productions(lhs=symbol)) - if not_allowed_productions is not None and len(not_allowed_productions) > 0: - productions = list( - filter( - lambda production: production not in not_allowed_productions, - productions, - ) - ) - - if len(productions) == 0: - raise Exception(f"There is no production possible for {symbol}") - - # sample - if user_priors and self._prior is not None: - probs = self._prior[str(symbol)] - if not_allowed_productions: - # remove prior probs if rule is not allowed - not_allowed_indices = [ - self.productions(lhs=symbol).index(nap) - for nap in not_allowed_productions - ] - probs = [p for i, p in enumerate(probs) if i not in not_allowed_indices] - # rescale s.t. probs sum up to one - cur_prob_sum = sum(probs) - probs = [x / cur_prob_sum for x in probs] - assert len(probs) == len(productions) - - production = np.random.choice(productions, p=probs) - else: - production = np.random.choice(productions) - counter = 0 - if self.constraint_is_class: - constraints = self.constraints.initialize_constraints(production.rhs()[0]) - else: - current_derivation = self.constraints(production.rhs()[0]) - for sym in production.rhs(): - if isinstance(sym, str): - # if terminal then add string to sequence - tree = tree + " " + sym - else: - if self.constraint_is_class: - not_allowed_productions = ( - constraints.get_not_allowed_productions( - productions=self.productions(lhs=sym) - ) - if constraints is not None - else [] - ) - else: - context_information = self.constraints( - production.rhs()[0], - current_derivation, - ) - if isinstance(context_information, list): - not_allowed_productions = self._get_not_allowed_productions( - self.productions(lhs=sym), context_information[counter] - ) - elif isinstance(context_information, bool): - not_allowed_productions = self._get_not_allowed_productions( - self.productions(lhs=sym), context_information - ) - else: - raise NotImplementedError - ret_val = self._constrained_sampler( - sym, not_allowed_productions, user_priors=user_priors - ) - tree = tree + " " + ret_val + ")" - if self.constraint_is_class: - constraints.update_context(ret_val + ")") - else: - current_derivation[counter] = ret_val + ")" - counter += 1 - return tree - - def compute_prior(self, string_tree: str, log: bool = True) -> float: - def skip_char(char: str) -> bool: - if char in [" ", "\t", "\n"]: - return True - # special case: "(" is (part of) a terminal - if ( - i != 0 - and char == "(" - and string_tree[i - 1] == " " - and string_tree[i + 1] == " " - ): - return False - return char == "(" - - def find_longest_match( - i: int, string_tree: str, symbols: list, max_match: int - ) -> int: - # search for longest matching symbol and add it - # assumes that the longest match is the true match - j = min(i + max_match, len(string_tree) - 1) - while j > i and j < len(string_tree): - if string_tree[i:j] in symbols: - break - j -= 1 - if j == i: - raise Exception(f"Terminal or nonterminal at position {i} does not exist") - return j - - prior_prob = 1.0 if not log else 0.0 - - symbols = self.nonterminals + self.terminals - max_match = max(map(len, symbols)) - find_longest_match_func = partial( - find_longest_match, - string_tree=string_tree, - symbols=symbols, - max_match=max_match, - ) - - q_production_rules: LifoQueue = LifoQueue() - current_derivations = {} - - i = 0 - while i < len(string_tree): - char = string_tree[i] - if skip_char(char): - pass - elif char == ")" and string_tree[i - 1] != " ": - # closing symbol of production - production = q_production_rules.get(block=False)[0][0] - idx = self.productions(production.lhs()).index(production) - prior = self.prior[str(production.lhs())] - if any( - prod.rhs()[0] == self.none_operation - for prod in self.productions(production.lhs()) - ): - outer_production = q_production_rules.queue[-1][0][0] - if len(q_production_rules.queue) not in current_derivations: - current_derivations[len(q_production_rules.queue)] = ( - self.constraints(outer_production.rhs()[0]) - ) - context_information = self.constraints( - outer_production.rhs()[0], - current_derivations[len(q_production_rules.queue)], - ) - if isinstance(context_information, list): - not_allowed_productions = self._get_not_allowed_productions( - self.productions(lhs=production.lhs()), - context_information[ - current_derivations[len(q_production_rules.queue)].index( - None - ) - ], - ) - elif isinstance(context_information, bool): - not_allowed_productions = self._get_not_allowed_productions( - self.productions(lhs=production.lhs()), - context_information, - ) - else: - raise NotImplementedError - current_derivations[len(q_production_rules.queue)][ - current_derivations[len(q_production_rules.queue)].index(None) - ] = production.rhs()[0] - if None not in current_derivations[len(q_production_rules.queue)]: - del current_derivations[len(q_production_rules.queue)] - if len(not_allowed_productions) > 0: - # remove prior prior if rule is not allowed - not_allowed_indices = [ - self.productions(lhs=production.lhs()).index(nap) - for nap in not_allowed_productions - ] - prior = [ - p for i, p in enumerate(prior) if i not in not_allowed_indices - ] - # rescale s.t. prior sum up to one - cur_prob_sum = sum(prior) - prior = [x / cur_prob_sum for x in prior] - idx -= sum(idx > i for i in not_allowed_indices) - - prior = prior[idx] - if log: - prior_prob += np.log(prior + 1e-12) - else: - prior_prob *= prior - else: - j = find_longest_match_func(i) - sym = string_tree[i:j] - i = j - 1 - - if sym in self.terminals: - q_production_rules.queue[-1][0] = [ - production - for production in q_production_rules.queue[-1][0] - if production.rhs()[q_production_rules.queue[-1][1]] == sym - ] - q_production_rules.queue[-1][1] += 1 - elif sym in self.nonterminals: - if not q_production_rules.empty(): - q_production_rules.queue[-1][0] = [ - production - for production in q_production_rules.queue[-1][0] - if str(production.rhs()[q_production_rules.queue[-1][1]]) - == sym - ] - q_production_rules.queue[-1][1] += 1 - q_production_rules.put([self.productions(lhs=Nonterminal(sym)), 0]) - else: - raise Exception(f"Unknown symbol {sym}") - i += 1 - - if not q_production_rules.empty(): - raise Exception(f"Error in prior computation for {string_tree}") - - return prior_prob - - def _compute_current_context(self, pre_subtree: str, post_subtree: str): - q_nonterminals: deque = deque() - for sym in pre_subtree.split(" "): - if sym == "": - continue - if sym[0] == "(": - sym = sym[1:] - if sym[-1] == ")": - for _ in range(sym.count(")")): - q_nonterminals.pop() - while sym[-1] == ")": - sym = sym[:-1] - if len(sym) == 1 and sym[0] in [" ", "\t", "\n", "[", "]"]: - continue - if sym in self.nonterminals: - q_nonterminals.append(sym) - - context_start_idx = pre_subtree.rfind(q_nonterminals[-1]) - pre_subtree_context = pre_subtree[context_start_idx:] - topology = pre_subtree_context[len(q_nonterminals[-1]) + 1 :].split(" ")[0] - productions = [ - prod - for prod in self.productions() - if pre_subtree_context[: len(q_nonterminals[-1])] == f"{prod.lhs()}" - and prod.rhs()[0] == topology - ] - if len(productions) == 0: - raise Exception("Cannot find corresponding production!") - - q_context: deque = deque() - current_derivation = [] - rhs_counter = 0 - tmp_str = "" - for s in pre_subtree_context[len(q_nonterminals[-1]) + 1 :].split(" "): - if s == "": - continue - if s[0] == "(": - if len(q_context) == 0 and len(s) > 1: - productions = [ - production - for production in productions - if [ - str(prod_sym) - for prod_sym in production.rhs() - if isinstance(prod_sym, Nonterminal) - ][rhs_counter] - == s[1:] - ] - rhs_counter += 1 - q_context.append(s) - tmp_str += " " + s - elif s[-1] == ")": - tmp_str += " " + s - while s[-1] == ")": - q_context.pop() - s = s[:-1] - if len(q_context) == 0: - tmp_str = tmp_str.strip() - current_derivation.append(tmp_str) - if len(productions) == 1 and len(current_derivation) == len( - self.constraints(productions[0].rhs()[0]) - ): - break - tmp_str = "" - elif len(q_context) > 0: - tmp_str += " " + s - current_derivation.append(None) # type: ignore[arg-type] - rhs_counter += 1 - q_context = deque() - if len(productions) == 1 and len(current_derivation) == len( - self.constraints(productions[0].rhs()[0]) - ): - pass - else: - for s in post_subtree.split(" "): - if s == "": - continue - elif s[0] == "(": - if len(q_context) == 0 and len(s) > 1: - productions = [ - production - for production in productions - if [ - str(prod_sym) - for prod_sym in production.rhs() - if isinstance(prod_sym, Nonterminal) - ][rhs_counter] - == s[1:] - ] - rhs_counter += 1 - q_context.append(s) - tmp_str += " " + s - elif s[-1] == ")": - tmp_str += " " + s - while s[-1] == ")": - if len(q_context) > 0: - q_context.pop() - s = s[:-1] - if len(q_context) == 0: - tmp_str = tmp_str.strip() - current_derivation.append(tmp_str) - if len(productions) == 1 and len(current_derivation) == len( - self.constraints(productions[0].rhs()[0]) - ): - break - tmp_str = "" - elif len(q_context) > 0: - tmp_str += " " + s - - return topology, current_derivation - - def mutate( - self, parent: str, subtree_index: int, subtree_node: str, patience: int = 50 - ): - # chop out subtree - pre, _, post = self.remove_subtree(parent, subtree_index) - if pre != " " and bool(post): - if self.constraint_is_class: - not_allowed_productions = self.constraints.mutate_not_allowed_productions( - subtree_node, - pre, - post, - possible_productions=self.productions(lhs=Nonterminal(subtree_node)), - ) - else: - rhs, current_derivation = self._compute_current_context(pre, post) - context_information = self.constraints( - rhs, - current_derivation, - ) - if isinstance(context_information, list): - not_allowed_productions = self._get_not_allowed_productions( - self.productions(lhs=Nonterminal(subtree_node)), - context_information[ - next( - i for i, cd in enumerate(current_derivation) if cd is None - ) - ], - ) - elif isinstance(context_information, bool): - not_allowed_productions = self._get_not_allowed_productions( - self.productions(lhs=Nonterminal(subtree_node)), - context_information, - ) - else: - raise NotImplementedError - else: - not_allowed_productions = [] - _patience = patience - while _patience > 0: - # only sample subtree -> avoids full sampling of large parse trees - new_subtree = self.sampler( - 1, - start_symbol=subtree_node, - not_allowed_productions=not_allowed_productions, - )[0] - child = pre + new_subtree + post - if parent != child: # ensure that parent is really mutated - break - if ( - len(self.productions(lhs=Nonterminal(subtree_node))) - - len(not_allowed_productions) - == 1 - ): - break - _patience -= 1 - return child.strip() diff --git a/neps/search_spaces/architecture/core_graph_grammar.py b/neps/search_spaces/architecture/core_graph_grammar.py deleted file mode 100644 index dfd39292a..000000000 --- a/neps/search_spaces/architecture/core_graph_grammar.py +++ /dev/null @@ -1,998 +0,0 @@ -import collections -import inspect -import queue -from abc import abstractmethod -from copy import deepcopy -from functools import partial - -import networkx as nx -import numpy as np - -from .cfg import Grammar -from .graph import Graph -from .primitives import AbstractPrimitive -from .topologies import AbstractTopology - - -def get_edge_lists_of_topologies(terminal_map: dict) -> dict: - topology_edge_lists = {} - for k, v in terminal_map.items(): - if inspect.isclass(v): - is_topology = issubclass(v, AbstractTopology) - elif isinstance(v, partial): - is_topology = issubclass(v.func, AbstractTopology) # type: ignore[arg-type] - else: - is_topology = False - if is_topology: - if isinstance(v, partial): - if hasattr(v.func, "get_edge_list"): - func_args = inspect.getfullargspec(v.func.get_edge_list).args # type: ignore[attr-defined] - kwargs = {k: v for k, v in v.keywords.items() if k in func_args} - topology_edge_lists[k] = v.func.get_edge_list(**kwargs) # type: ignore[attr-defined] - elif hasattr(v.func, "edge_list"): - topology_edge_lists[k] = v.func.edge_list # type: ignore[attr-defined] - else: - raise Exception( - f"Please implement a get_edge_list static method for {v.func.__name__} or set edge_list!" - ) - else: - topology_edge_lists[k] = v.edge_list - return topology_edge_lists - - -class CoreGraphGrammar(Graph): - def __init__( - self, - grammars: list[Grammar] | Grammar, - terminal_to_op_names: dict, - terminal_to_graph_edges: dict | None = None, - edge_attr: bool = True, - edge_label: str = "op_name", - zero_op: list | None = None, - identity_op: list | None = None, - name: str | None = None, - scope: str | None = None, - return_all_subgraphs: bool = False, - return_graph_per_hierarchy: bool = False, - ): - super().__init__(name, scope) - - self.grammars = [grammars] if isinstance(grammars, Grammar) else grammars - - self.terminal_to_op_names = terminal_to_op_names - - grammar_terminals = { - terminal for grammar in self.grammars for terminal in grammar.terminals - } - diff_terminals = grammar_terminals - set(self.terminal_to_op_names.keys()) - if len(diff_terminals) != 0: - raise Exception( - f"Terminals {diff_terminals} not defined in primitive mapping!" - ) - - if terminal_to_graph_edges is None: # only compute it once -> more efficient - self.terminal_to_graph_edges = get_edge_lists_of_topologies( - self.terminal_to_op_names - ) - else: - self.terminal_to_graph_edges = terminal_to_graph_edges - self.edge_attr = edge_attr - self.edge_label = edge_label - - self.zero_op = zero_op if zero_op is not None else [] - self.identity_op = identity_op if identity_op is not None else [] - - self.terminal_to_graph_nodes: dict = {} - - self.return_all_subgraphs = return_all_subgraphs - self.return_graph_per_hierarchy = return_graph_per_hierarchy - - def clear_graph(self): - while len(self.nodes()) != 0: - self.remove_node(next(iter(self.nodes()))) - - @abstractmethod - def id_to_string_tree(self, identifier: str): - raise NotImplementedError - - @abstractmethod - def string_tree_to_id(self, string_tree: str): - raise NotImplementedError - - @abstractmethod - def compute_prior(self, log: bool = True): - raise NotImplementedError - - @abstractmethod - def compose_functions(self, flatten_graph: bool = True): - raise NotImplementedError - - @staticmethod - def _check_graph(graph: nx.DiGraph): - if len(graph) == 0 or graph.number_of_edges() == 0: - raise ValueError("Invalid DAG") - - def update_op_names(self): - # update op names - for u, v in self.edges(): - try: - self.edges[u, v].update({"op_name": self.edges[u, v]["op"].get_op_name}) - except Exception: - self.edges[u, v].update({"op_name": self.edges[u, v]["op"].name}) - - def from_stringTree_to_graph_repr( - self, - string_tree: str, - grammar: Grammar, - valid_terminals: collections.abc.KeysView, - edge_attr: bool = True, - sym_name: str = "op_name", - prune: bool = True, - add_subtree_map: bool = False, - return_all_subgraphs: bool | None = None, - return_graph_per_hierarchy: bool | None = None, - ) -> nx.DiGraph | tuple[nx.DiGraph, collections.OrderedDict]: - """Generates graph from parse tree in string representation. - Note that we ignore primitive HPs! - - Args: - string_tree (str): parse tree. - grammar (Grammar): underlying grammar. - valid_terminals (list): list of keys. - edge_attr (bool, optional): Shoud graph be edge attributed (True) or node attributed (False). Defaults to True. - sym_name (str, optional): Attribute name of operation. Defaults to "op_name". - prune (bool, optional): Prune graph, e.g., None operations etc. Defaults to True. - add_subtree_map (bool, optional): Add attribute indicating to which subtrees of - the parse tree the specific part belongs to. Can only be true if you set prune=False! - TODO: Check if we really need this constraint or can also allow pruning. Defaults to False. - return_all_subgraphs (bool, optional): Additionally returns an hierarchical dictionary - containing all subgraphs. Defaults to False. - TODO: check if edge attr also works. - return_graph_per_hierarchy (bool, optional): Additionally returns a graph from each - each hierarchy. - - Returns: - nx.DiGraph: [description] - """ - - def get_node_labels(graph: nx.DiGraph): - return [ - (n, d[sym_name]) - for n, d in graph.nodes(data=True) - if d[sym_name] != "input" and d[sym_name] != "output" - ] - - def get_hierarchicy_dict( - string_tree: str, - subgraphs: dict, - hierarchy_dict: dict | None = None, - hierarchy_level_counter: int = 0, - ): - if hierarchy_dict is None: - hierarchy_dict = {} - if hierarchy_level_counter not in hierarchy_dict: - hierarchy_dict[hierarchy_level_counter] = [] - hierarchy_dict[hierarchy_level_counter].append(string_tree) - node_labels = get_node_labels(subgraphs[string_tree]) - for _, node_label in node_labels: - if node_label in subgraphs: - hierarchy_dict = get_hierarchicy_dict( - node_label, subgraphs, hierarchy_dict, hierarchy_level_counter + 1 - ) - return hierarchy_dict - - def get_graph_per_hierarchy(string_tree: str, subgraphs: dict): - hierarchy_dict = get_hierarchicy_dict( - string_tree=string_tree, subgraphs=subgraphs - ) - - graph_per_hierarchy = collections.OrderedDict() - for k, v in hierarchy_dict.items(): - if k == 0: - graph_per_hierarchy[k] = subgraphs[v[0]] - else: - subgraph_ = graph_per_hierarchy[k - 1].copy() - node_labels = get_node_labels(subgraph_) - for node, node_label in node_labels: - if node_label in list(subgraphs.keys()): - in_nodes = list(subgraph_.predecessors(node)) - out_nodes = list(subgraph_.successors(node)) - node_offset = max(subgraph_.nodes) + 1 - - new_subgraph = nx.relabel.relabel_nodes( - subgraphs[node_label], - mapping={ - n: n + node_offset - for n in subgraphs[node_label].nodes - }, - copy=True, - ) - first_nodes = {e[0] for e in new_subgraph.edges} - second_nodes = {e[1] for e in new_subgraph.edges} - (begin_node,) = first_nodes - second_nodes - (end_node,) = second_nodes - first_nodes - successors = list(new_subgraph.successors(begin_node)) - predecessors = list(new_subgraph.predecessors(end_node)) - new_subgraph.remove_nodes_from([begin_node, end_node]) - edges = [] - added_identities = False - for in_node in in_nodes: - for succ in successors: - if succ == end_node: - if not added_identities: - edges.extend( - [ - (inn, onn) - for inn in in_nodes - for onn in out_nodes - ] - ) - added_identities = True - else: - edges.append((in_node, succ)) - for out_node in out_nodes: - for pred in predecessors: - if pred != begin_node: - edges.append((pred, out_node)) - - subgraph_ = nx.compose(new_subgraph, subgraph_) - subgraph_.add_edges_from(edges) - subgraph_.remove_node(node) - - graph_per_hierarchy[k] = subgraph_ - return graph_per_hierarchy - - def to_node_attributed_edge_list( - edge_list: list[tuple], - ) -> tuple[list[tuple[int, int]], dict]: - node_offset = 2 - edge_to_node_map = {e: i + node_offset for i, e in enumerate(edge_list)} - first_nodes = {e[0] for e in edge_list} - second_nodes = {e[1] for e in edge_list} - (src,) = first_nodes - second_nodes - (tgt,) = second_nodes - first_nodes - node_list = [] - for e in edge_list: - ni = edge_to_node_map[e] - u, v = e - if u == src: - node_list.append((0, ni)) - if v == tgt: - node_list.append((ni, 1)) - - for e_ in filter(lambda e: (e[1] == u), edge_list): - node_list.append((edge_to_node_map[e_], ni)) - - return node_list, edge_to_node_map - - def skip_char(char: str) -> bool: - return char in [" ", "\t", "\n", "[", "]"] - - if prune: - add_subtree_map = False - - if return_all_subgraphs is None: - return_all_subgraphs = self.return_all_subgraphs - if return_graph_per_hierarchy is None: - return_graph_per_hierarchy = self.return_graph_per_hierarchy - compute_subgraphs = return_all_subgraphs or return_graph_per_hierarchy - - G = nx.DiGraph() - if add_subtree_map: - q_nonterminals: collections.deque = collections.deque() - if compute_subgraphs: - q_subtrees: collections.deque = collections.deque() - q_subgraphs: collections.deque = collections.deque() - subgraphs_dict = collections.OrderedDict() - if edge_attr: - node_offset = 0 - q_el: collections.deque = collections.deque() # edge-attr - terminal_to_graph = self.terminal_to_graph_edges - else: # node-attributed - G.add_node(0, **{sym_name: "input"}) - G.add_node(1, **{sym_name: "output"}) - node_offset = 2 - if bool(self.terminal_to_graph_nodes): - terminal_to_graph_nodes = self.terminal_to_graph_nodes - else: - terminal_to_graph_nodes = { - k: to_node_attributed_edge_list(edge_list) if edge_list else [] - for k, edge_list in self.terminal_to_graph_edges.items() - } - self.terminal_to_graph_nodes = terminal_to_graph_nodes - terminal_to_graph = { - k: v[0] if v else [] for k, v in terminal_to_graph_nodes.items() - } - q_el = collections.deque() # node-attr - - # pre-compute stuff - begin_end_nodes = {} - for sym, g in terminal_to_graph.items(): - if g: - first_nodes = {e[0] for e in g} - second_nodes = {e[1] for e in g} - (begin_node,) = first_nodes - second_nodes - (end_node,) = second_nodes - first_nodes - begin_end_nodes[sym] = (begin_node, end_node) - else: - begin_end_nodes[sym] = (None, None) - - for split_idx, sym in enumerate(string_tree.split(" ")): - is_nonterminal = False - if sym == "": - continue - if compute_subgraphs: - new_sym = True - sym_copy = sym[:] - if sym[0] == "(": - sym = sym[1:] - is_nonterminal = True - if sym[-1] == ")": - if add_subtree_map: - for _ in range(sym.count(")")): - q_nonterminals.pop() - if compute_subgraphs: - new_sym = False - while sym[-1] == ")" and sym not in valid_terminals: - sym = sym[:-1] - - if compute_subgraphs and new_sym: - if sym in grammar.nonterminals: - # need dict as a graph can have multiple subgraphs - q_subtrees.append(sym_copy[:]) - else: - q_subtrees[-1] += f" {sym_copy}" - - if len(sym) == 1 and skip_char(sym[0]): - continue - - if add_subtree_map and sym in grammar.nonterminals: - q_nonterminals.append((sym, split_idx)) - elif sym in valid_terminals and not is_nonterminal: # terminal symbol - if sym in self.terminal_to_graph_edges: - if len(q_el) == 0: - if edge_attr: - edges = [ - tuple(t + node_offset for t in e) - for e in self.terminal_to_graph_edges[sym] - ] - else: # node-attr - edges = [ - tuple(t for t in e) - for e in terminal_to_graph_nodes[sym][0] - ] - nodes = [ - terminal_to_graph_nodes[sym][1][e] - for e in self.terminal_to_graph_edges[sym] - ] - if add_subtree_map: - subtrees = [] - first_nodes = {e[0] for e in edges} - second_nodes = {e[1] for e in edges} - (src_node,) = first_nodes - second_nodes - (sink_node,) = second_nodes - first_nodes - else: - begin_node, end_node = begin_end_nodes[sym] - el = q_el.pop() - if edge_attr: - u, v = el - if add_subtree_map: - subtrees = G[u][v]["subtrees"] - G.remove_edge(u, v) - edges = [ - tuple( - u - if t == begin_node - else v - if t == end_node - else t + node_offset - for t in e - ) - for e in self.terminal_to_graph_edges[sym] - ] - else: # node-attr - n = el - if add_subtree_map: - subtrees = G.nodes[n]["subtrees"] - in_nodes = list(G.predecessors(n)) - out_nodes = list(G.successors(n)) - G.remove_node(n) - edges = [] - for e in terminal_to_graph_nodes[sym][0]: - if not (e[0] == begin_node or e[1] == end_node): - edges.append((e[0] + node_offset, e[1] + node_offset)) - elif e[0] == begin_node: - for nin in in_nodes: - edges.append((nin, e[1] + node_offset)) - elif e[1] == end_node: - for nout in out_nodes: - edges.append((e[0] + node_offset, nout)) - nodes = [ - terminal_to_graph_nodes[sym][1][e] + node_offset - for e in self.terminal_to_graph_edges[sym] - ] - - G.add_edges_from(edges) - - if compute_subgraphs: - subgraph = nx.DiGraph() - subgraph.add_edges_from(edges) - q_subgraphs.append( - { - "graph": subgraph, - "atoms": collections.OrderedDict( - (atom, None) - for atom in (edges if edge_attr else nodes) - ), - } - ) - - if add_subtree_map: - if edge_attr: - subtrees.append(q_nonterminals[-1]) - for u, v in edges: - G[u][v]["subtrees"] = subtrees.copy() - else: # node-attr - subtrees.append(q_nonterminals[-1]) - for n in nodes: - G.nodes[n]["subtrees"] = subtrees.copy() - - q_el.extend(reversed(edges if edge_attr else nodes)) - if edge_attr: - node_offset += max(max(self.terminal_to_graph_edges[sym])) - else: - node_offset += max(terminal_to_graph_nodes[sym][1].values()) - else: # primitive operations - el = q_el.pop() - if edge_attr: - u, v = el - if prune and sym in self.zero_op: - G.remove_edge(u, v) - if compute_subgraphs: - q_subgraphs[-1]["graph"].remove_edge(u, v) - del q_subgraphs[-1]["atoms"][(u, v)] - else: - G[u][v][sym_name] = sym - if compute_subgraphs: - q_subgraphs[-1]["graph"][u][v][sym_name] = sym - if add_subtree_map: - G[u][v]["subtrees"].append(q_nonterminals[-1]) - q_nonterminals.pop() - else: # node-attr - n = el - if prune and sym in self.zero_op: - G.remove_node(n) - if compute_subgraphs: - q_subgraphs[-1]["graph"].remove_node(n) - del q_subgraphs[-1]["atoms"][n] - elif prune and sym in self.identity_op: - G.add_edges_from( - [ - (n_in, n_out) - for n_in in G.predecessors(n) - for n_out in G.successors(n) - ] - ) - G.remove_node(n) - if compute_subgraphs: - q_subgraphs[-1]["graph"].add_edges_from( - [ - (n_in, n_out) - for n_in in q_subgraphs[-1]["graph"].predecessors( - n - ) - for n_out in q_subgraphs[-1]["graph"].successors( - n - ) - ] - ) - q_subgraphs[-1]["graph"].remove_node(n) - del q_subgraphs[-1]["atoms"][n] - else: - G.nodes[n][sym_name] = sym - if compute_subgraphs: - q_subgraphs[-1]["graph"].nodes[n][sym_name] = sym - q_subgraphs[-1]["atoms"][ - next( - filter( - lambda x: x[1] is None, - q_subgraphs[-1]["atoms"].items(), - ) - )[0] - ] = sym - if add_subtree_map: - G.nodes[n]["subtrees"].append(q_nonterminals[-1]) - q_nonterminals.pop() - if compute_subgraphs and sym_copy[-1] == ")": - q_subtrees[-1] += f" {sym_copy}" - for _ in range(sym_copy.count(")")): - subtree_identifier = q_subtrees.pop() - if len(q_subtrees) > 0: - q_subtrees[-1] += f" {subtree_identifier}" - if len(q_subtrees) == len(q_subgraphs) - 1: - difference = subtree_identifier.count( - "(" - ) - subtree_identifier.count(")") - if difference < 0: - subtree_identifier = subtree_identifier[:difference] - subgraph_dict = q_subgraphs.pop() - subgraph = subgraph_dict["graph"] - atoms = subgraph_dict["atoms"] - if len(q_subtrees) > 0: - # subtree_identifier is subgraph graph at [-1] - # (and sub-...-subgraph currently in q_subgraphs) - q_subgraphs[-1]["atoms"][ - next( - filter( - lambda x: x[1] is None, - q_subgraphs[-1]["atoms"].items(), - ) - )[0] - ] = subtree_identifier - - for atom in filter(lambda x: x[1] is not None, atoms.items()): - if edge_attr: - subgraph[atom[0][0]][atom[0][1]][sym_name] = atom[1] - else: # node-attr - subgraph.nodes[atom[0]][sym_name] = atom[1] - - if not edge_attr: # node-attr - # ensure there is actually one input and output node - first_nodes = {e[0] for e in subgraph.edges} - second_nodes = {e[1] for e in subgraph.edges} - new_src_node = max(subgraph.nodes) + 1 - src_nodes = first_nodes - second_nodes - subgraph.add_edges_from( - [ - (new_src_node, successor) - for src_node in src_nodes - for successor in subgraph.successors(src_node) - ] - ) - subgraph.add_node(new_src_node, **{sym_name: "input"}) - subgraph.remove_nodes_from(src_nodes) - new_sink_node = max(subgraph.nodes) + 1 - sink_nodes = second_nodes - first_nodes - subgraph.add_edges_from( - [ - (predecessor, new_sink_node) - for sink_node in sink_nodes - for predecessor in subgraph.predecessors(sink_node) - ] - ) - subgraph.add_node(new_sink_node, **{sym_name: "output"}) - subgraph.remove_nodes_from(sink_nodes) - subgraphs_dict[subtree_identifier] = subgraph - - if len(q_el) != 0: - raise Exception("Invalid string_tree") - - if prune: - G = self.prune_unconnected_parts(G, src_node, sink_node) - self._check_graph(G) - - if return_all_subgraphs or return_graph_per_hierarchy: - return_val = [G] - subgraphs_dict = collections.OrderedDict( - reversed(list(subgraphs_dict.items())) - ) - if prune: - for v in subgraphs_dict.values(): - first_nodes = {e[0] for e in v.edges} - second_nodes = {e[1] for e in v.edges} - (vG_src_node,) = first_nodes - second_nodes - (vG_sink_node,) = second_nodes - first_nodes - v = self.prune_unconnected_parts(v, vG_src_node, vG_sink_node) - self._check_graph(v) - if return_all_subgraphs: - return_val.append(subgraphs_dict) - if return_graph_per_hierarchy: - graph_per_hierarchy = get_graph_per_hierarchy(string_tree, subgraphs_dict) - _ = ( - graph_per_hierarchy.popitem() - ) # remove last graph since it is equal to full graph - return_val.append(graph_per_hierarchy) - return return_val - return G - - def get_graph_representation( - self, - identifier: str, - grammar: Grammar, - edge_attr: bool, - ) -> nx.DiGraph: - """This functions takes an identifier and constructs the - (multi-variate) composition of the functions it describes. - - Args: - identifier (str): identifier - grammar (Grammar): grammar - flatten_graph (bool, optional): Whether to flatten the graph. Defaults to True. - - Returns: - nx.DiGraph: (multi-variate) composition of functions. - """ - - def _skip_char(char: str) -> bool: - return char in [" ", "\t", "\n", "[", "]"] - - def _get_sym_from_split(split: str) -> str: - start_idx, end_idx = 0, len(split) - while start_idx < end_idx and split[start_idx] == "(": - start_idx += 1 - while start_idx < end_idx and split[end_idx - 1] == ")": - end_idx -= 1 - return split[start_idx:end_idx] - - def to_node_attributed_edge_list( - edge_list: list[tuple], - ) -> tuple[list[tuple[int, int]], dict]: - first_nodes = {e[0] for e in edge_list} - second_nodes = {e[1] for e in edge_list} - src = first_nodes - second_nodes - tgt = second_nodes - first_nodes - node_offset = len(src) - edge_to_node_map = {e: i + node_offset for i, e in enumerate(edge_list)} - node_list = [] - for e in edge_list: - ni = edge_to_node_map[e] - u, v = e - if u in src: - node_list.append((u, ni)) - if v in tgt: - node_list.append((ni, v)) - - for e_ in filter(lambda e: (e[1] == u), edge_list): - node_list.append((edge_to_node_map[e_], ni)) - - return node_list, edge_to_node_map - - descriptor = self.id_to_string_tree(identifier) - - if edge_attr: - terminal_to_graph = self.terminal_to_graph_edges - else: # node-attr - terminal_to_graph_nodes = { - k: to_node_attributed_edge_list(edge_list) if edge_list else (None, None) - for k, edge_list in self.terminal_to_graph_edges.items() - } - terminal_to_graph = {k: v[0] for k, v in terminal_to_graph_nodes.items()} - # edge_to_node_map = {k: v[1] for k, v in terminal_to_graph_nodes.items()} - - q_nonterminals: queue.LifoQueue = queue.LifoQueue() - q_topologies: queue.LifoQueue = queue.LifoQueue() - q_primitives: queue.LifoQueue = queue.LifoQueue() - - G = nx.DiGraph() - for _, split in enumerate(descriptor.split(" ")): - if _skip_char(split): - continue - sym = _get_sym_from_split(split) - - if sym in grammar.terminals: - is_topology = False - if ( - inspect.isclass(self.terminal_to_op_names[sym]) - and issubclass(self.terminal_to_op_names[sym], AbstractTopology) - or isinstance(self.terminal_to_op_names[sym], partial) - and issubclass(self.terminal_to_op_names[sym].func, AbstractTopology) - ): - is_topology = True - - if is_topology: - q_topologies.put([self.terminal_to_op_names[sym], 0]) - else: # is primitive operation - q_primitives.put(self.terminal_to_op_names[sym]) - q_topologies.queue[-1][1] += 1 # count number of primitives - elif sym in grammar.nonterminals: - q_nonterminals.put(sym) - else: - raise Exception(f"Unknown symbol {sym}") - - if ")" in split: - # closing symbol of production - while ")" in split: - if q_nonterminals.qsize() == q_topologies.qsize(): - topology, number_of_primitives = q_topologies.get(block=False) - primitives = [ - q_primitives.get(block=False) - for _ in range(number_of_primitives) - ][::-1] - if ( - topology in terminal_to_graph - and terminal_to_graph[topology] is not None - ) or isinstance(topology, partial): - raise NotImplementedError - else: - composed_function = topology(*primitives) - node_attr_dag = composed_function.get_node_list_and_ops() - G = node_attr_dag # TODO only works for DARTS for now - - if not q_topologies.empty(): - q_primitives.put(composed_function) - q_topologies.queue[-1][1] += 1 - - _ = q_nonterminals.get(block=False) - split = split[:-1] - - if not q_topologies.empty(): - raise Exception("Invalid descriptor") - - # G = self.prune_unconnected_parts(G, src_node, sink_node) - # self._check_graph(G) - return G - - def prune_graph(self, graph: nx.DiGraph | Graph = None, edge_attr: bool = True): - use_self = graph is None - if use_self: - graph = self - - in_degree = [n for n in graph.nodes() if graph.in_degree(n) == 0] - if len(in_degree) != 1: - raise Exception(f"Multiple in degree nodes: {in_degree}") - else: - src_node = in_degree[0] - out_degree = [n for n in graph.nodes() if graph.out_degree(n) == 0] - if len(out_degree) != 1: - raise Exception(f"Multiple out degree nodes: {out_degree}") - else: - tgt_node = out_degree[0] - - if edge_attr: - # remove edges with none - remove_edge_list = [] - for u, v, edge_data in graph.edges.data(): - if isinstance(edge_data.op, Graph): - self.prune_graph(edge_data.op, edge_attr=edge_attr) - elif isinstance(edge_data.op, list): - for op in edge_data.op: - if isinstance(op, Graph): - self.prune_graph(op, edge_attr=edge_attr) - elif isinstance(edge_data.op, AbstractPrimitive) or issubclass( - edge_data.op, AbstractPrimitive - ): - try: - if any(zero_op in edge_data.op_name for zero_op in self.zero_op): - remove_edge_list.append((u, v)) - except TypeError: - if any( - zero_op in edge_data.op.get_op_name - for zero_op in self.zero_op - ): - remove_edge_list.append((u, v)) - elif inspect.isclass(edge_data.op): - assert not issubclass( - edge_data.op, Graph - ), "Found non-initialized graph. Abort." - # we look at an uncomiled op - else: - raise ValueError(f"Unknown format of op: {edge_data.op}") - # remove_edge_list = [ - # e for e in graph.edges(data=True) if e[-1]["op_name"] in self.zero_op - # ] - graph.remove_edges_from(remove_edge_list) - else: - for n in list(nx.topological_sort(graph)): - if n in graph.nodes() and ( - graph.nodes[n]["op_name"] in self.zero_op - or graph.nodes[n]["op_name"] in self.identity_op - ): - if graph.nodes[n]["op_name"] in self.identity_op: - # reconnect edges for removed nodes with 'skip_connect' - graph.add_edges_from( - [ - (e_i[0], e_o[1]) - for e_i in graph.in_edges(n) - for e_o in graph.out_edges(n) - ] - ) - # remove nodes with 'skip_connect' or 'none' label - graph.remove_node(n) - - graph = self.prune_unconnected_parts(graph, src_node, tgt_node) - - if not use_self: - return graph - return None - - @staticmethod - def prune_unconnected_parts(graph, src_node, tgt_node): - def _backtrack_remove(graph, node: int): - predecessors = collections.deque(graph.predecessors(node)) - graph.remove_node(node) - while len(predecessors) > 0: - u = predecessors.pop() - if u not in graph.nodes(): # if it is already removed skip - continue - if ( - len(list(graph.successors(u))) > 0 - ): # there are more edges that could be valid paths - continue - graph = _backtrack_remove(graph, u) - return graph - - # after removal, some op nodes have no input nodes and some have no output nodes - # --> remove these redundant nodes - # O(|V|^2), but mostly O(|V|) (no zero op) - for n in list(nx.topological_sort(graph)): - if n in graph.nodes(): - predecessors = list(graph.predecessors(n)) - successors = list(graph.successors(n)) - if n != src_node and len(predecessors) == 0: - graph.remove_node(n) - elif n != tgt_node and len(successors) == 0: - graph = _backtrack_remove(graph, n) - return graph - - @staticmethod - def flatten_graph( - graph: nx.DiGraph, - flattened_graph: Graph = None, - start_node: int | None = None, - end_node: int | None = None, - ): - if flattened_graph is None: - flattened_graph = Graph() - nodes: dict = {} - for u, v, data in graph.edges(data=True): - if u in nodes: - _u = nodes[u] - else: - _u = ( - 1 - if len(flattened_graph.nodes.keys()) == 0 # type: ignore[union-attr] - else max(flattened_graph.nodes.keys()) + 1 # type: ignore[union-attr] - ) - _u = ( - start_node - if graph.in_degree(u) == 0 and start_node is not None - else _u - ) - nodes[u] = _u - if _u not in flattened_graph.nodes: # type: ignore[union-attr] - flattened_graph.add_node(_u) # type: ignore[union-attr] - flattened_graph.nodes[_u].update(graph.nodes[u]) # type: ignore[union-attr] - - if v in nodes: - _v = nodes[v] - else: - _v = max(flattened_graph.nodes.keys()) + 1 # type: ignore[union-attr] - _v = end_node if graph.out_degree(v) == 0 and end_node is not None else _v - nodes[v] = _v - if _v not in flattened_graph.nodes: # type: ignore[union-attr] - flattened_graph.add_node(_v) # type: ignore[union-attr] - flattened_graph.nodes[_v].update( # type: ignore[union-attr] - graph.nodes[v] - ) # last time node is called combo op is used - - if isinstance(data["op"], Graph): - flattened_graph = CoreGraphGrammar.flatten_graph( - data["op"], flattened_graph, start_node=_u, end_node=_v - ) - else: - flattened_graph.add_edge(_u, _v) # type: ignore[union-attr] - flattened_graph.edges[_u, _v].update(data) # type: ignore[union-attr] - - return flattened_graph - - def _compose_functions( - self, identifier: str, grammar: Grammar, flatten_graph: bool = True - ) -> nx.DiGraph: - """This functions takes an identifier and constructs the - (multi-variate) composition of the functions it describes. - - Args: - identifier (str): identifier - grammar (Grammar): grammar - flatten_graph (bool, optional): Whether to flatten the graph. Defaults to True. - - Returns: - nx.DiGraph: (multi-variate) composition of functions - """ - descriptor = self.id_to_string_tree(identifier) - - symbols = grammar.nonterminals + grammar.terminals - max_match = max(map(len, symbols)) - find_longest_match_func = partial( - find_longest_match, - descriptor=descriptor, - symbols=symbols, - max_match=max_match, - ) - - q_nonterminals: queue.LifoQueue = queue.LifoQueue() - q_topologies: queue.LifoQueue = queue.LifoQueue() - q_primitives: queue.LifoQueue = queue.LifoQueue() - i = 0 - while i < len(descriptor): - char = descriptor[i] - if skip_char(char, descriptor, i): - pass - elif char == ")" and descriptor[i - 1] != " ": - # closing symbol of production - if q_nonterminals.qsize() == q_topologies.qsize(): - topology, number_of_primitives = q_topologies.get(block=False) - primitives = [ - q_primitives.get(block=False) for _ in range(number_of_primitives) - ][::-1] - composed_function = topology(*primitives) - if not q_topologies.empty(): - q_primitives.put(composed_function) - q_topologies.queue[-1][1] += 1 - _ = q_nonterminals.get(block=False) - else: - j = find_longest_match_func(i) - sym = descriptor[i:j] - i = j - 1 - - if sym in grammar.terminals and descriptor[i - 1] != "(": - is_topology = False - if ( - inspect.isclass(self.terminal_to_op_names[sym]) - and issubclass(self.terminal_to_op_names[sym], AbstractTopology) - or isinstance(self.terminal_to_op_names[sym], partial) - and issubclass( - self.terminal_to_op_names[sym].func, AbstractTopology - ) - ): - is_topology = True - - if is_topology: - q_topologies.put([self.terminal_to_op_names[sym], 0]) - else: # is primitive operation - q_primitives.put(self.terminal_to_op_names[sym]) - q_topologies.queue[-1][1] += 1 # count number of primitives - elif sym in grammar.nonterminals: - q_nonterminals.put(sym) - else: - raise Exception(f"Unknown symbol {sym}") - - i += 1 - - if not q_topologies.empty(): - raise Exception("Invalid descriptor") - - self._check_graph(composed_function) - - if flatten_graph: - composed_function = self.flatten_graph(composed_function) - - return composed_function - - def graph_to_self(self, graph: nx.DiGraph, clear_self: bool = True) -> None: - """Copies graph to self. - - Args: - graph (nx.DiGraph): graph - """ - if clear_self: - self.clear() - for u, v, data in graph.edges(data=True): - self.add_edge(u, v) # type: ignore[union-attr] - self.edges[u, v].update(data) # type: ignore[union-attr] - for n, data in graph.nodes(data=True): - self.nodes[n].update(**data) - - -def skip_char(char: str, descriptor: str, i: int) -> bool: - if char in [" ", "\t", "\n"]: - return True - # special case: "(" is (part of) a terminal - if i != 0 and char == "(" and descriptor[i - 1] == " " and descriptor[i + 1] == " ": - return False - return char == "(" - - -def find_longest_match( - i: int, descriptor: str, symbols: list[str], max_match: int -) -> int: - # search for longest matching symbol and add it - # assumes that the longest match is the true match - j = min(i + max_match, len(descriptor) - 1) - while j > i and j < len(descriptor): - if descriptor[i:j] in symbols: - break - j -= 1 - if j == i: - raise Exception(f"Terminal or nonterminal at position {i} does not exist") - return j diff --git a/neps/search_spaces/architecture/graph.py b/neps/search_spaces/architecture/graph.py deleted file mode 100644 index b7dd5b5ec..000000000 --- a/neps/search_spaces/architecture/graph.py +++ /dev/null @@ -1,690 +0,0 @@ -from __future__ import annotations - -import copy -import inspect -import logging -import random -import types -from pathlib import Path -from more_itertools import collapse - -import networkx as nx -import torch -from networkx.algorithms.dag import lexicographical_topological_sort -from torch import nn - -from .primitives import AbstractPrimitive, Identity - -logger = logging.getLogger(__name__) - - -class Graph(torch.nn.Module, nx.DiGraph): - """Base class for defining a search space. Add nodes and edges - as for a directed acyclic graph in `networkx`. Nodes can contain - graphs as children, also edges can contain graphs as operations. - - Note, if a graph is copied, the shared attributes of its edges are - shallow copies whereas the private attributes are deep copies. - - To differentiate copies of the same graph you can define a `scope` - with `set_scope()`. - - **Graph at nodes:** - >>> graph = Graph() - >>> graph.add_node(1, subgraph=Graph()) - - If the node has more than one input use `set_input()` to define the - routing to the input nodes of the subgraph. - - **Graph at edges:** - >>> graph = Graph() - >>> graph.add_nodes_from([1, 2]) - >>> graph.add_edge(1, 2, EdgeData({'op': Graph()})) - - **Modify the graph after definition** - - If you want to modify the graph e.g. in an optimizer once - it has been defined already use the function `update_edges()` - or `update_nodes()`. - - **Use as pytorch module** - If you want to learn the weights of the operations or any - other parameters of the graph you have to parse it first. - >>> graph = getFancySearchSpace() - >>> graph.parse() - >>> logits = graph(data) - >>> optimizer.min(objective_to_minimize(logits, target)) - - To update the pytorch module representation (e.g. after removing or adding - some new edges), you have to unparse. Beware that this is not fast, so it should - not be done on each batch or epoch, rather once after discretizising. If you - want to change the representation of the graph use rather some shared operation - indexing at the edges. - >>> graph.update(remove_random_edges) - >>> graph.unparse() - >>> graph.parse() - >>> logits = graph(data) - - """ - - """ - Usually the optimizer does not operate on the whole graph, e.g. preprocessing - and post-processing are excluded. Scope can be used to define that or to - differentate instances of the "same" graph. - """ - OPTIMIZER_SCOPE = "all" - - """ - Whether the search space has an interface to one of the tabular benchmarks which - can then be used to query architecture performances. - - If this is set to true then `query()` should be implemented. - """ - QUERYABLE = False - - def __init__(self, name: str | None = None, scope: str | None = None): - """Initialise a graph. The edges are automatically filled with an EdgeData object - which defines the default operation as Identity. The default combination operation - is set as sum. - - Note: - When inheriting form `Graph` note that `__init__()` cannot take any - parameters. This is due to the way how networkx is implemented, i.e. graphs - are reconstructed internally and no parameters for init are considered. - - Our recommended solution is to create static attributes before initialization - and then load them dynamically in `__init__()`. - - >>> def __init__(self): - >>> num_classes = self.NUM_CLASSES - >>> MyGraph.NUM_CLASSES = 42 - >>> my_graph_42_classes = MyGraph() - - """ - # super().__init__() - nx.DiGraph.__init__(self) - torch.nn.Module.__init__(self) - - # Make DiGraph a member and not inherit. This is because when inheriting from - # `Graph` note that `__init__()` cannot take any parameters. This is due to - # the way how networkx is implemented, i.e. graphs are reconstructed internally - # and no parameters for init are considered. - # Therefore __getattr__ and __iter__ forward the DiGraph methods for straight-forward - # usage as if we would inherit. - - # self._nxgraph = nx.DiGraph() - - # Replace the default dicts at the edges with `EdgeData` objects - # `EdgeData` can be easily customized and allow shared parameters - # across different Graph instances. - - # self._nxgraph.edge_attr_dict_factory = lambda: EdgeData() - self.edge_attr_dict_factory = lambda: EdgeData() - - # Replace the default dicts at the nodes to include `input` from the beginning. - # `input` is required for storing the results of incoming edges. - - # self._nxgraph.node_attr_dict_factory = lambda: dict({'input': {}, 'comb_op': sum}) - self.node_attr_dict_factory = lambda: {"input": {}, "comb_op": sum} - - # remember to add all members also in `unparse()` - self.name = name - self.scope = scope - self.input_node_idxs = None - self.is_parsed = False - self._id = random.random() # pytorch expects unique modules in `add_module()` - - def __eq__(self, other): - return self.name == other.name and self.scope == other.scope - - def __hash__(self): - """As it is very complicated to compare graphs (i.e. check all edge - attributes, do the have shared attributes, ...) use just the name - for comparison. - - This is used when determining whether two instances are copies. - """ - h = 0 - h += hash(self.name) - h += hash(self.scope) if self.scope else 0 - h += hash(self._id) - return h - - def __repr__(self): - return f"Graph {self.name}-{self._id:.07f}, scope {self.scope}, {self.number_of_nodes()} nodes" - - def set_scope(self, scope: str, recursively=True): - """Sets the scope of this instance of the graph. - - The function should be used in a builder-like pattern - `'subgraph'=Graph().set_scope("scope")`. - - Args: - scope (str): the scope - recursively (bool): Also set the scope for all child graphs. - default True - - Returns: - Graph: self with the setted scope. - """ - self.scope = scope - if recursively: - for g in self._get_child_graphs(single_instances=False): - g.scope = scope - return self - - def add_node(self, node_index, **attr): - """Adds a node to the graph. - - Note that adding a node using an index that has been used already - will override its attributes. - - Args: - node_index (int): The index for the node. Expect to be >= 1. - **attr: The attributes which can be added in a dict like form. - """ - assert node_index >= 1, "Expecting the node index to be greater or equal 1" - nx.DiGraph.add_node(self, node_index, **attr) - - def copy(self): - """Copy as defined in networkx, i.e. a shallow copy. - - Just handling recursively nested graphs seperately. - """ - - def copy_dict(d): - copied_dict = d.copy() - for k, v in d.items(): - if isinstance(v, Graph): - copied_dict[k] = v.copy() - elif isinstance(v, list): - copied_dict[k] = [i.copy() if isinstance(i, Graph) else i for i in v] - elif isinstance(v, (AbstractPrimitive, torch.nn.Module)): - copied_dict[k] = copy.deepcopy(v) - return copied_dict - - G = self.__class__() - G.graph.update(self.graph) - G.add_nodes_from((n, copy_dict(d)) for n, d in self._node.items()) - G.add_edges_from( - (u, v, datadict.copy()) - for u, nbrs in self._adj.items() - for v, datadict in nbrs.items() - ) - G.scope = self.scope - G.name = self.name - return G - - def to_pytorch(self, **kwargs) -> nn.Module: - return self._to_pytorch(**kwargs) - - def _to_pytorch(self, write_out: bool = False) -> nn.Module: - def _import_code(code: str, name: str): - module = types.ModuleType(name) - exec(code, module.__dict__) - return module - - if not self.is_parsed: - self.parse() - - input_node = next(n for n in self.nodes if self.in_degree(n) == 0) - input_name = "x0" - self.nodes[input_node]["input"] = {0: input_name} - - forward_f = [] - used_input_names = [int(input_name[1:])] - submodule_list = [] - for node_idx in lexicographical_topological_sort(self): - node = self.nodes[node_idx] - if "subgraph" in node: - # TODO implementation not checked yet! - max_xidx = max(used_input_names) - submodule = node["subgraph"].to_pytorch(write_out=write_out) - submodule_list.append(submodule) - _forward_f = f"x{max_xidx + 1}=self.module_list[{len(submodule_list) - 1}]({node['input']})" - input_name = f"x{max_xidx + 1}" - used_input_names.append(max_xidx + 1) - forward_f.append(_forward_f) - x = f"x{max_xidx + 1}" - else: - if len(node["input"].values()) == 1: - x = next(iter(node["input"].values())) - else: - max_xidx = max(used_input_names) - if ( - "__name__" in dir(node["comb_op"]) - and node["comb_op"].__name__ == "sum" - ): - _forward_f = f"x{max_xidx + 1}=sum([" - elif isinstance(node["comb_op"], torch.nn.Module): - submodule_list.append(node["comb_op"]) - _forward_f = f"x{max_xidx + 1}=self.module_list[{len(submodule_list) - 1}]([" - else: - raise NotImplementedError - - for inp in node["input"].values(): - _forward_f += inp + "," - _forward_f = _forward_f[:-1] + "])" - forward_f.append(_forward_f) - x = f"x{max_xidx + 1}" - if int(x[1:]) not in used_input_names: - used_input_names.append(int(x[1:])) - node["input"] = {} # clear the input as we have processed it - if ( - len(list(self.neighbors(node_idx))) == 0 - and node_idx < list(lexicographical_topological_sort(self))[-1] - ): - # We have more than one output node. This is e.g. the case for - # auxillary losses. Attach them to the graph, handling must done - # by the user. - raise NotImplementedError - else: - # outgoing edges: process all outgoing edges - for neigbor_idx in self.neighbors(node_idx): - max_xidx = max(used_input_names) - edge_data = self.get_edge_data(node_idx, neigbor_idx) - # inject edge data only for AbstractPrimitive, not Graphs - if isinstance(edge_data.op, Graph): - submodule = edge_data.op.to_pytorch(write_out=write_out) - submodule_list.append(submodule) - _forward_f = f"x{max_xidx + 1}=self.module_list[{len(submodule_list) - 1}]({x})" - input_name = f"x{max_xidx + 1}" - used_input_names.append(max_xidx + 1) - forward_f.append(_forward_f) - elif isinstance(edge_data.op, AbstractPrimitive): - # edge_data.op.forward = partial( # type: ignore[assignment] - # edge_data.op.forward, edge_data=edge_data - # ) - submodule_list.append(edge_data.op) - _forward_f = f"x{max_xidx + 1}=self.module_list[{len(submodule_list) - 1}]({x})" - input_name = f"x{max_xidx + 1}" - used_input_names.append(max_xidx + 1) - forward_f.append(_forward_f) - else: - raise ValueError( - f"Unknown class as op: {edge_data.op}. Expected either Graph or AbstactPrimitive" - ) - self.nodes[neigbor_idx]["input"].update({node_idx: input_name}) - - forward_f.append(f"return {x}") - - model_file = "# Auto generated\nimport torch\nimport torch.nn\n\nclass Model(torch.nn.Module):\n\tdef __init__(self):\n" - model_file += "\t\tsuper().__init__()\n" - model_file += "\t\tself.module_list=torch.nn.ModuleList()\n" - model_file += "\n\tdef set_module_list(self,module_list):\n" - model_file += "\t\tself.module_list=torch.nn.ModuleList(module_list)\n" - model_file += "\n\tdef forward(self,x0,*args):\n" - for forward_lines in forward_f: - for forward_line in ( - [forward_lines] if isinstance(forward_lines, str) else forward_lines - ): - model_file += f"\t\t{forward_line}\n" - - try: - module_model = _import_code(model_file, "model") - model = module_model.Model() - except Exception as e: - raise Exception(e) from e - - model.set_module_list(submodule_list) - - if write_out: - tmp_path = Path(__file__).parent.resolve() / "model.py" - with open(tmp_path, "w", encoding="utf-8") as outfile: - outfile.write(model_file) - - return model - - def parse(self): - """Convert the graph into a neural network which can then - be optimized by pytorch. - """ - for node_idx in lexicographical_topological_sort(self): - if "subgraph" in self.nodes[node_idx]: - self.nodes[node_idx]["subgraph"].parse() - self.add_module( - f"{self.name}-subgraph_at({node_idx})", - self.nodes[node_idx]["subgraph"], - ) - elif isinstance(self.nodes[node_idx]["comb_op"], torch.nn.Module): - self.add_module( - f"{self.name}-comb_op_at({node_idx})", - self.nodes[node_idx]["comb_op"], - ) - - for neigbor_idx in self.neighbors(node_idx): - edge_data = self.get_edge_data(node_idx, neigbor_idx) - if isinstance(edge_data.op, Graph): - edge_data.op.parse() - elif edge_data.op.get_embedded_ops(): - for primitive in edge_data.op.get_embedded_ops(): - if isinstance(primitive, Graph): - primitive.parse() - - self.add_module( - f"{self.name}-edge({node_idx},{neigbor_idx})", - edge_data.op, - ) - self.is_parsed = True - - def unparse(self): - """Undo the pytorch parsing by reconstructing the graph uusing the - networkx data structures. - - This is done recursively also for child graphs. - - Returns: - Graph: An unparsed shallow copy of the graph. - """ - g = self.__class__() - g.clear() - - graph_nodes = self.nodes - graph_edges = self.edges - - # unparse possible child graphs - # be careful with copying/deepcopying here cause of shared edge data - for _, data in graph_nodes.data(): - if "subgraph" in data: - data["subgraph"] = data["subgraph"].unparse() - for _, _, data in graph_edges.data(): - if isinstance(data.op, Graph): - data.set("op", data.op.unparse()) - - # create the new graph - # Remember to add all members here to update. I know it is ugly but don't know better - g.add_nodes_from(graph_nodes.data()) - g.add_edges_from(graph_edges.data()) - g.graph.update(self.graph) - g.name = self.name - g.input_node_idxs = self.input_node_idxs - g.scope = self.scope - g.is_parsed = False - g._id = self._id - g.OPTIMIZER_SCOPE = self.OPTIMIZER_SCOPE - g.QUERYABLE = self.QUERYABLE - - return g - - def _get_child_graphs(self, single_instances: bool = False) -> list: - """Get all child graphs of the current graph. - - Args: - single_instances (bool): Whether to return multiple instances - (i.e. copies) of the same graph. When changing shared data - this should be set to True. - - Returns: - list: A list of all child graphs (can be empty) - """ - graphs = [] - for node_idx in lexicographical_topological_sort(self): - node_data = self.nodes[node_idx] - if "subgraph" in node_data: - graphs.append(node_data["subgraph"]) - graphs.append(node_data["subgraph"]._get_child_graphs()) - - for _, _, edge_data in self.edges.data(): - if isinstance(edge_data.op, Graph): - graphs.append(edge_data.op) - graphs.append(edge_data.op._get_child_graphs()) - elif isinstance(edge_data.op, list): - for op in edge_data.op: - if isinstance(op, Graph): - graphs.append(op) - graphs.append(op._get_child_graphs()) - elif isinstance(edge_data.op, AbstractPrimitive): - # maybe it is an embedded op? - embedded_ops = edge_data.op.get_embedded_ops() - if embedded_ops is not None: - if isinstance(embedded_ops, Graph): - graphs.append(embedded_ops) - graphs.append(embedded_ops._get_child_graphs()) - elif isinstance(embedded_ops, list): - for child_op in edge_data.op.get_embedded_ops(): - if isinstance(child_op, Graph): - graphs.append(child_op) - graphs.append(child_op._get_child_graphs()) - else: - logger.debug( - f"Got embedded op, but is neither a graph nor a list: {embedded_ops}" - ) - elif inspect.isclass(edge_data.op): - assert not issubclass( - edge_data.op, Graph - ), "Found non-initialized graph. Abort." - # we look at an uncomiled op - elif callable(edge_data.op): - pass - else: - raise ValueError(f"Unknown format of op: {edge_data.op}") - - graphs = list(collapse(graphs)) - - if single_instances: - single: list = [] - for g in graphs: - if g.name not in [sg.name for sg in single]: - single.append(g) - return sorted(single, key=lambda g: g.name) - else: - return sorted(graphs, key=lambda g: g.name) - - def compile(self): - """Instanciates the ops at the edges using the arguments specified at the edges.""" - for graph in [*self._get_child_graphs(single_instances=False), self]: - logger.debug(f"Compiling graph {graph.name}") - for _, v, edge_data in graph.edges.data(): - if not edge_data.is_final(): - attr = edge_data.to_dict() - op = attr.pop("op") - - if isinstance(op, list): - compiled_ops = [] - for i, o in enumerate(op): - if inspect.isclass(o): - # get the relevant parameter if there are more. - a = { - k: v[i] if isinstance(v, list) else v - for k, v in attr.items() - } - compiled_ops.append(o(**a)) - else: - logger.debug(f"op {o} already compiled. Skipping") - edge_data.set("op", compiled_ops) - elif isinstance(op, AbstractPrimitive): - logger.debug(f"op {op} already compiled. Skipping") - elif inspect.isclass(op) and issubclass(op, AbstractPrimitive): - # Init the class - if "op_name" in attr: - del attr["op_name"] - edge_data.set("op", op(**attr)) - elif isinstance(op, Graph): - pass # This is already covered by _get_child_graphs - else: - raise ValueError(f"Unkown format of op: {op}") - - def clone(self): - """Deep copy of the current graph. - - Returns: - Graph: Deep copy of the graph. - """ - return copy.deepcopy(self) - - -class EdgeData: - """Class that holds data for each edge. - Data can be shared between instances of the graph - where the edges lives in. - - Also defines the default key 'op', which is `Identity()`. It must - be private always. - - Items can be accessed directly as attributes with `.key` or - in a dict-like fashion with `[key]`. To set a new item use `.set()`. - """ - - def __init__(self, data: dict | None = None): - """Initializes a new EdgeData object. - 'op' is set as Identity() and private by default. - - Args: - data (dict): Inject some initial data. Will be always private. - """ - if data is None: - data = {} - self._private = {} - self._shared = {} - - # set internal attributes - self._shared["_deleted"] = False - self._private["_final"] = False - - # set defaults and potential input - self.set("op", Identity(), shared=False) - for k, v in data.items(): - self.set(k, v, shared=False) - - def __getitem__(self, key: str): - assert not str(key).startswith("_"), "Access to private keys not allowed!" - return self.__getattr__(str(key)) - - def get(self, key: str, default): - try: - return self.__getattr__(key) - except AttributeError: - return default - - def __getattr__(self, key: str): - if key.startswith("__"): # Required for deepcopy, not sure why - raise AttributeError(key) - assert not key.startswith("_"), "Access to private keys not allowed!" - if key in self._private: - return self._private[key] - elif key in self._shared: - return self._shared[key] - else: - raise AttributeError(f"Cannot find field '{key}' in the given EdgeData!") - - def __setattr__(self, name: str, val): - if name.startswith("_"): - super().__setattr__(name, val) - else: - raise ValueError("not allowed. use set().") - - def __str__(self): - return f"private: <{self._private!s}>, shared: <{self._shared!s}>" - - def __repr__(self): - return self.__str__() - - def update(self, data): - """Update the data in here. If the data is added as dict, - then all variables will be handled as private. - - Args: - data (EdgeData or dict): If dict, then values will be set as - private. If EdgeData then all entries will be replaced. - """ - if isinstance(data, dict): - for k, v in data.items(): - self.set(k, v) - elif isinstance(data, EdgeData): - # TODO: do update and not replace! - self.__dict__.update(data.__dict__) - else: - raise ValueError(f"Unsupported type {data}") - - def remove(self, key: str): - """Removes an item from the EdgeData. - - Args: - key (str): The key for the item to be removed. - """ - if key in self._private: - del self._private[key] - elif key in self._shared: - del self._shared[key] - else: - raise KeyError(f"Tried to delete unkown key {key}") - - def copy(self): - """When a graph is copied to get multiple instances (e.g. when - reusing subgraphs at more than one location) then - this function will be called for all edges. - - It will create a deep copy for the private entries but - only a shallow copy for the shared entries. E.g. architectural - weights should be shared, but parameters of a 3x3 convolution not. - - Therefore 'op' must be always private. - - Returns: - EdgeData: A new EdgeData object with independent private - items, but shallow shared items. - """ - new_self = EdgeData() - new_self._private = copy.deepcopy(self._private) - new_self._shared = self._shared - - # we need to handle copy of graphs seperately - for k, v in self._private.items(): - if isinstance(v, Graph): - new_self._private[k] = v.copy() - elif isinstance(v, list): - new_self._private[k] = [ - i.copy() if isinstance(i, Graph) else i for i in v - ] - - return new_self - - def set(self, key: str, value, shared=False): - """Used to assign a new item to the EdgeData object. - - Args: - key (str): The key. - value (object): The value to store - shared (bool): Default: False. Whether the item should - be a shallow copy between different instances of EdgeData - (and consequently between different instances of Graph). - """ - assert isinstance(key, str), f"Accepting only string keys, got {type(key)}" - assert not key.startswith("_"), "Access to private keys not allowed!" - assert not self.is_final(), "Trying to change finalized edge!" - if shared: - if key in self._private: - raise ValueError("Key {} alredy defined as non-shared") - else: - self._shared[key] = value - elif key in self._shared: - raise ValueError(f"Key {key} alredy defined as shared") - else: - self._private[key] = value - - def clone(self): - """Return a true deep copy of EdgeData. Even shared - items are not shared anymore. - - Returns: - EdgeData: New independent instance. - """ - return copy.deepcopy(self) - - def is_final(self): - """Returns: - bool: True if the edge was finalized, False else. - """ - return self._private["_final"] - - def to_dict(self, subset="all"): - if subset == "shared": - return {k: v for k, v in self._shared.items() if not k.startswith("_")} - elif subset == "private": - return {k: v for k, v in self._private.items() if not k.startswith("_")} - elif subset == "all": - d = self.to_dict("private") - d.update(self.to_dict("shared")) - return d - else: - raise ValueError(f"Unknown subset {subset}") diff --git a/neps/search_spaces/architecture/graph_grammar.py b/neps/search_spaces/architecture/graph_grammar.py deleted file mode 100644 index e8ce9e906..000000000 --- a/neps/search_spaces/architecture/graph_grammar.py +++ /dev/null @@ -1,308 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from copy import deepcopy -from typing import Any, ClassVar, Mapping -from typing_extensions import override, Self -from neps.utils.types import NotSet -from typing import TYPE_CHECKING, Any, ClassVar, Mapping -from typing_extensions import Self, override - -import networkx as nx - -from neps.search_spaces.parameter import ParameterWithPrior -from neps.utils.types import NotSet - -from .core_graph_grammar import CoreGraphGrammar -from .mutations import bananas_mutate, simple_mutate - -if TYPE_CHECKING: - from .cfg import Grammar - - -# TODO(eddiebergman): This is a halfway solution, but essentially a lot -# of things `Parameter` does, does not fit nicely with a Graph based -# parameters, in the future we probably better just have these as two seperate -# classes. For now, this class sort of captures the overlap between -# `Parameter` and Graph based parameters. -# The problem here is that the `Parameter` expects the `load_from` -# and the `.value` to be the same type, which is not the case for -# graph based parameters. -class GraphParameter( # noqa: D101 - ParameterWithPrior[nx.DiGraph, str] -): - # NOTE(eddiebergman): What I've managed to learn so far is that - # these hyperparameters work mostly with strings externally, - # i.e. setting the value through `load_from` or `set_value` should be a string. - # At that point, the actual `.value` is a graph object created from said - # string. This would most likely break with a few things in odd places - # and I'm surprised it's lasted this long. - # At serialization time, it doesn't actually serialize the .value but instead - # relies on the string it was passed initially, I'm not actually sure if there's - # a way to go from the graph object to the string in this code... - # Essentially on the outside, we need to ensure we don't pass ih the graph object - # itself - DEFAULT_CONFIDENCE_SCORES: ClassVar[Mapping[str, float]] = {"not_in_use": 1.0} - prior_confidence_choice = "not_in_use" - has_prior: bool - input_kwargs: dict[str, Any] - - @property - @abstractmethod - def id(self) -> str: ... - - # NOTE(eddiebergman): Unlike traditional parameters, it seems - @property - @abstractmethod - def value(self) -> nx.DiGraph: ... - - # NOTE(eddiebergman): This is a function common to the three graph - # parameters that is used for `load_from` - @abstractmethod - def create_from_id(self, value: str) -> None: ... - - # NOTE(eddiebergman): Function shared between graph parameters. - # Used to `set_value()` - @abstractmethod - def reset(self) -> None: ... - - @override - def __eq__(self, other: Any) -> bool: - if not isinstance(other, GraphGrammar): - return NotImplemented - - return self.id == other.id - - @abstractmethod - def compute_prior(self, normalized_value: float) -> float: ... - - @override - def set_value(self, value: str | None) -> None: - # NOTE(eddiebergman): Not entirely sure how this should be done - # as previously this would have just overwritten a property method - # `self.value = None` - if not isinstance(value, str): - raise ValueError( - "Expected a string for setting value a `GraphParameter`", - f" got {type(value)}", - ) - self.reset() - self.normalized_value = value - - if value is None: - return - - self.create_from_id(value) - - @override - def sample_value(self, *, user_priors: bool = False) -> nx.DiGraph: - # TODO(eddiebergman): This could definitely be optimized - # Right now it copies the entire object just to get a value out - # of it. - return self.sample(user_priors=user_priors).value - - @override - def load_from(self, value: Any) -> None: - match value: - case GraphParameter(): - value = value.id - case str(): - self.create_from_id(value) - case _: - raise TypeError(f"Unrecognized type {type(value)}") - - @abstractmethod - def mutate( # noqa: D102 - self, parent: Self | None = None, *, mutation_strategy: str = "bananas" - ) -> Self: ... - - def value_to_normalized(self, value: nx.DiGraph) -> float: # noqa: D102 - raise NotImplementedError - - def normalized_to_value(self, normalized_value: float) -> nx.DiGraph: # noqa: D102 - raise NotImplementedError - - @override - def clone(self) -> Self: - new_self = self.__class__(**self.input_kwargs) - - # HACK(eddiebergman): It seems the subclasses all have these and - # so we just copy over those attributes, deepcloning anything that is mutable - if self._value is not None: - _attrs_that_subclasses_use_to_reoresent_a_value = ( - ("_value", True), - ("string_tree", False), - ("string_tree_list", False), - ("_function_id", False), - ) - for _attr, is_mutable in _attrs_that_subclasses_use_to_reoresent_a_value: - retrieved_attr = getattr(self, _attr, NotSet) - if retrieved_attr is NotSet: - continue - - if is_mutable: - setattr(new_self, _attr, deepcopy(retrieved_attr)) - else: - setattr(new_self, _attr, retrieved_attr) - - return new_self - - -class GraphGrammar(GraphParameter, CoreGraphGrammar): - hp_name = "graph_grammar" - - def __init__( # noqa: D107, PLR0913 - self, - grammar: Grammar, - terminal_to_op_names: dict, - prior: dict | None = None, - terminal_to_graph_edges: dict | None = None, - edge_attr: bool = True, # noqa: FBT001, FBT002 - edge_label: str = "op_name", - zero_op: list | None = None, - identity_op: list | None = None, - new_graph_repr_func: bool = False, # noqa: FBT001, FBT002 - name: str | None = None, - scope: str | None = None, - **kwargs, - ): - if identity_op is None: - identity_op = ["Identity", "id"] - if zero_op is None: - zero_op = ["Zero", "zero"] - if isinstance(grammar, list) and len(grammar) != 1: - raise NotImplementedError("Does not support multiple grammars") - - CoreGraphGrammar.__init__( - self, - grammars=grammar, - terminal_to_op_names=terminal_to_op_names, - terminal_to_graph_edges=terminal_to_graph_edges, - edge_attr=edge_attr, - edge_label=edge_label, - zero_op=zero_op, - identity_op=identity_op, - name=name, - scope=scope, - **kwargs, - ) - GraphParameter.__init__(self, value=None, prior=None, is_fidelity=False) - - self.string_tree: str = "" - self._function_id: str = "" - self.new_graph_repr_func = new_graph_repr_func - - if prior is not None: - self.grammars[0].prior = prior - self.has_prior = prior is not None - - @override - def sample(self, *, user_priors: bool = False) -> Self: - copy_self = self.clone() - copy_self.reset() - copy_self.string_tree = copy_self.grammars[0].sampler(1, user_priors=user_priors)[ - 0 - ] - _ = copy_self.value # required for checking if graph is valid! - return copy_self - - @property - @override - def value(self) -> nx.DiGraph: - if self._value is None: - if self.new_graph_repr_func: - self._value = self.get_graph_representation( - self.id, - self.grammars[0], - edge_attr=self.edge_attr, - ) - assert isinstance(self._value, nx.DiGraph) - else: - _value = self.from_stringTree_to_graph_repr( - self.string_tree, - self.grammars[0], - valid_terminals=self.terminal_to_op_names.keys(), - edge_attr=self.edge_attr, - ) - # NOTE: This asumption was not true but I don't really know - # how to handle it otherwise, will just leave it as is for now - # -x- assert isinstance(_value, nx.DiGraph), _value - self._value = _value - return self._value - - @override - def mutate( - self, - parent: GraphGrammar | None = None, - mutation_rate: float = 1.0, - mutation_strategy: str = "bananas", - ) -> Self: - if parent is None: - parent = self - parent_string_tree = parent.string_tree - - if mutation_strategy == "bananas": - child_string_tree, is_same = bananas_mutate( - parent_string_tree=parent_string_tree, - grammar=self.grammars[0], - mutation_rate=mutation_rate, - ) - else: - child_string_tree, is_same = simple_mutate( - parent_string_tree=parent_string_tree, - grammar=self.grammars[0], - ) - - if is_same: - raise Exception("Parent is the same as child!") - - return parent.create_new_instance_from_id( - self.string_tree_to_id(child_string_tree) - ) - - @override - def compute_prior(self, *, log: bool = True) -> float: - return self.grammars[0].compute_prior(self.string_tree, log=log) - - @property - def id(self) -> str: # noqa: D102 - if self._function_id is None or self._function_id == "": - if self.string_tree == "": - raise ValueError("Cannot infer identifier!") - self._function_id = self.string_tree_to_id(self.string_tree) - return self._function_id - - @id.setter - def id(self, value: str) -> None: - self._function_id = value - - def create_from_id(self, identifier: str) -> None: # noqa: D102 - self.reset() - self._function_id = identifier - self.id = identifier - self.string_tree = self.id_to_string_tree(self.id) - _ = self.value # required for checking if graph is valid! - - @staticmethod - def id_to_string_tree(identifier: str) -> str: # noqa: D102 - return identifier - - @staticmethod - def string_tree_to_id(string_tree: str) -> str: # noqa: D102 - return string_tree - - @abstractmethod - def create_new_instance_from_id(self, identifier: str): # noqa: D102 - raise NotImplementedError - - def reset(self) -> None: # noqa: D102 - self.clear_graph() - self.string_tree = "" - self._value = None - self._function_id = "" - - def compose_functions( # noqa: D102 - self, - flatten_graph: bool = True, # noqa: FBT001, FBT002 - ) -> nx.DiGraph: - return self._compose_functions(self.id, self.grammars[0], flatten_graph) diff --git a/neps/search_spaces/architecture/mutations.py b/neps/search_spaces/architecture/mutations.py deleted file mode 100644 index 21660afe1..000000000 --- a/neps/search_spaces/architecture/mutations.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import annotations - -import random -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .cfg import Grammar - - -def simple_mutate(parent_string_tree: str, grammar: Grammar) -> tuple[str, bool]: # noqa: D103 - # works if there is only one grammar - # randomly choose a subtree from the parent and replace - # with a new randomly generated subtree - - # choose subtree to delete - subtree_node, subtree_idx = grammar.rand_subtree(parent_string_tree) - child_string_tree = grammar.mutate( - parent=parent_string_tree, - subtree_index=subtree_idx, - subtree_node=subtree_node, - ) - return child_string_tree, parent_string_tree == child_string_tree - - -def bananas_mutate( # noqa: D103 - parent_string_tree: str, - grammar: Grammar, - mutation_rate: float = 1.0, - mutation_prob: float | None = None, - patience: int = 50, -) -> tuple[str, bool]: - split_tree = parent_string_tree.split(" ") - swappable_indices = [ - i - for i in range(len(split_tree)) - if split_tree[i][1:] in grammar.swappable_nonterminals - ] - _mutation_prob = ( - mutation_rate / len(swappable_indices) if mutation_prob is None else mutation_prob - ) - child_string_tree = parent_string_tree - - idx = 0 - while idx < len(swappable_indices): - swap_idx = swappable_indices[idx] - if random.random() < _mutation_prob: # noqa: S311 - subtree_node = split_tree[swap_idx][1:] - subtree_idx = swap_idx - child_string_tree = grammar.mutate( - parent=child_string_tree, - subtree_index=subtree_idx, - subtree_node=subtree_node, - patience=patience, - ) - - # update swappable indices - split_tree = child_string_tree.split(" ") - swappable_indices = [ - i - for i in range(len(split_tree)) - if split_tree[i][1:] in grammar.swappable_nonterminals - ] - _mutation_prob = ( - mutation_rate / len(swappable_indices) - if mutation_prob is None - else mutation_prob - ) - idx += 1 - - return child_string_tree, child_string_tree == parent_string_tree diff --git a/neps/search_spaces/architecture/primitives.py b/neps/search_spaces/architecture/primitives.py deleted file mode 100644 index 18c11d283..000000000 --- a/neps/search_spaces/architecture/primitives.py +++ /dev/null @@ -1,499 +0,0 @@ -from __future__ import annotations # noqa: D100 - -from abc import ABCMeta, abstractmethod - -import torch -from torch import nn - - -class _AbstractPrimitive(nn.Module, metaclass=ABCMeta): - """Use this class when creating new operations for edges. - - This is required because we are agnostic to operations - at the edges. As a consequence, they can contain subgraphs - which requires naslib to detect and properly process them. - """ - - @abstractmethod - def forward(self, x): - """The forward processing of the operation.""" - raise NotImplementedError - - @abstractmethod - def get_embedded_ops(self): - """Return any embedded ops so that they can be - analysed whether they contain a child graph, e.g. - a 'motif' in the hierachical search space. - - If there are no embedded ops, then simply return - `None`. Should return a list otherwise. - """ - raise NotImplementedError - - @property - def get_op_name(self): - return type(self).__name__ - - -class AbstractPrimitive(_AbstractPrimitive): # noqa: D101 - def forward(self, x): # noqa: D102 - raise NotImplementedError - - def get_embedded_ops(self): # noqa: D102 - return None - - -class Identity(AbstractPrimitive): - """An implementation of the Identity operation.""" - - def __init__(self, **kwargs): # noqa: D107 - super().__init__(locals()) - - def forward(self, x: object) -> object: # noqa: D102 - return x - - -class Zero(AbstractPrimitive): - """Implementation of the zero operation. It removes - the connection by multiplying its input with zero. - """ - - def __init__(self, stride, **kwargs): - """When setting stride > 1 then it is assumed that the - channels must be doubled. - """ - super().__init__(locals()) - self.stride = int(stride) - - def forward(self, x): # noqa: D102 - if self.stride == 1: - return x.mul(0.0) - - return x[:, :, :: self.stride, :: self.stride].mul(0.0) - - def __repr__(self): - return f"" - - -class Zero1x1(AbstractPrimitive): - """Implementation of the zero operation. It removes - the connection by multiplying its input with zero. - """ - - def __init__(self, stride, **kwargs): - """When setting stride > 1 then it is assumed that the - channels must be doubled. - """ - super().__init__(locals()) - self.stride = int(stride) - - def forward(self, x): # noqa: D102 - if self.stride == 1: - return x.mul(0.0) - - x = x[:, :, :: self.stride, :: self.stride].mul(0.0) - return torch.cat([x, x], dim=1) # double the channels TODO: ugly as hell - - def __repr__(self): - return f"" - - -class SepConv(AbstractPrimitive): - """Implementation of Separable convolution operation as - in the DARTS paper, i.e. 2 sepconv directly after another. - """ - - def __init__( # noqa: D107 - self, - c_in: int, - c_out: int, - kernel_size: int, - stride: int, - padding: int, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - - c_in = int(c_in) - c_out = int(c_out) - kernel_size = int(kernel_size) - stride = int(stride) - padding = int(padding) - affine = bool(affine) - - self.kernel_size = kernel_size - self.op = nn.Sequential( - nn.ReLU(inplace=False), - nn.Conv2d( - c_in, - c_in, - kernel_size=kernel_size, - stride=stride, - padding=padding, - groups=c_in, - bias=False, - ), - nn.Conv2d(c_in, c_in, kernel_size=1, padding=0, bias=False), - nn.BatchNorm2d(c_in, affine=affine), - nn.ReLU(inplace=False), - nn.Conv2d( - c_in, - c_in, - kernel_size=kernel_size, - stride=1, - padding=padding, - groups=c_in, - bias=False, - ), - nn.Conv2d(c_in, c_out, kernel_size=1, padding=0, bias=False), - nn.BatchNorm2d(c_out, affine=affine), - ) - - def forward(self, x): # noqa: D102 - return self.op(x) - - @property - def get_op_name(self): # noqa: D102 - op_name = super().get_op_name - op_name += f"{self.kernel_size}x{self.kernel_size}" - return op_name - - -class DilConv(AbstractPrimitive): - """Implementation of a dilated separable convolution as - used in the DARTS paper. - """ - - def __init__( # noqa: D107 - self, - c_in: int, - c_out: int, - kernel_size: int, - stride: int, - padding: int, - dilation: int, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - - c_in = int(c_in) - c_out = int(c_out) - kernel_size = int(kernel_size) - stride = int(stride) - padding = int(padding) - dilation = int(dilation) - affine = bool(affine) - - self.kernel_size = kernel_size - self.op = nn.Sequential( - nn.ReLU(inplace=False), - nn.Conv2d( - c_in, - c_in, - kernel_size=kernel_size, - stride=stride, - padding=padding, - dilation=dilation, - groups=c_in, - bias=False, - ), - nn.Conv2d(c_in, c_out, kernel_size=1, padding=0, bias=False), - nn.BatchNorm2d(c_out, affine=affine), - ) - - def forward(self, x): # noqa: D102 - return self.op(x) - - @property - def get_op_name(self): # noqa: D102 - op_name = super().get_op_name - op_name += f"{self.kernel_size}x{self.kernel_size}" - return op_name - - -class Stem(AbstractPrimitive): - """This is used as an initial layer directly after the - image input. - """ - - def __init__(self, c_out: int, c_in: int = 3, **kwargs): # noqa: D107 - super().__init__(locals()) - - c_out = int(c_out) - - self.seq = nn.Sequential( - nn.Conv2d(c_in, c_out, 3, padding=1, bias=False), nn.BatchNorm2d(c_out) - ) - - def forward(self, x): # noqa: D102 - return self.seq(x) - - -class Sequential(AbstractPrimitive): - """Implementation of `torch.nn.Sequential` to be used - as op on edges. - """ - - def __init__(self, *args, **kwargs): # noqa: D107 - super().__init__(locals()) - self.primitives = args - self.op = nn.Sequential(*args) - - def forward(self, x): # noqa: D102 - return self.op(x) - - def get_embedded_ops(self): # noqa: D102 - return list(self.primitives) - - -class MaxPool(AbstractPrimitive): # noqa: D101 - def __init__(self, kernel_size: int, stride: int, **kwargs): # noqa: D107 - super().__init__(locals()) - - kernel_size = int(kernel_size) - stride = int(stride) - - self.maxpool = nn.MaxPool2d(kernel_size, stride=stride, padding=1) - - def forward(self, x): # noqa: D102 - return self.maxpool(x) - - -class MaxPool1x1(AbstractPrimitive): - """Implementation of MaxPool with an optional 1x1 convolution - in case stride > 1. The 1x1 convolution is required to increase - the number of channels. - """ - - def __init__( # noqa: D107 - self, - kernel_size: int, - stride: int, - c_in: int, - c_out: int, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - - kernel_size = int(kernel_size) - stride = int(stride) - c_in = int(c_in) - c_out = int(c_out) - affine = bool(affine) - - self.stride = stride - self.maxpool = nn.MaxPool2d(kernel_size, stride=stride, padding=1) - if stride > 1: - assert c_in is not None - assert c_out is not None - self.conv = nn.Conv2d(c_in, c_out, 1, stride=1, padding=0, bias=False) - self.bn = nn.BatchNorm2d(c_out, affine=affine) - - def forward(self, x): # noqa: D102 - x = self.maxpool(x) - if self.stride > 1: - x = self.conv(x) - x = self.bn(x) - return x - - -class AvgPool(AbstractPrimitive): - """Implementation of Avergae Pooling.""" - - def __init__(self, kernel_size: int, stride: int, **kwargs): # noqa: D107 - stride = int(stride) - super().__init__(locals()) - self.avgpool = nn.AvgPool2d( - kernel_size=3, stride=stride, padding=1, count_include_pad=False - ) - - def forward(self, x): # noqa: D102 - return self.avgpool(x) - - -class AvgPool1x1(AbstractPrimitive): - """Implementation of Avergae Pooling with an optional - 1x1 convolution afterwards. The convolution is required - to increase the number of channels if stride > 1. - """ - - def __init__( # noqa: D107 - self, - kernel_size: int, - stride: int, - c_in: int, - c_out: int, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - stride = int(stride) - self.stride = int(stride) - self.avgpool = nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False) - if stride > 1: - assert c_in is not None - assert c_out is not None - self.conv = nn.Conv2d(c_in, c_out, 1, stride=1, padding=0, bias=False) - self.bn = nn.BatchNorm2d(c_out, affine=affine) - - def forward(self, x): # noqa: D102 - x = self.avgpool(x) - if self.stride > 1: - x = self.conv(x) - x = self.bn(x) - return x - - -class ReLUConvBN(AbstractPrimitive): # noqa: D101 - def __init__( # noqa: D107 - self, - c_in: int, - c_out: int, - kernel_size: int, - stride: int = 1, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - kernel_size = int(kernel_size) - stride = int(stride) - - self.kernel_size = kernel_size - pad = 0 if int(stride) == 1 and kernel_size == 1 else 1 - self.op = nn.Sequential( - nn.ReLU(inplace=False), - nn.Conv2d(c_in, c_out, kernel_size, stride=stride, padding=pad, bias=False), - nn.BatchNorm2d(c_out, affine=affine), - ) - - def forward(self, x): # noqa: D102 - return self.op(x) - - @property - def get_op_name(self): # noqa: D102 - op_name = super().get_op_name - op_name += f"{self.kernel_size}x{self.kernel_size}" - return op_name - - -class ConvBnReLU(AbstractPrimitive): - """Implementation of 2d convolution, followed by 2d batch normalization and - ReLU activation. - """ - - def __init__( # noqa: D107 - self, - c_in: int, - c_out: int, - kernel_size: int, - stride: int = 1, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - self.kernel_size = kernel_size - pad = 0 if stride == 1 and kernel_size == 1 else 1 - self.op = nn.Sequential( - nn.Conv2d(c_in, c_out, kernel_size, stride=stride, padding=pad, bias=False), - nn.BatchNorm2d(c_out, affine=affine), - nn.ReLU(inplace=False), - ) - - def forward(self, x): # noqa: D102 - return self.op(x) - - @property - def get_op_name(self): # noqa: D102 - op_name = super().get_op_name - op_name += f"{self.kernel_size}x{self.kernel_size}" - return op_name - - -class ConvBn(AbstractPrimitive): - """Implementation of 2d convolution, followed by 2d batch normalization and ReLU - activation. - """ - - def __init__( # noqa: D107 - self, - c_in: int, - c_out: int, - kernel_size: int, - stride=1, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - self.kernel_size = kernel_size - pad = 0 if stride == 1 and kernel_size == 1 else 1 - self.op = nn.Sequential( - nn.Conv2d(c_in, c_out, kernel_size, stride=stride, padding=pad, bias=False), - nn.BatchNorm2d(c_out, affine=affine), - ) - - def forward(self, x): # noqa: D102 - return self.op(x) - - @property - def get_op_name(self): # noqa: D102 - op_name = super().get_op_name - op_name += f"{self.kernel_size}x{self.kernel_size}" - return op_name - - -class Concat1x1(AbstractPrimitive): - """Implementation of the channel-wise concatination followed by a 1x1 convolution - to retain the channel dimension. - """ - - def __init__( # noqa: D107 - self, - num_in_edges: int, - c_out: int, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - self.conv = nn.Conv2d( - num_in_edges * c_out, c_out, kernel_size=1, stride=1, padding=0, bias=False - ) - self.bn = nn.BatchNorm2d(c_out, affine=affine) - - def forward(self, x): - """Expecting a list of input tensors. Stacking them channel-wise - and applying 1x1 conv. - """ - x = torch.cat(x, dim=1) - x = self.conv(x) - return self.bn(x) - - -class ResNetBasicblock(AbstractPrimitive): # noqa: D101 - def __init__( # noqa: D107 - self, - c_in: int, - c_out: int, - stride: int, - affine: bool = True, # noqa: FBT001, FBT002 - **kwargs, - ): - super().__init__(locals()) - assert stride in (1, 2), f"invalid stride {stride}" - self.conv_a = ReLUConvBN(c_in, c_out, 3, stride) - self.conv_b = ReLUConvBN(c_out, c_out, 3) - if stride == 2: - self.downsample = nn.Sequential( - nn.Conv2d(c_in, c_out, kernel_size=1, stride=2, padding=0, bias=False), - nn.BatchNorm2d(c_out), - ) - else: - self.downsample = None - - def forward(self, x): # noqa: D102 - basicblock = self.conv_a(x) - basicblock = self.conv_b(basicblock) - residual = self.downsample(x) if self.downsample is not None else x - return residual + basicblock diff --git a/neps/search_spaces/architecture/topologies.py b/neps/search_spaces/architecture/topologies.py deleted file mode 100644 index 431b3d3c7..000000000 --- a/neps/search_spaces/architecture/topologies.py +++ /dev/null @@ -1,187 +0,0 @@ -from __future__ import annotations # noqa: D100 - -import inspect -import queue -from abc import ABCMeta -from functools import partial -from typing import Callable - -from .graph import Graph - - -class AbstractTopology(Graph, metaclass=ABCMeta): # noqa: D101 - edge_list: list = [] # noqa: RUF012 - - def __init__( # noqa: D107 - self, name: str | None = None, scope: str | None = None, merge_fn: Callable = sum - ): - super().__init__(name=name, scope=scope) - - self.merge_fn = merge_fn - - def mutate(self): # noqa: D102 - pass - - def sample(self): # noqa: D102 - pass - - def create_graph(self, vals: dict): # noqa: C901, D102 - def get_args_and_defaults(func): - signature = inspect.signature(func) - return list(signature.parameters.keys()), { - k: v.default - for k, v in signature.parameters.items() - if v.default is not inspect.Parameter.empty - } - - def get_op_name_from_dict(val: dict): - # currently assumes that missing args are ints! - op = val["op"] - args: dict = {} - arg_names, default_args = get_args_and_defaults(op) - for arg_name in arg_names: - if arg_name in ("self", "kwargs") or arg_name in args: - continue - if arg_name in val: - args[arg_name] = val[arg_name] - elif arg_name in default_args: - args[arg_name] = default_args[arg_name] - else: - args[arg_name] = 42 - - if "groups" in args and args["groups"] != 1: - args["c_in"] = args["groups"] - args["c_out"] = args["groups"] - - return op(**args).get_op_name - - assert isinstance(vals, dict) - for (u, v), val in vals.items(): - self.add_edge(u, v) - if isinstance(val, dict): - _val = val - _val["op_name"] = get_op_name_from_dict(val) - elif isinstance(val, int): # for synthetic benchmarks - _val = {"op": val, "op_name": val} - elif hasattr(val, "get_op_name"): - _val = {"op": val, "op_name": val.get_op_name} - elif callable(val): - _val = {"op": val, "op_name": val.__name__} - else: - raise Exception(f"Cannot extract op name from {val}") - - self.edges[u, v].update(_val) - - @property - def get_op_name(self): # noqa: D102 - return type(self).__name__ - - def __call__(self, x): # noqa: D102 - cur_node_idx = next(node for node in self.nodes if self.in_degree(node) == 0) - predecessor_inputs = {cur_node_idx: [x]} - next_successors = queue.Queue() - next_successors.put(cur_node_idx) - cur_successors = queue.Queue() - inputs = None - while not cur_successors.empty() or not next_successors.empty(): - if not cur_successors.empty(): - next_node_idx = cur_successors.get(block=False) - if next_node_idx not in next_successors.queue: - next_successors.put(next_node_idx) - if next_node_idx not in predecessor_inputs: - predecessor_inputs[next_node_idx] = [] - predecessor_inputs[next_node_idx].append( - self.edges[(cur_node_idx, next_node_idx)].op(inputs) - ) - else: - cur_node_idx = next_successors.get(block=False) - if self.out_degree(cur_node_idx) > 0: - for successor in self.successors(cur_node_idx): - cur_successors.put(successor) - - if len(predecessor_inputs[cur_node_idx]) == 1: - inputs = predecessor_inputs[cur_node_idx][0] - else: - inputs = self.merge_fn(predecessor_inputs[cur_node_idx]) - return inputs - - -class _SequentialNEdge(AbstractTopology): - edge_list: list = [] # noqa: RUF012 - - def __init__(self, *edge_vals, number_of_edges: int, **kwargs): - super().__init__(**kwargs) - - self.name = f"sequential_{number_of_edges}_edges" - self.edge_list = self.get_edge_list(number_of_edges=number_of_edges) - self.create_graph(dict(zip(self.edge_list, edge_vals))) - self.set_scope(self.name) - - @staticmethod - def get_edge_list(number_of_edges: int): - return [(i + 1, i + 2) for i in range(number_of_edges)] - - -LinearNEdge = _SequentialNEdge - - -def get_sequential_n_edge(number_of_edges: int): # noqa: D103 - return partial(_SequentialNEdge, number_of_edges=number_of_edges) - - -class Residual(AbstractTopology): # noqa: D101 - edge_list = [ # noqa: RUF012 - (1, 2), - (1, 3), - (2, 3), - ] - - def __init__(self, *edge_vals, **kwargs): # noqa: D107 - super().__init__(**kwargs) - - self.name = "residual" - self.create_graph(dict(zip(self.edge_list, edge_vals))) - self.set_scope(self.name) - - -class Diamond(AbstractTopology): # noqa: D101 - edge_list = [(1, 2), (1, 3), (2, 4), (3, 4)] # noqa: RUF012 - - def __init__(self, *edge_vals, **kwargs): # noqa: D107 - super().__init__(**kwargs) - - self.name = "diamond" - self.create_graph(dict(zip(self.edge_list, edge_vals))) - self.set_scope(self.name) - - -class DiamondMid(AbstractTopology): # noqa: D101 - edge_list = [(1, 2), (1, 3), (2, 3), (2, 4), (3, 4)] # noqa: RUF012 - - def __init__(self, *edge_vals, **kwargs): # noqa: D107 - super().__init__(**kwargs) - - self.name = "diamond_mid" - self.create_graph(dict(zip(self.edge_list, edge_vals))) - self.set_scope(self.name) - - -class _DenseNNodeDAG(AbstractTopology): - edge_list: list = [] # noqa: RUF012 - - def __init__(self, *edge_vals, number_of_nodes: int, **kwargs): - super().__init__(**kwargs) - - self.edge_list = self.get_edge_list(number_of_nodes=number_of_nodes) - - self.name = f"dense_{number_of_nodes}_node_dag" - self.create_graph(dict(zip(self.edge_list, edge_vals))) - self.set_scope(self.name) - - @staticmethod - def get_edge_list(number_of_nodes: int): - return [(i + 1, j + 1) for j in range(number_of_nodes) for i in range(j)] - - -def get_dense_n_node_dag(number_of_nodes: int): # noqa: D103 - return partial(_DenseNNodeDAG, number_of_nodes=number_of_nodes) diff --git a/neps/search_spaces/hyperparameters/__init__.py b/neps/search_spaces/hyperparameters/__init__.py deleted file mode 100644 index 14e7ce792..000000000 --- a/neps/search_spaces/hyperparameters/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from neps.search_spaces.hyperparameters.categorical import ( - Categorical, - CategoricalParameter, -) -from neps.search_spaces.hyperparameters.constant import Constant, ConstantParameter -from neps.search_spaces.hyperparameters.float import Float, FloatParameter -from neps.search_spaces.hyperparameters.integer import Integer, IntegerParameter -from neps.search_spaces.hyperparameters.numerical import Numerical, NumericalParameter - -__all__ = [ - "Categorical", - "CategoricalParameter", - "Constant", - "ConstantParameter", - "Float", - "FloatParameter", - "Integer", - "IntegerParameter", - "Numerical", - "NumericalParameter", -] diff --git a/neps/search_spaces/hyperparameters/categorical.py b/neps/search_spaces/hyperparameters/categorical.py deleted file mode 100644 index aa407d357..000000000 --- a/neps/search_spaces/hyperparameters/categorical.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Categorical hyperparameter for search spaces.""" - -from __future__ import annotations - -from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeAlias -from typing_extensions import Self, override - -import numpy as np -import numpy.typing as npt -from more_itertools import all_unique - -from neps.search_spaces.domain import Domain -from neps.search_spaces.parameter import ParameterWithPrior - -if TYPE_CHECKING: - from neps.utils.types import f64 - -CategoricalTypes: TypeAlias = float | int | str - - -class Categorical(ParameterWithPrior[CategoricalTypes, CategoricalTypes]): - """A list of **unordered** choices for a parameter. - - This kind of [`Parameter`][neps.search_spaces.parameter] is used - to represent hyperparameters that can take on a discrete set of unordered - values. For example, the `optimizer` hyperparameter in a neural network - search space can be a `Categorical` with choices like - `#!python ["adam", "sgd", "rmsprop"]`. - - ```python - import neps - - optimizer_choice = neps.Categorical( - ["adam", "sgd", "rmsprop"], - default="adam" - ) - ``` - - Please see the [`Parameter`][neps.search_spaces.parameter], - [`ParameterWithPrior`][neps.search_spaces.parameter.ParameterWithPrior], - for more details on the methods available for this class. - """ - - PRIOR_CONFIDENCE_SCORES: ClassVar[Mapping[str, Any]] = { - "low": 2, - "medium": 4, - "high": 6, - } - - def __init__( - self, - choices: Iterable[float | int | str], - *, - prior: float | int | str | None = None, - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Create a new `Categorical`. - - Args: - choices: choices for the hyperparameter. - prior: prior value for the hyperparameter, must be in `choices=` - if provided. - prior_confidence: confidence score for the prior value, used when - condsider prior based optimization. - """ - choices = list(choices) - if len(choices) <= 1: - raise ValueError("Categorical choices must have more than one value.") - - super().__init__(value=None, is_fidelity=False, prior=prior) - - for choice in choices: - if not isinstance(choice, float | int | str): - raise TypeError( - f'Choice "{choice}" is not of a valid type (float, int, str)' - ) - - if not all_unique(choices): - raise ValueError(f"Choices must be unique but got duplicates.\n{choices}") - - if prior is not None and prior not in choices: - raise ValueError( - f"Default value {prior} is not in the provided choices {choices}" - ) - - self.choices = list(choices) - - # NOTE(eddiebergman): If there's ever a very large categorical, - # then it would be beneficial to have a lookup table for indices as - # currently we do a list.index() operation which is O(n). - # However for small sized categoricals this is likely faster than - # a lookup table. - # For now we can just cache the index of the value and prior. - self._value_index: int | None = None - - self.prior_confidence_choice = prior_confidence - self.prior_confidence_score = self.PRIOR_CONFIDENCE_SCORES[prior_confidence] - self.has_prior = self.prior is not None - self._prior_index: int | None = ( - self.choices.index(prior) if prior is not None else None - ) - self.domain = Domain.indices(len(self.choices)) - - @override - def clone(self) -> Self: - clone = self.__class__( - choices=self.choices, - prior=self.prior, - prior_confidence=self.prior_confidence_choice, # type: ignore - ) - if self.value is not None: - clone.set_value(self.value) - - return clone - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, self.__class__): - return False - - return ( - self.choices == other.choices - and self.value == other.value - and self.is_fidelity == other.is_fidelity - and self.prior == other.prior - and self.has_prior == other.has_prior - and self.prior_confidence_score == other.prior_confidence_score - ) - - def __repr__(self) -> str: - return f"" - - def _compute_user_prior_probabilities(self) -> npt.NDArray[f64]: - # The prior value should have "prior_confidence_score" more probability - # than all the other values. - assert self._prior_index is not None - probabilities = np.ones(len(self.choices)) - probabilities[self._prior_index] = self.prior_confidence_score - return probabilities / np.sum(probabilities) - - @override - def sample_value(self, *, user_priors: bool = False) -> Any: - indices = np.arange(len(self.choices)) - if user_priors and self.prior is not None: - probabilities = self._compute_user_prior_probabilities() - return self.choices[np.random.choice(indices, p=probabilities)] - - return self.choices[np.random.choice(indices)] - - @override - def value_to_normalized(self, value: Any) -> float: - return float(self.choices.index(value)) - - @override - def normalized_to_value(self, normalized_value: float) -> Any: - return self.choices[int(np.rint(normalized_value))] - - @override - def set_value(self, value: Any | None) -> None: - if value is None: - self._value = None - self._value_index = None - self.normalized_value = None - return - - self._value = value - self._value_index = self.choices.index(value) - self.normalized_value = float(self._value_index) - - -class CategoricalParameter(Categorical): - """Deprecated: Use `Categorical` instead of `CategoricalParameter`. - - This class remains for backward compatibility and will raise a deprecation - warning if used. - """ - - def __init__( - self, - choices: Iterable[float | int | str], - *, - prior: float | int | str | None = None, - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Initialize a deprecated `CategoricalParameter`. - - Args: - choices: choices for the hyperparameter. - prior: prior value for the hyperparameter, must be in `choices=` - if provided. - prior_confidence: confidence score for the prior value, used when - condsider prior based optimization. - - Raises: - DeprecationWarning: A warning indicating that `neps.CategoricalParameter` is - deprecated and `neps.Categorical` should be used instead. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.CategoricalParameter' is deprecated and will be removed " - "in future releases. Please use 'neps.Categorical' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - super().__init__( - choices=choices, - prior=prior, - prior_confidence=prior_confidence, - ) diff --git a/neps/search_spaces/hyperparameters/constant.py b/neps/search_spaces/hyperparameters/constant.py deleted file mode 100644 index 155ddc9e2..000000000 --- a/neps/search_spaces/hyperparameters/constant.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Constant hyperparameter for search spaces.""" - -from __future__ import annotations - -from typing import Any, TypeVar -from typing_extensions import Self, override - -from neps.search_spaces.parameter import Parameter - -T = TypeVar("T", int, float, str) - - -class Constant(Parameter[T, T]): - """A constant value for a parameter. - - This kind of [`Parameter`][neps.search_spaces.parameter] is used - to represent hyperparameters with values that should not change during - optimization. For example, the `batch_size` hyperparameter in a neural - network search space can be a `Constant` with a value of `32`. - - ```python - import neps - - batch_size = neps.Constant(32) - ``` - - !!! note - - As the name suggests, the value of a `Constant` only have one - value and so its [`.prior`][neps.search_spaces.parameter.Parameter.prior] - and [`.value`][neps.search_spaces.parameter.Parameter.value] should always be - the same. - - This also implies that the - [`.prior`][neps.search_spaces.parameter.Parameter.prior] can never be `None`. - - Please use - [`.set_constant_value()`][neps.search_spaces.hyperparameters.constant.Constant.set_constant_value] - if you need to change the value of the constant parameter. - """ - - def __init__(self, value: T): - """Create a new `Constant`. - - Args: - value: value for the hyperparameter. - """ - super().__init__(value=value, prior=value, is_fidelity=False) # type: ignore - self._value: T = value # type: ignore - - @override - def clone(self) -> Self: - return self.__class__(value=self.value) - - @property - @override - def value(self) -> T: - """Get the value of the constant parameter.""" - return self._value - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, self.__class__): - return NotImplemented - - return self.value == other.value and self.is_fidelity == other.is_fidelity - - def __repr__(self) -> str: - return f"" - - @override - def sample_value(self) -> T: - return self.value - - @override - def set_value(self, value: T | None) -> None: - """Set the value of the constant parameter. - - !!! note - - This method is a no-op but will raise a `ValueError` if the value - is different from the current value. - - Please see - [`.set_constant_value()`][neps.search_spaces.hyperparameters.constant.Constant.set_constant_value] - which can be used to set both the - [`.value`][neps.search_spaces.parameter.Parameter.value] - and the [`.prior`][neps.search_spaces.parameter.Parameter.prior] at once - - Args: - value: value to set the parameter to. - - Raises: - ValueError: if the value is different from the current value. - """ - if value != self._value: - raise ValueError( - f"Constant does not allow changing the set value. " - f"Tried to set value to {value}, but it is already {self.value}" - ) - - @override - def value_to_normalized(self, value: T) -> float: - return 1.0 if value == self._value else 0.0 - - @override - def normalized_to_value(self, normalized_value: float) -> T: - return self._value - - -class ConstantParameter(Constant): - """Deprecated: Use `Constant` instead of `ConstantParameter`. - - This class remains for backward compatibility and will raise a deprecation - warning if used. - """ - - def __init__(self, value: T): - """Initialize a deprecated `ConstantParameter`. - - Args: - value: value for the hyperparameter. - - Raises: - DeprecationWarning: A warning indicating that `neps.ConstantParameter` is - deprecated and `neps.Constant` should be used instead. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.ConstantParameter' is deprecated and will be removed in" - " future releases. Please use 'neps.Constant' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - super().__init__(value=value) diff --git a/neps/search_spaces/hyperparameters/float.py b/neps/search_spaces/hyperparameters/float.py deleted file mode 100644 index fef72f684..000000000 --- a/neps/search_spaces/hyperparameters/float.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Float hyperparameter for search spaces.""" - -from __future__ import annotations - -import math -from collections.abc import Mapping -from typing import TYPE_CHECKING, ClassVar, Literal -from typing_extensions import Self, override - -import numpy as np - -from neps.search_spaces.domain import Domain -from neps.search_spaces.hyperparameters.numerical import Numerical - -if TYPE_CHECKING: - from neps.utils.types import Number - - -class Float(Numerical[float]): - """A float value for a parameter. - - This kind of [`Parameter`][neps.search_spaces.parameter] is used - to represent hyperparameters with continuous float values, optionally specifying if - it exists - on a log scale. - For example, `l2_norm` could be a value in `(0.1)`, while the `learning_rate` - hyperparameter in a neural network search space can be a `Float` - with a range of `(0.0001, 0.1)` but on a log scale. - - ```python - import neps - - l2_norm = neps.Float(0, 1) - learning_rate = neps.Float(1e-4, 1e-1, log=True) - ``` - - Please see the [`Numerical`][neps.search_spaces.numerical.Numerical] - class for more details on the methods available for this class. - """ - - DEFAULT_CONFIDENCE_SCORES: ClassVar[Mapping[str, float]] = { - "low": 0.5, - "medium": 0.25, - "high": 0.125, - } - - def __init__( - self, - lower: Number, - upper: Number, - *, - log: bool = False, - is_fidelity: bool = False, - prior: Number | None = None, - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Create a new `Float`. - - Args: - lower: lower bound for the hyperparameter. - upper: upper bound for the hyperparameter. - log: whether the hyperparameter is on a log scale. - is_fidelity: whether the hyperparameter is fidelity. - prior: prior value for the hyperparameter. - prior_confidence: confidence score for the prior value, used when - condsidering prior based optimization.. - """ - super().__init__( - lower=float(lower), - upper=float(upper), - log=log, - prior=float(prior) if prior is not None else None, - prior_confidence=prior_confidence, - is_fidelity=is_fidelity, - domain=Domain.floating(lower, upper, log=log), - ) - - @override - def clone(self) -> Self: - clone = self.__class__( - lower=self.lower, - upper=self.upper, - log=self.log, - is_fidelity=self.is_fidelity, - prior=self.prior, - prior_confidence=self.prior_confidence_choice, - ) - if self.value is not None: - clone.set_value(self.value) - - return clone - - @override - def set_value(self, value: float | None) -> None: - if value is None: - self._value = None - self.normalized_value = None - return - - if not self.lower <= value <= self.upper: - cls_name = self.__class__.__name__ - raise ValueError( - f"{cls_name} parameter: prior bounds error. Expected lower <= prior" - f" <= upper, but got lower={self.lower}, value={value}," - f" upper={self.upper}" - ) - - value = float(value) - self._value = value - self.normalized_value = self.value_to_normalized(value) - - @override - def sample_value(self, *, user_priors: bool = False) -> float: - if self.log: - assert self.log_bounds is not None - low, high = self.log_bounds - prior = self.log_prior - else: - low, high, prior = self.lower, self.upper, self.prior - - if user_priors and self.has_prior: - dist, std = self._get_truncnorm_prior_and_std() - value = dist.rvs() * std + prior - else: - value = np.random.uniform(low=low, high=high) - - if self.log: - value = math.exp(value) - - return float(min(self.upper, max(self.lower, value))) - - @override - def value_to_normalized(self, value: float) -> float: - if self.log: - assert self.log_bounds is not None - low, high = self.log_bounds - else: - low, high = self.lower, self.upper - - value = np.log(value) if self.log else value - return float((value - low) / (high - low)) - - @override - def normalized_to_value(self, normalized_value: float) -> float: - if self.log: - assert self.log_bounds is not None - low, high = self.log_bounds - else: - low, high = self.lower, self.upper - - normalized_value = normalized_value * (high - low) + low - _value = np.exp(normalized_value) if self.log else normalized_value - return float(_value) - - def __repr__(self) -> str: - float_repr = f"{self.value:.07f}" if self.value is not None else "None" - return f"" - - -class FloatParameter(Float): - """Deprecated: Use `Float` instead of `FloatParameter`. - - This class remains for backward compatibility and will raise a deprecation - warning if used. - """ - - def __init__( - self, - lower: Number, - upper: Number, - *, - log: bool = False, - is_fidelity: bool = False, - prior: Number | None = None, - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Initialize a deprecated `FloatParameter`. - - Args: - lower: lower bound for the hyperparameter. - upper: upper bound for the hyperparameter. - log: whether the hyperparameter is on a log scale. - is_fidelity: whether the hyperparameter is fidelity. - prior: prior value for the hyperparameter. - prior_confidence: confidence score for the prior value, used when - condsidering prior based optimization.. - - Raises: - DeprecationWarning: A warning indicating that `neps.FloatParameter` is - deprecated and `neps.Float` should be used instead. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.FloatParameter' is deprecated and will be removed in" - " future releases. Please use 'neps.Float' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - super().__init__( - lower=lower, - upper=upper, - log=log, - is_fidelity=is_fidelity, - prior=prior, - prior_confidence=prior_confidence, - ) diff --git a/neps/search_spaces/hyperparameters/integer.py b/neps/search_spaces/hyperparameters/integer.py deleted file mode 100644 index 0386032d9..000000000 --- a/neps/search_spaces/hyperparameters/integer.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Float hyperparameter for search spaces.""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, ClassVar, Literal -from typing_extensions import Self, override - -import numpy as np - -from neps.search_spaces.domain import Domain -from neps.search_spaces.hyperparameters.float import Float -from neps.search_spaces.hyperparameters.numerical import Numerical - -if TYPE_CHECKING: - from neps.utils.types import Number - - -class Integer(Numerical[int]): - """An integer value for a parameter. - - This kind of [`Parameter`][neps.search_spaces.parameter] is used - to represent hyperparameters with continuous integer values, optionally specifying - f it exists on a log scale. - For example, `batch_size` could be a value in `(32, 128)`, while the `num_layers` - hyperparameter in a neural network search space can be a `Integer` - with a range of `(1, 1000)` but on a log scale. - - ```python - import neps - - batch_size = neps.Integer(32, 128) - num_layers = neps.Integer(1, 1000, log=True) - ``` - """ - - DEFAULT_CONFIDENCE_SCORES: ClassVar[Mapping[str, float]] = { - "low": 0.5, - "medium": 0.25, - "high": 0.125, - } - - def __init__( - self, - lower: Number, - upper: Number, - *, - log: bool = False, - is_fidelity: bool = False, - prior: Number | None = None, - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Create a new `Integer`. - - Args: - lower: lower bound for the hyperparameter. - upper: upper bound for the hyperparameter. - log: whether the hyperparameter is on a log scale. - is_fidelity: whether the hyperparameter is fidelity. - prior: prior value for the hyperparameter. - prior_confidence: confidence score for the prior value, used when - condsider prior based optimization. - """ - lower = int(np.rint(lower)) - upper = int(np.rint(upper)) - _size = upper - lower + 1 - if _size <= 1: - raise ValueError( - f"Integer: expected at least 2 possible values in the range," - f" got upper={upper}, lower={lower}." - ) - - super().__init__( - lower=int(np.rint(lower)), - upper=int(np.rint(upper)), - log=log, - is_fidelity=is_fidelity, - prior=int(np.rint(prior)) if prior is not None else None, - prior_confidence=prior_confidence, - domain=Domain.integer(lower, upper, log=log), - ) - - # We subtract/add 0.499999 from lower/upper bounds respectively, such that - # sampling in the float space gives equal probability for all integer values, - # i.e. [x - 0.499999, x + 0.499999] - self.float_hp = Float( - lower=self.lower - 0.499999, - upper=self.upper + 0.499999, - log=self.log, - is_fidelity=is_fidelity, - prior=prior, - prior_confidence=prior_confidence, - ) - - def __repr__(self) -> str: - return f"" - - @override - def clone(self) -> Self: - clone = self.__class__( - lower=self.lower, - upper=self.upper, - log=self.log, - is_fidelity=self.is_fidelity, - prior=self.prior, - prior_confidence=self.prior_confidence_choice, - ) - if self.value is not None: - clone.set_value(self.value) - - return clone - - @override - def load_from(self, value: Any) -> None: - self._value = int(np.rint(value)) - - @override - def set_value(self, value: int | None) -> None: - if value is None: - self._value = None - self.normalized_value = None - self.float_hp.set_value(None) - return - - if not self.lower <= value <= self.upper: - cls_name = self.__class__.__name__ - raise ValueError( - f"{cls_name} parameter: prior bounds error. Expected lower <= prior" - f" <= upper, but got lower={self.lower}, value={value}," - f" upper={self.upper}" - ) - - value = int(np.rint(value)) - - self.float_hp.set_value(value) - self._value = value - self.normalized_value = self.value_to_normalized(value) - - @override - def sample_value(self, *, user_priors: bool = False) -> int: - val = self.float_hp.sample_value(user_priors=user_priors) - return int(np.rint(val)) - - @override - def value_to_normalized(self, value: int) -> float: - return self.float_hp.value_to_normalized(float(np.rint(value))) - - @override - def normalized_to_value(self, normalized_value: float) -> int: - return int(np.rint(self.float_hp.normalized_to_value(normalized_value))) - - -class IntegerParameter(Integer): - """Deprecated: Use `Integer` instead of `IntegerParameter`. - - This class remains for backward compatibility and will raise a deprecation - warning if used. - """ - - def __init__( - self, - lower: Number, - upper: Number, - *, - log: bool = False, - is_fidelity: bool = False, - prior: Number | None = None, - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Initialize a deprecated `IntegerParameter`. - - Args: - lower: lower bound for the hyperparameter. - upper: upper bound for the hyperparameter. - log: whether the hyperparameter is on a log scale. - is_fidelity: whether the hyperparameter is fidelity. - prior: prior value for the hyperparameter. - prior_confidence: confidence score for the prior value, used when - condsider prior based optimization. - - Raises: - DeprecationWarning: A warning indicating that `neps.IntegerParameter` is - deprecated and `neps.Integer` should be used instead. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.IntegerParameter' is deprecated and will be removed in" - " future releases. Please use 'neps.Integer' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - super().__init__( - lower=lower, - upper=upper, - log=log, - is_fidelity=is_fidelity, - prior=prior, - prior_confidence=prior_confidence, - ) diff --git a/neps/search_spaces/hyperparameters/numerical.py b/neps/search_spaces/hyperparameters/numerical.py deleted file mode 100644 index 037f0be97..000000000 --- a/neps/search_spaces/hyperparameters/numerical.py +++ /dev/null @@ -1,238 +0,0 @@ -"""The [`Numerical`][neps.search_spaces.Numerical] is -a [`Parameter`][neps.search_spaces.Parameter] that represents a numerical -range. - -The two primary numerical hyperparameters are: - -* [`Float`][neps.search_spaces.Float] for continuous - float values. -* [`Integer`][neps.search_spaces.Integer] for discrete - integer values. - -The [`Numerical`][neps.search_spaces.Numerical] is a -base class for both of these hyperparameters, and includes methods from -both [`ParameterWithPrior`][neps.search_spaces.ParameterWithPrior], -allowing you to set a confidence along with a -[`.prior`][neps.search_spaces.Parameter.prior] that can be used -with certain algorithms. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from functools import lru_cache -from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar -from typing_extensions import override - -import numpy as np -import scipy - -from neps.search_spaces.parameter import ParameterWithPrior - -if TYPE_CHECKING: - from neps.search_spaces.domain import Domain - from neps.utils.types import TruncNorm - -T = TypeVar("T", int, float) - - -# OPTIM(eddiebergman): When calculating priors over and over, -# creating this scipy.rvs is surprisingly slow. Since we do not -# mutate them, we just cache them. This is done across instances so -# we also can access this cache with new copies of the hyperparameters. -@lru_cache(maxsize=128, typed=False) -def _get_truncnorm_prior_and_std( - low: int | float, - high: int | float, - prior: int | float, - confidence_score: float, -) -> tuple[TruncNorm, float]: - std = (high - low) * confidence_score - a, b = (low - prior) / std, (high - prior) / std - return scipy.stats.truncnorm(a, b), float(std) - - -class Numerical(ParameterWithPrior[T, T]): - """A numerical hyperparameter is bounded by a lower and upper value. - - Attributes: - lower: The lower bound of the numerical hyperparameter. - upper: The upper bound of the numerical hyperparameter. - log: Whether the hyperparameter is in log space. - log_bounds: The log bounds of the hyperparameter, if `log=True`. - log_prior: The log prior value of the hyperparameter, if `log=True` - and a `prior` is set. - prior_confidence_choice: The prior confidence choice. - prior_confidence_score: The prior confidence score. - has_prior: Whether the hyperparameter has a prior. - """ - - DEFAULT_CONFIDENCE_SCORES: ClassVar[Mapping[str, float]] - - def __init__( - self, - lower: T, - upper: T, - *, - log: bool = False, - prior: T | None, - is_fidelity: bool, - domain: Domain[T], - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Initialize the numerical hyperparameter. - - Args: - lower: The lower bound of the numerical hyperparameter. - upper: The upper bound of the numerical hyperparameter. - log: Whether the hyperparameter is in log space. - prior: The prior value of the hyperparameter. - is_fidelity: Whether the hyperparameter is a fidelity parameter. - domain: The domain of the hyperparameter. - prior_confidence: The prior confidence choice. - """ - super().__init__(value=None, prior=prior, is_fidelity=is_fidelity) # type: ignore - _cls_name = self.__class__.__name__ - if lower >= upper: - raise ValueError( - f"{_cls_name} parameter: bounds error (lower >= upper). Actual values: " - f"lower={lower}, upper={upper}" - ) - - if log and (lower <= 0 or upper <= 0): - raise ValueError( - f"{_cls_name} parameter: bounds error (log scale cant have bounds <= 0)." - f" Actual values: lower={lower}, upper={upper}" - ) - - if prior is not None and not lower <= prior <= upper: - raise ValueError( - f"Float parameter: prior bounds error. Expected lower <= prior" - f" <= upper, but got lower={lower}, prior={prior}," - f" upper={upper}" - ) - - if prior_confidence not in self.DEFAULT_CONFIDENCE_SCORES: - raise ValueError( - f"{_cls_name} parameter: prior confidence score error. Expected one of " - f"{list(self.DEFAULT_CONFIDENCE_SCORES.keys())}, but got " - f"{prior_confidence}" - ) - - if is_fidelity and (lower <= 0 or upper <= 0): - raise ValueError( - f"{_cls_name} parameter: fidelity parameter bounds error (log scale " - f"can't have bounds <= 0). Actual values: lower={lower}, upper={upper}" - ) - - # Validate 'log' and 'is_fidelity' types to prevent configuration errors - # from the YAML input - for param, value in {"log": log, "is_fidelity": is_fidelity}.items(): - if not isinstance(value, bool): - raise TypeError( - f"Expected '{param}' to be a boolean, but got type: " - f"{type(value).__name__}" - ) - - self.lower: T = lower - self.upper: T = upper - self.log: bool = log - self.domain: Domain[T] = domain - self.log_bounds: tuple[float, float] | None = None - self.log_prior: float | None = None - if self.log: - self.log_bounds = (float(np.log(lower)), float(np.log(upper))) - self.log_prior = float(np.log(self.prior)) if self.prior is not None else None - - self.prior_confidence_choice: Literal["low", "medium", "high"] = prior_confidence - - self.prior_confidence_score: float = self.DEFAULT_CONFIDENCE_SCORES[ - prior_confidence - ] - self.has_prior: bool = self.prior is not None - - @override - def __eq__(self, other: Any) -> bool: - if not isinstance(other, self.__class__): - return False - - return ( - self.lower == other.lower - and self.upper == other.upper - and self.log == other.log - and self.is_fidelity == other.is_fidelity - and self.value == other.value - and self.prior == other.prior - and self.prior_confidence_score == other.prior_confidence_score - ) - - def _get_truncnorm_prior_and_std(self) -> tuple[TruncNorm, float]: - if self.log: - assert self.log_bounds is not None - low, high = self.log_bounds - prior = self.log_prior - else: - low, high = self.lower, self.upper - prior = self.prior - - assert prior is not None - return _get_truncnorm_prior_and_std( - low=low, - high=high, - prior=prior, - confidence_score=self.prior_confidence_score, - ) - - -class NumericalParameter(Numerical): - """Deprecated: Use `Numerical` instead of `NumericalParameter`. - - This class remains for backward compatibility and will raise a deprecation - warning if used. - """ - - def __init__( - self, - lower: T, - upper: T, - *, - log: bool = False, - prior: T | None, - is_fidelity: bool, - domain: Domain[T], - prior_confidence: Literal["low", "medium", "high"] = "low", - ): - """Initialize a deprecated `NumericalParameter`. - - Args: - lower: The lower bound of the numerical hyperparameter. - upper: The upper bound of the numerical hyperparameter. - log: Whether the hyperparameter is in log space. - prior: The prior value of the hyperparameter. - is_fidelity: Whether the hyperparameter is a fidelity parameter. - domain: The domain of the hyperparameter. - prior_confidence: The prior confidence choice. - - Raises: - DeprecationWarning: A warning indicating that `neps.NumericalParameter` is - deprecated and `neps.Numerical` should be used instead. - """ - import warnings - - warnings.warn( - ( - "Usage of 'neps.NumericalParameter' is deprecated and will be removed in" - " future releases. Please use 'neps.Numerical' instead." - ), - DeprecationWarning, - stacklevel=2, - ) - super().__init__( - lower=lower, - upper=upper, - log=log, - prior=prior, - is_fidelity=is_fidelity, - domain=domain, - prior_confidence=prior_confidence, - ) diff --git a/neps/search_spaces/parameter.py b/neps/search_spaces/parameter.py deleted file mode 100644 index f8b763cba..000000000 --- a/neps/search_spaces/parameter.py +++ /dev/null @@ -1,216 +0,0 @@ -"""The base [`Parameter`][neps.search_spaces.Parameter] class. - -The `Parameter` refers to both the hyperparameter definition but also -holds a [`.value`][neps.search_spaces.Parameter.value] which can be -set or empty, in which case it is `None`. - -!!! tip - - A `Parameter` which allows for defining a - [`.default`][neps.search_spaces.Parameter.default] and some prior, - i.e. some default value along with a confidence that this is a good setting, - should implement the [`ParameterWithPrior`][neps.search_spaces.ParameterWithPrior] - class. - - This is utilized by certain optimization routines to inform the search process. -""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import Mapping -from typing import Any, ClassVar, Generic, TypeVar -from typing_extensions import Self - -ValueT = TypeVar("ValueT") -SerializedT = TypeVar("SerializedT") - - -class Parameter(ABC, Generic[ValueT, SerializedT]): - """A base class for hyperparameters. - - Attributes: - prior: default value for the hyperparameter. This value - is used as a prior to inform algorithms about a decent - default value for the hyperparameter, as well as use - attributes from [`ParameterWithPrior`][neps.search_spaces.ParameterWithPrior], - to aid in optimization. - is_fidelity: whether the hyperparameter is fidelity. - value: value for the hyperparameter, if any. - normalized_value: normalized value for the hyperparameter. - """ - - def __init__( - self, - *, - value: ValueT | None, - prior: ValueT | None, - is_fidelity: bool, - ): - """Create a new `Parameter`. - - Args: - value: value for the hyperparameter. - prior: default value for the hyperparameter. - is_fidelity: whether the hyperparameter is fidelity. - """ - self.prior = prior - self.is_fidelity = is_fidelity - - # TODO(eddiebergman): The reason to have this not as a straight alone - # attribute is that the graph parameters currently expose there own - # way of calculating a value on demand. - # To fix this would mean to essentially decouple GraphParameter entirely - # from Parameter as it's less of a heirarchy and more of just a small overlap - # of functionality. - self._value = value - self.normalized_value = ( - self.value_to_normalized(value) if value is not None else None - ) - - # TODO: Pass in through subclasses - self.prior_confidence_score: float - - # TODO(eddiebergman): All this does is just check values which highly unlikely - # what we want. However this needs to be tackled in a seperate PR. - # - # > The Princess is in another castle. - # - def __eq__(self, other: Any) -> bool: - # Assuming that two different classes should represent two different parameters - if not isinstance(other, self.__class__): - return NotImplemented - - if self.value is not None and other.value is not None: - return self.value == other.value - - return False - - @abstractmethod - def clone(self) -> Self: - """Create a copy of the `Parameter`.""" - - @property - def value(self) -> ValueT | None: - """Get the value of the hyperparameter, or `None` if not set.""" - return self._value - - def sample(self) -> Self: - """Sample a new version of this `Parameter` with a random value. - - Will set the [`.value`][neps.search_spaces.Parameter.value] to the - sampled value. - - Returns: - A new `Parameter` with a sampled value. - """ - value = self.sample_value() - copy_self = self.clone() - copy_self.set_value(value) - return copy_self - - @abstractmethod - def sample_value(self) -> ValueT: - """Sample a new value.""" - - @abstractmethod - def set_value(self, value: ValueT | None) -> None: - """Set the value for the hyperparameter. - - Args: - value: value for the hyperparameter. - """ - - @abstractmethod - def value_to_normalized(self, value: ValueT) -> float: - """Convert a value to a normalized value. - - Normalization is different per hyperparameter type, - but roughly refers to numeric values. - - * `(0, 1)` scaling in the case of - a [`Numerical`][neps.search_spaces.Numerical], - * `{0.0, 1.0}` for a [`Constant`][neps.search_spaces.Constant], - * `[0, 1, ..., n]` for a - [`Categorical`][neps.search_spaces.Categorical]. - - Args: - value: value to convert. - - Returns: - The normalized value. - """ - - @abstractmethod - def normalized_to_value(self, normalized_value: float) -> ValueT: - """Convert a normalized value back to value in the defined hyperparameter range. - - Args: - normalized_value: normalized value to convert. - - Returns: - The value. - """ - - def load_from(self, value: Any) -> None: - """Load a serialized value into the hyperparameter's value. - - Args: - value: value to load. - """ - self.set_value(value) - - -class ParameterWithPrior(Parameter[ValueT, SerializedT]): - """A base class for hyperparameters with priors. - - Attributes: - prior_confidence_choice: The choice of how confident any algorithm should - be in the prior value being a good value. - prior_confidence_score: A score used by algorithms to utilize the prior value. - has_prior: whether the hyperparameter has a prior that can be used by an - algorithm. In many cases, this refers to having a prior value. - """ - - DEFAULT_CONFIDENCE_SCORES: ClassVar[Mapping[str, float]] - prior_confidence_choice: str - prior_confidence_score: float - has_prior: bool - - # NOTE(eddiebergman): Like the normal `Parameter.sample` but with `user_priors`. - @abstractmethod - def sample_value(self, *, user_priors: bool = False) -> ValueT: - """Sample a new value. - - Similar to - [`Parameter.sample_value()`][neps.search_spaces.Parameter.sample_value], - but a `ParameterWithPrior` can use the confidence score by setting - `user_priors=True`. - - Args: - user_priors: whether to use the confidence score - when sampling a value. - - Returns: - The sampled value. - """ - - def sample(self, *, user_priors: bool = False) -> Self: - """Sample a new version of this `Parameter` with a random value. - - Similar to - [`Parameter.sample()`][neps.search_spaces.Parameter.sample], - but a `ParameterWithPrior` can use the confidence score by setting - `user_priors=True`. - - Args: - user_priors: whether to use the confidence score - when sampling a value. - - Returns: - A new `Parameter` with a sampled value. - """ - value = self.sample_value(user_priors=user_priors) - copy_self = self.clone() - copy_self.set_value(value) - return copy_self diff --git a/neps/search_spaces/search_space.py b/neps/search_spaces/search_space.py deleted file mode 100644 index 3cb69b855..000000000 --- a/neps/search_spaces/search_space.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Contains the [`SearchSpace`][neps.search_spaces.search_space.SearchSpace] class -which is a container for hyperparameters that can be sampled, mutated, and crossed over. -""" - -from __future__ import annotations - -import logging -import pprint -from collections.abc import Iterator, Mapping -from pathlib import Path -from typing import Any - -import ConfigSpace as CS -import yaml - -from neps.search_spaces.architecture.graph_grammar import GraphParameter -from neps.search_spaces.domain import UNIT_FLOAT_DOMAIN -from neps.search_spaces.hyperparameters import ( - Categorical, - Constant, - Float, - Integer, - Numerical, -) -from neps.search_spaces.parameter import Parameter, ParameterWithPrior -from neps.search_spaces.yaml_search_space_utils import ( - SearchSpaceFromYamlFileError, - deduce_type, - formatting_cat, - formatting_const, - formatting_float, - formatting_int, -) - -logger = logging.getLogger(__name__) - - -def pipeline_space_from_configspace( - configspace: CS.ConfigurationSpace, -) -> dict[str, Parameter]: - """Constructs the [`Parameter`][neps.search_spaces.parameter.Parameter] objects - from a [`ConfigurationSpace`][ConfigSpace.configuration_space.ConfigurationSpace]. - - Args: - configspace: The configuration space to construct the pipeline space from. - - Returns: - A dictionary where keys are parameter names and values are parameter objects. - """ - pipeline_space = {} - parameter: Parameter - if any(configspace.get_conditions()) or any(configspace.get_forbiddens()): - raise NotImplementedError( - "The ConfigurationSpace has conditions or forbidden clauses, " - "which are not supported by neps." - ) - - for hyperparameter in configspace.get_hyperparameters(): - if isinstance(hyperparameter, CS.Constant): - parameter = Constant(value=hyperparameter.value) - elif isinstance(hyperparameter, CS.CategoricalHyperparameter): - parameter = Categorical( - hyperparameter.choices, - prior=hyperparameter.default_value, - ) - elif isinstance(hyperparameter, CS.OrdinalHyperparameter): - parameter = Categorical( - hyperparameter.sequence, - prior=hyperparameter.default_value, - ) - elif isinstance(hyperparameter, CS.UniformIntegerHyperparameter): - parameter = Integer( - lower=hyperparameter.lower, - upper=hyperparameter.upper, - log=hyperparameter.log, - prior=hyperparameter.default_value, - ) - elif isinstance(hyperparameter, CS.UniformFloatHyperparameter): - parameter = Float( - lower=hyperparameter.lower, - upper=hyperparameter.upper, - log=hyperparameter.log, - prior=hyperparameter.default_value, - ) - else: - raise ValueError(f"Unknown hyperparameter type {hyperparameter}") - pipeline_space[hyperparameter.name] = parameter - return pipeline_space - - -def pipeline_space_from_yaml( # noqa: C901 - config: str | Path | dict, -) -> dict[str, Parameter]: - """Reads configuration details from a YAML file or a dictionary and constructs a - pipeline space dictionary. - - Args: - config: Path to the YAML file or a dictionary containing parameter configurations. - - Returns: - A dictionary where keys are parameter names and values are parameter objects. - - Raises: - SearchSpaceFromYamlFileError: Raised if there are issues with the YAML file's - format, contents, or if the dictionary is invalid. - """ - try: - if isinstance(config, str | Path): - # try to load the YAML file - try: - yaml_file_path = Path(config) - with yaml_file_path.open("r") as file: - config = yaml.safe_load(file) - if not isinstance(config, dict): - raise ValueError( - "The loaded pipeline_space is not a valid dictionary. Please " - "ensure that you use a proper structure. See the documentation " - "for more details." - ) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Unable to find the specified file for 'pipeline_space' at " - f"'{config}'. Please verify the path specified in the " - f"'pipeline_space' argument and try again." - ) from e - except yaml.YAMLError as e: - raise ValueError(f"The file at {config} is not a valid YAML file.") from e - - pipeline_space: dict[str, Parameter] = {} - - if len(config) == 1 and "pipeline_space" in config: - config = config["pipeline_space"] - for name, details in config.items(): # type: ignore - param_type = deduce_type(name, details) - - if param_type in ("int", "integer"): - formatted_details = formatting_int(name, details) - pipeline_space[name] = Integer(**formatted_details) - elif param_type == "float": - formatted_details = formatting_float(name, details) - pipeline_space[name] = Float(**formatted_details) - elif param_type in ("cat", "categorical"): - formatted_details = formatting_cat(name, details) - pipeline_space[name] = Categorical(**formatted_details) - elif param_type == "const": - const_details = formatting_const(details) - pipeline_space[name] = Constant(const_details) # type: ignore - else: - # Handle unknown parameter type - raise TypeError( - f"Unsupported parameter with details: {details} for '{name}'.\n" - f"Supported Types for argument type are:\n" - "For integer parameter: int, integer\n" - "For float parameter: float\n" - "For categorical parameter: cat, categorical\n" - "Constant parameter was not detect\n" - ) - except (KeyError, TypeError, ValueError, FileNotFoundError) as e: - raise SearchSpaceFromYamlFileError(e) from e - - return pipeline_space - - -class SearchSpace(Mapping[str, Any]): - """A container for hyperparameters that can be sampled, mutated, and crossed over. - - Provides operations for operating on and generating new configurations from the - hyperparameters. - - !!! note - - The `SearchSpace` class is both the definition of the search space and also - a configuration at the same time. - - When refering to the `SearchSpace` as a configuration, the documentation will - refer to it as a `configuration` or `config`. Otherwise, it will be referred to - as a `search space`. - - !!! note "TODO" - - This documentation is WIP. If you have any questions, please reach out so we can - know better what to document. - """ - - def __init__(self, **hyperparameters: Parameter): # noqa: C901, PLR0912 - """Initialize the SearchSpace with hyperparameters. - - Args: - **hyperparameters: The hyperparameters that define the search space. - """ - # Ensure a consistent ordering for uses throughout the lib - _hyperparameters = sorted(hyperparameters.items(), key=lambda x: x[0]) - _fidelity_param: Numerical | None = None - _fidelity_name: str | None = None - _has_prior: bool = False - - for name, hp in _hyperparameters: - if hp.is_fidelity: - if _fidelity_param is not None: - raise ValueError( - "neps only supports one fidelity parameter in the pipeline space," - " but multiple were given. (Hint: check you pipeline space for " - "multiple is_fidelity=True)" - ) - - if not isinstance(hp, Numerical): - raise ValueError( - f"Only float and integer fidelities supported, got {hp}" - ) - - _fidelity_param = hp - _fidelity_name = name - - if isinstance(hp, ParameterWithPrior) and hp.has_prior: - _has_prior = True - - self.hyperparameters: dict[str, Parameter] = dict(_hyperparameters) - self.fidelity: Numerical | None = _fidelity_param - self.fidelity_name: str | None = _fidelity_name - self.has_prior: bool = _has_prior - - self.prior_config = {} - for name, hp in _hyperparameters: - if hp.prior is not None: - self.prior_config[name] = hp.prior - continue - - match hp: - case Categorical(): - first_choice = hp.choices[0] - self.prior_config[name] = first_choice - case Integer() | Float(): - if hp.is_fidelity: - self.prior_config[name] = hp.upper - continue - - midpoint = hp.domain.cast_one(0.5, frm=UNIT_FLOAT_DOMAIN) - self.prior_config[name] = midpoint - case Constant(): - self.prior_config[name] = hp.value - case GraphParameter(): - self.prior_config[name] = hp.prior - case _: - raise TypeError(f"Unknown hyperparameter type {hp}") - - self.categoricals: Mapping[str, Categorical] = { - k: hp for k, hp in _hyperparameters if isinstance(hp, Categorical) - } - self.numerical: Mapping[str, Integer | Float] = { - k: hp - for k, hp in _hyperparameters - if isinstance(hp, Integer | Float) and not hp.is_fidelity - } - self.graphs: Mapping[str, GraphParameter] = { - k: hp for k, hp in _hyperparameters if isinstance(hp, GraphParameter) - } - self.constants: Mapping[str, Any] = { - k: hp.value for k, hp in _hyperparameters if isinstance(hp, Constant) - } - # NOTE: For future of multiple fidelities - self.fidelities: Mapping[str, Integer | Float] = {} - if _fidelity_param is not None and _fidelity_name is not None: - assert isinstance(_fidelity_param, Integer | Float) - self.fidelities = {_fidelity_name: _fidelity_param} - - # TODO: Deprecate out, ideally configs are just dictionaries, - # not attached to this space object - self._values = { - hp_name: hp if isinstance(hp, GraphParameter) else hp.value - for hp_name, hp in self.hyperparameters.items() - } - - # TODO: Deprecate and remove - def from_dict(self, config: Mapping[str, Any | GraphParameter]) -> SearchSpace: - """Create a new instance of this search space with parameters set from the config. - - Args: - config: The dictionary of hyperparameters to set with values. - """ - new = self.clone() - for name, val in config.items(): - new.hyperparameters[name].load_from(val) - new._values[name] = new.hyperparameters[name].value - - return new - - def clone(self) -> SearchSpace: - """Create a copy of the search space.""" - return self.__class__(**{k: v.clone() for k, v in self.hyperparameters.items()}) - - def __getitem__(self, key: str) -> Parameter: - return self.hyperparameters[key] - - def __iter__(self) -> Iterator[str]: - return iter(self.hyperparameters) - - def __len__(self) -> int: - return len(self.hyperparameters) - - def __str__(self) -> str: - return pprint.pformat(self.hyperparameters) diff --git a/neps/search_spaces/yaml_search_space_utils.py b/neps/search_spaces/yaml_search_space_utils.py deleted file mode 100644 index ff6d72ad8..000000000 --- a/neps/search_spaces/yaml_search_space_utils.py +++ /dev/null @@ -1,385 +0,0 @@ -import logging -import re -from typing import Literal, overload - -logger = logging.getLogger("neps") - - -@overload -def convert_scientific_notation( - value: str | int | float, show_usage_flag: Literal[False] = False -) -> float: ... - - -@overload -def convert_scientific_notation( - value: str | int | float, show_usage_flag: Literal[True] -) -> tuple[float, bool]: ... - - -def convert_scientific_notation( - value: str | int | float, show_usage_flag: bool = False -) -> float | tuple[float, bool]: - """ - Convert a given value to a float if it's a string that matches scientific e notation. - This is especially useful for numbers like "3.3e-5" which YAML parsers may not - directly interpret as floats. - - If the 'show_usage_flag' is set to True, the function returns a tuple of the float - conversion and a boolean flag indicating whether scientific notation was detected. - - Args: - value (str | int | float): The value to convert. Can be an integer, float, - or a string representing a number, possibly in - scientific notation. - show_usage_flag (bool): Optional; defaults to False. If True, the function - also returns a flag indicating whether scientific - notation was detected in the string. - - Returns: - float: The value converted to float if 'show_usage_flag' is False. - (float, bool): A tuple containing the value converted to float and a flag - indicating scientific notation detection if 'show_usage_flag' - is True. - - Raises: - ValueError: If the value is a string and does not represent a valid number. - """ - - e_notation_pattern = r"^-?\d+(\.\d+)?[eE]-?\d+$" - - flag = False # Flag if e notation was detected - - if isinstance(value, str): - # Remove all whitespace from the string - value_no_space = value.replace(" ", "") - - # check for e notation - if re.match(e_notation_pattern, value_no_space): - flag = True - - if show_usage_flag is True: - return float(value), flag - else: - return float(value) - - -class SearchSpaceFromYamlFileError(Exception): - """ - Exception raised for errors occurring during the initialization of the search space - from a YAML file. - - Attributes: - exception_type (str): The type of the original exception. - message (str): A detailed message that includes the type of the original exception - and the error description. - - Args: - exception (Exception): The original exception that was raised during the - initialization of the search space from the YAML file. - - Example Usage: - try: - # Code to initialize search space from YAML file - except (KeyError, TypeError, ValueError) as e: - raise SearchSpaceFromYamlFileError(e) - """ - - def __init__(self, exception: Exception) -> None: - self.exception_type = type(exception).__name__ - self.message = ( - f"Error occurred during initialization of search space from " - f"YAML file.\n {self.exception_type}: {exception}" - ) - super().__init__(self.message) - - -def deduce_type( - name: str, details: dict[str, str | int | float] | str | int | float -) -> str: - """Deduces the parameter type from details. - - Args: - name: The name of the parameter. - details: A dictionary containing parameter specifications or - a direct value (string, integer, or float). - - Returns: - The deduced parameter type ('int', 'float', 'categorical', or 'constant'). - - Raises: - TypeError: If the type cannot be deduced or the details don't align with expected - constraints. - """ - if isinstance(details, (str, int, float)): - return "const" - - if isinstance(details, dict): - if "type" in details: - param_type = details.pop("type") - assert isinstance(param_type, str) - return param_type.lower() - - return deduce_param_type(name, details) - - raise TypeError( - f"Unable to deduce parameter type for '{name}' with details '{details}'." - ) - - -def deduce_param_type(name: str, details: dict[str, int | str | float]) -> str: - """Deduces the parameter type based on the provided details. - - The function interprets the 'details' dictionary to determine the parameter type. - The dictionary should include key-value pairs that describe the parameter's - characteristics, such as lower, upper and choices. - - - Args: - name (str): The name of the parameter. - details ((dict[str, int | str | float])): A dictionary containing parameter - specifications. - - Returns: - str: The deduced parameter type ('int', 'float' or 'categorical'). - - Raises: - TypeError: If the parameter type cannot be deduced from the details, or if the - provided details have inconsistent types for expected keys. - - Example: - param_type = deduce_param_type('example_param', {'lower': 0, 'upper': 10})""" - # Logic to deduce type from details - - # check for int and float conditions - if "lower" in details and "upper" in details: - # Determine if it's an integer or float range parameter - if isinstance(details["lower"], int) and isinstance(details["upper"], int): - param_type = "int" - elif isinstance(details["lower"], float) and isinstance(details["upper"], float): - param_type = "float" - else: - try: - details["lower"], flag_lower = convert_scientific_notation( - details["lower"], show_usage_flag=True - ) - details["upper"], flag_upper = convert_scientific_notation( - details["upper"], show_usage_flag=True - ) - except ValueError as e: - raise TypeError( - f"Inconsistent types for 'lower' and 'upper' in '{name}'. " - f"Both must be either integers or floats." - ) from e - - # check if one value is e notation and if so convert it to float - if flag_lower or flag_upper: - logger.info( - f"Because of e notation, Parameter {name} gets " - f"interpreted as float" - ) - param_type = "float" - else: - raise TypeError( - f"Inconsistent types for 'lower' and 'upper' in '{name}'. " - f"Both must be either integers or floats." - ) - # check for categorical condition - elif "choices" in details: - param_type = "categorical" - else: - raise KeyError( - f"Unable to deduce parameter type from {name} " - f"with details {details}\n" - "Supported parameters:\n" - "Float and Integer: Expected keys: 'lower', 'upper'\n" - "Categorical: Expected keys: 'choices'\n" - ) - return param_type - - -def formatting_int(name: str, details: dict[str, str | int | float]) -> dict: - """ - Converts scientific notation values to integers. - - This function converts the 'lower' and 'upper' bounds, as well as the 'default' - value (if present), from scientific notation to integers. - - Args: - name (str): The name of the integer parameter. - details (dict[str, str | int | float]): A dictionary containing the parameter's - specifications. Expected keys include - 'lower', 'upper', and optionally 'default'. - - Raises: - TypeError: If 'lower', 'upper', or 'default' cannot be converted from scientific - notation to integers. - - Returns: - The dictionary with the converted integer parameter details. - """ - if not isinstance(details["lower"], int) or not isinstance(details["upper"], int): - try: - # for numbers like 1e2 and 10^ - lower, flag_lower = convert_scientific_notation( - details["lower"], show_usage_flag=True - ) - upper, flag_upper = convert_scientific_notation( - details["upper"], show_usage_flag=True - ) - # check if one value format is e notation and if it's an integer - if flag_lower or flag_upper: - if lower == int(lower) and upper == int(upper): - details["lower"] = int(lower) - details["upper"] = int(upper) - else: - raise TypeError() - else: - raise TypeError() - except (ValueError, TypeError) as e: - raise TypeError( - f"'lower' and 'upper' must be integer for " f"integer parameter '{name}'." - ) from e - if "default" in details: - if not isinstance(details["default"], int): - try: - # convert value can raise ValueError - default = convert_scientific_notation(details["default"]) - if default == int(default): - details["default"] = int(default) - else: - raise TypeError() # type of value is not int - except (ValueError, TypeError) as e: - raise TypeError( - f"default value {details['default']} " - f"must be integer for integer parameter {name}" - ) from e - return details - - -def formatting_float(name: str, details: dict[str, str | int | float]) -> dict: - """ - Converts scientific notation values to floats. - - This function converts the 'lower' and 'upper' bounds, as well as the 'default' - value (if present), from scientific notation to floats. - - Args: - name: The name of the float parameter. - details: A dictionary containing the parameter's specifications. Expected keys - include 'lower', 'upper', and optionally 'default'. - - Raises: - TypeError: If 'lower', 'upper', or 'default' cannot be converted from scientific - notation to floats. - - Returns: - The dictionary with the converted float parameter details. - """ - - if not isinstance(details["lower"], float) or not isinstance(details["upper"], float): - try: - # for numbers like 1e-5 and 10^ - details["lower"] = convert_scientific_notation(details["lower"]) - details["upper"] = convert_scientific_notation(details["upper"]) - except ValueError as e: - raise TypeError( - f"'lower' and 'upper' must be float for " f"float parameter '{name}'." - ) from e - if "default" in details: - if not isinstance(details["default"], float): - try: - details["default"] = convert_scientific_notation(details["default"]) - except ValueError as e: - raise TypeError( - f" default'{details['default']}' must be float for float " - f"parameter {name} " - ) from e - return details - - -def formatting_cat(name: str, details: dict[str, list | str | int | float]) -> dict: - """ - This function ensures that the 'choices' key in the details is a list and attempts - to convert any elements expressed in scientific notation to floats. It also handles - the 'default' value, converting it from scientific notation if necessary. - - Args: - name: The name of the categorical parameter. - details: A dictionary containing the parameter's specifications. The required key - is 'choices', which must be a list. The 'default' key is optional. - - Raises: - TypeError: If 'choices' is not a list. - - Returns: - The validated and possibly converted categorical parameter details. - """ - if not isinstance(details["choices"], list): - raise TypeError(f"The 'choices' for '{name}' must be a list.") - - for i, element in enumerate(details["choices"]): - try: - converted_value, e_flag = convert_scientific_notation( - element, show_usage_flag=True - ) - - if e_flag: - # Replace the element at the same position - details["choices"][i] = converted_value - except ValueError: - pass # If a ValueError occurs, simply continue to the next element - - if "default" in details: - e_flag = False - extracted_default = details["default"] - if not isinstance(extracted_default, (str, int, float)): - raise TypeError( - f"The 'default' value for '{name}' must be a string, integer, or float." - f" Got {type(extracted_default).__name__}." - ) - - try: - # check if e notation, if then convert to number - default, e_flag = convert_scientific_notation( - extracted_default, show_usage_flag=True - ) - except ValueError: - pass # if default value is not in a numeric format, Value Error occurs - - if e_flag is True: - details["default"] = default - - return details - - -def formatting_const(details: str | int | float) -> str | int | float: - """Validates and converts a constant parameter. - - This function checks if the 'details' parameter contains a value expressed in - scientific notation and converts it to a float. It ensures that the input - is appropriately formatted, either as a string, integer, or float. - - Args: - details: A constant parameter that can be a string, integer, or float. - If the value is in scientific notation, it will be converted to a float. - - Returns: - The validated and possibly converted constant parameter. - """ - - # check for e notation and convert it to float - e_flag = False - try: - converted_value, e_flag = convert_scientific_notation( - details, show_usage_flag=True - ) - except ValueError: - # if the value is not able to convert to float a ValueError get raised by - # convert_scientific_notation function - pass - - if e_flag: - details = converted_value - - return details diff --git a/neps/space/__init__.py b/neps/space/__init__.py new file mode 100644 index 000000000..f2bbc55ca --- /dev/null +++ b/neps/space/__init__.py @@ -0,0 +1,15 @@ +from neps.space.domain import Domain +from neps.space.encoding import ConfigEncoder +from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter +from neps.space.search_space import SearchSpace + +__all__ = [ + "Categorical", + "ConfigEncoder", + "Constant", + "Domain", + "Float", + "Integer", + "Parameter", + "SearchSpace", +] diff --git a/neps/search_spaces/domain.py b/neps/space/domain.py similarity index 93% rename from neps/search_spaces/domain.py rename to neps/space/domain.py index 5d1a76286..34374b4c3 100644 --- a/neps/search_spaces/domain.py +++ b/neps/space/domain.py @@ -8,7 +8,7 @@ * Whether the domain is split into bins. With that, the primary method of a domain is to be able to -[`cast()`][neps.search_spaces.domain.Domain.cast] a tensor of +[`cast()`][neps.space.domain.Domain.cast] a tensor of values from one to domain to another, e.g. `values_a = domain_a.cast(values_b, frm=domain_b)`. @@ -16,28 +16,27 @@ to log space, etc. The core method to do so is to be able to cast -[`to_unit()`][neps.search_spaces.domain.Domain.to_unit] which takes +[`to_unit()`][neps.space.domain.Domain.to_unit] which takes values to a unit interval [0, 1], and then to be able to cast values in [0, 1] -to the new domain with [`from_unit()`][neps.search_spaces.domain.Domain.from_unit]. +to the new domain with [`from_unit()`][neps.space.domain.Domain.from_unit]. There are some shortcuts implemented in `cast`, such as skipping going through the unit interval if the domains are the same, as no transformation is needed. The primary methods for creating a domain are -* [`Domain.float(l, u, ...)`][neps.search_spaces.domain.Domain.float] - +* [`Domain.floating(l, u, ...)`][neps.space.domain.Domain.floating] - Used for modelling float ranges -* [`Domain.int(l, u, ...)`][neps.search_spaces.domain.Domain.int] - +* [`Domain.integer(l, u, ...)`][neps.space.domain.Domain.integer] - Used for modelling integer ranges -* [`Domain.indices(n)`][neps.search_spaces.domain.Domain.indices] - +* [`Domain.indices(n)`][neps.space.domain.Domain.indices] - Primarly used to model categorical choices If you have a tensor of values, where each column corresponds to a different domain, -you can take a look at [`Domain.translate()`][neps.search_spaces.domain.Domain.translate] +you can take a look at [`Domain.translate()`][neps.space.domain.Domain.translate] If you need a unit-interval domain, please use the -[`Domain.unit_float()`][neps.search_spaces.domain.Domain.unit_float] -or `UNIT_FLOAT_DOMAIN` constant. +[`Domain.unit_float()`][neps.space.domain.Domain.unit_float]. """ from __future__ import annotations @@ -51,7 +50,7 @@ from torch import Tensor if TYPE_CHECKING: - from neps.search_spaces.encoding import ConfigEncoder + from neps.space.encoding import ConfigEncoder Number = int | float V = TypeVar("V", int, float) @@ -63,11 +62,11 @@ class Domain(Generic[V]): The primary methods for creating a domain are - * [`Domain.float(l, u, ...)`][neps.search_spaces.domain.Domain.float] - + * [`Domain.floating(l, u, ...)`][neps.space.domain.Domain.floating] - Used for modelling float ranges - * [`Domain.int(l, u, ...)`][neps.search_spaces.domain.Domain.int] - + * [`Domain.integer(l, u, ...)`][neps.space.domain.Domain.integer] - Used for modelling integer ranges - * [`Domain.indices(n)`][neps.search_spaces.domain.Domain.indices] - + * [`Domain.indices(n)`][neps.space.domain.Domain.indices] - Primarly used to model categorical choices """ @@ -288,7 +287,7 @@ def cast(self, x: Tensor, frm: Domain, *, dtype: torch.dtype | None = None) -> T """Cast a tensor of values frm the domain `frm` to this domain. If you need to cast a tensor of mixed domains, use - [`Domain.translate()`][neps.search_spaces.domain.Domain.translate]. + [`Domain.translate()`][neps.space.domain.Domain.translate]. Args: x: Tensor of values in the `frm` domain to cast to this domain. @@ -377,7 +376,7 @@ def translate( if isinstance(frm, Domain) and isinstance(to, Domain): return to.cast(x, frm=frm, dtype=dtype) - from neps.search_spaces.encoding import ConfigEncoder + from neps.space.encoding import ConfigEncoder frm = ( [frm] * ndims diff --git a/neps/search_spaces/encoding.py b/neps/space/encoding.py similarity index 82% rename from neps/search_spaces/encoding.py rename to neps/space/encoding.py index d95464993..1cdc7b1f9 100644 --- a/neps/search_spaces/encoding.py +++ b/neps/space/encoding.py @@ -1,7 +1,7 @@ """Encoding of hyperparameter configurations into tensors. For the most part, you can just use -[`ConfigEncoder.from_space()`][neps.search_spaces.encoding.ConfigEncoder.from_space] +[`ConfigEncoder.from_space()`][neps.space.encoding.ConfigEncoder.from_space] to create an encoder over a list of hyperparameters, along with any constants you want to include when decoding configurations. """ @@ -15,14 +15,11 @@ import torch -from neps.search_spaces.domain import UNIT_FLOAT_DOMAIN, Domain -from neps.search_spaces.hyperparameters.categorical import Categorical -from neps.search_spaces.hyperparameters.float import Float -from neps.search_spaces.hyperparameters.integer import Integer +from neps.space.domain import Domain +from neps.space.parameters import Categorical, Float, Integer if TYPE_CHECKING: - from neps.search_spaces.parameter import Parameter - from neps.search_spaces.search_space import SearchSpace + from neps.space.search_space import SearchSpace V = TypeVar("V", int, float) @@ -177,7 +174,7 @@ class MinMaxNormalizer(TensorTransformer, Generic[V]): def __post_init__(self) -> None: if self.bins is None: - self.domain = UNIT_FLOAT_DOMAIN + self.domain = Domain.unit_float() else: self.domain = Domain.floating(0.0, 1.0, bins=self.bins) @@ -219,23 +216,23 @@ class ConfigEncoder: tensors. The primary methods/properties to be aware of are: - * [`from_space()`](neps.search_spaces.encoding.ConfigEncoder.default]: Create a + * [`from_space()`][neps.space.encoding.ConfigEncoder.from_space]: Create a default encoder over a list of hyperparameters. Please see the method docs for more details on how it encodes different types of hyperparameters. - * [`encode()`]]neps.search_spaces.encoding.ConfigEncoder.encode]: Encode a list of + * [`encode()`]]neps.space.encoding.ConfigEncoder.encode]: Encode a list of configurations into a single tensor using the transforms of the encoder. - * [`decode()`][neps.search_spaces.encoding.ConfigEncoder.decode]: Decode a 2d tensor + * [`decode()`][neps.space.encoding.ConfigEncoder.decode]: Decode a 2d tensor of length `N` into a list of `N` configurations. - * [`domains`][neps.search_spaces.encoding.ConfigEncoder.domains): The - [`Domain`][neps.search_spaces.domain.Domain] that each hyperparameter is encoded + * [`domains`][neps.space.encoding.ConfigEncoder.domains]: The + [`Domain`][neps.space.domain.Domain] that each hyperparameter is encoded into. This is useful in combination with classes like [`Sampler`][neps.sampling.samplers.Sampler], - [`Prior`][neps.sampling.priors.Prior], and + [`Prior`][neps.sampling.Prior], and [`TorchDistributionWithDomain`][neps.sampling.distributions.TorchDistributionWithDomain], which require knowledge of the domains of each column for the tensor, for example, to sample values directly into the encoded space, getting log probabilities of the encoded values. - * [`ncols`][neps.search_spaces.encoding.ConfigEncoder.ncols]: The number of columns + * [`ndim`][neps.space.encoding.ConfigEncoder.ndim]: The number of columns in the encoded tensor, useful for initializing some `Sampler`s. """ @@ -348,7 +345,7 @@ def pdist( numericals = Domain.translate( numericals, frm=self.numerical_domains, - to=UNIT_FLOAT_DOMAIN, + to=Domain.unit_float(), ) dists = torch.nn.functional.pdist(numericals, p=numerical_ord) @@ -375,7 +372,7 @@ def pdist( return sq @property - def ncols(self) -> int: + def ndim(self) -> int: """The number of columns in the encoded tensor.""" return len(self.transformers) @@ -467,7 +464,6 @@ def from_space( space: SearchSpace, *, include_fidelity: bool = False, - include_constants_when_decoding: bool = True, custom_transformers: dict[str, TensorTransformer] | None = None, ) -> ConfigEncoder: """Create a default encoder over a list of hyperparameters. @@ -481,11 +477,6 @@ def from_space( Args: space: The search space to build an encoder for - include_constants_when_decoding: Whether to include constants in the encoder. - These will not be present in the encoded tensors obtained in - [`encode()`][neps.search_spaces.encoding.ConfigEncoder.encode] - but will present when using - [`decode()`][neps.search_spaces.encoding.ConfigEncoder.decode]. include_fidelity: Whether to include fidelities in the encoding custom_transformers: A mapping of hyperparameter names to custom transformers to use @@ -493,57 +484,17 @@ def from_space( Returns: A `ConfigEncoder` instance """ - parameters = {**space.numerical, **space.categoricals} - if include_fidelity: - parameters.update(space.fidelities) - - return ConfigEncoder.from_parameters( - parameters=parameters, - constants=space.constants if include_constants_when_decoding else None, - custom_transformers=custom_transformers, - ) - - @classmethod - def from_parameters( - cls, - parameters: Mapping[str, Parameter], - constants: Mapping[str, Any] | None = None, - *, - custom_transformers: dict[str, TensorTransformer] | None = None, - ) -> ConfigEncoder: - """Create a default encoder over a list of hyperparameters. - - This method creates a default encoder over a list of hyperparameters. It - automatically creates transformers for each hyperparameter based on its type. - The transformers are as follows: - - * `Float` and `Integer` are normalized to the unit interval. - * `Categorical` is transformed into an integer. - - Args: - parameters: A mapping of hyperparameter names to hyperparameters. - constants: A mapping of constant hyperparameters to include when decoding. - custom_transformers: A mapping of hyperparameter names to custom transformers. - - Returns: - A `ConfigEncoder` instance - """ - if constants is not None: - overlap = set(parameters) & set(constants) + # Sanity check we do not apply custom transformers to constants + if len(space.constants) and custom_transformers is not None: + overlap = set(custom_transformers) & set(space.constants) if any(overlap): raise ValueError( - "`constants=` and `parameters=` cannot have overlapping" - f" keys: {overlap=}" + f"Can not apply `custom_transformers= to `constants=`: {overlap=}" ) - if custom_transformers is not None: - overlap = set(custom_transformers) & set(constants) - if any(overlap): - raise ValueError( - f"Can not apply `custom_transformers=`" - f" to `constants=`: {overlap=}" - ) - else: - constants = {} + + parameters = {**space.numerical, **space.categoricals} + if include_fidelity: + parameters.update(space.fidelities) custom = custom_transformers or {} transformers: dict[str, TensorTransformer] = {} @@ -563,4 +514,4 @@ def from_parameters( " please provide it as `constants=`." ) - return cls(transformers, constants=constants) + return cls(transformers, constants=space.constants) diff --git a/neps/search_spaces/functions.py b/neps/space/functions.py similarity index 55% rename from neps/search_spaces/functions.py rename to neps/space/functions.py index ba8b6bc9c..e0b83c248 100644 --- a/neps/search_spaces/functions.py +++ b/neps/space/functions.py @@ -7,12 +7,10 @@ import torch -from neps.search_spaces.domain import UNIT_FLOAT_DOMAIN, Domain -from neps.search_spaces.parameter import Parameter, ParameterWithPrior -from neps.search_spaces.search_space import SearchSpace +from neps.space.domain import Domain if TYPE_CHECKING: - from neps.search_spaces.encoding import ConfigEncoder + from neps.space.encoding import ConfigEncoder logger = logging.getLogger(__name__) @@ -55,7 +53,7 @@ def pairwise_dist( numericals = Domain.translate( numericals, frm=encoder.numerical_domains, - to=UNIT_FLOAT_DOMAIN, + to=Domain.unit_float(), ) dists = torch.nn.functional.pdist(numericals, p=numerical_ord) @@ -81,53 +79,3 @@ def pairwise_dist( sq[row_ix, col_ix] = dists sq[col_ix, row_ix] = dists return sq - - -def sample_one_old( - space: SearchSpace, - *, - user_priors: bool = False, - patience: int = 1, - ignore_fidelity: bool = True, -) -> SearchSpace: - """Sample a configuration from the search space. - - Args: - space: The search space to sample from. - user_priors: Whether to use user priors when sampling. - patience: The number of times to try to sample a valid value for a - hyperparameter. - ignore_fidelity: Whether to ignore the fidelity parameter when sampling. - - Returns: - A sampled configuration from the search space. - """ - sampled_hps: dict[str, Parameter] = {} - - for name, hp in space.hyperparameters.items(): - if hp.is_fidelity and ignore_fidelity: - sampled_hps[name] = hp.clone() - continue - - for attempt in range(patience): - try: - if user_priors and isinstance(hp, ParameterWithPrior): - sampled_hps[name] = hp.sample(user_priors=user_priors) - else: - sampled_hps[name] = hp.sample() - break - except Exception as e: # noqa: BLE001 - logger.warning( - f"Attempt {attempt + 1}/{patience} failed for" - f" sampling {name}: {e!s}" - ) - else: - logger.error( - f"Failed to sample valid value for {name} after {patience} attempts" - ) - raise ValueError( - f"Could not sample valid value for hyperparameter {name}" - f" in {patience} tries!" - ) - - return SearchSpace(**sampled_hps) diff --git a/neps/space/parameters.py b/neps/space/parameters.py new file mode 100644 index 000000000..f4f1d38d0 --- /dev/null +++ b/neps/space/parameters.py @@ -0,0 +1,237 @@ +"""A module of all the parameters for the search space.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal, TypeAlias + +import numpy as np +from more_itertools import all_unique + +from neps.space.domain import Domain + + +@dataclass +class Float: + """A float value for a parameter. + + This kind of parameter is used to represent hyperparameters with continuous float + values, optionally specifying if it exists on a log scale. + + For example, `l2_norm` could be a value in `(0.1)`, while the `learning_rate` + hyperparameter in a neural network search space can be a `Float` + with a range of `(0.0001, 0.1)` but on a log scale. + + ```python + import neps + + l2_norm = neps.Float(0, 1) + learning_rate = neps.Float(1e-4, 1e-1, log=True) + ``` + """ + + lower: float + """The lower bound of the numerical hyperparameter.""" + + upper: float + """The upper bound of the numerical hyperparameter.""" + + log: bool = False + """Whether the hyperparameter is in log space.""" + + prior: float | None = None + """Prior value for the hyperparameter.""" + + prior_confidence: Literal["low", "medium", "high"] = "low" + """Confidence score for the prior value when considering prior based optimization.""" + + is_fidelity: bool = False + """Whether the hyperparameter is fidelity.""" + + domain: Domain[float] = field(init=False) + + def __post_init__(self) -> None: + if self.lower >= self.upper: + raise ValueError( + f"Float parameter: bounds error (lower >= upper). Actual values: " + f"lower={self.lower}, upper={self.upper}" + ) + + if self.log and (self.lower <= 0 or self.upper <= 0): + raise ValueError( + f"Float parameter: bounds error (log scale cant have bounds <= 0). " + f"Actual values: lower={self.lower}, upper={self.upper}" + ) + + if self.prior is not None and not self.lower <= self.prior <= self.upper: + raise ValueError( + f"Float parameter: prior bounds error. Expected lower <= prior <= upper, " + f"but got lower={self.lower}, prior={self.prior}, upper={self.upper}" + ) + + self.lower = float(self.lower) + self.upper = float(self.upper) + + if np.isnan(self.lower): + raise ValueError("Can not have lower bound that is nan") + + if np.isnan(self.upper): + raise ValueError("Can not have upper bound that is nan") + + self.domain = Domain.floating(self.lower, self.upper, log=self.log) + + +@dataclass +class Integer: + """An integer value for a parameter. + + This kind of parameter is used to represent hyperparameters with + continuous integer values, optionally specifying f it exists on a log scale. + + For example, `batch_size` could be a value in `(32, 128)`, while the `num_layers` + hyperparameter in a neural network search space can be a `Integer` + with a range of `(1, 1000)` but on a log scale. + + ```python + import neps + + batch_size = neps.Integer(32, 128) + num_layers = neps.Integer(1, 1000, log=True) + ``` + """ + + lower: int + """The lower bound of the numerical hyperparameter.""" + + upper: int + """The upper bound of the numerical hyperparameter.""" + + log: bool = False + """Whether the hyperparameter is in log space.""" + + prior: int | None = None + """Prior value for the hyperparameter.""" + + prior_confidence: Literal["low", "medium", "high"] = "low" + """Confidence score for the prior value when considering prior based optimization.""" + + is_fidelity: bool = False + """Whether the hyperparameter is fidelity.""" + + def __post_init__(self) -> None: + if self.lower >= self.upper: + raise ValueError( + f"Integer parameter: bounds error (lower >= upper). Actual values: " + f"lower={self.lower}, upper={self.upper}" + ) + + # We could get scientific notation such as 1e3 which would be a float. + # However we can safely cast this to `int` so we do an equality check + # to see if a possible float value matches its int value, before raising + # about floats. + lower_int = int(self.lower) + upper_int = int(self.upper) + if lower_int != self.lower or upper_int != self.upper: + raise ValueError( + f"Integer parameter: bounds error (lower and upper must be integers). " + f"Actual values: lower={self.lower}, upper={self.upper}" + ) + + self.lower = lower_int + self.upper = upper_int + + if self.log and (self.lower <= 0 or self.upper <= 0): + raise ValueError( + f"Integer parameter: bounds error (log scale cant have bounds <= 0). " + f"Actual values: lower={self.lower}, upper={self.upper}" + ) + + if self.prior is not None and not self.lower <= self.prior <= self.upper: + raise ValueError( + f"Integer parameter: Expected lower <= prior <= upper," + f"but got lower={self.lower}, prior={self.prior}, upper={self.upper}" + ) + + self.domain = Domain.integer(self.lower, self.upper, log=self.log) + + +@dataclass +class Categorical: + """A list of **unordered** choices for a parameter. + + This kind of parameter is used to represent hyperparameters that can take on a + discrete set of unordered values. For example, the `optimizer` hyperparameter + in a neural network search space can be a `Categorical` with choices like + `#!python ["adam", "sgd", "rmsprop"]`. + + ```python + import neps + + optimizer_choice = neps.Categorical( + ["adam", "sgd", "rmsprop"], + prior="adam" + ) + ``` + """ + + choices: list[float | int | str] + """The list of choices for the categorical hyperparameter.""" + + prior: float | int | str | None = None + """The default value for the categorical hyperparameter.""" + + prior_confidence: Literal["low", "medium", "high"] = "low" + """Confidence score for the prior value when considering prior based optimization.""" + + def __post_init__(self) -> None: + self.choices = list(self.choices) + + if len(self.choices) <= 1: + raise ValueError("Categorical choices must have more than one value.") + + for choice in self.choices: + if not isinstance(choice, float | int | str): + raise TypeError( + f'Choice "{choice}" is not of a valid type (float, int, str)' + ) + + if not all_unique(self.choices): + raise ValueError(f"Choices must be unique, got duplicates.\n{self.choices}") + + if self.prior is not None and self.prior not in self.choices: + raise ValueError( + f"Default value {self.prior} is not in the provided" + f" choices {self.choices}" + ) + + self.domain = Domain.indices(len(self.choices)) + + +@dataclass +class Constant: + """A constant value for a parameter. + + This kind of parameter is used to represent hyperparameters with values that + should not change during optimization. + + For example, the `batch_size` hyperparameter in a neural network search space + can be a `Constant` with a value of `32`. + + ```python + import neps + + batch_size = neps.Constant(32) + ``` + """ + + value: Any + + +Parameter: TypeAlias = Float | Integer | Categorical | Constant +"""A type alias for all the parameter types. + +* [`Float`][neps.space.Float] +* [`Integer`][neps.space.Integer] +* [`Categorical`][neps.space.Categorical] +* [`Constant`][neps.space.Constant] +""" diff --git a/neps/space/parsing.py b/neps/space/parsing.py new file mode 100644 index 000000000..82dffe6cc --- /dev/null +++ b/neps/space/parsing.py @@ -0,0 +1,293 @@ +"""This module contains functions for parsing search spaces.""" + +from __future__ import annotations + +import dataclasses +import logging +import re +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, Any, TypeAlias + +from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter +from neps.space.search_space import SearchSpace + +if TYPE_CHECKING: + from ConfigSpace import ConfigurationSpace + +logger = logging.getLogger("neps") + +E_NOTATION_PATTERN = r"^-?\d+(\.\d+)?[eE]-?\d+$" + + +def scientific_parse(value: str | int | float) -> str | int | float: + """Parse a value that may be scientific notation.""" + if not isinstance(value, str): + return value + + value_no_space = value.replace(" ", "") + is_scientific = re.match(E_NOTATION_PATTERN, value_no_space) + + if not is_scientific: + return value + + # We know there's an 'e' in the string, + # Now we need to check if its an integer or float + # `int` wont parse scientific notation so we first cast to float + # and see if it's the same as the int cast + float_val = float(value_no_space) + int_val = int(float_val) + if float_val == int_val: + return int_val + + return float_val + + +SerializedParameter: TypeAlias = ( + Mapping[str, Any] # {"type": "int", ...} + | str # const + | int # const + | float # const + | tuple[int, int] # int + | tuple[float, float] # float + | tuple[int | float | str, int | float | str] # bounds (with scientific not.) + | list[int | str | float] # categorical +) + + +def as_parameter(details: SerializedParameter) -> Parameter: # noqa: C901, PLR0911, PLR0912 + """Deduces the parameter type from details. + + Args: + details: A dictionary containing parameter specifications or + a direct value (string, integer, or float). + + Returns: + The deduced parameter type ('int', 'float', 'categorical', or 'constant'). + + Raises: + TypeError: If the type cannot be deduced or the details don't align with expected + constraints. + """ + match details: + # Constant + case str() | int() | float(): + val = scientific_parse(details) + return Constant(val) + + # Bounds of float or int + case tuple((x, y)): + _x = scientific_parse(x) + _y = scientific_parse(y) + match (_x, _y): + case (int(), int()): + return Integer(_x, _y) + case (float(), float()): + return Float(_x, _y) + case _: + raise ValueError( + f"Expected both 'int' or 'float' for bounds but got {type(_x)=}" + f" and {type(_y)=}." + ) + # Matches any sequence of length 2. We could have the issue that the user + # deserializes a yaml tuple pair which gets converted to a list. + # We interpret this as bounds if: + # 1. There are 2 elements + # 2. Both elements are co-ercible to the same number type + # 3. They are ordered + case (x, y): # 1. + _x = scientific_parse(x) + _y = scientific_parse(y) + match (_x, _y): + case (int(), int()) if _x <= _y: # 2./3. + return Integer(_x, _y) + case (float(), float()) if _x <= _y: # 2./3. + return Float(_x, _y) + + # Error case: + # We do have two numbers, but of different types. This could + # be user error so rather than guess, we raise an error. + case (int(), float()) | (float(), int()): + raise ValueError( + f"Got a mix of a float and an int with {details=}," + " tried to interpret these as bounds but found them to be" + " different types." + "\nIf you wanted to specify a categorical, i.e. a discrete" + f" choices between the values {x=} and {y=}, then you can use" + " the more verbose syntax of specifying 'type: cat'." + "\nIf you did intend to specify bounds, then ensure that" + " the values are both of the same type." + ) + # At least one of them is a string, so we treat is as categorical. + case _: + return Categorical(choices=[_x, _y]) + + ## Categorical list of choices (tuple is reserved for bounds) + case Sequence() if not isinstance(details, tuple): + # It's unlikely that if we find an element that can be converted to + # scientific notation that we wouldn't want to do so, for example, + # when specifying a grid. Hence, we map over the list and convert + # what we can + details = [scientific_parse(d) for d in details] + return Categorical(details) + + # Categorical dict declartion + case {"choices": choices, **rest}: + _type = rest.pop("type", None) + if _type is not None and _type not in ("cat", "categorical"): + raise ValueError(f"Unrecognized type '{_type}' with 'choices' set.") + + # See note above about scientific notation elements + choices = [scientific_parse(c) for c in choices] + return Categorical(choices, **rest) # type: ignore + + # Constant dict declartion + case {"value": v, **_rest}: + _type = _rest.pop("type", None) + if _type is not None and _type not in ("const", "constant"): + raise ValueError( + f"Unrecognized type '{_type}' with 'value' set," + f" which indicates to treat value `{v}` a constant." + ) + + return Constant(v, **_rest) # type: ignore + + # Bounds dict declartion + case {"lower": l, "upper": u, **rest}: + _x = scientific_parse(l) + _y = scientific_parse(u) + + _type = rest.pop("type", None) + match _type: + case "int" | "integer": + return Integer(_x, _y, **rest) # type: ignore + case "float" | "floating": + return Float(_x, _y, **rest) # type: ignore + case None: + match (_x, _y): + case (int(), int()): + return Integer(_x, _y, **rest) # type: ignore + case (float(), float()): + return Float(_x, _y, **rest) # type: ignore + case _: + raise ValueError( + f"Expected both 'int' or 'float' for bounds but" + f" got {type(_x)=} and {type(_y)=}." + ) + case _: + raise ValueError( + f"Unrecognized type '{_type}' with both a 'lower'" + " and 'upper' set." + ) + case _: + raise ValueError( + f"Unable to deduce parameter with details '{details}'." + " Please see our documentation for details." + ) + + +def convert_mapping(pipeline_space: Mapping[str, Any]) -> SearchSpace: + """Converts a dictionary to a SearchSpace object.""" + parameters: dict[str, Parameter] = {} + for name, details in pipeline_space.items(): + match details: + case Float() | Integer() | Categorical() | Constant(): + parameters[name] = dataclasses.replace(details) # copy + case str() | int() | float() | Mapping(): + try: + parameters[name] = as_parameter(details) + except (TypeError, ValueError) as e: + raise ValueError(f"Error parsing parameter '{name}'") from e + case _: + raise ValueError( + f"Unrecognized parameter type '{type(details)}' for '{name}'." + ) + + return SearchSpace(parameters) + + +def convert_configspace(configspace: ConfigurationSpace) -> SearchSpace: + """Constructs a [`SearchSpace`][neps.space.SearchSpace] + from a [`ConfigurationSpace`](https://automl.github.io/ConfigSpace/latest/). + + Args: + configspace: The configuration space to construct the pipeline space from. + + Returns: + A dictionary where keys are parameter names and values are parameter objects. + """ + import ConfigSpace as CS + + space = {} + parameter: Parameter + if any(configspace.get_conditions()) or any(configspace.get_forbiddens()): + raise NotImplementedError( + "The ConfigurationSpace has conditions or forbidden clauses, " + "which are not supported by neps." + ) + + for hyperparameter in configspace.get_hyperparameters(): + if isinstance(hyperparameter, CS.Constant): + parameter = Constant(value=hyperparameter.value) + elif isinstance(hyperparameter, CS.CategoricalHyperparameter): + parameter = Categorical( + hyperparameter.choices, + prior=hyperparameter.default_value, + ) + elif isinstance(hyperparameter, CS.OrdinalHyperparameter): + parameter = Categorical( + hyperparameter.sequence, + prior=hyperparameter.default_value, + ) + elif isinstance(hyperparameter, CS.UniformIntegerHyperparameter): + parameter = Integer( + lower=hyperparameter.lower, + upper=hyperparameter.upper, + log=hyperparameter.log, + prior=hyperparameter.default_value, + ) + elif isinstance(hyperparameter, CS.UniformFloatHyperparameter): + parameter = Float( + lower=hyperparameter.lower, + upper=hyperparameter.upper, + log=hyperparameter.log, + prior=hyperparameter.default_value, + ) + else: + raise ValueError(f"Unknown hyperparameter type {hyperparameter}") + space[hyperparameter.name] = parameter + return SearchSpace(space) + + +def convert_to_space( + space: ( + Mapping[str, dict | str | int | float | Parameter] + | SearchSpace + | ConfigurationSpace + ), +) -> SearchSpace: + """Converts a search space to a SearchSpace object. + + Args: + space: The search space to convert. + + Returns: + The SearchSpace object representing the search space. + """ + # We quickly check ConfigSpace becuse it inherits from Mapping + try: + from ConfigSpace import ConfigurationSpace + + if isinstance(space, ConfigurationSpace): + return convert_configspace(space) + except ImportError: + pass + + match space: + case SearchSpace(): + return space + case Mapping(): + return convert_mapping(space) + case _: + raise ValueError( + f"Unsupported type '{type(space)}' for conversion to SearchSpace." + ) diff --git a/neps/space/search_space.py b/neps/space/search_space.py new file mode 100644 index 000000000..9b2e62ca0 --- /dev/null +++ b/neps/space/search_space.py @@ -0,0 +1,80 @@ +"""Contains the [`SearchSpace`][neps.space.search_space.SearchSpace] class +which is a container for hyperparameters that can be sampled, mutated, and crossed over. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from neps.space.parameters import Categorical, Constant, Float, Integer, Parameter + + +@dataclass +class SearchSpace: + """A container for parameters.""" + + parameters: dict[str, Parameter] + """The parameters which define the search space.""" + + fidelity: tuple[str, Float | Integer] | None = field(init=False, default=None) + """The fidelity parameter for the search space.""" + + prior: dict[str, Any] = field(init=False, default_factory=dict) + """The prior configuration for the search space.""" + + categoricals: dict[str, Categorical] = field(init=False, default_factory=dict) + """The categorical hyperparameters in the search space.""" + + searchables: dict[str, Integer | Float | Categorical] = field( + init=False, default_factory=dict + ) + + numerical: dict[str, Integer | Float] = field(init=False, default_factory=dict) + """The numerical hyperparameters in the search space.""" + + constants: dict[str, Any] = field(init=False, default_factory=dict) + """The constant hyperparameters in the search space.""" + + fidelities: dict[str, Integer | Float] = field(init=False, default_factory=dict) + """The fidelity hyperparameters in the search space. + + Currently no optimizer supports multiple fidelities but it is defined here incase. + """ + + def __post_init__(self) -> None: + self.parameters = dict(sorted(self.parameters.items(), key=lambda x: x[0])) + + # Process the hyperparameters + for name, hp in self.parameters.items(): + match hp: + case Float() | Integer(): + if hp.is_fidelity: + if self.fidelity is not None: + raise ValueError( + "neps only supports one fidelity parameter in the" + " pipeline space, but multiple were given." + f" Other fidelity: {self.fidelity[0]}, new: {name}" + ) + self.fidelity = (name, hp) + self.fidelities[name] = hp + + self.numerical[name] = hp + self.searchables[name] = hp + + if hp.prior is not None: + self.prior[name] = hp.prior + + case Categorical(): + self.categoricals[name] = hp + self.searchables[name] = hp + + if hp.prior is not None: + self.prior[name] = hp.prior + + case Constant(): + self.constants[name] = hp.value + self.prior[name] = hp.value + + case _: + raise ValueError(f"Unknown hyperparameter type: {hp}") diff --git a/neps/state/__init__.py b/neps/state/__init__.py index b8eb55af3..921088f75 100644 --- a/neps/state/__init__.py +++ b/neps/state/__init__.py @@ -1,11 +1,17 @@ +from neps.state.neps_state import NePSState from neps.state.optimizer import BudgetInfo, OptimizationState, OptimizerInfo from neps.state.seed_snapshot import SeedSnapshot +from neps.state.settings import DefaultReportValues, OnErrorPossibilities, WorkerSettings from neps.state.trial import Trial __all__ = [ "BudgetInfo", + "DefaultReportValues", + "NePSState", + "OnErrorPossibilities", "OptimizationState", "OptimizerInfo", "SeedSnapshot", "Trial", + "WorkerSettings", ] diff --git a/neps/state/_eval.py b/neps/state/_eval.py index 915e38235..ce8f75b17 100644 --- a/neps/state/_eval.py +++ b/neps/state/_eval.py @@ -104,7 +104,7 @@ def _eval_trial( logger.exception(e) report = trial.set_complete( report_as="crashed", - objective_to_minimize=default_report_values.objective_to_minimize_value_on_error, + objective_to_minimize=default_report_values.objective_value_on_error, cost=default_report_values.cost_value_on_error, learning_curve=default_report_values.learning_curve_on_error, extra=None, diff --git a/neps/state/filebased.py b/neps/state/filebased.py index 6ea08bdf5..7c74e6bfc 100644 --- a/neps/state/filebased.py +++ b/neps/state/filebased.py @@ -1,4 +1,4 @@ -"""TODO.""" +"""Contains reading and writing of various aspects of NePS.""" from __future__ import annotations @@ -162,13 +162,7 @@ def write(cls, err_dump: ErrDump, path: Path) -> None: @dataclass class FileLocker: - """File-based locker using `portalocker`. - - [`FileLocker`][neps.state.locker.file.FileLocker] implements - the [`Locker`][neps.state.locker.locker.Locker] protocol using - `portalocker` to lock a file between processes with a shared - filesystem. - """ + """File-based locker using `portalocker`.""" lock_path: Path poll: float diff --git a/neps/state/neps_state.py b/neps/state/neps_state.py index 92bd2e5ad..a14c0fc7b 100644 --- a/neps/state/neps_state.py +++ b/neps/state/neps_state.py @@ -5,7 +5,7 @@ it without having to worry about locking or out-dated information. For an actual instantiation of this object, see -[`create_or_load_filebased_neps_state`][neps.state.filebased.create_or_load_filebased_neps_state]. +[`create_or_load_filebased_neps_state()`][neps.state.neps_state.NePSState.create_or_load]. """ from __future__ import annotations @@ -14,10 +14,11 @@ import logging import pickle import time -from collections.abc import Callable, Iterable +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path from typing import ( + TYPE_CHECKING, Literal, TypeAlias, TypeVar, @@ -34,7 +35,6 @@ TRIAL_FILELOCK_TIMEOUT, ) from neps.exceptions import NePSError, TrialAlreadyExistsError, TrialNotFoundError -from neps.optimizers.base_optimizer import BaseOptimizer from neps.state.err_dump import ErrDump from neps.state.filebased import ( FileLocker, @@ -46,6 +46,9 @@ from neps.state.trial import Report, Trial from neps.utils.files import atomic_write, deserialize, serialize +if TYPE_CHECKING: + from neps.optimizers.optimizer import AskFunction + logger = logging.getLogger(__name__) @@ -260,15 +263,15 @@ def lock_and_read_trials(self) -> dict[str, Trial]: @overload def lock_and_sample_trial( - self, optimizer: BaseOptimizer, *, worker_id: str, n: None = None + self, optimizer: AskFunction, *, worker_id: str, n: None = None ) -> Trial: ... @overload def lock_and_sample_trial( - self, optimizer: BaseOptimizer, *, worker_id: str, n: int + self, optimizer: AskFunction, *, worker_id: str, n: int ) -> list[Trial]: ... def lock_and_sample_trial( - self, optimizer: BaseOptimizer, *, worker_id: str, n: int | None = None + self, optimizer: AskFunction, *, worker_id: str, n: int | None = None ) -> Trial | list[Trial]: """Acquire the state lock and sample a trial.""" with self._optimizer_lock.lock(): @@ -301,33 +304,30 @@ def lock_and_report_trial_evaluation( @overload def _sample_trial( self, - optimizer: BaseOptimizer, + optimizer: AskFunction, *, worker_id: str, trials: dict[str, Trial], n: int, - _sample_hooks: list[Callable] | None = ..., ) -> list[Trial]: ... @overload def _sample_trial( self, - optimizer: BaseOptimizer, + optimizer: AskFunction, *, worker_id: str, trials: dict[str, Trial], n: None, - _sample_hooks: list[Callable] | None = ..., ) -> Trial: ... def _sample_trial( self, - optimizer: BaseOptimizer, + optimizer: AskFunction, *, worker_id: str, trials: dict[str, Trial], n: int | None, - _sample_hooks: list[Callable] | None = None, ) -> Trial | list[Trial]: """Sample a new trial from the optimizer. @@ -340,7 +340,6 @@ def _sample_trial( worker_id: The worker that is sampling the trial. n: The number of trials to sample. trials: The current trials. - _sample_hooks: A list of hooks to apply to the optimizer before sampling. Returns: The new trial. @@ -350,13 +349,7 @@ def _sample_trial( opt_state.seed_snapshot.set_as_global_seed_state() - # TODO: Not sure if any existing pre_load hooks required - # it to be done after `load_results`... I hope not. - if _sample_hooks is not None: - for hook in _sample_hooks: - optimizer = hook(optimizer) # type: ignore - - assert isinstance(optimizer, BaseOptimizer) + assert callable(optimizer) if opt_state.budget is not None: # NOTE: All other values of budget are ones that should remain # constant, there are currently only these two which are dynamic as @@ -368,11 +361,11 @@ def _sample_trial( ) opt_state.budget.used_evaluations = len(trials) - sampled_configs = optimizer.ask( + sampled_configs = optimizer( trials=trials, - budget_info=opt_state.budget.clone() - if opt_state.budget is not None - else None, + budget_info=( + opt_state.budget.clone() if opt_state.budget is not None else None + ), n=n, ) @@ -396,7 +389,7 @@ def _sample_trial( trial = Trial.new( trial_id=sampled_config.id, - location="", # HACK: This will be set by the `TrialRepo` in `put_new` + location=str(self._trial_repo.directory / f"config_{sampled_config.id}"), config=sampled_config.config, previous_trial=sampled_config.previous_config_id, previous_trial_location=previous_trial_location, diff --git a/neps/state/settings.py b/neps/state/settings.py index 148d8c277..bbc7c0498 100644 --- a/neps/state/settings.py +++ b/neps/state/settings.py @@ -11,7 +11,7 @@ class DefaultReportValues: """Values to use when an error occurs.""" - objective_to_minimize_value_on_error: float | None = None + objective_value_on_error: float | None = None """The value to use for the objective_to_minimize when an error occurs.""" cost_value_on_error: float | None = None diff --git a/neps/state/trial.py b/neps/state/trial.py index 0ead5a21c..2d501ce99 100644 --- a/neps/state/trial.py +++ b/neps/state/trial.py @@ -3,20 +3,16 @@ from __future__ import annotations import logging -from collections.abc import Callable, Mapping -from dataclasses import asdict, dataclass +from collections.abc import Mapping +from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, Any, ClassVar, Literal +from typing import Any, ClassVar, Literal from typing_extensions import Self import numpy as np from neps.exceptions import NePSError -if TYPE_CHECKING: - from neps.search_spaces import SearchSpace - from neps.utils.types import ERROR, ConfigResult, RawConfig - logger = logging.getLogger(__name__) @@ -76,25 +72,6 @@ def __post_init__(self) -> None: if isinstance(self.err, str): self.err = Exception(self.err) # type: ignore - def to_deprecate_result_dict(self) -> dict[str, Any] | ERROR: - """Return the report as a dictionary.""" - if self.reported_as == "success": - d = { - "objective_to_minimize": self.objective_to_minimize, - "cost": self.cost, - **self.extra, - } - - # HACK: Backwards compatibility. Not sure how much this is needed - # but it should be removed once optimizers stop calling the - # `get_objective_to_minimize`, `get_cost`, `get_learning_curve` methods of - # `BaseOptimizer` and just use the `Report` directly. - if "info_dict" not in d or "learning_curve" not in d["info_dict"]: - d.setdefault("info_dict", {})["learning_curve"] = self.learning_curve - return d - - return "error" - def __eq__(self, value: Any, /) -> bool: # HACK : Since it could be probably that one of objective_to_minimize or cost is # nan, we need a custom comparator for this object @@ -168,33 +145,7 @@ def new( @property def id(self) -> str: """Return the id of the trial.""" - return self.metadata.id - - def into_config_result( - self, - config_to_search_space: Callable[[RawConfig], SearchSpace], - ) -> ConfigResult: - """Convert the trial and report to a `ConfigResult` object.""" - if self.report is None: - raise self.NotReportedYetError("The trial has not been reported yet.") - from neps.utils.types import ConfigResult - - result: dict[str, Any] | ERROR - if self.report.reported_as == "success": - result = { - **self.report.extra, - "objective_to_minimize": self.report.objective_to_minimize, - "cost": self.report.cost, - } - else: - result = "error" - - return ConfigResult( - self.id, - config=config_to_search_space(self.config), - result=result, - metadata=asdict(self.metadata), - ) + return self.metadata.id # type: ignore def set_submitted(self, *, time_submitted: float) -> None: """Set the trial as submitted.""" diff --git a/neps/status/status.py b/neps/status/status.py index 2283730bb..68239f5fb 100644 --- a/neps/status/status.py +++ b/neps/status/status.py @@ -3,23 +3,40 @@ # ruff: noqa: T201 from __future__ import annotations -from dataclasses import asdict +from collections.abc import Mapping +from dataclasses import asdict, dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import pandas as pd from neps.runtime import get_workers_neps_state -from neps.state.filebased import FileLocker -from neps.state.neps_state import NePSState -from neps.state.trial import Trial -from neps.utils.types import ConfigID, _ConfigResultForStats +from neps.state.neps_state import FileLocker, NePSState, Report, Trial if TYPE_CHECKING: - from neps.search_spaces.search_space import SearchSpace + from neps.space import SearchSpace -def get_summary_dict( +# TODO(eddiebergman): This is a hack because status.py expects a `ConfigResult` +# where the `config` is a dict config (`RawConfig`), while all the optimizers +# expect a `ConfigResult` where the `config` is a `SearchSpace`. Ideally we +# just rework status to use `Trial` and `Report` directly as they contain a lot more +# information. +@dataclass +class _ConfigResultForStats: + id: str + config: Mapping[str, Any] + result: Mapping[str, Any] | Literal["error"] + metadata: dict + + @property + def objective_to_minimize(self) -> float | Literal["error"]: + if isinstance(self.result, dict): + return float(self.result["objective_to_minimize"]) + return "error" + + +def get_summary_dict( # noqa: C901 root_directory: str | Path, *, add_details: bool = False, @@ -36,6 +53,21 @@ def get_summary_dict( """ root_directory = Path(root_directory) + def _to_deprecate_result_dict(report: Report) -> dict[str, Any] | Literal["error"]: + """Return the report as a dictionary.""" + if report.reported_as == "success": + d = { + "objective_to_minimize": report.objective_to_minimize, + "cost": report.cost, + **report.extra, + } + + if "info_dict" not in d or "learning_curve" not in d["info_dict"]: + d.setdefault("info_dict", {})["learning_curve"] = report.learning_curve + return d + + return "error" + # NOTE: We don't lock the shared state since we are just reading and don't need to # make decisions based on the state try: @@ -45,7 +77,7 @@ def get_summary_dict( trials = shared_state.lock_and_read_trials() - evaluated: dict[ConfigID, _ConfigResultForStats] = {} + evaluated: dict[str, _ConfigResultForStats] = {} for trial in trials.values(): if trial.report is None: @@ -54,7 +86,7 @@ def get_summary_dict( _result_for_stats = _ConfigResultForStats( id=trial.id, config=trial.config, - result=trial.report.to_deprecate_result_dict(), + result=_to_deprecate_result_dict(trial.report), metadata=asdict(trial.metadata), ) evaluated[trial.id] = _result_for_stats diff --git a/neps/utils/cli.py b/neps/utils/cli.py index 901b2ac75..ee3dbed89 100644 --- a/neps/utils/cli.py +++ b/neps/utils/cli.py @@ -2,8 +2,8 @@ """This module provides a command-line interface (CLI) for NePS.""" from __future__ import annotations + import warnings -from typing import Tuple from datetime import timedelta, datetime import seaborn as sns import matplotlib.pyplot as plt @@ -15,11 +15,10 @@ from pathlib import Path from typing import Optional, List import neps -from neps.api import Default from neps.state.seed_snapshot import SeedSnapshot from neps.status.status import post_run_csv import pandas as pd -from neps.utils.run_args import ( +from neps.utils.yaml_loading import ( RUN_ARGS, EVALUATE_PIPELINE, ROOT_DIRECTORY, @@ -28,24 +27,20 @@ MAX_EVALUATIONS_TOTAL, MAX_COST_TOTAL, PIPELINE_SPACE, - DEVELOPMENT_STAGE_ID, - TASK_ID, - SEARCHER, - SEARCHER_KWARGS, + OPTIMIZER, IGNORE_ERROR, - OBJECTIVE_TO_MINIMIZE_VALUE_ON_ERROR, + OBJECTIVE_VALUE_ON_ERROR, COST_VALUE_ON_ERROR, CONTINUE_UNTIL_MAX_EVALUATION_COMPLETED, OVERWRITE_WORKING_DIRECTORY, - get_run_args_from_yaml, + load_yaml_config, ) -from neps.optimizers.base_optimizer import BaseOptimizer -from neps.utils.run_args import load_and_return_object +from neps.utils.yaml_loading import load_and_return_object from neps.state.neps_state import NePSState from neps.state.trial import Trial from neps.exceptions import TrialNotFoundError from neps.status.status import get_summary_dict -from neps.api import _run_args +from neps.optimizers import load_optimizer from neps.state.optimizer import BudgetInfo, OptimizationState, OptimizerInfo # Suppress specific warnings @@ -123,13 +118,13 @@ def init_config(args: argparse.Namespace) -> None: if args.database: if config_path.exists(): - run_args = get_run_args_from_yaml(config_path) + run_args = load_yaml_config(config_path) max_cost_total = run_args.get(MAX_COST_TOTAL) # Create the optimizer - _, optimizer_info = load_optimizer(run_args) - if optimizer_info is None: - return - + _, optimizer_info = load_optimizer( + optimizer=run_args.get(OPTIMIZER), # type: ignore + space=run_args.get(PIPELINE_SPACE), # type: ignore + ) try: directory = run_args.get(ROOT_DIRECTORY) if directory is None: @@ -221,23 +216,18 @@ def init_config(args: argparse.Namespace) -> None: # Debug and Monitoring overwrite_working_directory: false post_run_summary: true -development_stage_id: -task_id: # Parallelization Setup max_evaluations_per_run: continue_until_max_evaluation_completed: true # Error Handling -objective_to_minimize_value_on_error: +objective_value_on_error: cost_value_on_error: ignore_errors: # Customization Options -searcher: hyperband # Internal key to select a NePS optimizer. - -# Hooks -pre_load_hooks: +optimizer: hyperband # Internal key to select a NePS optimizer. """ ) else: @@ -280,11 +270,12 @@ def run_optimization(args: argparse.Namespace) -> None: """Collects arguments from the parser and runs the NePS optimization. Args: args (argparse.Namespace): Parsed command-line arguments. """ - if isinstance(args.run_args, Default): + if args.run_args is None: run_args = Path("run_config.yaml") else: run_args = args.run_args - if not isinstance(args.evaluate_pipeline, Default): + + if isinstance(args.evaluate_pipeline, str): module_path, function_name = args.evaluate_pipeline.split(":") evaluate_pipeline = load_and_return_object( module_path, function_name, EVALUATE_PIPELINE @@ -294,8 +285,8 @@ def run_optimization(args: argparse.Namespace) -> None: evaluate_pipeline = args.evaluate_pipeline kwargs = {} - if args.searcher_kwargs: - kwargs = parse_kv_pairs(args.searcher_kwargs) # convert kwargs + if args.optimizer_kwargs: + kwargs = parse_kv_pairs(args.optimizer_kwargs) # convert kwargs # Collect arguments from args and prepare them for neps.run options = { @@ -305,8 +296,6 @@ def run_optimization(args: argparse.Namespace) -> None: ROOT_DIRECTORY: args.root_directory, OVERWRITE_WORKING_DIRECTORY: args.overwrite_working_directory, POST_RUN_SUMMARY: args.post_run_summary, - DEVELOPMENT_STAGE_ID: args.development_stage_id, - TASK_ID: args.task_id, MAX_EVALUATIONS_TOTAL: args.max_evaluations_total, MAX_EVALUATIONS_PER_RUN: args.max_evaluations_per_run, CONTINUE_UNTIL_MAX_EVALUATION_COMPLETED: ( @@ -314,9 +303,9 @@ def run_optimization(args: argparse.Namespace) -> None: ), MAX_COST_TOTAL: args.max_cost_total, IGNORE_ERROR: args.ignore_errors, - OBJECTIVE_TO_MINIMIZE_VALUE_ON_ERROR: args.objective_to_minimize_value_on_error, + OBJECTIVE_VALUE_ON_ERROR: args.objective_value_on_error, COST_VALUE_ON_ERROR: args.cost_value_on_error, - SEARCHER: args.searcher, + OPTIMIZER: args.optimizer, **kwargs, } logging.basicConfig(level=logging.INFO) @@ -413,7 +402,7 @@ def sample_config(args: argparse.Namespace) -> None: print(f"Error: run_args file {run_args_path} does not exist.") return - run_args = get_run_args_from_yaml(run_args_path) + run_args = load_yaml_config(run_args_path) # Get root_directory from the run_args root_directory = run_args.get(ROOT_DIRECTORY) @@ -434,9 +423,10 @@ def sample_config(args: argparse.Namespace) -> None: worker_id = args.worker_id num_configs = args.number_of_configs if args.number_of_configs else 1 - optimizer, _ = load_optimizer(run_args) - if optimizer is None: - return + optimizer, _ = load_optimizer( + optimizer=run_args.get(OPTIMIZER), # type: ignore + space=run_args.get(PIPELINE_SPACE), # type: ignore + ) # Sample trials for _ in range(num_configs): @@ -502,7 +492,7 @@ def status(args: argparse.Namespace) -> None: # Print summary print("NePS Status:") print("-----------------------------") - print(f"Optimizer: {neps_state.lock_and_get_optimizer_info().info['searcher_alg']}") + print(f"Optimizer: {neps_state.lock_and_get_optimizer_info().info['optimizer_alg']}") print(f"Succeeded Trials: {succeeded_trials_count}") print(f"Failed Trials (Errors): {failed_trials_count}") print(f"Active Trials: {evaluating_trials_count}") @@ -597,16 +587,16 @@ def status(args: argparse.Namespace) -> None: # Display optimizer information optimizer_info = neps_state.lock_and_get_optimizer_info().info - searcher_name = optimizer_info.get("searcher_name", "N/A") - searcher_alg = optimizer_info.get("searcher_alg", "N/A") - searcher_args = optimizer_info.get("searcher_args", {}) + optimizer_name = optimizer_info.get("optimizer_name", "N/A") + optimizer_alg = optimizer_info.get("optimizer_alg", "N/A") + optimizer_args = optimizer_info.get("optimizer_args", {}) print("\nOptimizer Information:") print("-----------------------------") - print(f"Name: {searcher_name}") - print(f"Algorithm: {searcher_alg}") + print(f"Name: {optimizer_name}") + print(f"Algorithm: {optimizer_alg}") print("Parameter:") - for arg, value in searcher_args.items(): + for arg, value in optimizer_args.items(): print(f" {arg}: {value}") print("-----------------------------") @@ -885,8 +875,6 @@ def print_help(args: Optional[argparse.Namespace] = None) -> None: --root-directory (Optional: Directory for saving progress and synchronization. Default is 'root_directory' from run_config.yaml if not provided.) --overwrite-working-directory (Deletes the working directory at the start of the run.) - --development-stage-id (Identifier for the development stage.) - --task-id (Identifier for the task.) --post-run-summary/--no-post-run-summary (Toggle summary after running.) --max-evaluations-total (Total number of evaluations to run.) --max-evaluations-per-run (Max evaluations per run call.) @@ -895,8 +883,8 @@ def print_help(args: Optional[argparse.Namespace] = None) -> None: --ignore-errors (Ignore errors during optimization.) --objective_to_minimize-value-on-error (Assumed objective_to_minimize value on error.) --cost-value-on-error (Assumed cost value on error.) - --searcher (Searcher algorithm key for optimization.) - --searcher-kwargs ... (Additional kwargs for the searcher.) + --optimizer (optimizer algorithm key for optimization.) + --optimizer-kwargs ... (Additional kwargs for the optimizer.) neps info-config [OPTIONS] Provides detailed information about a specific configuration by its ID. @@ -1026,7 +1014,7 @@ def handle_report_config(args: argparse.Namespace) -> None: print(f"Error: run_args file {run_args_path} does not exist.") return - run_args = get_run_args_from_yaml(run_args_path) + run_args = load_yaml_config(run_args_path) # Get root_directory from run_args root_directory = run_args.get("root_directory") @@ -1103,36 +1091,6 @@ def handle_report_config(args: argparse.Namespace) -> None: print("----------------------\n") -def load_optimizer(run_args: dict) -> Tuple[Optional[BaseOptimizer], Optional[dict]]: - """Create an optimizer""" - try: - searcher_info = { - "searcher_name": "", - "searcher_alg": "", - "searcher_selection": "", - "neps_decision_tree": True, - "searcher_args": {}, - } - - # Call _run_args() to create the optimizer - optimizer, searcher_info = _run_args( - searcher_info=searcher_info, - pipeline_space=run_args.get(PIPELINE_SPACE), - max_cost_total=run_args.get(MAX_COST_TOTAL, None), - ignore_errors=run_args.get(IGNORE_ERROR, False), - objective_to_minimize_value_on_error=run_args.get( - OBJECTIVE_TO_MINIMIZE_VALUE_ON_ERROR, None - ), - cost_value_on_error=run_args.get(COST_VALUE_ON_ERROR, None), - searcher=run_args.get(SEARCHER, "default"), - **run_args.get(SEARCHER_KWARGS, {}), - ) - return optimizer, searcher_info - except Exception as e: - print(f"Error creating optimizer: {e}") - return None, None - - def parse_time_end(time_str: str) -> float: """Parses a UNIX timestamp or a human-readable time string and returns a UNIX timestamp.""" @@ -1166,7 +1124,7 @@ def main() -> None: ) # Subparser for "init" command - parser_init = subparsers.add_parser("init", help="Generate 'run_args' " "YAML file") + parser_init = subparsers.add_parser("init", help="Generate 'run_args' YAML file") parser_init.add_argument( "--config-path", type=str, @@ -1193,25 +1151,17 @@ def main() -> None: parser_run = subparsers.add_parser("run", help="Run a neural pipeline search.") # Adding arguments to the 'run' subparser with defaults parser_run.add_argument( - "--run-args", - type=str, - help="Path to the YAML configuration file.", - default=Default(None), - ) - parser_run.add_argument( - "--run-pipeline", + "--run", type=str, - help="Optional: Provide the path to a Python file and a function name separated " - "by a colon, e.g., 'path/to/module.py:function_name'. " - "If provided, it overrides the evaluate_pipeline setting from the YAML " - "configuration.", - default=Default(None), + help="Path to the YAML configuration file(s).", + nargs="*", + default=None, ) parser_run.add_argument( "--pipeline-space", type=str, - default=Default(None), + default=None, help="Path to the YAML file defining the search space for the optimization. " "This can be provided here or defined within the 'run_args' YAML file. " "(default: %(default)s)", @@ -1219,38 +1169,24 @@ def main() -> None: parser_run.add_argument( "--root-directory", type=str, - default=Default(None), + default=None, help="The directory to save progress to. This is also used to synchronize " "multiple calls for parallelization. (default: %(default)s)", ) parser_run.add_argument( "--overwrite-working-directory", action="store_true", - default=Default(False), # noqa: FBT003 + default=False, # noqa: FBT003 help="If set, deletes the working directory at the start of the run. " "This is useful, for example, when debugging a evaluate_pipeline function. " "(default: %(default)s)", ) - parser_run.add_argument( - "--development-stage-id", - type=str, - default=Default(None), - help="Identifier for the current development stage, used in multi-stage " - "projects. (default: %(default)s)", - ) - parser_run.add_argument( - "--task-id", - type=str, - default=Default(None), - help="Identifier for the current task, useful in projects with multiple tasks. " - "(default: %(default)s)", - ) # Create a mutually exclusive group for post-run summary flags summary_group = parser_run.add_mutually_exclusive_group(required=False) summary_group.add_argument( "--post-run-summary", action="store_true", - default=Default(True), # noqa: FBT003 + default=True, # noqa: FBT003 help="Provide a summary of the results after running. (default: %(default)s)", ) summary_group.add_argument( @@ -1262,27 +1198,27 @@ def main() -> None: parser_run.add_argument( "--max-evaluations-total", type=int, - default=Default(None), + default=None, help="Total number of evaluations to run. (default: %(default)s)", ) parser_run.add_argument( "--max-evaluations-per-run", type=int, - default=Default(None), + default=None, help="Number of evaluations a specific call should maximally do. " "(default: %(default)s)", ) parser_run.add_argument( "--continue-until-max-evaluation-completed", action="store_true", - default=Default(False), # noqa: FBT003 + default=False, # noqa: FBT003 help="If set, only stop after max-evaluations-total have been completed. This " "is only relevant in the parallel setting. (default: %(default)s)", ) parser_run.add_argument( "--max-cost-total", type=float, - default=Default(None), + default=None, help="No new evaluations will start when this cost is exceeded. Requires " "returning a cost in the evaluate_pipeline function, e.g., `return dict(" "objective_to_minimize=objective_to_minimize, cost=cost)`. (default: %(default)s)", @@ -1290,43 +1226,43 @@ def main() -> None: parser_run.add_argument( "--ignore-errors", action="store_true", - default=Default(False), # noqa: FBT003 + default=False, # noqa: FBT003 help="If set, ignore errors during the optimization process. (default: %(" "default)s)", ) parser_run.add_argument( "--objective_to_minimize-value-on-error", type=float, - default=Default(None), + default=None, help="Loss value to assume on error. (default: %(default)s)", ) parser_run.add_argument( "--cost-value-on-error", type=float, - default=Default(None), + default=None, help="Cost value to assume on error. (default: %(default)s)", ) parser_run.add_argument( - "--searcher", + "--optimizer", type=str, - default=Default("default"), - help="String key of searcher algorithm to use for optimization. (default: %(" + default="default", + help="String key of optimizer algorithm to use for optimization. (default: %(" "default)s)", ) parser_run.add_argument( - "--searcher-kwargs", + "--optimizer-kwargs", type=str, nargs="+", - help="Additional keyword arguments as key=value pairs for the searcher.", + help="Additional keyword arguments as key=value pairs for the optimizer.", ) parser_run.set_defaults(func=run_optimization) # Subparser for "info-config" command parser_info_config = subparsers.add_parser( - "info-config", help="Provides information about " "specific config." + "info-config", help="Provides information about specific config." ) parser_info_config.add_argument( "id", type=str, help="The configuration ID to be used." diff --git a/neps/utils/common.py b/neps/utils/common.py index 6fd4a10ab..7abfe448c 100644 --- a/neps/utils/common.py +++ b/neps/utils/common.py @@ -3,15 +3,30 @@ from __future__ import annotations import gc +import importlib.util import inspect -from collections.abc import Iterator, Mapping, Sequence +import os +import sys +from collections.abc import Callable, Iterator from contextlib import contextmanager from functools import partial from pathlib import Path from typing import Any import torch -import yaml + + +def extract_keyword_defaults(f: Callable) -> dict[str, Any]: + """Extracts the keywords from a function, if any.""" + if isinstance(f, partial): + return dict(f.keywords) + + signature = inspect.signature(f) + return { + k: v.default + for k, v in signature.parameters.items() + if v.default is not inspect.Parameter.empty + } # TODO(eddiebergman): I feel like this function should throw an error if it can't @@ -189,62 +204,22 @@ def get_initial_directory(pipeline_directory: Path | str | None = None) -> Path: return path -def get_searcher_data( - searcher: str | Path, *, loading_custom_searcher: bool = False -) -> tuple[dict[str, Any], str]: - """Returns the data from the YAML file associated with the specified searcher. +def capture_function_arguments(the_locals: dict, func: Callable) -> dict: + """Capture the function arguments and their values from the locals dictionary. Args: - searcher: The name of the searcher. - loading_custom_searcher: Flag if searcher contains a custom yaml + the_locals: The locals dictionary of the function. + func: The function to capture arguments from. Returns: - The content of the YAML file and searcher name. + A dictionary of function arguments and their values. """ - if loading_custom_searcher: - user_yaml_path = Path(searcher).with_suffix(".yaml") - - if not user_yaml_path.exists(): - raise FileNotFoundError( - "Failed to get info for searcher from user-defined YAML file. " - f"File '{searcher}.yaml' does not exist at '{user_yaml_path}'" - ) - - with user_yaml_path.open("r") as file: - data = yaml.safe_load(file) - - file_name = user_yaml_path.stem - searcher = data.pop("name", file_name) - - else: - # TODO(eddiebergman): This is a bad idea as it relies on folder structure to be - # correct, we should either have a dedicated resource folder or at least have - # this defined as a constant somewhere, incase we access elsewhere. - # Seems like we could just include this as a method on `SearcherConfigs` class. - # TODO(eddiebergman): Need to make sure that these yaml files are actually - # included in a source dist when published to PyPI. - - # This is pointing to yaml file directory elsewhere in the source code. - resource_path = ( - Path(__file__).parent.parent.absolute() - / "optimizers" - / "default_searchers" - / searcher - ).with_suffix(".yaml") - - from neps.optimizers.info import SearcherConfigs - - searchers = SearcherConfigs.get_searchers() - - if not resource_path.exists(): - raise FileNotFoundError( - f"Searcher '{searcher}' not in:\n{', '.join(searchers)}" - ) - - with resource_path.open() as file: - data = yaml.safe_load(file) - - return data, searcher # type: ignore + signature = inspect.signature(func) + return { + key: the_locals[key] + for key in signature.parameters + if key in the_locals and key != "self" + } # TODO(eddiebergman): This seems like a bad function name, I guess this is used for a @@ -270,88 +245,65 @@ def is_partial_class(obj: Any) -> bool: return inspect.isclass(obj) -def instance_from_map( # noqa: C901 - mapping: dict[str, Any], - request: str | list | tuple | type, - name: str = "mapping", - *, - allow_any: bool = True, - as_class: bool = False, - kwargs: dict | None = None, -) -> Any: - """Get an instance of an class from a mapping. - - Arguments: - mapping: Mapping from string keys to classes or instances - request: A key from the mapping. If allow_any is True, could also be an - object or a class, to use a custom object. - name: Name of the mapping used in error messages - allow_any: If set to True, allows using custom classes/objects. - as_class: If the class should be returned without beeing instanciated - kwargs: Arguments used for the new instance, if created. Its purpose is - to serve at default arguments if the user doesn't built the object. +@contextmanager +def gc_disabled() -> Iterator[None]: + """Context manager to disable garbage collection for a block. - Raises: - ValueError: if the request is invalid (not a string if allow_any is False), - or invalid key. + We specifically put this around file I/O operations to minimize the time + spend garbage collecting while having the file handle open. """ - # Split arguments of the form (request, kwargs) - args_dict = kwargs or {} - if isinstance(request, Sequence) and not isinstance(request, str): - if len(request) != 2: - raise ValueError( - "When building an instance and specifying arguments, " - "you should give a pair (class, arguments)" - ) - request, req_args_dict = request - - if not isinstance(req_args_dict, Mapping): - raise ValueError("The arguments should be given as a dictionary") + gc.disable() + try: + yield + finally: + gc.enable() - args_dict = {**args_dict, **req_args_dict} - # Then, get the class/instance from the request - if isinstance(request, str): - if request not in mapping: - raise ValueError(f"{request} doesn't exists for {name}") +def dynamic_load_object(path: str, object_name: str) -> object: + """Dynamically loads an object from a given module file path. - instance = mapping[request] - elif allow_any: - instance = request - else: - raise ValueError(f"Object {request} invalid key for {name}") + Args: + path: File system path or module path to the Python module. + object_name: Name of the object to import from the module. - # Check if the request is a class if it is mandatory - if (args_dict or as_class) and not is_partial_class(instance): - raise ValueError( - f"{instance} is not a class and can't be used with additional arguments" - ) + Returns: + object: The imported object from the module. - # Give the arguments to the class - if args_dict: - instance = partial(instance, **args_dict) # type: ignore + Raises: + ImportError: If the module or object cannot be found. + """ + # file system path + if os.sep in path: + _path = Path(path).with_suffix(".py") + if not _path.exists(): + raise ImportError( + f"Failed to import '{object_name}'. File '{path}' does not exist." + ) + module_path = path.replace(os.sep, ".").replace(".py", "") - if as_class: - return instance + # module path + else: + module_path = path - if is_partial_class(instance): - try: - instance = instance() # type: ignore - except TypeError as e: - raise TypeError(f"{e} when calling {instance} with {args_dict}") from e + # Dynamically import the module. + spec = importlib.util.spec_from_file_location(module_path, path) - return instance + if spec is None or spec.loader is None: + raise ImportError( + f"Failed to import '{object_name}'." + f" Spec or loader is None for module '{module_path}'." + ) + module = importlib.util.module_from_spec(spec) + sys.modules[module_path] = module + spec.loader.exec_module(module) -@contextmanager -def gc_disabled() -> Iterator[None]: - """Context manager to disable garbage collection for a block. + # Retrieve the object. + imported_object = getattr(module, object_name, None) + if imported_object is None: + raise ImportError( + f"Failed to import '{object_name}'." + f"Object does not exist in module '{module_path}'." + ) - We specifically put this around file I/O operations to minimize the time - spend garbage collecting while having the file handle open. - """ - gc.disable() - try: - yield - finally: - gc.enable() + return imported_object diff --git a/neps/utils/files.py b/neps/utils/files.py index b49c53011..d79e43047 100644 --- a/neps/utils/files.py +++ b/neps/utils/files.py @@ -122,3 +122,29 @@ def deserialize( ) return data + + +def load_and_merge_yamls(*paths: str | Path | IO[str]) -> dict[str, Any]: + """Load and merge yaml files into a single dictionary. + + Raises: + ValueError: If there are duplicate keys in the yaml files. + """ + config: dict[str, Any] = {} + for path in paths: + match path: + case str() | Path(): + with Path(path).open("r") as file: + read_config = yaml.safe_load(file) + + case _: + read_config = yaml.safe_load(path) + + shared_keys = set(config) & set(read_config) + + if any(shared_keys): + raise ValueError(f"Duplicate key(s) {shared_keys} in {paths}") + + config.update(read_config) + + return config diff --git a/neps/utils/run_args.py b/neps/utils/run_args.py deleted file mode 100644 index 5a94ebc7e..000000000 --- a/neps/utils/run_args.py +++ /dev/null @@ -1,636 +0,0 @@ -"""This module provides utility functions for handling yaml content of run_args. -It includes functions for loading and processing configurations. -""" - -from __future__ import annotations - -import importlib.util -import logging -import sys -from collections.abc import Callable -from pathlib import Path -from typing import Any - -import yaml - -from neps.optimizers.base_optimizer import BaseOptimizer -from neps.search_spaces.search_space import pipeline_space_from_yaml - -logger = logging.getLogger("neps") - -# Define the name of the arguments as variables for easier code maintenance -RUN_ARGS = "run_args" -EVALUATE_PIPELINE = "evaluate_pipeline" -PIPELINE_SPACE = "pipeline_space" -ROOT_DIRECTORY = "root_directory" -MAX_EVALUATIONS_TOTAL = "max_evaluations_total" -MAX_COST_TOTAL = "max_cost_total" -OVERWRITE_WORKING_DIRECTORY = "overwrite_working_directory" -POST_RUN_SUMMARY = "post_run_summary" -DEVELOPMENT_STAGE_ID = "development_stage_id" -TASK_ID = "task_id" -CONTINUE_UNTIL_MAX_EVALUATION_COMPLETED = "continue_until_max_evaluation_completed" -OBJECTIVE_TO_MINIMIZE_VALUE_ON_ERROR = "objective_to_minimize_value_on_error" -COST_VALUE_ON_ERROR = "cost_value_on_error" -IGNORE_ERROR = "ignore_errors" -SEARCHER = "searcher" -PRE_LOAD_HOOKS = "pre_load_hooks" -# searcher_kwargs is used differently in yaml and just play a role for considering -# arguments of a custom searcher class (BaseOptimizer) -SEARCHER_KWARGS = "searcher_kwargs" -MAX_EVALUATIONS_PER_RUN = "max_evaluations_per_run" - - -def get_run_args_from_yaml(path: str | Path) -> dict: - """Load and validate NEPS run arguments from a specified YAML configuration file - provided via run_args. - - This function reads a YAML file, extracts the arguments required by NePS, - validates these arguments, and then returns them in a dictionary. It checks for the - presence and validity of expected parameters, and distinctively handles more complex - configurations, specifically those that are dictionaries(e.g. pipeline_space) or - objects(e.g. evaluate_pipeline) requiring loading. - - Args: - path (str): The file path to the YAML configuration file. - - Returns: - A dictionary of validated run arguments. - - Raises: - KeyError: If any parameter name is invalid. - """ - # Load the YAML configuration file - config = config_loader(path) - - # Initialize an empty dictionary to hold the extracted settings - settings = {} - - # List allowed NePS run arguments with simple types (e.g., string, int). Parameters - # like 'evaluate_pipeline', 'preload_hooks', 'pipeline_space', - # and 'searcher' are excluded due to needing specialized processing. - expected_parameters = [ - ROOT_DIRECTORY, - MAX_EVALUATIONS_TOTAL, - MAX_COST_TOTAL, - OVERWRITE_WORKING_DIRECTORY, - POST_RUN_SUMMARY, - DEVELOPMENT_STAGE_ID, - TASK_ID, - MAX_EVALUATIONS_PER_RUN, - CONTINUE_UNTIL_MAX_EVALUATION_COMPLETED, - OBJECTIVE_TO_MINIMIZE_VALUE_ON_ERROR, - COST_VALUE_ON_ERROR, - IGNORE_ERROR, - ] - - # Flatten the YAML file's structure to separate flat parameters (flat_config) and - # those needing special handling (special_configs). - flat_config, special_configs = extract_leaf_keys(config) - - # Check if flatten dict (flat_config) just contains the expected parameters - for parameter, value in flat_config.items(): - if parameter in expected_parameters: - settings[parameter] = value - else: - raise KeyError( - f"Parameter '{parameter}' is not an argument of neps.run() " - f"provided via run_args." - f"See here all valid arguments:" - f" {', '.join(expected_parameters)}, " - f"'evaluate_pipeline', 'preload_hooks', 'pipeline_space'" - ) - - # Process complex configurations (e.g., 'pipeline_space', 'searcher') and integrate - # them into 'settings'. - handle_special_argument_cases(settings, special_configs) - - # check if all provided arguments have legal types - check_run_args(settings) - - logger.debug( - f"The 'run_args' arguments: {settings} are now extracted and type-tested from " - f"referenced YAML." - ) - - return settings - - -def config_loader(path: str | Path) -> dict: - """Loads a YAML file and returns the contents under the 'run_args' key. - - Args: - path (str): Path to the YAML file. - - Returns: - Content of the yaml (dict) - - Raises: - FileNotFoundError: If the file at 'path' does not exist. - ValueError: If the file is not a valid YAML. - """ - try: - with open(path) as file: # noqa: PTH123 - config = yaml.safe_load(file) - except FileNotFoundError as e: - raise FileNotFoundError( - f"The specified file was not found: '{path}'." - f" Please make sure that the path is correct and " - f"try again." - ) from e - except yaml.YAMLError as e: - raise ValueError(f"The file at {path} is not a valid YAML file.") from e - - return config - - -def extract_leaf_keys(d: dict, special_keys: dict | None = None) -> tuple[dict, dict]: - """Recursive function to extract leaf keys and their values from a nested dictionary. - Special keys (e.g.'evaluate_pipeline') are also extracted if present - and their corresponding values (dict) at any level in the nested structure. - - Args: - d (dict): The dictionary to extract values from. - special_keys (dict|None): A dictionary to store values of special keys. - - Returns: - A tuple containing the leaf keys dictionary and the dictionary for - special keys. - """ - if special_keys is None: - special_keys = { - EVALUATE_PIPELINE: None, - PRE_LOAD_HOOKS: None, - SEARCHER: None, - PIPELINE_SPACE: None, - } - - leaf_keys = {} - for k, v in d.items(): - if k in special_keys and v != "None": - special_keys[k] = v - elif isinstance(v, dict): - # Recursively call to explore nested dictionaries - nested_leaf_keys, _ = extract_leaf_keys(v, special_keys) - leaf_keys.update(nested_leaf_keys) - elif v is not None and v != "None": - leaf_keys[k] = v - return leaf_keys, special_keys - - -def handle_special_argument_cases(settings: dict, special_configs: dict) -> None: - """Process and integrate special configuration cases into the 'settings' dictionary. - - This function updates 'settings' with values from 'special_configs'. It handles - specific keys that require more complex processing, such as 'pipeline_space' and - 'searcher', which may need to load a function/dict from paths. It also manages nested - configurations like 'pre_load_hooks' which need individual processing or function - loading. - - Args: - settings (dict): The dictionary to be updated with processed configurations. - special_configs (dict): A dictionary containing configuration keys and values - that require special processing. - - """ - # process special configs - process_evaluate_pipeline(EVALUATE_PIPELINE, special_configs, settings) - process_pipeline_space(PIPELINE_SPACE, special_configs, settings) - process_searcher(SEARCHER, special_configs, settings) - - if special_configs[PRE_LOAD_HOOKS] is not None: - # Loads the pre_load_hooks functions and add them in a list to settings. - settings[PRE_LOAD_HOOKS] = load_hooks_from_config(special_configs[PRE_LOAD_HOOKS]) - - -def process_pipeline_space(key: str, special_configs: dict, settings: dict) -> None: - """Process or load the pipeline space configuration. - - This function checks if the given key exists in the `special_configs` dictionary. - If it exists, it processes the associated value, which can be either a dictionary - or a string. Based on the keys of the dictionary it decides if the pipeline_space - have to be loaded or needs to be converted into a neps search_space structure. - The processed pipeline space is then stored in the `settings` - dictionary under the given key. - - Args: - key (str): The key to check in the `special_configs` dictionary. - special_configs (dict): The dictionary containing special configuration values. - settings (dict): The dictionary where the processed pipeline space will be stored. - - Raises: - TypeError: If the value associated with the key is neither a string nor a - dictionary. - """ - if special_configs.get(key) is not None: - pipeline_space = special_configs[key] - # Define the type of processed_pipeline_space to accommodate both situations - if isinstance(pipeline_space, dict): - # determine if dict contains path_loading or the actual search space - expected_keys = {"path", "name"} - actual_keys = set(pipeline_space.keys()) - if expected_keys != actual_keys: - # pipeline_space directly defined in run_args yaml - processed_pipeline_space = pipeline_space_from_yaml(pipeline_space) - else: - # pipeline_space stored in a python dict, not using a yaml - processed_pipeline_space = load_and_return_object( - pipeline_space["path"], pipeline_space["name"], key - ) # type: ignore - elif isinstance(pipeline_space, str): - # load yaml from path - processed_pipeline_space = pipeline_space_from_yaml(pipeline_space) - else: - raise TypeError( - f"Value for {key} must be a string or a dictionary, " - f"but got {type(pipeline_space).__name__}." - ) - settings[key] = processed_pipeline_space - - -def process_searcher(key: str, special_configs: dict, settings: dict) -> None: - """Processes the searcher configuration and updates the settings dictionary. - - Checks if the key exists in special_configs. If found, it processes the - value based on its type. Updates settings with the processed searcher. - - Args: - key (str): Key to look up in special_configs. - special_configs (dict): Dictionary of special configurations. - settings (dict): Dictionary to update with the processed searcher. - - Raises: - TypeError: If the value for the key is neither a string, Path, nor a dictionary. - """ - if special_configs.get(key) is not None: - searcher = special_configs[key] - if isinstance(searcher, dict): - # determine if dict contains path_loading or the actual searcher config - expected_keys = {"path", "name"} - actual_keys = set(searcher.keys()) - if expected_keys.issubset(actual_keys): - path = searcher.pop("path") - name = searcher.pop("name") - settings[SEARCHER_KWARGS] = searcher - searcher = load_and_return_object(path, name, key) - - elif isinstance(searcher, str | Path): - pass - else: - raise TypeError( - f"Value for {key} must be a string or a dictionary, " - f"but got {type(searcher).__name__}." - ) - settings[key] = searcher - - -def process_evaluate_pipeline(key: str, special_configs: dict, settings: dict) -> None: - """Processes the run pipeline configuration and updates the settings dictionary. - - Args: - key (str): Key to look up in special_configs. - special_configs (dict): Dictionary of special configurations. - settings (dict): Dictionary to update with the processed function. - - Raises: - KeyError: If required keys ('path' and 'name') are missing in the config. - """ - if special_configs.get(key) is not None: - config = special_configs[key] - try: - func = load_and_return_object(config["path"], config["name"], key) - settings[key] = func - except KeyError as e: - raise KeyError( - f"Missing key for argument {key}: {e}. Expect 'path' " - f"and 'name' as keys when loading '{key}' " - f"from 'run_args'" - ) from e - - -def load_and_return_object(module_path: str, object_name: str, key: str) -> object: - """Dynamically loads an object from a given module file path. - - This function attempts to dynamically import an object by its name from a specified - module path. If the initial import fails, it retries with a '.py' extension appended - to the path. - - Args: - module_path (str): File system path to the Python module. - object_name (str): Name of the object to import from the module. - key (str): Identifier for the argument causing the error, for enhanced error - feedback. - - Returns: - object: The imported object from the module. - - Raises: - ImportError: If the module or object cannot be found, with a message detailing - the issue. - """ - - def import_object(path: str) -> object | None: - try: - # Convert file system path to module path, removing '.py' if present. - module_name = ( - path[:-3].replace("/", ".") - if path.endswith(".py") - else path.replace("/", ".") - ) - - # Dynamically import the module. - spec = importlib.util.spec_from_file_location(module_name, path) - if spec is None or spec.loader is None: - return None # Failed to load module spec. - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - - # Retrieve the object. - imported_object = getattr(module, object_name, None) - if imported_object is None: - return None # Object not found in module. - return imported_object - except FileNotFoundError: - return None # File not found. - - # Attempt to import the object using the provided path. - imported_object = import_object(module_path) - if imported_object is None: - # If the object could not be imported, attempt again by appending '.py', - # if not already present. - if not module_path.endswith(".py"): - module_path += ".py" - imported_object = import_object(module_path) - - if imported_object is None: - raise ImportError( - f"Failed to import '{object_name}' for argument '{key}'. " - f"Module path '{module_path}' not found or object does not " - f"exist." - ) - - return imported_object - - -def load_hooks_from_config(pre_load_hooks_dict: dict) -> list: - """Loads hook functions from a dictionary of configurations. - - Args: - pre_load_hooks_dict (Dict): Dictionary with hook names as keys and paths as values - - Returns: - List: List of loaded hook functions. - """ - loaded_hooks = [] - for name, path in pre_load_hooks_dict.items(): - hook_func = load_and_return_object(path, name, PRE_LOAD_HOOKS) - loaded_hooks.append(hook_func) - return loaded_hooks - - -def check_run_args(settings: dict) -> None: - """Validates the types of NePS configuration settings. - - Checks that each setting's value type matches its expected type. Raises - TypeError for type mismatches. - - Args: - settings (dict): NePS configuration settings. - - Raises: - TypeError: For mismatched setting value types. - """ - # Mapping parameter names to their allowed types - # [task_id, development_stage_id, pre_load_hooks] require special handling of type, - # that's why they are not listed - expected_types = { - EVALUATE_PIPELINE: Callable, - ROOT_DIRECTORY: str, - # TODO: Support CS.ConfigurationSpace for pipeline_space - PIPELINE_SPACE: (str, dict), - OVERWRITE_WORKING_DIRECTORY: bool, - POST_RUN_SUMMARY: bool, - MAX_EVALUATIONS_TOTAL: int, - MAX_COST_TOTAL: (int, float), - MAX_EVALUATIONS_PER_RUN: int, - CONTINUE_UNTIL_MAX_EVALUATION_COMPLETED: bool, - OBJECTIVE_TO_MINIMIZE_VALUE_ON_ERROR: float, - COST_VALUE_ON_ERROR: float, - IGNORE_ERROR: bool, - SEARCHER_KWARGS: dict, - } - for param, value in settings.items(): - if param in (DEVELOPMENT_STAGE_ID, TASK_ID): - # this argument can be Any - continue - elif param == PRE_LOAD_HOOKS: # noqa: RET507 - # check if all items in pre_load_hooks are callable objects - if not all(callable(item) for item in value): - raise TypeError("All items in 'pre_load_hooks' must be callable.") - elif param == SEARCHER: - if not (isinstance(value, str | dict) or issubclass(value, BaseOptimizer)): - raise TypeError( - "Parameter 'searcher' must be a string or a class that is a subclass " - "of BaseOptimizer." - ) - else: - try: - expected_type = expected_types[param] - except KeyError as e: - raise KeyError(f"{param} is not a valid argument of neps") from e - if not isinstance(value, expected_type): # type: ignore - raise TypeError( - f"Parameter '{param}' expects a value of type {expected_type}, got " - f"{type(value)} instead." - ) - - -def check_essential_arguments( - evaluate_pipeline: Callable | None, - root_directory: str | None, - pipeline_space: dict | None, - max_cost_total: int | None, - max_evaluations_total: int | None, - searcher: BaseOptimizer | dict | str | None, -) -> None: - """Validates essential NePS configuration arguments. - - Ensures 'evaluate_pipeline', 'root_directory', 'pipeline_space', and either - 'max_cost_total' or 'max_evaluations_total' are provided for NePS execution. - Raises ValueError with missing argument details. Additionally, checks 'searcher' - is a BaseOptimizer if 'pipeline_space' is absent. - - Args: - evaluate_pipeline: Function for the pipeline execution. - root_directory (str): Directory path for data storage. - pipeline_space: search space for this run. - max_cost_total: Max allowed total cost for experiments. - max_evaluations_total: Max allowed evaluations. - searcher: Optimizer for the configuration space. - - Raises: - ValueError: Missing or invalid essential arguments. - """ - if not evaluate_pipeline: - raise ValueError("'evaluate_pipeline' is required but was not provided.") - if not root_directory: - raise ValueError("'root_directory' is required but was not provided.") - if not pipeline_space and not isinstance(searcher, BaseOptimizer): - # handling special case for searcher instance, in which user doesn't have to - # provide the search_space because it's the argument of the searcher. - raise ValueError("'pipeline_space' is required but was not provided.") - - if not max_evaluations_total and not max_cost_total: - raise ValueError( - "'max_evaluations_total' or 'max_cost_total' is required but " - "both were not provided." - ) - - -# Handle Settings - - -class Sentinel: - """Introduce a sentinel object as default value for checking variable assignment.""" - - def __repr__(self) -> str: - return "" - - -UNSET = Sentinel() - - -class Settings: - """Centralizes and manages configuration settings from various sources of NePS - arguments (run_args (yaml) and neps func_args). - """ - - def __init__(self, func_args: dict, yaml_args: Path | str | Default | None = None): - """Initializes the Settings object by merging function arguments with YAML - configuration settings and assigning them to class attributes. It checks for - necessary configurations and handles default values where specified. - - Args: - func_args (dict): The function arguments directly passed to NePS. - yaml_args (dict | None): Optional. YAML file arguments provided via run_args. - """ - self.evaluate_pipeline = UNSET - self.root_directory = UNSET - self.pipeline_space = UNSET - self.overwrite_working_directory = UNSET - self.post_run_summary = UNSET - self.development_stage_id = UNSET - self.task_id = UNSET - self.max_evaluations_total = UNSET - self.max_evaluations_per_run = UNSET - self.continue_until_max_evaluation_completed = UNSET - self.max_cost_total = UNSET - self.ignore_errors = UNSET - self.objective_to_minimize_value_on_error = UNSET - self.cost_value_on_error = UNSET - self.pre_load_hooks = UNSET - self.searcher = UNSET - self.sample_batch_size = UNSET - self.searcher_kwargs = UNSET - - if not isinstance(yaml_args, Default) and yaml_args is not None: - yaml_settings = get_run_args_from_yaml(yaml_args) - dict_settings = self.merge(func_args, yaml_settings) - else: - dict_settings = {} - for key, value in func_args.items(): - if isinstance(value, Default): - dict_settings[key] = value.value - else: - dict_settings[key] = value - - # drop run_args, not needed as a setting attribute - del dict_settings[RUN_ARGS] - self.assign(dict_settings) - self.check() - - def merge(self, func_args: dict, yaml_args: dict) -> dict: - """Merge func_args and yaml_args. func_args gets priority over yaml_args.""" - # Initialize with YAML settings - merged_settings = yaml_args.copy() - - # overwrite or merge keys - for key, value in func_args.items(): - # Handle searcher_kwargs for BaseOptimizer case - if key == SEARCHER_KWARGS: - merged_settings[SEARCHER_KWARGS] = { - **yaml_args.pop(SEARCHER_KWARGS, {}), - **func_args[SEARCHER_KWARGS], - } - elif not isinstance(value, Default): - merged_settings[key] = value - elif key not in yaml_args: - # If the key is not in yaml_args, set it from Default - merged_settings[key] = value.value - return merged_settings - - def assign(self, dict_settings: dict) -> None: - """Updates existing attributes with values from `dict_settings`. - Raises AttributeError if any attribute in `dict_settings` does not exist. - """ - for key, value in dict_settings.items(): - if hasattr(self, key): - setattr(self, key, value) - else: - raise AttributeError(f"'Settings' object has no attribute '{key}'") - - def check_unassigned_attributes(self) -> list: - """Check for UNSET and Default class.""" - return [ - key - for key, value in self.__dict__.items() - if value is UNSET or isinstance(value, Default) - ] - - def check(self) -> None: - """Check if all values are assigned and if the essentials are provided - correctly. - """ - unassigned_attributes = self.check_unassigned_attributes() - if unassigned_attributes: - raise ValueError( - f"Unassigned or default-initialized attributes detected: " - f"{', '.join(unassigned_attributes)}" - ) - check_essential_arguments( - self.evaluate_pipeline, # type: ignore - self.root_directory, # type: ignore - self.pipeline_space, # type: ignore - self.max_cost_total, # type: ignore - self.max_evaluations_total, # type: ignore - self.searcher, # type: ignore - ) - - -class Default: - """A class to enable default detection. - - Attributes: - value: The value to be stored as the default. - - Methods: - __init__(self, value): Initializes the Default object with a value. - __repr__(self): Returns a string representation of the Default object. - """ - - def __init__(self, value: Any): - """Initialize the Default object with the specified value. - - Args: - value: The value to store as default. Can be any data type. - """ - self.value = value - - def __repr__(self) -> str: - """Return the string representation of the Default object. - - Returns: - A string that represents the Default object in the format . - """ - return f"" diff --git a/neps/utils/types.py b/neps/utils/types.py deleted file mode 100644 index e5fd343a1..000000000 --- a/neps/utils/types.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Primitive types to be used in NePS or consumers of NePS.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Literal, TypeAlias - -import numpy as np - -if TYPE_CHECKING: - from neps.search_spaces.search_space import SearchSpace - from neps.state.trial import Report - -# TODO(eddiebergman): We can turn this to an enum at some -# point to prevent having to isinstance and str match -ERROR: TypeAlias = Literal["error"] -Number: TypeAlias = int | float -ConfigID: TypeAlias = str -RawConfig: TypeAlias = Mapping[str, Any] -ResultDict: TypeAlias = Mapping[str, Any] - -# NOTE(eddiebergman): Getting types for scipy distributions sucks -# this is more backwards compatible and easier to work with -TruncNorm: TypeAlias = Any - - -class _NotSet: - def __repr__(self) -> str: - return "NotSet" - - -NotSet = _NotSet() - -f64 = np.float64 - - -# TODO(eddiebergman): Ideally, use `Trial` objects which can carry a lot more -# useful information to optimizers than the below dataclass. Would be a follow up -# refactor. -@dataclass -class ConfigResult: - """Primary class through which optimizers recieve results.""" - - id: str - """Unique identifier for the configuration.""" - - config: SearchSpace - """Configuration that was evaluated.""" - - result: Report | ResultDict | ERROR - """Some dictionary of results.""" - - metadata: dict - """Any additional data to store with this config and result.""" - - -# TODO(eddiebergman): This is a hack because status.py expects a `ConfigResult` -# where the `config` is a dict config (`RawConfig`), while all the optimizers -# expect a `ConfigResult` where the `config` is a `SearchSpace`. Ideally we -# just rework status to use `Trial` and `Report` directly as they contain a lot more -# information. -@dataclass -class _ConfigResultForStats: - id: str - config: RawConfig - result: ResultDict | ERROR - metadata: dict - - @property - def objective_to_minimize(self) -> float | ERROR: - if isinstance(self.result, dict): - return float(self.result["objective_to_minimize"]) - return "error" diff --git a/neps_examples/README.md b/neps_examples/README.md index b0b642cee..b1613520e 100644 --- a/neps_examples/README.md +++ b/neps_examples/README.md @@ -7,6 +7,6 @@ Understand how to analyze runs on a basic level. 2. **Efficiency examples** showcase how to enhance efficiency in NePS. Learn about expert priors, multi-fidelity, and parallelization to streamline your pipeline and optimize search processes. -3. **Convenience examples** show tensorboard compatibility and its integration, explore the compatibility with PyTorch Lightning, see the declarative API, understand file management within the run pipeline function used in NePS. +3. **Convenience examples** show tensorboard compatibility and its integration, explore the compatibility with PyTorch Lightning, see the declarative API, understand file management within the evaluate pipeline function used in NePS. 4. **Experimental examples** tailored for NePS contributors. These examples provide insights and practices for experimental scenarios. diff --git a/neps_examples/basic_usage/architecture.py b/neps_examples/basic_usage/architecture.py deleted file mode 100644 index adca3544f..000000000 --- a/neps_examples/basic_usage/architecture.py +++ /dev/null @@ -1,130 +0,0 @@ -raise NotImplementedError( - "Support for graphs was temporarily removed, if you'd like to use a version" - " of NePS that supports graphs, please use version v0.12.2" -) - -import logging - -from torch import nn - -import neps -from neps.search_spaces.architecture import primitives as ops -from neps.search_spaces.architecture import topologies as topos -from neps.search_spaces.architecture.primitives import AbstractPrimitive - - -class DownSampleBlock(AbstractPrimitive): - def __init__(self, in_channels: int, out_channels: int): - super().__init__(locals()) - self.conv_a = ReLUConvBN( - in_channels, out_channels, kernel_size=3, stride=2, padding=1 - ) - self.conv_b = ReLUConvBN( - out_channels, out_channels, kernel_size=3, stride=1, padding=1 - ) - self.downsample = nn.Sequential( - nn.AvgPool2d(kernel_size=2, stride=2, padding=0), - nn.Conv2d( - in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False - ), - ) - - def forward(self, inputs): - basicblock = self.conv_a(inputs) - basicblock = self.conv_b(basicblock) - residual = self.downsample(inputs) - return residual + basicblock - - -class ReLUConvBN(AbstractPrimitive): - def __init__(self, in_channels, out_channels, kernel_size, stride, padding): - super().__init__(locals()) - - self.kernel_size = kernel_size - self.op = nn.Sequential( - nn.ReLU(inplace=False), - nn.Conv2d( - in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=1, - bias=False, - ), - nn.BatchNorm2d(out_channels, affine=True, track_running_stats=True), - ) - - def forward(self, x): - return self.op(x) - - -class AvgPool(AbstractPrimitive): - def __init__(self, **kwargs): - super().__init__(kwargs) - self.op = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False) - - def forward(self, x): - return self.op(x) - - -primitives = { - "Sequential15": topos.get_sequential_n_edge(15), - "DenseCell": topos.get_dense_n_node_dag(4), - "down": {"op": DownSampleBlock}, - "avg_pool": {"op": AvgPool}, - "id": {"op": ops.Identity}, - "conv3x3": {"op": ReLUConvBN, "kernel_size": 3, "stride": 1, "padding": 1}, - "conv1x1": {"op": ReLUConvBN, "kernel_size": 1, "stride": 1, "padding": 0}, -} - - -structure = { - "S": ["Sequential15(C, C, C, C, C, down, C, C, C, C, C, down, C, C, C, C, C)"], - "C": ["DenseCell(OPS, OPS, OPS, OPS, OPS, OPS)"], - "OPS": ["id", "conv3x3", "conv1x1", "avg_pool"], -} - - -def set_recursive_attribute(op_name, predecessor_values): - in_channels = 16 if predecessor_values is None else predecessor_values["out_channels"] - out_channels = in_channels * 2 if op_name == "DownSampleBlock" else in_channels - return dict(in_channels=in_channels, out_channels=out_channels) - - -def run_pipeline(architecture): - in_channels = 3 - base_channels = 16 - n_classes = 10 - out_channels_factor = 4 - - # E.g., in shape = (N, 3, 32, 32) => out shape = (N, 10) - model = architecture.to_pytorch() - model = nn.Sequential( - nn.Conv2d(in_channels, base_channels, 3, padding=1, bias=False), - nn.BatchNorm2d(base_channels), - model, - nn.BatchNorm2d(base_channels * out_channels_factor), - nn.ReLU(inplace=True), - nn.AdaptiveAvgPool2d(1), - nn.Flatten(), - nn.Linear(base_channels * out_channels_factor, n_classes), - ) - return 1 - - -pipeline_space = dict( - architecture=neps.Architecture( - set_recursive_attribute=set_recursive_attribute, - structure=structure, - primitives=primitives, - ) -) - -logging.basicConfig(level=logging.INFO) -neps.run( - run_pipeline=run_pipeline, - pipeline_space=pipeline_space, - root_directory="results/architecture", - max_evaluations_total=15, -) diff --git a/neps_examples/basic_usage/architecture_and_hyperparameters.py b/neps_examples/basic_usage/architecture_and_hyperparameters.py deleted file mode 100644 index b7f4bd637..000000000 --- a/neps_examples/basic_usage/architecture_and_hyperparameters.py +++ /dev/null @@ -1,127 +0,0 @@ -raise NotImplementedError( - "Support for graphs was temporarily removed, if you'd like to use a version" - " of NePS that supports graphs, please use version v0.12.2" -) - -import logging - -from torch import nn - -import neps -from neps.search_spaces.architecture import primitives as ops -from neps.search_spaces.architecture import topologies as topos -from neps.search_spaces.architecture.primitives import AbstractPrimitive - - -class DownSampleBlock(AbstractPrimitive): - def __init__(self, in_channels: int, out_channels: int): - super().__init__(locals()) - self.conv_a = ReLUConvBN( - in_channels, out_channels, kernel_size=3, stride=2, padding=1 - ) - self.conv_b = ReLUConvBN( - out_channels, out_channels, kernel_size=3, stride=1, padding=1 - ) - self.downsample = nn.Sequential( - nn.AvgPool2d(kernel_size=2, stride=2, padding=0), - nn.Conv2d( - in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False - ), - ) - - def forward(self, inputs): - basicblock = self.conv_a(inputs) - basicblock = self.conv_b(basicblock) - residual = self.downsample(inputs) - return residual + basicblock - - -class ReLUConvBN(AbstractPrimitive): - def __init__(self, in_channels, out_channels, kernel_size, stride, padding): - super().__init__(locals()) - - self.kernel_size = kernel_size - self.op = nn.Sequential( - nn.ReLU(inplace=False), - nn.Conv2d( - in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - dilation=1, - bias=False, - ), - nn.BatchNorm2d(out_channels, affine=True, track_running_stats=True), - ) - - def forward(self, x): - return self.op(x) - - -class AvgPool(AbstractPrimitive): - def __init__(self, **kwargs): - super().__init__(kwargs) - self.op = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False) - - def forward(self, x): - return self.op(x) - - -primitives = { - "Sequential15": topos.get_sequential_n_edge(15), - "DenseCell": topos.get_dense_n_node_dag(4), - "down": {"op": DownSampleBlock}, - "avg_pool": {"op": AvgPool}, - "id": {"op": ops.Identity}, - "conv3x3": {"op": ReLUConvBN, "kernel_size": 3, "stride": 1, "padding": 1}, - "conv1x1": {"op": ReLUConvBN, "kernel_size": 1, "stride": 1, "padding": 0}, -} - - -structure = { - "S": ["Sequential15(C, C, C, C, C, down, C, C, C, C, C, down, C, C, C, C, C)"], - "C": ["DenseCell(OPS, OPS, OPS, OPS, OPS, OPS)"], - "OPS": ["id", "conv3x3", "conv1x1", "avg_pool"], -} - - -def set_recursive_attribute(op_name, predecessor_values): - in_channels = 16 if predecessor_values is None else predecessor_values["out_channels"] - out_channels = in_channels * 2 if op_name == "DownSampleBlock" else in_channels - return dict(in_channels=in_channels, out_channels=out_channels) - - -def run_pipeline(**config): - optimizer = config["optimizer"] - learning_rate = config["learning_rate"] - model = config["architecture"].to_pytorch() - - target_params = 1531258 - number_of_params = sum(p.numel() for p in model.parameters()) - validation_error = abs(target_params - number_of_params) / target_params - - target_lr = 10e-3 - validation_error += abs(target_lr - learning_rate) / target_lr - validation_error += int(optimizer == "sgd") - - return validation_error - - -pipeline_space = dict( - architecture=neps.Architecture( - set_recursive_attribute=set_recursive_attribute, - structure=structure, - primitives=primitives, - ), - optimizer=neps.Categorical(choices=["sgd", "adam"]), - learning_rate=neps.Float(lower=10e-7, upper=10e-3, log=True), -) - -logging.basicConfig(level=logging.INFO) -neps.run( - run_pipeline=run_pipeline, - pipeline_space=pipeline_space, - root_directory="results/hyperparameters_architecture_example", - max_evaluations_total=15, -) diff --git a/neps_examples/basic_usage/hyperparameters.py b/neps_examples/basic_usage/hyperparameters.py index f86a5ae41..3940f2050 100644 --- a/neps_examples/basic_usage/hyperparameters.py +++ b/neps_examples/basic_usage/hyperparameters.py @@ -1,18 +1,14 @@ import logging -import time -from warnings import warn import numpy as np import neps -def evaluate_pipeline(float1, float2, categorical, integer1, integer2): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(float1, float2, categorical, integer1, integer2) def evaluate_pipeline(float1, float2, categorical, integer1, integer2): - objective_to_minimize = -float(np.sum([float1, float2, int(categorical), integer1, integer2])) - # time.sleep(0.7) # For demonstration purposes + objective_to_minimize = -float( + np.sum([float1, float2, int(categorical), integer1, integer2]) + ) return objective_to_minimize diff --git a/neps_examples/convenience/declarative_usage/config.yaml b/neps_examples/convenience/declarative_usage/config.yaml index 83776cc54..858eb6e52 100644 --- a/neps_examples/convenience/declarative_usage/config.yaml +++ b/neps_examples/convenience/declarative_usage/config.yaml @@ -3,7 +3,6 @@ experiment: max_evaluations_total: 20 overwrite_working_directory: true post_run_summary: true - development_stage_id: "beta" pipeline_space: epochs: 5 @@ -20,7 +19,7 @@ pipeline_space: lower: 64 upper: 128 -searcher: - strategy: "bayesian_optimization" +optimizer: + name: "bayesian_optimization" initial_design_size: 5 surrogate_model: gp diff --git a/neps_examples/convenience/logging_additional_info.py b/neps_examples/convenience/logging_additional_info.py index 70b2681c8..6756e03c7 100644 --- a/neps_examples/convenience/logging_additional_info.py +++ b/neps_examples/convenience/logging_additional_info.py @@ -6,13 +6,12 @@ import neps -def run_pipeline(float1, float2, categorical, integer1, integer2): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(float1, float2, categorical, integer1, integer2) def evaluate_pipeline(float1, float2, categorical, integer1, integer2): start = time.time() - objective_to_minimize = -float(np.sum([float1, float2, int(categorical), integer1, integer2])) + objective_to_minimize = -float( + np.sum([float1, float2, int(categorical), integer1, integer2]) + ) end = time.time() return { "objective_to_minimize": objective_to_minimize, diff --git a/neps_examples/convenience/neps_tblogger_tutorial.py b/neps_examples/convenience/neps_tblogger_tutorial.py index f3fd83220..c8737af7e 100644 --- a/neps_examples/convenience/neps_tblogger_tutorial.py +++ b/neps_examples/convenience/neps_tblogger_tutorial.py @@ -73,8 +73,8 @@ #2 Prepare the input data. #3 Design the model. #4 Design the pipeline search spaces. -#5 Design the run pipeline function. -#6 Use neps.run the run the entire search using your specified searcher. +#5 Design the evaluate pipeline function. +#6 Use neps.run the run the entire search using your specified optimizer. Each step will be covered in detail thourghout the code @@ -243,13 +243,6 @@ def pipeline_space() -> dict: ############################################################# # Implement the pipeline run search. - -def run_pipeline(lr, optim, weight_decay): - # Deprecated function, use evaluate_pipeline instead - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(lr, optim, weight_decay) - - def evaluate_pipeline(lr, optim, weight_decay): # Create the network model. model = MLP() @@ -337,7 +330,7 @@ def evaluate_pipeline(lr, optim, weight_decay): ############################################################# -# Running neps with BO as the searcher. +# Running neps with BO as the optimizer. if __name__ == "__main__": """ @@ -360,7 +353,7 @@ def evaluate_pipeline(lr, optim, weight_decay): evaluate_pipeline=evaluate_pipeline, pipeline_space=pipeline_space(), root_directory="results/neps_tblogger_example", - searcher="random_search", + optimizer="random_search", ) neps.run( diff --git a/neps_examples/convenience/neps_x_lightning.py b/neps_examples/convenience/neps_x_lightning.py index 8e019957d..c1482111a 100644 --- a/neps_examples/convenience/neps_x_lightning.py +++ b/neps_examples/convenience/neps_x_lightning.py @@ -37,6 +37,7 @@ These dependencies ensure you have everything you need for this tutorial. """ + import argparse import glob import logging @@ -136,7 +137,10 @@ def training_step( self.train_accuracy.update(preds, y) self.log_dict( - {"train_objective_to_minimize": objective_to_minimize, "train_acc": self.val_accuracy.compute()}, + { + "train_objective_to_minimize": objective_to_minimize, + "train_acc": self.val_accuracy.compute(), + }, on_epoch=True, on_step=False, prog_bar=True, @@ -151,15 +155,16 @@ def validation_step( self.val_accuracy.update(preds, y) self.log_dict( - {"val_objective_to_minimize": objective_to_minimize, "val_acc": self.val_accuracy.compute()}, + { + "val_objective_to_minimize": objective_to_minimize, + "val_acc": self.val_accuracy.compute(), + }, on_epoch=True, on_step=False, prog_bar=True, ) - def test_step( - self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int - ) -> None: + def test_step(self, batch: Tuple[torch.Tensor, torch.Tensor], batch_idx: int) -> None: _, preds, y = self.common_step(batch, batch_idx) self.test_accuracy.update(preds, y) @@ -253,9 +258,7 @@ def search_space() -> dict: data_dir=neps.Constant("./data"), batch_size=neps.Constant(64), lr=neps.Float(lower=1e-5, upper=1e-2, log=True, prior=1e-3), - weight_decay=neps.Float( - lower=1e-5, upper=1e-3, log=True, prior=5e-4 - ), + weight_decay=neps.Float(lower=1e-5, upper=1e-3, log=True, prior=5e-4), optimizer=neps.Categorical(choices=["Adam", "SGD"], prior="Adam"), epochs=neps.Integer(lower=1, upper=9, log=False, is_fidelity=True), ) @@ -263,18 +266,7 @@ def search_space() -> dict: ############################################################# -# Define the run pipeline function - -def run_pipeline(pipeline_directory, previous_pipeline_directory, **config): - # Deprecated function, use evaluate_pipeline instead - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline( - pipeline_directory, - previous_pipeline_directory, - **config, - ) - - +# Define the evaluate pipeline function def evaluate_pipeline(pipeline_directory, previous_pipeline_directory, **config) -> dict: # Initialize the first directory to store the event and checkpoints files init_dir = get_initial_directory(pipeline_directory) @@ -321,7 +313,9 @@ def evaluate_pipeline(pipeline_directory, previous_pipeline_directory, **config) trainer.fit(model) train_accuracy = trainer.logged_metrics.get("train_acc", None) - val_objective_to_minimize = trainer.logged_metrics.get("val_objective_to_minimize", None) + val_objective_to_minimize = trainer.logged_metrics.get( + "val_objective_to_minimize", None + ) val_accuracy = trainer.logged_metrics.get("val_acc", None) # Test the model and retrieve test metrics @@ -362,7 +356,7 @@ def evaluate_pipeline(pipeline_directory, previous_pipeline_directory, **config) pipeline_space=search_space(), root_directory="results/hyperband", max_evaluations_total=args.max_evaluations_total, - searcher="hyperband", + optimizer="hyperband", ) # Record the end time and calculate execution time diff --git a/neps_examples/convenience/running_on_slurm_scripts.py b/neps_examples/convenience/running_on_slurm_scripts.py index 7a52c21ba..86fe41ac2 100644 --- a/neps_examples/convenience/running_on_slurm_scripts.py +++ b/neps_examples/convenience/running_on_slurm_scripts.py @@ -1,11 +1,9 @@ -""" Example that shows HPO with NePS based on a slurm script. -""" +"""Example that shows HPO with NePS based on a slurm script.""" import logging import os import time from pathlib import Path -from warnings import warn import neps @@ -28,11 +26,6 @@ def _get_validation_error(pipeline_directory: Path): return float(validation_error_file.read_text()) return None -def run_pipeline_via_slurm( - pipeline_directory: Path, optimizer: str, learning_rate: float -): - warn("run_pipeline_via_slurm is deprecated, use evaluate_pipeline_via_slurm instead", DeprecationWarning) - return evaluate_pipeline_via_slurm(pipeline_directory, optimizer, learning_rate) def evaluate_pipeline_via_slurm( pipeline_directory: Path, optimizer: str, learning_rate: float diff --git a/neps_examples/convenience/working_directory_per_pipeline.py b/neps_examples/convenience/working_directory_per_pipeline.py index ce36e9296..cedde75bc 100644 --- a/neps_examples/convenience/working_directory_per_pipeline.py +++ b/neps_examples/convenience/working_directory_per_pipeline.py @@ -7,12 +7,8 @@ import neps -def run_pipeline(pipeline_directory: Path, float1, categorical, integer1): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(pipeline_directory, float1, categorical, integer1) - def evaluate_pipeline(pipeline_directory: Path, float1, categorical, integer1): - # When adding pipeline_directory to run_pipeline, neps detects its presence and + # When adding pipeline_directory to evaluate_pipeline, neps detects its presence and # passes a directory unique for each pipeline configuration. You can then use this # pipeline_directory to create / save files pertaining to a specific pipeline, e.g.: weight_file = pipeline_directory / "weight_file.txt" diff --git a/neps_examples/efficiency/expert_priors_for_hyperparameters.py b/neps_examples/efficiency/expert_priors_for_hyperparameters.py index e85802ba0..a78dad043 100644 --- a/neps_examples/efficiency/expert_priors_for_hyperparameters.py +++ b/neps_examples/efficiency/expert_priors_for_hyperparameters.py @@ -4,9 +4,6 @@ import neps -def run_pipeline(some_float, some_integer, some_cat): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(some_float, some_integer, some_cat) def evaluate_pipeline(some_float, some_integer, some_cat): start = time.time() @@ -28,13 +25,22 @@ def evaluate_pipeline(some_float, some_integer, some_cat): # that speeds up the search pipeline_space = dict( some_float=neps.Float( - lower=1, upper=1000, log=True, prior=900, prior_confidence="medium" + lower=1, + upper=1000, + log=True, + prior=900, + prior_confidence="medium", ), some_integer=neps.Integer( - lower=0, upper=50, prior=35, prior_confidence="low" + lower=0, + upper=50, + prior=35, + prior_confidence="low", ), some_cat=neps.Categorical( - choices=["a", "b", "c"], prior="a", prior_confidence="high" + choices=["a", "b", "c"], + prior="a", + prior_confidence="high", ), ) diff --git a/neps_examples/efficiency/multi_fidelity.py b/neps_examples/efficiency/multi_fidelity.py index 8f0eedc66..326e2f286 100644 --- a/neps_examples/efficiency/multi_fidelity.py +++ b/neps_examples/efficiency/multi_fidelity.py @@ -2,6 +2,7 @@ from warnings import warn import numpy as np +from pathlib import Path import torch import torch.nn.functional as F from torch import nn, optim @@ -39,15 +40,17 @@ def get_model_and_optimizer(learning_rate): # Important: Include the "pipeline_directory" and "previous_pipeline_directory" arguments -# in your run_pipeline function. This grants access to NePS's folder system and is +# in your evaluate_pipeline function. This grants access to NePS's folder system and is # critical for leveraging efficient multi-fidelity optimization strategies. -def run_pipeline(pipeline_directory, previous_pipeline_directory, learning_rate, epoch): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(pipeline_directory, previous_pipeline_directory, learning_rate, epoch) - -def evaluate_pipeline(pipeline_directory, previous_pipeline_directory, learning_rate, epoch): +def evaluate_pipeline( + pipeline_directory: Path, # The path associated with this configuration + previous_pipeline_directory: Path + | None, # The path associated with any previous config + learning_rate: float, + epoch: int, +) -> dict: model, optimizer = get_model_and_optimizer(learning_rate) checkpoint_name = "checkpoint.pth" @@ -74,7 +77,9 @@ def evaluate_pipeline(pipeline_directory, previous_pipeline_directory, learning_ objective_to_minimize = np.log(learning_rate / epoch) # Replace with actual error epochs_spent_in_this_call = epoch - epochs_previously_spent # Optional for stopping - return dict(objective_to_minimize=objective_to_minimize, cost=epochs_spent_in_this_call) + return dict( + objective_to_minimize=objective_to_minimize, cost=epochs_spent_in_this_call + ) pipeline_space = dict( diff --git a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py index 6a7655b0d..f056e95dc 100644 --- a/neps_examples/efficiency/multi_fidelity_and_expert_priors.py +++ b/neps_examples/efficiency/multi_fidelity_and_expert_priors.py @@ -6,10 +6,6 @@ import neps -def run_pipeline(float1, float2, integer1, fidelity): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(float1, float2, integer1, fidelity) - def evaluate_pipeline(float1, float2, integer1, fidelity): objective_to_minimize = -float(np.sum([float1, float2, integer1])) / fidelity return objective_to_minimize @@ -17,15 +13,29 @@ def evaluate_pipeline(float1, float2, integer1, fidelity): pipeline_space = dict( float1=neps.Float( - lower=1, upper=1000, log=False, prior=600, prior_confidence="medium" + lower=1, + upper=1000, + log=False, + prior=600, + prior_confidence="medium", ), float2=neps.Float( - lower=-10, upper=10, prior=0, prior_confidence="medium" + lower=-10, + upper=10, + prior=0, + prior_confidence="medium", ), integer1=neps.Integer( - lower=0, upper=50, prior=35, prior_confidence="low" + lower=0, + upper=50, + prior=35, + prior_confidence="low", + ), + fidelity=neps.Integer( + lower=1, + upper=10, + is_fidelity=True, ), - fidelity=neps.Integer(lower=1, upper=10, is_fidelity=True), ) logging.basicConfig(level=logging.INFO) diff --git a/neps_examples/experimental/expert_priors_for_architecture_and_hyperparameters.py b/neps_examples/experimental/expert_priors_for_architecture_and_hyperparameters.py deleted file mode 100644 index 073f69925..000000000 --- a/neps_examples/experimental/expert_priors_for_architecture_and_hyperparameters.py +++ /dev/null @@ -1,139 +0,0 @@ -import logging -import time -from warnings import warn - -from torch import nn - -import neps -from neps.search_spaces.architecture import primitives as ops -from neps.search_spaces.architecture import topologies as topos - -primitives = { - "id": ops.Identity(), - "conv3x3": {"op": ops.ReLUConvBN, "kernel_size": 3, "stride": 1, "padding": 1}, - "conv1x1": {"op": ops.ReLUConvBN, "kernel_size": 1}, - "avg_pool": {"op": ops.AvgPool1x1, "kernel_size": 3, "stride": 1}, - "downsample": {"op": ops.ResNetBasicblock, "stride": 2}, - "residual": topos.Residual, - "diamond": topos.Diamond, - "linear": topos.get_sequential_n_edge(2), - "diamond_mid": topos.DiamondMid, -} - -structure = { - "S": [ - "diamond D2 D2 D1 D1", - "diamond D1 D2 D2 D1", - "diamond D1 D1 D2 D2", - "linear D2 D1", - "linear D1 D2", - "diamond_mid D1 D2 D1 D2 D1", - "diamond_mid D2 D2 Cell D1 D1", - ], - "D2": [ - "diamond D1 D1 D1 D1", - "linear D1 D1", - "diamond_mid D1 D1 Cell D1 D1", - ], - "D1": [ - "diamond D1Helper D1Helper Cell Cell", - "diamond Cell Cell D1Helper D1Helper", - "diamond D1Helper Cell Cell D1Helper", - "linear D1Helper Cell", - "linear Cell D1Helper", - "diamond_mid D1Helper D1Helper Cell Cell Cell", - "diamond_mid Cell D1Helper D1Helper D1Helper Cell", - ], - "D1Helper": ["linear Cell downsample"], - "Cell": [ - "residual OPS OPS OPS", - "diamond OPS OPS OPS OPS", - "linear OPS OPS", - "diamond_mid OPS OPS OPS OPS OPS", - ], - "OPS": ["conv3x3", "conv1x1", "avg_pool", "id"], -} - -prior_distr = { - "S": [1 / 7 for _ in range(7)], - "D2": [1 / 3 for _ in range(3)], - "D1": [1 / 7 for _ in range(7)], - "D1Helper": [1], - "Cell": [1 / 4 for _ in range(4)], - "OPS": [1 / 4 for _ in range(4)], -} - - -def set_recursive_attribute(op_name, predecessor_values): - in_channels = 64 if predecessor_values is None else predecessor_values["c_out"] - out_channels = in_channels * 2 if op_name == "ResNetBasicblock" else in_channels - return dict(c_in=in_channels, c_out=out_channels) - - -def run_pipeline(some_architecture, some_float, some_integer, some_cat): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning) - return evaluate_pipeline(some_architecture, some_float, some_integer, some_cat) - -def evaluate_pipeline(some_architecture, some_float, some_integer, some_cat): - start = time.time() - - in_channels = 3 - n_classes = 20 - base_channels = 64 - out_channels = 512 - - model = some_architecture.to_pytorch() - model = nn.Sequential( - ops.Stem(base_channels, c_in=in_channels), - model, - nn.AdaptiveAvgPool2d(1), - nn.Flatten(), - nn.Linear(out_channels, n_classes), - ) - - number_of_params = sum(p.numel() for p in model.parameters()) - y = abs(1.5e7 - number_of_params) - - if some_cat != "a": - y *= some_float + some_integer - else: - y *= -some_float - some_integer - - end = time.time() - - return { - "objective_to_minimize": y, - "info_dict": { - "test_score": y, - "train_time": end - start, - }, - } - - -pipeline_space = dict( - some_architecture=neps.Function( - set_recursive_attribute=set_recursive_attribute, - structure=structure, - primitives=primitives, - name="pibo", - prior=prior_distr, - ), - some_float=neps.Float( - lower=1, upper=1000, log=True, prior=900, prior_confidence="medium" - ), - some_integer=neps.Integer( - lower=0, upper=50, prior=35, prior_confidence="low" - ), - some_cat=neps.Categorical( - choices=["a", "b", "c"], prior="a", prior_confidence="high" - ), -) - -logging.basicConfig(level=logging.INFO) -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, - root_directory="results/user_priors_with_graphs", - max_evaluations_total=15, - use_priors=True, -) diff --git a/neps_examples/experimental/freeze_thaw.py b/neps_examples/experimental/freeze_thaw.py index 9c63d109e..93007523a 100644 --- a/neps_examples/experimental/freeze_thaw.py +++ b/neps_examples/experimental/freeze_thaw.py @@ -35,7 +35,7 @@ def training_pipeline( num_neurons, epochs, learning_rate, - weight_decay + weight_decay, ): """ Trains and validates a simple neural network on the MNIST dataset. @@ -75,16 +75,18 @@ def training_pipeline( criterion = nn.CrossEntropyLoss() # Select optimizer - optimizer = optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=weight_decay) + optimizer = optim.AdamW( + model.parameters(), lr=learning_rate, weight_decay=weight_decay + ) # Loading potential checkpoint start_epoch = 1 if previous_pipeline_directory is not None: - if (Path(previous_pipeline_directory) / "checkpoint.pt").exists(): - states = torch.load(Path(previous_pipeline_directory) / "checkpoint.pt") - model = states["model"] - optimizer = states["optimizer"] - start_epoch = states["epochs"] + if (Path(previous_pipeline_directory) / "checkpoint.pt").exists(): + states = torch.load(Path(previous_pipeline_directory) / "checkpoint.pt") + model = states["model"] + optimizer = states["optimizer"] + start_epoch = states["epochs"] # Training loop for epoch in range(start_epoch, epochs + 1): @@ -118,9 +120,9 @@ def training_pipeline( # Saving checkpoint states = { - "model": model, - "optimizer": optimizer, - "epochs": epochs, + "model": model, + "optimizer": optimizer, + "epochs": epochs, } torch.save(states, Path(pipeline_directory) / "checkpoint.pt") @@ -136,7 +138,9 @@ def training_pipeline( writer_config_hparam=True, # Appending extra data extra_data={ - "train_objective_to_minimize": tblogger.scalar_logging(objective_to_minimize.item()), + "train_objective_to_minimize": tblogger.scalar_logging( + objective_to_minimize.item() + ), "val_err": tblogger.scalar_logging(val_err), }, ) @@ -158,17 +162,10 @@ def training_pipeline( neps.run( pipeline_space=pipeline_space, evaluate_pipeline=training_pipeline, - searcher="ifbo", + optimizer="ifbo", max_evaluations_total=50, root_directory="./debug/ifbo-mnist/", overwrite_working_directory=False, # set to False for a multi-worker run - # (optional) ifbo hyperparameters - step_size=1, - # (optional) ifbo surrogate model hyperparameters (for FT-PFN) - surrogate_model_args=dict( - version="0.0.1", - target_path=None, - ), ) # NOTE: this is `experimental` and may not work as expected diff --git a/neps_examples/experimental/hierarchical_architecture.py b/neps_examples/experimental/hierarchical_architecture.py deleted file mode 100644 index 440b116aa..000000000 --- a/neps_examples/experimental/hierarchical_architecture.py +++ /dev/null @@ -1,103 +0,0 @@ -raise NotImplementedError( - "Support for graphs was temporarily removed, if you'd like to use a version" - " of NePS that supports graphs, please use version v0.12.2" -) - -import logging - -from torch import nn - -import neps -from neps.search_spaces.architecture import primitives as ops -from neps.search_spaces.architecture import topologies as topos - -primitives = { - "id": ops.Identity(), - "conv3x3": {"op": ops.ReLUConvBN, "kernel_size": 3, "stride": 1, "padding": 1}, - "conv1x1": {"op": ops.ReLUConvBN, "kernel_size": 1}, - "avg_pool": {"op": ops.AvgPool1x1, "kernel_size": 3, "stride": 1}, - "downsample": {"op": ops.ResNetBasicblock, "stride": 2}, - "residual": topos.Residual, - "diamond": topos.Diamond, - "linear": topos.get_sequential_n_edge(2), - "diamond_mid": topos.DiamondMid, -} - -structure = { - "S": [ - "diamond D2 D2 D1 D1", - "diamond D1 D2 D2 D1", - "diamond D1 D1 D2 D2", - "linear D2 D1", - "linear D1 D2", - "diamond_mid D1 D2 D1 D2 D1", - "diamond_mid D2 D2 Cell D1 D1", - ], - "D2": [ - "diamond D1 D1 D1 D1", - "linear D1 D1", - "diamond_mid D1 D1 Cell D1 D1", - ], - "D1": [ - "diamond D1Helper D1Helper Cell Cell", - "diamond Cell Cell D1Helper D1Helper", - "diamond D1Helper Cell Cell D1Helper", - "linear D1Helper Cell", - "linear Cell D1Helper", - "diamond_mid D1Helper D1Helper Cell Cell Cell", - "diamond_mid Cell D1Helper D1Helper D1Helper Cell", - ], - "D1Helper": ["linear Cell downsample"], - "Cell": [ - "residual OPS OPS OPS", - "diamond OPS OPS OPS OPS", - "linear OPS OPS", - "diamond_mid OPS OPS OPS OPS OPS", - ], - "OPS": ["conv3x3", "conv1x1", "avg_pool", "id"], -} - - -def set_recursive_attribute(op_name, predecessor_values): - in_channels = 64 if predecessor_values is None else predecessor_values["c_out"] - out_channels = in_channels * 2 if op_name == "ResNetBasicblock" else in_channels - return dict(c_in=in_channels, c_out=out_channels) - - -def run_pipeline(architecture: neps.Function): - in_channels = 3 - n_classes = 20 - base_channels = 64 - out_channels = 512 - - model = architecture.to_pytorch() - model = nn.Sequential( - ops.Stem(base_channels, c_in=in_channels), - model, - nn.AdaptiveAvgPool2d(1), - nn.Flatten(), - nn.Linear(out_channels, n_classes), - ) - - number_of_params = sum(p.numel() for p in model.parameters()) - validation_error = abs(1.5e7 - number_of_params) - - return validation_error - - -pipeline_space = dict( - architecture=neps.Function( - set_recursive_attribute=set_recursive_attribute, - structure=structure, - primitives=primitives, - name="makrograph", - ) -) - -logging.basicConfig(level=logging.INFO) -neps.run( - run_pipeline=run_pipeline, - pipeline_space=pipeline_space, - root_directory="results/hierarchical_architecture_example", - max_evaluations_total=15, -) diff --git a/neps/optimizers/random_search/__init__.py b/neps_examples/template/__init__.py similarity index 100% rename from neps/optimizers/random_search/__init__.py rename to neps_examples/template/__init__.py diff --git a/neps_examples/template/basic.py b/neps_examples/template/basic.py new file mode 100644 index 000000000..3ebaa98cb --- /dev/null +++ b/neps_examples/template/basic.py @@ -0,0 +1,78 @@ +""" +NOTE!!! This code is not meant to be executed. +It is only to serve as a template to help interface NePS with an existing ML/DL pipeline. + +The following script is designed as a template for using NePS. +It describes the crucial components that a user needs to provide in order to interface +a NePS optimizer. + +The 2 crucial components are: + +* The search space, called the `pipeline_space` in NePS + * This defines the set of hyperparameters that the optimizer will search over + * This declaration also allows injecting priors in the form of defaults per hyperparameter + +* The `evaluate_pipeline` function + * This function is called by the optimizer and is responsible for running the pipeline + * The function should at the minimum expect the hyperparameters as keyword arguments + * The function should return the loss of the pipeline as a float + * If the return value is a dictionary, it should have a key called "objective_value_to_minimize" with the loss as a float + + +Overall, running an optimizer from NePS involves 4 clear steps: +1. Importing neccessary packages including neps. +2. Designing the search space as a dictionary. +3. Creating the evaluate_pipeline and returning the loss and other wanted metrics. +4. Using neps run with the optimizer of choice. +""" + +import logging + +import neps + + +logger = logging.getLogger("neps_template.run") + + +def pipeline_space() -> dict: + # Create the search space based on NEPS parameters and return the dictionary. + # Example: + space = dict( + lr=neps.Float( + lower=1e-5, + upper=1e-2, + log=True, # If True, the search space is sampled in log space + prior=1e-3, # a non-None value here acts as the mode of the prior distribution + ), + ) + return space + + +def evaluate_pipeline(lr: float) -> dict | float: + # Evaluate pipeline should include the following steps: + + # 1. Defining the model. + # 1.1 Load any checkpoint if necessary + # 2. Each optimization variable should get its values from the pipeline space, i.e. "lr". + # 3. The training loop + # 3.1 Save any checkpoint if necessary + # 4. Returning the loss, which can be either as a single float or as part of + # an info dictionary containing other metrics. + + # Can use global logger to log any information + logger.info(f"Running pipeline with learning_rate {lr}") + + return dict or float + + +if __name__ == "__main__": + # 1. Creating the logger + + # 2. Passing the correct arguments to the neps.run function + + neps.run( + evaluate_pipeline=evaluate_pipeline, # User TODO (defined above) + pipeline_space=pipeline_space(), # User TODO (defined above) + root_directory="results", + max_evaluations_total=10, + ) diff --git a/neps_examples/template/ifbo.py b/neps_examples/template/ifbo.py new file mode 100644 index 000000000..8676dec23 --- /dev/null +++ b/neps_examples/template/ifbo.py @@ -0,0 +1,125 @@ +import numpy as np +from pathlib import Path + +ASSUMED_MAX_LOSS = 10 + + +def pipeline_space() -> dict: + # Create the search space based on NEPS parameters and return the dictionary. + # IMPORTANT: + space = dict( + lr=neps.Float( + lower=1e-5, + upper=1e-2, + log=True, # If True, the search space is sampled in log space + prior=1e-3, # a non-None value here acts as the mode of the prior distribution + ), + wd=neps.Float( + lower=0, + upper=1e-1, + log=True, + prior=1e-3, + ), + epoch=neps.Integer( + lower=1, + upper=10, + is_fidelity=True, # IMPORTANT to set this to True for the fidelity parameter + ), + ) + return space + + +def evaluate_pipeline( + pipeline_directory: Path, # The directory where the config is saved + previous_pipeline_directory: Path + | None, # The directory of the config's immediate lower fidelity + **config, # The hyperparameters to be used in the pipeline +) -> dict | float: + # Defining the model + # Can define outside the function or import from a file, package, etc. + class my_model(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear1 = nn.Linear(in_features=224, out_features=512) + self.linear2 = nn.Linear(in_features=512, out_features=10) + + def forward(self, x): + x = F.relu(self.linear1(x)) + x = self.linear2(x) + return x + + # Instantiates the model + model = my_model() + + # IMPORTANT: Extracting hyperparameters from passed config + learning_rate = config["lr"] + weight_decay = config["wd"] + + # Initializing the optimizer + optimizer = torch.optim.Adam( + model.parameters(), lr=learning_rate, weight_decay=weight_decay + ) + + ## Checkpointing + # loading the checkpoint if it exists + previous_state = load_checkpoint( # predefined function from neps + directory=previous_pipeline_directory, + model=model, # relies on pass-by-reference + optimizer=optimizer, # relies on pass-by-reference + ) + # adjusting run budget based on checkpoint + if previous_state is not None: + epoch_already_trained = previous_state["epochs"] + # + Anything else saved in the checkpoint. + else: + epoch_already_trained = 0 + # + Anything else with default value. + + # Extracting target epochs from config + max_epochs = config["epoch"] + + # User TODO: + # Load relevant data for training and validation + + # Actual model training + for epoch in range(epoch_already_trained, max_epochs): + # Training loop + ... + # Validation loop + ... + logger.info(f"Epoch: {epoch}, Loss: {...}, Val. acc.: {...}") + loss = ... + + # Save the checkpoint data in the current directory + save_checkpoint( + directory=pipeline_directory, + values_to_save={"epochs": max_epochs}, + model=model, + optimizer=optimizer, + ) + + # NOTE: Normalize the loss to be between 0 and 1 + ## crucial for ifBO's FT-PFN surrogate to work as expected + loss = np.clip(loss, 0, ASSUMED_MAX_LOSS) / ASSUMED_MAX_LOSS + + # Return a dictionary with the results, or a single float value (loss) + return { + "objective_value_to_minimize": loss, + "info_dict": { + "train_accuracy": ..., + "test_accuracy": ..., + }, + } + + +if __name__ == "__main__": + import neps + + neps.run( + evaluate_pipeline=evaluate_pipeline, + pipeline_space=pipeline_space(), + root_directory="results", + max_evaluations_total=50, + optimizer="ifbo", + ) +# end of evaluate_pipeline diff --git a/neps_examples/template/priorband.py b/neps_examples/template/priorband.py new file mode 100644 index 000000000..aa30354ed --- /dev/null +++ b/neps_examples/template/priorband.py @@ -0,0 +1,154 @@ +"""Boilerplate code to optimize a simple PyTorch model using PriorBand. + +NOTE!!! This code is not meant to be executed. +It is only to serve as a template to help interface NePS with an existing ML/DL pipeline. + + +The following script is designed as a template for using `PriorBand` from NePS. +It describes the crucial components that a user needs to provide in order to interface PriorBand. + +The 2 crucial components are: +* The search space, called the `pipeline_space` in NePS + * This defines the set of hyperparameters that the optimizer will search over + * This declaration also allows injecting priors in the form of defaults per hyperparameter +* The `evaluate_pipeline` function + * This function is called by the optimizer and is responsible for running the pipeline + * The function should at the minimum expect the hyperparameters as keyword arguments + * The function should return the loss of the pipeline as a float + * If the return value is a dictionary, it should have a key called "objective_value_to_minimize" with the loss as a float + + +Overall, running an optimizer from NePS involves 4 clear steps: +1. Importing neccessary packages including neps. +2. Designing the search space as a dictionary. +3. Creating the evaluate_pipeline and returning the loss and other wanted metrics. +4. Using neps run with the optimizer of choice. +""" + +import logging + +import torch +import torch.nn as nn +import torch.nn.functional as F +from pathlib import Path + +import neps +from neps.utils.common import load_checkpoint, save_checkpoint + +logger = logging.getLogger("neps_template.run") + + +def pipeline_space() -> dict: + # Create the search space based on NEPS parameters and return the dictionary. + # IMPORTANT: + space = dict( + lr=neps.Float( + lower=1e-5, + upper=1e-2, + log=True, # If True, the search space is sampled in log space + prior=1e-3, # a non-None value here acts as the mode of the prior distribution + ), + wd=neps.Float( + lower=0, + upper=1e-1, + log=True, + prior=1e-3, + ), + epoch=neps.Integer( + lower=1, + upper=10, + is_fidelity=True, # IMPORTANT to set this to True for the fidelity parameter + ), + ) + return space + + +def evaluate_pipeline( + pipeline_directory: Path, # The directory where the config is saved + previous_pipeline_directory: Path + | None, # The directory of the config's immediate lower fidelity + **config, # The hyperparameters to be used in the pipeline +) -> dict | float: + # Defining the model + # Can define outside the function or import from a file, package, etc. + class my_model(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear1 = nn.Linear(in_features=224, out_features=512) + self.linear2 = nn.Linear(in_features=512, out_features=10) + + def forward(self, x): + x = F.relu(self.linear1(x)) + x = self.linear2(x) + return x + + # Instantiates the model + model = my_model() + + # IMPORTANT: Extracting hyperparameters from passed config + learning_rate = config["lr"] + weight_decay = config["wd"] + + # Initializing the optimizer + optimizer = torch.optim.Adam( + model.parameters(), lr=learning_rate, weight_decay=weight_decay + ) + + ## Checkpointing + # loading the checkpoint if it exists + previous_state = load_checkpoint( # predefined function from neps + directory=previous_pipeline_directory, + model=model, # relies on pass-by-reference + optimizer=optimizer, # relies on pass-by-reference + ) + # adjusting run budget based on checkpoint + if previous_state is not None: + epoch_already_trained = previous_state["epochs"] + # + Anything else saved in the checkpoint. + else: + epoch_already_trained = 0 + # + Anything else with default value. + + # Extracting target epochs from config + max_epochs = config["epoch"] + + # User TODO: + # Load relevant data for training and validation + + # Actual model training + for epoch in range(epoch_already_trained, max_epochs): + # Training loop + ... + # Validation loop + ... + logger.info(f"Epoch: {epoch}, Loss: {...}, Val. acc.: {...}") + + # Save the checkpoint data in the current directory + save_checkpoint( + directory=pipeline_directory, + values_to_save={"epochs": max_epochs}, + model=model, + optimizer=optimizer, + ) + + # Return a dictionary with the results, or a single float value (loss) + return { + "objective_value_to_minimize": ..., + "info_dict": { + "train_accuracy": ..., + "test_accuracy": ..., + }, + } + + +# end of evaluate_pipeline + + +if __name__ == "__main__": + neps.run( + evaluate_pipeline=evaluate_pipeline, # User TODO (defined above) + pipeline_space=pipeline_space(), # User TODO (defined above) + root_directory="results", + max_evaluations_total=25, # total number of times `evaluate_pipeline` is called + optimizer="priorband", # "priorband_bo" for longer budgets, and set `initial_design_size`` + ) diff --git a/neps_examples/template/pytorch-lightning.py b/neps_examples/template/pytorch-lightning.py new file mode 100644 index 000000000..02573d3de --- /dev/null +++ b/neps_examples/template/pytorch-lightning.py @@ -0,0 +1,176 @@ +"""Boilerplate code to optimize a simple PyTorch Lightning model. + +NOTE!!! This code is not meant to be executed. +It is only to serve as a template to help interface NePS with an existing ML/DL pipeline. + + +The following script describes the crucial components that a user needs to provide +in order to interface with Lightning. + +The 3 crucial components are: +* The search space, called the `pipeline_space` in NePS + * This defines the set of hyperparameters that the optimizer will search over + * This declaration also allows injecting priors in the form of defaults per hyperparameter +* The `lightning module` + * This defines the training, validation, and testing of the model + * This distributes the hyperparameters + * This can be used to create the Dataloaders for training, validation, and testing +* The `evaluate_pipeline` function + * This function is called by the optimizer and is responsible for running the pipeline + * The function should at the minimum expect the hyperparameters as keyword arguments + * The function should return the loss of the pipeline as a float + * If the return value is a dictionary, it should have a key called "objective_value_to_minimize" with the loss as a float + +Overall, running an optimizer from NePS with Lightning involves 5 clear steps: +1. Importing neccessary packages including NePS and Lightning. +2. Designing the search space as a dictionary. +3. Creating the LightningModule with the required parameters +4. Creating the evaluate_pipeline and returning the loss and other wanted metrics. +5. Using neps run with the optimizer of choice. + +For a more detailed guide, please refer to: +https://github.com/automl/neps/blob/master/neps_examples/convenience/neps_x_lightning.py +""" + +import logging + +import lightning as L +import torch +from lightning.pytorch.callbacks import ModelCheckpoint +from lightning.pytorch.loggers import TensorBoardLogger + +import neps +from neps.utils.common import get_initial_directory, load_lightning_checkpoint + +logger = logging.getLogger("neps_template.run") + + +def pipeline_space() -> dict: + # Create the search space based on NEPS parameters and return the dictionary. + # IMPORTANT: + space = dict( + lr=neps.Float( + lower=1e-5, + upper=1e-2, + log=True, # If True, the search space is sampled in log space + prior=1e-3, # a non-None value here acts as the mode of the prior distribution + ), + optimizer=neps.Categorical(choices=["Adam", "SGD"], prior="Adam"), + epochs=neps.Integer( + lower=1, + upper=9, + is_fidelity=True, # IMPORTANT to set this to True for the fidelity parameter + ), + ) + return space + + +class LitModel(L.LightningModule): + def __init__(self, configuration: dict): + super().__init__() + + self.save_hyperparameters(configuration) + + # You can now define your criterion, data transforms, model layers, and + # metrics obtained during training + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Forward pass function + pass + + def training_step(self, batch: torch.Tensor, batch_idx: int) -> torch.Tensor: + # Training step function + # Training metric of choice + pass + + def validation_step(self, batch: torch.Tensor, batch_idx: int) -> torch.Tensor: + # Validation step function + # Validation metric of choice + pass + + def test_step(self, batch: torch.Tensor, batch_idx: int) -> torch.Tensor: + # Test step function + # Test metric of choice + pass + + def configure_optimizers(self) -> torch.optim.Optimizer: + # Define the optimizer base on the configuration + if self.hparams.optimizer == "Adam": + optimizer = torch.optim.Adam(self.parameters(), lr=self.hparams.lr) + elif self.hparams.optimizer == "SGD": + optimizer = torch.optim.SGD(self.parameters(), lr=self.hparams.lr) + else: + raise ValueError(f"{self.hparams.optimizer} is not a valid optimizer") + return optimizer + + # Here one can now configure the dataloaders for the model + # Further details can be found here: + # https://lightning.ai/docs/pytorch/stable/data/datamodule.html + # https://github.com/automl/neps/blob/master/neps_examples/convenience/neps_x_lightning.py + + +def evaluate_pipeline( + pipeline_directory, # The directory where the config is saved + previous_pipeline_directory, # The directory of the config's immediate lower fidelity + **config, # The hyperparameters to be used in the pipeline +) -> dict | float: + # Start by getting the initial directory which will be used to store tensorboard + # event files and checkpoint files + init_dir = get_initial_directory(pipeline_directory) + checkpoint_dir = init_dir / "checkpoints" + tensorboard_dir = init_dir / "tensorboard" + + # Create the model + model = LitModel(config) + + # Create the TensorBoard logger and the checkpoint callback + logger = TensorBoardLogger( + save_dir=tensorboard_dir, name="data", version="logs", default_hp_metric=False + ) + checkpoint_callback = ModelCheckpoint(dirpath=checkpoint_dir) + + # Checking for any checkpoint files and checkpoint data, returns None if + # no checkpoint files exist. + checkpoint_path, checkpoint_data = load_lightning_checkpoint( + previous_pipeline_directory=previous_pipeline_directory, + checkpoint_dir=checkpoint_dir, + ) + + # Create a PyTorch Lightning Trainer + epochs = config["epochs"] + + trainer = L.Trainer( + logger=logger, + max_epochs=epochs, + callbacks=[checkpoint_callback], + ) + + # Train, test, and get their corresponding metrics + if checkpoint_path: + trainer.fit(model, ckpt_path=checkpoint_path) + else: + trainer.fit(model) + val_loss = trainer.logged_metrics.get("val_loss", None) + + trainer.test(model) + test_loss = trainer.logged_metrics.get("test_loss", None) + + # Return a dictionary with the results, or a single float value (loss) + return { + "objective_value_to_minimize": val_loss, + "info_dict": { + "test_loss": test_loss, + }, + } + + +# end of evaluate_pipeline + +if __name__ == "__main__": + neps.run( + evaluate_pipeline=evaluate_pipeline, # User TODO (defined above) + pipeline_space=pipeline_space(), # User TODO (defined above) + root_directory="results", + max_evaluations_total=25, # total number of times `evaluate_pipeline` is called + optimizer="priorband", # "priorband_bo" for longer budgets, and set `initial_design_size`` + ) diff --git a/pyproject.toml b/pyproject.toml index 978c5da47..855efe181 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,24 +102,13 @@ requires = [ build-backend = "setuptools.build_meta" -# TODO(eddiebergman): Include more of these as we go on in migration -# "tests", -# "neps_examples", [tool.ruff] target-version = "py310" output-format = "full" line-length = 90 src = ["neps"] -# TODO(eddiebergman): Include more of these as we go on in migration exclude = [ - "neps/optimizers/multi_fidelity_prior/utils.py", - "neps/search_spaces/architecture/**/*.py", - "neps/search_spaces/yaml_search_space_utils.py", - "neps/search_spaces/architecture", - "neps/utils/run_args_from_yaml.py", - "neps/api.py", - "tests", "neps_examples", ".bzr", ".direnv", @@ -226,6 +215,7 @@ ignore = [ "NPY002", # Replace legacy `np.random.choice` call with `np.random.Generator` "N803", # Arguments should start with a lower case letter. "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes + "E741", # `l` is an ambiguous variable name ] @@ -244,6 +234,8 @@ ignore = [ "TCH", "N803", "C901", # Too complex + "PT011", # Catch value error to broad + "ARG001", # unused param ] "__init__.py" = ["I002"] "neps_examples/*" = [ @@ -287,7 +279,6 @@ addopts = "--basetemp ./tests_tmpdir -m 'not ci_examples'" markers = [ "ci_examples", "core_examples", - "regression_all", "runtime", "neps_api", "summary_csv", @@ -318,23 +309,9 @@ disallow_incomplete_defs = true # ...all types no_implicit_optional = true check_untyped_defs = true -# TODO(eddiebergman): Improve coverage on these modules [[tool.mypy.overrides]] -module = [ - "neps.api", - "neps.search_spaces.architecture.*", - "neps.utils.run_args_from_yaml", - "neps.optimizers.multi_fidelity.successive_halving", - "neps.optimizers.multi_fidelity.sampling_policy", - "neps.optimizers.multi_fidelity.promotion_policy", - "neps.optimizers.bayesian_optimization.acquisition_functions.ei", - "neps.optimizers.bayesian_optimization.acquisition_functions.prior_weighted", - "neps.optimizers.bayesian_optimization.acquisition_functions.ucb", - "neps.optimizers.bayesian_optimization.acquisition_functions.base_acquisition", - "neps.optimizers.bayesian_optimization.acquisition_functions.weighted_acquisition", - "requests.*", -] -ignore_errors = true +module = ["requests"] +ignore_missing_imports = true [tool.bumpversion] current_version = "0.12.2" diff --git a/tests/joint_config_space.py b/tests/joint_config_space.py deleted file mode 100644 index 1e5b2b62e..000000000 --- a/tests/joint_config_space.py +++ /dev/null @@ -1,113 +0,0 @@ -from __future__ import annotations - -import ConfigSpace as CS -from jahs_bench.lib.core.constants import Activations - -joint_config_space = CS.ConfigurationSpace("jahs_bench_config_space") -# noinspection PyPep8 -joint_config_space.add_hyperparameters( - [ - # CS.OrdinalHyperparameter("N", sequence=[1, 3, 5], default_value=1, - # meta=dict(help="Number of cell repetitions")), - # CS.OrdinalHyperparameter("W", sequence=[4, 8, 16], default_value=4, - # meta=dict(help="The width of the first channel in the cell. Each of the " - # "subsequent cell's first channels is twice as wide as the " - # "previous cell's, thus, for a value 4 (default) of W, the first " - # "channel widths are [4, 8, 16].")), - CS.CategoricalHyperparameter( - "Op1", - choices=list(range(5)), - default_value=0, - meta={"help": "The operation on the first edge of the cell."}, - ), - CS.CategoricalHyperparameter( - "Op2", - choices=list(range(5)), - default_value=0, - meta={"help": "The operation on the second edge of the cell."}, - ), - CS.CategoricalHyperparameter( - "Op3", - choices=list(range(5)), - default_value=0, - meta={"help": "The operation on the third edge of the cell."}, - ), - CS.CategoricalHyperparameter( - "Op4", - choices=list(range(5)), - default_value=0, - meta={"help": "The operation on the fourth edge of the cell."}, - ), - CS.CategoricalHyperparameter( - "Op5", - choices=list(range(5)), - default_value=0, - meta={"help": "The operation on the fifth edge of the cell."}, - ), - CS.CategoricalHyperparameter( - "Op6", - choices=list(range(5)), - default_value=0, - meta={"help": "The operation on the sixth edge of the cell."}, - ), - # CS.OrdinalHyperparameter("Resolution", sequence=[0.25, 0.5, 1.], default_value=1., - # meta=dict(help="The sample resolution of the input images w.r.t. one side of the " - # "actual image size, assuming square images, i.e. for a dataset " - # "with 32x32 images, specifying a value of 0.5 corresponds to " - # "using downscaled images of size 16x16 as inputs.")), - CS.CategoricalHyperparameter( - "TrivialAugment", - choices=[True, False], - default_value=False, - meta={ - "help": "Controls whether or not TrivialAugment is used for pre-processing " - "data. If False (default), a set of manually chosen transforms is " - "applied during pre-processing. If True, these are skipped in favor of " - "applying random transforms selected by TrivialAugment." - }, - ), - CS.CategoricalHyperparameter( - "Activation", - choices=list(Activations.__members__.keys()), - default_value="ReLU", - meta={ - "help": "Which activation function is to be used for the network. " - "Default is ReLU." - }, - ), - ] -) - -# Add Optimizer related HyperParamters -optimizers = CS.CategoricalHyperparameter( - "Optimizer", - choices=["SGD"], - default_value="SGD", - meta={ - "help": "Which optimizer to use for training this model. " - "This is just a placeholder for now, to be used " - "properly in future versions." - }, -) -lr = CS.UniformFloatHyperparameter( - "LearningRate", - lower=1e-3, - upper=1e0, - default_value=1e-1, - log=True, - meta={ - "help": "The learning rate for the optimizer used during model training. In the " - "case of adaptive learning rate optimizers such as Adam, this is the " - "initial learning rate." - }, -) -weight_decay = CS.UniformFloatHyperparameter( - "WeightDecay", - lower=1e-5, - upper=1e-2, - default_value=5e-4, - log=True, - meta={"help": "Weight decay to be used by the " "optimizer during model training."}, -) - -joint_config_space.add_hyperparameters([optimizers, lr, weight_decay]) diff --git a/tests/losses.json b/tests/losses.json deleted file mode 100644 index 9c1e161c6..000000000 --- a/tests/losses.json +++ /dev/null @@ -1 +0,0 @@ -{"random_search": {"cifar10": [10.570075988769531, 11.034774780273438, 10.087379455566406, 10.419334411621094, 10.432853698730469, 8.5814208984375, 10.457664489746094, 11.290725708007812, 10.5799560546875, 9.727294921875, 9.501747131347656, 10.170425415039062, 9.806816101074219, 10.66925048828125, 10.491233825683594, 9.634124755859375, 9.191375732421875, 8.844978332519531, 10.2298583984375, 10.434898376464844, 9.487800598144531, 10.427787780761719, 10.502761840820312, 10.279136657714844, 10.1964111328125, 11.07977294921875, 9.425407409667969, 9.701408386230469, 10.23583984375, 10.708969116210938, 9.558792114257812, 9.986930847167969, 10.56585693359375, 10.63482666015625, 11.0650634765625, 10.207893371582031, 9.922348022460938, 11.085418701171875, 11.045547485351562, 10.198143005371094, 10.31964111328125, 10.305580139160156, 10.344978332519531, 9.250556945800781, 10.117431640625, 9.896835327148438, 9.52008056640625, 9.566474914550781, 10.341537475585938, 9.719619750976562, 9.478080749511719, 9.690483093261719, 10.247886657714844, 10.712760925292969, 10.926887512207031, 10.650840759277344, 10.041717529296875, 11.061965942382812, 10.502098083496094, 10.246772766113281, 10.305030822753906, 9.34393310546875, 9.596229553222656, 10.688194274902344, 10.123970031738281, 10.976806640625, 9.254158020019531, 10.040168762207031, 9.82464599609375, 10.861183166503906, 10.738700866699219, 9.88568115234375, 10.434127807617188, 9.848808288574219, 10.386619567871094, 10.578804016113281, 9.660018920898438, 10.445465087890625, 10.763755798339844, 10.146713256835938, 10.194534301757812, 9.678909301757812, 10.699859619140625, 9.986000061035156, 9.880363464355469, 10.01849365234375, 10.228996276855469, 10.0753173828125, 10.508392333984375, 10.601631164550781, 11.081802368164062, 10.485763549804688, 9.727066040039062, 10.158500671386719, 9.9129638671875, 11.635810852050781, 10.416252136230469, 11.731185913085938, 10.537567138671875, 11.287857055664062], "fashion_mnist": [5.09051513671875, 5.3495025634765625, 5.090606689453125, 5.306755065917969, 4.944160461425781, 5.066337585449219, 5.340538024902344, 5.605537414550781, 5.409492492675781, 4.850059509277344, 5.271759033203125, 4.999168395996094, 5.353126525878906, 5.377349853515625, 5.094085693359375, 4.8185882568359375, 5.007720947265625, 5.248565673828125, 4.989860534667969, 5.30499267578125, 4.9860076904296875, 5.429695129394531, 5.023948669433594, 5.251014709472656, 5.251373291015625, 5.018852233886719, 5.361701965332031, 5.115943908691406, 5.258811950683594, 5.1905975341796875, 5.2493438720703125, 4.862884521484375, 5.296844482421875, 5.2973480224609375, 5.001739501953125, 5.132057189941406, 5.379150390625, 5.0308380126953125, 5.3088226318359375, 5.2285919189453125, 4.874839782714844, 4.875190734863281, 4.905540466308594, 5.091346740722656, 5.354927062988281, 5.405769348144531, 4.9038238525390625, 5.291114807128906, 5.022491455078125, 5.3507843017578125, 4.900177001953125, 5.125740051269531, 4.790794372558594, 4.908744812011719, 5.427764892578125, 4.928062438964844, 5.122749328613281, 5.211883544921875, 4.912879943847656, 5.304801940917969, 5.118843078613281, 5.316253662109375, 5.3155670166015625, 4.891822814941406, 5.075309753417969, 5.0142974853515625, 5.219169616699219, 4.976043701171875, 5.457160949707031, 5.560661315917969, 5.253791809082031, 5.599967956542969, 5.248786926269531, 4.922248840332031, 5.365226745605469, 4.915000915527344, 5.304725646972656, 5.5551300048828125, 5.2591552734375, 5.234474182128906, 5.337677001953125, 5.391693115234375, 4.871734619140625, 5.320770263671875, 4.729896545410156, 5.177436828613281, 5.51544189453125, 5.3778228759765625, 5.18963623046875, 5.084320068359375, 5.321952819824219, 5.127876281738281, 5.319007873535156, 5.161949157714844, 5.094940185546875, 5.2720794677734375, 5.098762512207031, 5.069129943847656, 5.495567321777344, 5.208320617675781], "hartmann3": [-3.7983644860179906, -3.4647375444431776, -3.724169378773501, -3.504482275855162, -3.5785228865447265, -3.2721948147194295, -3.424084036888103, -3.7072770292237176, -3.809994424648415, -3.6122056535539344, -3.5854465335983425, -3.844968774956946, -3.7347735478598283, -3.3691644158659364, -3.5412900367276676, -3.475734478264119, -3.8200439315391703, -3.83138442558726, -3.477400690334344, -3.4059398111190027, -3.7302972540667945, -3.1786964283460017, -3.5846499739643733, -3.256067420434629, -3.7187271090335243, -3.7826435340290523, -3.6342242353280025, -3.7605790715819465, -3.84909095948994, -3.5053069712058815, -3.7184922209021334, -3.800851244225936, -3.640804694679649, -3.3445513278128844, -3.8286547389644268, -3.573509742962088, -3.3190981974729334, -3.7609483078955943, -3.7744748548210194, -3.740621329285652, -3.6575432714187976, -3.686124319032891, -3.2467145074719324, -3.613123426231481, -3.8021768045842785, -3.621244873548109, -3.6192377014972266, -3.790856298972633, -3.548585974001048, -3.5592414866476254, -3.748377631924132, -3.7640467078382986, -3.7261908737840193, -3.581735983650286, -3.497987958071237, -3.720547971292352, -3.5412710119799535, -3.7391815825328405, -3.707270453492867, -3.666516126665965, -3.6723740963465534, -3.860704880219101, -3.3901487007606894, -3.6526042397434133, -3.821241682607301, -3.7313455393716906, -3.617482549039436, -3.8578496678333662, -3.6637739511791554, -3.7972669220230264, -3.5790734458422886, -3.7957396427176944, -3.4887409887061613, -3.778853898995849, -3.681495553302428, -3.692316299676422, -3.6735452557842283, -3.5961136884767786, -3.6657513195141593, -3.40861050776996, -3.8221486334983172, -3.767732293800785, -3.1725058921931346, -3.7536159697200975, -3.732562952763116, -3.3891581960764374, -3.4043726566370287, -3.569473096316691, -3.5687144990134247, -3.614801958410597, -3.799566283154234, -3.7199199442907918, -3.7185932588354116, -3.7187450018200634, -3.7366497076525995, -3.7475969099503734, -3.8002552452531257, -3.4060344407612617, -3.6688420010180787, -3.8308094701889], "hartmann6": [-2.942062167483819, -1.4270925701133137, -2.1670911981727583, -2.1391854030934256, -1.9703696121582686, -2.594122312444073, -2.765667751003329, -1.8110638451435552, -2.6196122976169867, -2.2412214116916607, -2.452084960741839, -1.8427121515062914, -2.608997350701581, -2.9014141339786677, -1.890320716157041, -2.545449071520854, -2.6583290500643884, -1.6054330860086385, -2.5408742738896675, -1.7074266404496088, -1.85969822199324, -2.65282030100074, -2.372449979122238, -1.4817042674371381, -2.533274868880178, -2.6426776279101176, -2.8844629338229093, -2.1509993233474045, -2.4576802851489497, -2.754241097009174, -2.8614515046033975, -3.1393263421404116, -2.5080712124365503, -1.9081559295607045, -2.144271742789371, -2.3086881255087097, -2.4972047804985724, -2.4153105932057017, -2.4135416612728533, -2.1755330721224277, -2.8801109890706234, -2.532838834617856, -3.0013330323245184, -2.845572945180195, -2.200193341199692, -1.7219543694068289, -2.6192546058487793, -2.2359822745696376, -2.1176613414017162, -1.8822616390525695, -2.4086827634039714, -2.2816243000743723, -2.165480067594273, -2.551451690430731, -2.638452146179405, -2.49884193760565, -2.357393575663534, -1.8583791294401601, -1.9636624434435468, -1.9951930274271241, -1.886440016674821, -2.432228785576797, -2.624508437250958, -2.7519888636176346, -2.9086333455217708, -2.0741067316762414, -2.446973157988633, -1.6462176720172292, -2.774427246629535, -2.201021955667192, -2.8070888873925735, -2.9702639208221466, -2.5146096807555276, -2.4388073485628983, -2.711317839707429, -2.4530651355252613, -2.734370008195378, -2.5801823248997273, -2.617856907872104, -2.0293432923318777, -1.93754359750606, -2.404251688411716, -2.0994030071591565, -1.9350609878168923, -2.263173223412282, -2.1345613646378223, -2.4387632069846914, -2.0504854619835515, -1.4384752897760327, -2.1621792846176895, -1.9256183341026185, -1.6618419098621113, -2.567175842548426, -1.8326320955843476, -2.0687530228373707, -1.8359872962404584, -2.444666881848311, -2.1568004943308154, -2.8749755249504685, -1.7131984047333892]}, "mf_bayesian_optimization": {"cifar10": [8.102218627929688, 9.120758056640625, 9.197235107421875, 9.314567565917969, 10.190299987792969, 9.6614990234375, 8.895378112792969, 9.573394775390625, 9.210296630859375, 8.729545593261719, 8.708778381347656, 10.21875, 10.008346557617188, 8.446937561035156, 9.194610595703125, 9.100334167480469, 10.321441650390625, 8.973121643066406, 9.705284118652344, 9.438888549804688, 9.307945251464844, 8.909683227539062, 9.4857177734375, 9.239250183105469, 8.85089111328125, 10.386962890625, 9.804962158203125, 10.164108276367188, 8.109199523925781, 8.016578674316406, 8.767509460449219, 8.346847534179688, 8.876449584960938, 9.234138488769531, 9.700332641601562, 8.373954772949219, 9.015975952148438, 8.562347412109375, 9.981231689453125, 8.512466430664062, 9.041938781738281, 9.176300048828125, 8.693832397460938, 9.337608337402344, 9.373138427734375, 8.25970458984375, 11.478584289550781, 9.039764404296875, 8.903793334960938, 7.851806640625, 9.292510986328125, 10.145538330078125, 8.894271850585938, 9.318794250488281, 9.97802734375, 10.720413208007812, 9.400299072265625, 9.318153381347656, 10.211288452148438, 8.429229736328125, 8.929550170898438, 9.603477478027344, 9.132301330566406, 8.73175048828125, 9.69183349609375, 9.170875549316406, 10.596893310546875, 8.982208251953125, 8.338356018066406, 10.031143188476562, 9.769561767578125, 9.627120971679688, 8.767288208007812, 9.834907531738281, 8.803001403808594, 8.625770568847656, 9.299758911132812, 9.298690795898438, 9.5645751953125, 9.03472900390625, 9.215667724609375, 9.008460998535156, 9.05419921875, 9.662025451660156, 8.660446166992188, 9.04095458984375, 10.133522033691406, 9.288856506347656, 10.933906555175781, 7.94622802734375, 8.634315490722656, 8.298187255859375, 9.50299072265625, 9.299980163574219, 9.237464904785156, 8.387069702148438, 9.435394287109375, 9.357254028320312, 9.201988220214844, 8.980323791503906], "fashion_mnist": [4.912925720214844, 4.808631896972656, 5.056343078613281, 5.1057891845703125, 4.89794921875, 4.8985443115234375, 4.901756286621094, 4.851066589355469, 5.0069122314453125, 4.854034423828125, 4.8776397705078125, 5.075859069824219, 4.946296691894531, 5.324104309082031, 5.109672546386719, 5.163368225097656, 4.86181640625, 5.0963134765625, 4.693626403808594, 4.952384948730469, 5.1372528076171875, 4.749778747558594, 4.839080810546875, 4.9800872802734375, 4.949943542480469, 4.892311096191406, 4.92108154296875, 5.054450988769531, 4.80377197265625, 5.033592224121094, 4.9016876220703125, 4.927215576171875, 5.0565948486328125, 5.106781005859375, 4.75128173828125, 4.719673156738281, 4.8021240234375, 5.003562927246094, 4.785224914550781, 5.1726531982421875, 5.156364440917969, 5.115470886230469, 4.777351379394531, 4.98150634765625, 4.730522155761719, 5.6309814453125, 5.045310974121094, 4.917938232421875, 5.200675964355469, 4.863861083984375, 4.795806884765625, 5.223625183105469, 5.415069580078125, 5.372398376464844, 4.945793151855469, 5.2683258056640625, 5.101051330566406, 5.241966247558594, 4.903839111328125, 4.976020812988281, 4.9986724853515625, 5.24957275390625, 5.062644958496094, 5.031547546386719, 5.14990234375, 4.78924560546875, 5.056243896484375, 4.898658752441406, 5.126441955566406, 4.981636047363281, 5.2509307861328125, 4.970909118652344, 5.175140380859375, 5.048194885253906, 5.134803771972656, 5.205291748046875, 4.92401123046875, 5.131980895996094, 5.33258056640625, 5.019996643066406, 5.0273895263671875, 5.010467529296875, 5.144783020019531, 5.032691955566406, 4.8980255126953125, 5.2896270751953125, 5.2210845947265625, 5.11920166015625, 5.291419982910156, 5.0905609130859375, 5.156303405761719, 4.8618011474609375, 4.871925354003906, 4.887908935546875, 4.993476867675781, 5.145660400390625, 4.9629974365234375, 5.022300720214844, 4.946128845214844, 5.3415069580078125]}, "bayesian_optimization": {"cifar10": [11.834663391113281, 10.850135803222656, 13.106803894042969, 10.621971130371094, 12.089996337890625, 11.989456176757812, 9.944725036621094, 12.151741027832031, 9.771446228027344, 9.653892517089844, 14.155509948730469, 9.849746704101562, 11.660682678222656, 9.726371765136719, 8.373817443847656, 8.547760009765625, 8.521583557128906, 8.890823364257812, 9.660530090332031, 10.417236328125, 10.407752990722656, 10.489616394042969, 8.149627685546875, 9.338088989257812, 9.880485534667969, 9.053581237792969, 10.286293029785156, 9.050323486328125, 12.33587646484375, 9.067794799804688, 10.680534362792969, 10.400833129882812, 7.941017150878906, 8.674705505371094, 8.025360107421875, 8.910812377929688, 9.311843872070312, 9.874847412109375, 7.759437561035156, 11.285186767578125, 8.689544677734375, 10.093734741210938, 9.649566650390625, 12.401290893554688, 9.348533630371094, 8.556083679199219, 9.835929870605469, 11.356254577636719, 11.3424072265625, 9.509292602539062, 9.295768737792969, 11.967460632324219, 10.924751281738281, 13.746360778808594, 10.807502746582031, 12.209007263183594, 11.160293579101562, 8.994827270507812, 10.222831726074219, 10.316673278808594, 8.706809997558594, 8.860237121582031, 8.495071411132812, 10.071884155273438, 11.452072143554688, 11.503829956054688, 10.215606689453125, 8.048088073730469, 8.259941101074219, 11.198013305664062, 10.408485412597656, 11.677360534667969, 10.564231872558594, 8.570747375488281, 8.69677734375, 8.675621032714844, 11.110565185546875, 9.985069274902344, 12.029739379882812, 8.316459655761719, 9.269187927246094, 15.092262268066406, 9.952056884765625, 9.06463623046875, 9.109573364257812, 9.456016540527344, 8.761322021484375, 12.986724853515625, 8.250045776367188, 8.607147216796875, 8.646743774414062, 8.980308532714844, 13.667716979980469, 11.633316040039062, 10.320816040039062, 12.442756652832031, 8.036582946777344, 8.479621887207031, 9.891937255859375, 11.2366943359375], "fashion_mnist": [4.9603424072265625, 5.071281433105469, 4.962150573730469, 4.827796936035156, 5.347526550292969, 4.6628875732421875, 4.999702453613281, 4.886085510253906, 5.3385467529296875, 4.770286560058594, 4.6191864013671875, 4.685035705566406, 4.7356719970703125, 4.6202850341796875, 5.357818603515625, 5.702720642089844, 5.026557922363281, 4.743721008300781, 4.7940521240234375, 4.773796081542969, 5.129631042480469, 4.84552001953125, 5.792655944824219, 5.2164764404296875, 4.673927307128906, 4.660346984863281, 4.624336242675781, 5.1123504638671875, 5.088066101074219, 5.051849365234375, 4.8744964599609375, 5.350379943847656, 4.981048583984375, 4.9394378662109375, 4.8551177978515625, 4.895240783691406, 5.573570251464844, 5.437744140625, 5.227935791015625, 4.942718505859375, 4.915061950683594, 4.697944641113281, 5.247138977050781, 4.7621612548828125, 4.958221435546875, 5.188468933105469, 5.064666748046875, 4.766532897949219, 5.066398620605469, 4.884849548339844, 4.673484802246094, 4.997337341308594, 4.749031066894531, 4.78302001953125, 4.9005889892578125, 4.967002868652344, 4.698448181152344, 4.86181640625, 5.0426483154296875, 4.780479431152344, 4.6356658935546875, 4.8165435791015625, 5.2922515869140625, 4.7222900390625, 5.3287200927734375, 4.797332763671875, 4.7588958740234375, 5.6631011962890625, 5.182411193847656, 5.007575988769531, 4.9674072265625, 4.693756103515625, 4.718101501464844, 5.490745544433594, 4.863746643066406, 5.2880859375, 5.3683624267578125, 4.564300537109375, 4.8800506591796875, 5.093727111816406, 5.62841796875, 4.820304870605469, 4.636566162109375, 5.5658416748046875, 5.124908447265625, 5.017478942871094, 5.150848388671875, 4.7201080322265625, 5.049140930175781, 4.790504455566406, 5.594146728515625, 5.1763458251953125, 5.105926513671875, 4.721961975097656, 5.21624755859375, 4.770225524902344, 5.327949523925781, 4.7087249755859375, 4.6690673828125, 5.120002746582031], "hartmann3": [-3.8178953072486967, -3.704775176517175, -3.8616219953873934, -3.5335807178711205, -3.7453646895666224, -3.724349458315775, -3.8082700218796206, -3.8112266622014936, -3.6381864416516088, -3.757543399794309, -3.717965871204767, -3.857327714892407, -3.755620258329924, -3.7764124673312653, -3.8166786066128644, -3.3266149308162176, -3.6318030080195567, -3.6783783725352532, -3.828153557778992, -3.7445079370049683, -3.7748854966315606, -3.768296234547888, -3.5998019566930024, -3.396255586934601, -3.8565237529701757, -3.6704283751493643, -3.8143244378302397, -1.9604315298529318, -3.613903912451665, -3.7866736749598773, -3.772166915524116, -3.7557386131680284, -3.75795099598279, -3.795670231190154, -3.8618487074128836, -3.8318902221548905, -3.6560392853114103, -3.7088067903799655, -3.6229461526417706, -3.7796465581026544, -3.789455489311304, -3.364116534127566, -3.7297619176629064, -3.4017365238028496, -3.6427163466230876, -3.3534894320962003, -3.6923051184426523, -3.7716634314504374, -3.7845428160866206, -3.753420518879679, -3.784187054931985, -3.5838034135058146, -3.8502436470074795, -2.597000906824191, -3.453444993373902, -3.785393240641991, -3.529974457228139, -3.740708317652978, -3.734122308582061, -3.67567657536361, -3.488047110341496, -3.735042140290241, -3.596701110723103, -3.730395457081181, -3.7041648761040484, -3.5465548528547504, -3.796429126486114, -3.6896101771297314, -3.66816253719168, -3.809565241525358, -3.6070273168821254, -3.743868657747066, -3.637445738600241, -3.726638981152361, -3.329452580660945, -3.769717580846608, -3.590972402696094, -3.7411588951193324, -3.78491914351238, -3.7754830039322345, -3.646186867377315, -3.6787080372572842, -3.5105613248742977, -3.801374009126365, -3.71401997938509, -3.7338092770206477, -3.7977390131650894, -3.530603761892602, -3.647723502655943, -3.7781449708359878, -3.6322055033004457, -3.1523083833378727, -3.412512781728038, -3.8303062762884994, -3.7226096486393296, -3.553696760208801, -3.6001168164084145, -3.581568826546383, -3.794874288307295, -3.581545967706646], "hartmann6": [-1.4549353278454038, -3.017174282345673, -2.9924333806676526, -2.721419510024227, -3.2973752131191634, -1.8341551991045113, -3.2668843561655243, -2.8347365300591334, -3.282790730047836, -3.1865756593491024, -2.982051709701689, -2.7485144603759237, -3.24173025958149, -3.134187749287507, -2.986062914159029, -2.932255875192143, -2.54217381789621, -3.1846937051481965, -2.902783188254062, -3.155614111164215, -2.899345934137389, -2.983779033637031, -3.2899323827033413, -3.1264045197377137, -3.142967961445229, -2.9324723031348694, -3.190017392761569, -3.292192486034387, -2.762574857954942, -3.2702396728108596, -2.6880860918265923, -2.989670689749928, -3.253750888535704, -2.8542984600236787, -3.252832804604744, -2.5442162213989556, -2.6052102152992065, -3.067795857145709, -2.8822447448882658, -3.140518761414347, -3.075813039891277, -3.03146184308219, -3.2979506275466615, -2.7197729302344933, -2.641564083074762, -2.702144998419366, -3.1670226043156444, -3.2932413787494665, -3.2923375595533435, -2.8727455141505818, -3.148190457843324, -2.820802211825644, -3.063369160676188, -3.0529188332750943, -3.0680238909470483, -3.2818403924328803, -3.2660725750584554, -3.131088071126102, -3.2890769050177533, -1.5829467114483784, -2.810513145953112, -3.312869701840981, -3.2124587900344044, -1.165456499117023, -1.6041283006949516, -3.0070669378242534, -2.7050223399135094, -2.708042778787764, -3.184524278644351, -3.1294539190549733, -2.9347834341589607, -2.7914377384651723, -3.2807067958203495, -1.6031238415481386, -2.9332137593080883, -2.897540573469963, -2.898552345504198, -2.63074616349976, -2.712932420759872, -2.4988020509933175, -3.259261535112579, -3.286437497756355, -3.092165340365627, -3.196218552395296, -2.916500796349388, -2.957113417613621, -3.303814006958171, -3.078648692296201, -1.4767638288161589, -3.1955443506729577, -3.17131783842418, -2.720688718208119, -3.254299537458789, -2.9759483354354503, -2.9408882599451656, -2.8628376552784673, -2.819730502978512, -2.92169177708724, -3.0304590769661273, -3.0737396309589915]}} diff --git a/tests/regression_objectives.py b/tests/regression_objectives.py deleted file mode 100644 index 97adb6874..000000000 --- a/tests/regression_objectives.py +++ /dev/null @@ -1,295 +0,0 @@ -from __future__ import annotations - -import warnings -from collections.abc import Callable -from pathlib import Path -from typing import Any, Literal - -import numpy as np - -import neps -from neps.search_spaces.search_space import SearchSpace, pipeline_space_from_configspace - - -class RegressionObjectiveBase: - """Base class for creating new synthetic or real objectives for the regression tests - Regression runner uses properties defined here, - each property should be appropriately defined by the subclasses. - """ - - def __init__(self, optimizer: str, task: str): - self.optimizer = optimizer - self.task = task - self.has_fidelity = self.optimizer != "random_search" - self._run_pipeline: Callable | None = None - self._pipeline_space: SearchSpace | dict[str, Any] = {} - - @property - def pipeline_space(self) -> SearchSpace | dict[str, Any]: - if not self._pipeline_space: - raise NotImplementedError( - f"pipeline_space can not be {self._pipeline_space}," - f" the subclass {type(self)} must implement " - f"a pipeline_space attribute" - ) - return self._pipeline_space - - @pipeline_space.setter - def pipeline_space(self, value): - self._pipeline_space = value - - @property - def run_pipeline(self) -> Callable: - warnings.warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - if self._run_pipeline is None: - raise NotImplementedError( - f"run_pipeline can not be None, " - f"the subclass {type(self)} must " - f"implement a run_pipeline Callable" - ) - return self._run_pipeline - - @property - def evaluate_pipeline(self) -> Callable: - if self._run_pipeline is None: - raise NotImplementedError( - f"evaluate_pipeline can not be None, " - f"the subclass {type(self)} must " - f"implement a evaluate_pipeline Callable" - ) - return self._run_pipeline - - @run_pipeline.setter - def run_pipeline(self, value): - warnings.warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - self._run_pipeline = value - - @evaluate_pipeline.setter - def evaluate_pipeline(self, value): - self._run_pipeline = value - - def __call__(self, *args, **kwargs) -> dict[str, Any]: - return self.run_pipeline(*args, **kwargs) - - -class JAHSObjective(RegressionObjectiveBase): - def evaluation_func(self): - """If the optimizer is cost aware, return the evaluation function with cost.""" - import jahs_bench - - self.benchmark = jahs_bench.Benchmark( - task=self.task, kind="surrogate", download=True, save_dir=self.save_dir - ) - - def cost_evaluation(**joint_configuration): - epoch = joint_configuration.pop("epoch") - joint_configuration.update({"N": 5, "W": 16, "Resolution": 1.0}) - - results = self.benchmark(joint_configuration, nepochs=epoch) - return { - "objective_to_minimize": 100 - results[epoch]["valid-acc"], - "cost": results[epoch]["runtime"], - } - - def objective_to_minimize_evaluation(**joint_configuration): - epoch = joint_configuration.pop("epoch") - joint_configuration.update({"N": 5, "W": 16, "Resolution": 1.0}) - - results = self.benchmark(joint_configuration, nepochs=epoch) - return 100 - results[epoch]["valid-acc"] - - if "cost" in self.optimizer: - return cost_evaluation - return objective_to_minimize_evaluation - - def __init__( - self, - optimizer: str = "mf_bayesian_optimization", - task: ( - Literal["cifar10", "colorectal_histology", "fashion_mnist"] | str - ) = "cifar10", - save_dir: str | Path = "jahs_bench_data", - **kwargs, - ): - """Download benchmark, initialize Pipeline space and evaluation function. - - Args: - optimizer: The optimizer that will be run, this is used to determine the - fidelity parameter of the pipeline space and whether to return the cost value - in the run_pipeline function - task: the dataset name for jahs_bench - save_dir: The (absolute or relative) path to a directory where the data - required for the benchmark to run will be read from. - """ - super().__init__(optimizer=optimizer, task=task) - from tests.joint_config_space import joint_config_space - - self.save_dir = Path(save_dir) - self.benchmark = None - - self.pipeline_space = pipeline_space_from_configspace(joint_config_space) - - self.pipeline_space["epoch"] = neps.Integer( - lower=1, upper=200, is_fidelity=self.has_fidelity - ) - self.run_pipeline = self.evaluation_func() - - self.surrogate_model = "gp" if self.optimizer != "random_search" else None - self.surrogate_model_args = kwargs.get("surrogate_model_args") - - -class HartmannObjective(RegressionObjectiveBase): - z_min = 3 - z_max = 100 - - def evaluation_fn(self) -> Callable: - def hartmann3(**z_nX): - z = z_nX.get("z") if self.has_fidelity else self.z_max - - X_0 = z_nX.get("X_0") - X_1 = z_nX.get("X_1") - X_2 = z_nX.get("X_2") - Xs = (X_0, X_1, X_2) - - log_z = np.log(z) - log_lb, log_ub = np.log(self.z_min), np.log(self.z_max) - log_z_scaled = (log_z - log_lb) / (log_ub - log_lb) - - # Highest fidelity (1) accounts for the regular Hartmann - X = np.array([X_0, X_1, X_2]).reshape(1, -1) - alpha = np.array([1.0, 1.2, 3.0, 3.2]) - - alpha_prime = alpha - self.bias * np.power(1 - log_z_scaled, 1) - A = np.array([[3.0, 10, 30], [0.1, 10, 35], [3.0, 10, 30], [0.1, 10, 35]]) - P = np.array( - [ - [3689, 1170, 2673], - [4699, 4387, 7470], - [1091, 8732, 5547], - [381, 5743, 8828], - ] - ) - - inner_sum = np.sum(A * (X[:, np.newaxis, :] - 0.0001 * P) ** 2, axis=-1) - H = -(np.sum(alpha_prime * np.exp(-inner_sum), axis=-1)) - - # and add some noise - with warnings.catch_warnings(): - warnings.simplefilter("ignore") # Seed below will overflow - rng = np.random.default_rng(seed=abs(self.seed * z * hash(Xs))) - - noise = np.abs(rng.normal(size=H.size)) * self.noise * (1 - log_z_scaled) - - objective_to_minimize = float((H + noise)[0]) - cost = 0.05 + (1 - 0.05) * (z / self.z_max) ** 2 - - result = {"objective_to_minimize": objective_to_minimize} - if "cost" in self.optimizer: - result.update({"cost": cost}) - - return result - - def hartmann6(**z_nX): - z = z_nX.get("z") if self.has_fidelity else self.z_max - - X_0 = z_nX.get("X_0") - X_1 = z_nX.get("X_1") - X_2 = z_nX.get("X_2") - X_3 = z_nX.get("X_3") - X_4 = z_nX.get("X_4") - X_5 = z_nX.get("X_5") - Xs = (X_0, X_1, X_2, X_3, X_4, X_5) - - # Change by Carl - z now comes in normalized - log_z = np.log(z) - log_lb, log_ub = np.log(self.z_min), np.log(self.z_max) - log_z_scaled = (log_z - log_lb) / (log_ub - log_lb) - - # Highest fidelity (1) accounts for the regular Hartmann - X = np.array([X_0, X_1, X_2, X_3, X_4, X_5]).reshape(1, -1) - alpha = np.array([1.0, 1.2, 3.0, 3.2]) - alpha_prime = alpha - self.bias * np.power(1 - log_z_scaled, 1) - A = np.array( - [ - [10, 3, 17, 3.5, 1.7, 8], - [0.05, 10, 17, 0.1, 8, 14], - [3, 3.5, 1.7, 10, 17, 8], - [17, 8, 0.05, 10, 0.1, 14], - ] - ) - P = np.array( - [ - [1312, 1696, 5569, 124, 8283, 5886], - [2329, 4135, 8307, 3736, 1004, 9991], - [2348, 1451, 3522, 2883, 3047, 6650], - [4047, 8828, 8732, 5743, 1091, 381], - ] - ) - - inner_sum = np.sum(A * (X[:, np.newaxis, :] - 0.0001 * P) ** 2, axis=-1) - H = -(np.sum(alpha_prime * np.exp(-inner_sum), axis=-1)) - - # and add some noise - with warnings.catch_warnings(): - warnings.simplefilter("ignore") # Seed below will overflow - rng = np.random.default_rng(seed=abs(self.seed * z * hash(Xs))) - - noise = np.abs(rng.normal(size=H.size)) * self.noise * (1 - log_z_scaled) - - objective_to_minimize = float((H + noise)[0]) - cost = 0.05 + (1 - 0.05) * (z / self.z_max) ** 2 - - result = {"objective_to_minimize": objective_to_minimize} - if "cost" in self.optimizer: - result.update({"cost": cost}) - - return result - - return hartmann3 if self.dim == 3 else hartmann6 - - - def __init__( - self, - optimizer: str, - task: Literal["hartmann3", "hartmann6"], - bias: float = 0.5, - noise: float = 0.1, - seed: int = 1337, - **kwargs, - ): - """Initialize Pipeline space and evaluation function. - - Args: - optimizer: The optimizer that will be run, this is used to determine the - fidelity parameter of the pipeline space and whether to return the cost value - in the run_pipeline function - task: the type of hartmann function used - """ - super().__init__(optimizer=optimizer, task=task) - if task == "hartmann3": - self.dim = 3 - elif self.task == "hartmann6": - self.dim = 6 - else: - raise ValueError( - "Hartmann objective is only defined for 'hartmann3' and 'hartmann6' " - ) - - self.pipeline_space: dict[str, Any] = { - f"X_{i}": neps.Float(lower=0.0, upper=1.0) for i in range(self.dim) - } - - if self.has_fidelity: - self.pipeline_space["z"] = neps.Integer( - lower=self.z_min, upper=self.z_max, is_fidelity=self.has_fidelity - ) - - self.bias = bias - self.noise = noise - self.seed = seed - self.random_state = np.random.default_rng(seed) - - self.surrogate_model = "gp" if self.optimizer != "random_search" else None - self.surrogate_model_args = kwargs.get("surrogate_model_args") - - self.run_pipeline = self.evaluation_fn() diff --git a/tests/regression_runner.py b/tests/regression_runner.py deleted file mode 100644 index 7cba1c65c..000000000 --- a/tests/regression_runner.py +++ /dev/null @@ -1,276 +0,0 @@ -# mypy: disable-error-code = union-attr -from __future__ import annotations - -import json -import logging -from collections.abc import Callable -from pathlib import Path - -import numpy as np -from scipy.stats import kstest - -import neps -from tests.regression_objectives import ( - HartmannObjective, - JAHSObjective, - RegressionObjectiveBase, -) -from tests.settings import ITERATIONS, LOSS_FILE, MAX_EVALUATIONS_TOTAL, OPTIMIZERS, TASKS - -TASK_OBJECTIVE_MAPPING = { - "cifar10": JAHSObjective, - "fashion_mnist": JAHSObjective, - "hartmann3": HartmannObjective, - "hartmann6": HartmannObjective, -} - -logging.basicConfig(level=logging.INFO) - - -def incumbent_at(root_directory: str | Path, step: int): - """Return the incumbent of the run at step n. - - Args: - root_directory: root directory of the optimization run - step: step n at which to return the incumbent - """ - log_file = Path(root_directory, "all_losses_and_configs.txt") - losses = [ - float(line[6:]) - for line in log_file.read_text(encoding="utf-8").splitlines() - if "Loss: " in line - ] - return min(losses[:step]) - - -class RegressionRunner: - """This class runs the optimization algorithms and stores the results in separate files.""" - - def __init__( - self, - objective: RegressionObjectiveBase | Callable, - iterations: int = 100, - max_evaluations: int = 150, - max_cost_total: int = 10000, - experiment_name: str = "", - **kwargs, - ): - """Download benchmark, initialize Pipeline space, evaluation function and set paths,. - - Args: - objective: callable that takes a configuration as input and evaluates it - iterations: number of times to record the whole optimization process - max_evaluations: maximum number of total evaluations for each optimization process - max_cost_total: budget for cost aware optimizers - experiment_name: string to identify different experiments - """ - self.objective = objective - if isinstance(objective, RegressionObjectiveBase): - self.task = self.objective.task - self.optimizer = self.objective.optimizer - self.pipeline_space = self.objective.pipeline_space - else: - self.task = kwargs.get("task") - if self.task is None: - raise AttributeError( - f"self.task can not be {self.task}, " - f"please provide a task argument" - ) - - self.optimizer = kwargs.get("optimizer") - if self.optimizer is None: - raise AttributeError( - f"self.optimizer can not be {self.optimizer}, " - f"please provide an optimizer argument" - ) - - self.pipeline_space = kwargs.get("pipeline_space") - if self.pipeline_space is None: - raise AttributeError( - f"self.pipeline_space can not be {self.pipeline_space}, " - f"please provide an pipeline_space argument" - ) - if experiment_name: - experiment_name += "_" - self.name = f"{self.optimizer}_{self.task}_{experiment_name}runs" - self.iterations = iterations - self.benchmark = None - - # Cost cooling optimizer expects budget but none of the others does - self.max_cost_total = max_cost_total if "cost" in self.optimizer else None - self.max_evaluations = max_evaluations - - self.final_losses: list[float] = [] - - # Number of samples for testing - self.sample_size = 10 - - @property - def root_directory(self): - return f"./{self.name}" - - @property - def final_losses_path(self): - return Path(self.root_directory, self.objective_to_minimize_file_name) - - @property - def objective_to_minimize_file_name(self): - return f"final_losses_{self.max_evaluations}_.txt" - - def save_losses(self): - if not self.final_losses_path.parent.exists(): - Path(self.root_directory).mkdir() - with self.final_losses_path.open(mode="w+", encoding="utf-8") as f: - f.writelines([str(objective_to_minimize) + "\n" for objective_to_minimize in self.final_losses]) - logging.info( - f"Saved the results of {len(self.final_losses)} " - f"runs of {self.max_evaluations} " - f"max evaluations into the file: {self.final_losses_path}" - ) - - def neps_run(self, working_directory: Path): - neps.run( - evaluate_pipeline=self.objective, - pipeline_space=self.pipeline_space, - searcher=self.optimizer, - max_cost_total=self.max_cost_total, - root_directory=working_directory, - max_evaluations_total=self.max_evaluations, - ) - - return incumbent_at(working_directory, self.max_evaluations) - - def run_regression(self, save=False): - """Run iterations number of neps runs.""" - for i in range(self.iterations): - working_directory = Path(self.root_directory, "results/test_run_" + str(i)) - - best_error = self.neps_run(working_directory) - - self.final_losses.append(float(best_error)) - - if save: - self.save_losses() - - return np.array(self.final_losses) - - def read_results(self): - """Read the results of the last run. - Either returns results of the most recent run, or - return the values from LOSS_FILE. - """ - if self.final_losses: - return np.array(self.final_losses) - if self.final_losses_path.exists(): - # Read from final_losses_path for each regression run - self.final_losses = [ - float(objective_to_minimize) - for objective_to_minimize in self.final_losses_path.read_text( - encoding="utf-8" - ).splitlines()[: self.iterations] - ] - else: - # Read from the results of previous runs if final_losses_path is not saved - try: - for i in range(self.iterations): - working_directory = Path( - self.root_directory, "results/test_run_" + str(i) - ) - best_error = incumbent_at(working_directory, self.max_evaluations) - self.final_losses.append(float(best_error)) - except FileNotFoundError as not_found: - # Try reading from the LOSS_FILE in the worst case - if LOSS_FILE.exists(): - with LOSS_FILE.open(mode="r", encoding="utf-8") as f: - objective_to_minimize_dict = json.load(f) - self.final_losses = objective_to_minimize_dict[self.optimizer][self.task] - else: - raise FileNotFoundError( - f"Results from the previous runs are not " - f"found, and {LOSS_FILE} does not exist" - ) from not_found - return np.array(self.final_losses) - - def test(self): - """Target run for the regression test, keep all the parameters same. - - Args: - max_evaluations: Number of evaluations after which to terminate optimization. - """ - # Sample losses of self.sample_size runs - samples = [] - for i in range(self.sample_size): - working_directory = Path(self.root_directory, f"results/test_run_target_{i}") - best_error = self.neps_run(working_directory) - samples.append(best_error) - - # Run tests - target = self.read_results() - - threshold = self.median_threshold(target) - - ks_result = kstest(samples, target) - median_dist = np.median(samples) - np.median(target) - ks_test = 0 if ks_result.pvalue < 0.1 else 1 - median_test = 0 if abs(median_dist) > threshold else 1 - median_improvement = 1 if median_dist < 0 else 0 - - return ks_test, median_test, median_improvement - - @staticmethod - def median_threshold( - target: np.ndarray, percentile: float | int = 92.5, sample_size: int = 10 - ): - stat_size = 1000 - p_index = int(stat_size * percentile / 100) - distances = np.zeros(stat_size) - for i, _ in enumerate(distances): - _sample = np.random.choice(target, size=sample_size, replace=False) - median_dist = np.median(_sample) - np.median(target) - distances[i] = median_dist - distances.sort() - return distances[p_index] - - -if __name__ == "__main__": - # Collect samples for each optimizer and store the data in the LOSS_FILE - json_file = Path("losses.json") - if json_file.exists(): - with json_file.open(mode="r", encoding="utf-8") as f: - losses_dict = json.load(f) - else: - losses_dict = {} - - for optimizer in OPTIMIZERS: - if optimizer in losses_dict: - pass - for task in TASKS: - if ( - isinstance(losses_dict.get(optimizer, None), dict) - and len(losses_dict[optimizer].get(task, [])) == ITERATIONS - ): - continue - else: - runner = RegressionRunner( - objective=TASK_OBJECTIVE_MAPPING[task]( - optimizer=optimizer, task=task - ), - max_evaluations=MAX_EVALUATIONS_TOTAL, - ) - runner.run_regression(save=True) - best_results = runner.read_results().tolist() - minv, maxv = min(best_results), max(best_results) - if isinstance(losses_dict.get(optimizer, None), dict) and isinstance( - losses_dict[optimizer].get(task, None), list - ): - losses_dict[optimizer][task] = best_results - elif isinstance(losses_dict.get(optimizer, None), dict): - update_dict = {task: best_results} - losses_dict[optimizer].update(update_dict) - else: - update_dict = {optimizer: {task: best_results}} - losses_dict.update(update_dict) - - # print(losses_dict) - with json_file.open(mode="w", encoding="utf-8") as f: - json.dump(losses_dict, f) diff --git a/tests/settings.py b/tests/settings.py deleted file mode 100644 index fc2be3fef..000000000 --- a/tests/settings.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -ITERATIONS = 100 -MAX_EVALUATIONS_TOTAL = 150 - -OPTIMIZERS = [ - "random_search", - # "mf_bayesian_optimization", - "bayesian_optimization", -] - -TASKS = [ - # "cifar10", "fashion_mnist", - "hartmann3", - "hartmann6", -] - -LOSS_FILE = Path(Path(__file__).parent, "losses.json") diff --git a/tests/test_config_encoder.py b/tests/test_config_encoder.py index c07500bbe..9a80b856d 100644 --- a/tests/test_config_encoder.py +++ b/tests/test_config_encoder.py @@ -1,93 +1,31 @@ from __future__ import annotations -import pytest import torch -from neps.search_spaces.domain import Domain -from neps.search_spaces.encoding import ( - CategoricalToIntegerTransformer, - ConfigEncoder, - MinMaxNormalizer, -) -from neps.search_spaces.hyperparameters import Categorical, Float, Integer - - -def test_config_encoder_default() -> None: - parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(5, 6), - "c": Float(5, 6), - } - - encoder = ConfigEncoder.from_parameters(parameters) - - # Numericals first, alphabetic - # Categoricals last, alphabetic - assert encoder.transformers == { - "b": MinMaxNormalizer(parameters["b"].domain), - "c": MinMaxNormalizer(parameters["c"].domain), - "a": CategoricalToIntegerTransformer(parameters["a"].choices), - } - - # Domains, (of each column) match those of the transformers - assert encoder.domains == [ - Domain.unit_float(), - Domain.unit_float(), - Domain.indices(n=len(parameters["a"].choices), is_categorical=True), - ] - - assert encoder.ncols == len(parameters) - assert encoder.n_numerical == 2 - assert encoder.n_categorical == 1 - assert encoder.numerical_slice == slice(0, 2) - assert encoder.categorical_slice == slice(2, 3) - assert encoder.index_of == {"a": 2, "b": 0, "c": 1} - assert encoder.domain_of == { - "b": Domain.unit_float(), - "c": Domain.unit_float(), - "a": Domain.indices(n=len(parameters["a"].choices), is_categorical=True), - } - assert encoder.constants == {} - - configs = [ - {"c": 5.5, "b": 5, "a": "cat"}, - {"c": 5.5, "b": 5, "a": "dog"}, - {"c": 6, "b": 6, "a": "mouse"}, - ] - encoded = encoder.encode(configs) - expcected_encoding = torch.tensor( - [ - # b, c, a - [0.0, 0.5, 0.0], # config 1 - [0.0, 0.5, 2.0], # config 2 - [1.0, 1.0, 1.0], # config 3 - ], - dtype=torch.float64, - ) - torch.testing.assert_close(encoded, expcected_encoding, check_dtype=True) - - decoded = encoder.decode(encoded) - assert decoded == configs +from neps.space import Categorical, ConfigEncoder, Float, Integer, SearchSpace +from neps.space.functions import pairwise_dist def test_config_encoder_pdist_calculation() -> None: - parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(1, 10), - "c": Float(1, 10), - } - encoder = ConfigEncoder.from_parameters(parameters) + parameters = SearchSpace( + { + "a": Categorical(["cat", "mouse", "dog"]), + "b": Integer(1, 10), + "c": Float(1, 10), + } + ) + encoder = ConfigEncoder.from_space(parameters) config1 = {"a": "cat", "b": 1, "c": 1.0} config2 = {"a": "mouse", "b": 10, "c": 10.0} # Same config, no distance x = encoder.encode([config1, config1]) - dist = encoder.pdist(x, square_form=False) + dist = pairwise_dist(x, encoder=encoder, square_form=False) assert dist.item() == 0.0 # Opposite configs, max distance x = encoder.encode([config1, config2]) - dist = encoder.pdist(x, square_form=False) + dist = pairwise_dist(x, encoder=encoder, square_form=False) # The first config should have it's p2 euclidean distance as the norm # of the distances between these two configs, i.e. the distance along the @@ -107,19 +45,21 @@ def test_config_encoder_pdist_calculation() -> None: def test_config_encoder_pdist_squareform() -> None: - parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(1, 10), - "c": Float(1, 10), - } - encoder = ConfigEncoder.from_parameters(parameters) + parameters = SearchSpace( + { + "a": Categorical(["cat", "mouse", "dog"]), + "b": Integer(1, 10), + "c": Float(1, 10), + } + ) + encoder = ConfigEncoder.from_space(parameters) config1 = {"a": "cat", "b": 1, "c": 1.0} config2 = {"a": "dog", "b": 5, "c": 5} config3 = {"a": "mouse", "b": 10, "c": 10.0} # Same config, no distance x = encoder.encode([config1, config2, config3]) - dist = encoder.pdist(x, square_form=False) + dist = pairwise_dist(x, encoder=encoder, square_form=False) # 3 possible distances assert dist.shape == (3,) @@ -130,7 +70,7 @@ def test_config_encoder_pdist_squareform() -> None: rtol=1e-4, ) - dist_sq = encoder.pdist(x, square_form=True) + dist_sq = pairwise_dist(x, encoder=encoder, square_form=True) assert dist_sq.shape == (3, 3) # Distance to self along diagonal should be 0 @@ -138,78 +78,3 @@ def test_config_encoder_pdist_squareform() -> None: # Should be symmetric torch.testing.assert_close(dist_sq, dist_sq.T) - - -def test_config_encoder_accepts_custom_transformers() -> None: - parameters = { - "b": Integer(5, 6), - "a": Float(5, 6), - "c": Categorical(["cat", "mouse", "dog"]), - } - encoder = ConfigEncoder.from_parameters( - parameters, - custom_transformers={ - "c": CategoricalToIntegerTransformer(parameters["c"].choices) - }, - ) - assert encoder.transformers["c"] == CategoricalToIntegerTransformer( - parameters["c"].choices - ) - - -def test_config_encoder_removes_constants_in_encoding_and_includes_in_decoding() -> None: - parameters = { - "b": Integer(5, 6), - "a": Float(5, 6), - "c": Categorical(["cat", "mouse", "dog"]), - } - - x = "raspberry" - - encoder = ConfigEncoder.from_parameters(parameters, constants={"x": x}) - assert encoder.constants == {"x": x} - - enc_x = encoder.encode([{"a": 5.5, "b": 5, "c": "cat", "x": x}]) - - assert enc_x.shape == (1, 3) # No x, just a, b, c - - dec_x = encoder.decode(enc_x) - assert dec_x == [{"a": 5.5, "b": 5, "c": "cat", "x": x}] - - # This doesn't have to hold true, but it's our current behaviour, we could make - # weaker gaurantees but then we'd have to clone the constants, even if it's very large - assert dec_x[0]["x"] is x - - -def test_config_encoder_complains_if_missing_entry_in_config() -> None: - parameters = { - "b": Integer(5, 6), - "a": Float(5, 6), - "c": Categorical(["cat", "mouse", "dog"]), - } - - encoder = ConfigEncoder.from_parameters(parameters) - - with pytest.raises(KeyError): - encoder.encode([{"a": 5.5, "b": 5}]) - - -def test_config_encoder_sorts_parameters_by_name_for_consistent_ordering() -> None: - parameters = { - "a": Categorical([0, 1]), - "b": Integer(0, 1), - "c": Float(0, 1), - } - p1 = dict(sorted(parameters.items())) - p2 = dict(sorted(parameters.items(), reverse=True)) - - encoder_1 = ConfigEncoder.from_parameters(p1) - encoder_2 = ConfigEncoder.from_parameters(p2) - - assert encoder_1.index_of["a"] == 2 - assert encoder_1.index_of["b"] == 0 - assert encoder_1.index_of["c"] == 1 - - assert encoder_2.index_of["a"] == 2 - assert encoder_2.index_of["b"] == 0 - assert encoder_2.index_of["c"] == 1 diff --git a/tests/test_domain.py b/tests/test_domain.py index ab3d3894d..762f07a7e 100644 --- a/tests/test_domain.py +++ b/tests/test_domain.py @@ -4,7 +4,7 @@ import torch from pytest_cases import parametrize -from neps.search_spaces.domain import Domain +from neps.space import Domain T = torch.tensor @@ -170,7 +170,7 @@ def test_domain_casting( "x, frm, to, expected", [ ( - # This test combines all the previous cast domains in one go as a single tensor + # Combines all the previous cast domains in one go as a single tensor T( [ [1e-2, 1e-1, 1e0, 1e1, 1e2], diff --git a/tests/test_neps_api/__init__.py b/tests/test_neps_api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_neps_api/solution_yamls/bo_custom_created.yaml b/tests/test_neps_api/solution_yamls/bo_custom_created.yaml deleted file mode 100644 index c808c6995..000000000 --- a/tests/test_neps_api/solution_yamls/bo_custom_created.yaml +++ /dev/null @@ -1,5 +0,0 @@ -searcher_name: baseoptimizer -searcher_alg: BayesianOptimization -searcher_selection: user-instantiation -neps_decision_tree: false -searcher_args: {} diff --git a/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml deleted file mode 100644 index f2ca84725..000000000 --- a/tests/test_neps_api/solution_yamls/bo_neps_decided.yaml +++ /dev/null @@ -1,10 +0,0 @@ -searcher_name: bayesian_optimization -searcher_alg: bayesian_optimization -searcher_selection: neps-default -neps_decision_tree: true -searcher_args: - initial_design_size: null - use_priors: false - use_cost: false - sample_prior_first: false - device: null diff --git a/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml b/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml deleted file mode 100644 index 602b965c9..000000000 --- a/tests/test_neps_api/solution_yamls/hyperband_custom_created.yaml +++ /dev/null @@ -1,5 +0,0 @@ -searcher_name: baseoptimizer -searcher_alg: Hyperband -searcher_selection: user-instantiation -neps_decision_tree: false -searcher_args: {} diff --git a/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml b/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml deleted file mode 100644 index dbd7723f5..000000000 --- a/tests/test_neps_api/solution_yamls/hyperband_neps_decided.yaml +++ /dev/null @@ -1,11 +0,0 @@ -searcher_name: hyperband -searcher_alg: hyperband -searcher_selection: neps-default -neps_decision_tree: true -searcher_args: - eta: 3 - initial_design_type: max_budget - use_priors: false - random_interleave_prob: 0.0 - sample_prior_first: false - sample_prior_at_target: false diff --git a/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml b/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml deleted file mode 100644 index d94ea209f..000000000 --- a/tests/test_neps_api/solution_yamls/pibo_neps_decided.yaml +++ /dev/null @@ -1,10 +0,0 @@ -searcher_name: pibo -searcher_alg: pibo -searcher_selection: neps-default -neps_decision_tree: true -searcher_args: - initial_design_size: null - use_priors: true - use_cost: false - sample_prior_first: true - device: null diff --git a/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml b/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml deleted file mode 100644 index c0a98cb37..000000000 --- a/tests/test_neps_api/solution_yamls/priorband_bo_user_decided.yaml +++ /dev/null @@ -1,22 +0,0 @@ -searcher_name: priorband_bo -searcher_alg: priorband -searcher_selection: neps-default -neps_decision_tree: false -searcher_args: - eta: 3 - initial_design_type: max_budget - prior_confidence: medium - random_interleave_prob: 0.0 - sample_prior_first: true - sample_prior_at_target: false - prior_weight_type: geometric - inc_sample_type: mutation - inc_mutation_rate: 0.5 - inc_mutation_std: 0.25 - inc_style: dynamic - model_based: true - modelling_type: joint - initial_design_size: 5 - surrogate_model: gp - acquisition: EI - log_prior_weighted: false diff --git a/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml b/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml deleted file mode 100644 index 6899bd00a..000000000 --- a/tests/test_neps_api/solution_yamls/priorband_neps_decided.yaml +++ /dev/null @@ -1,17 +0,0 @@ -searcher_name: priorband -searcher_alg: priorband -searcher_selection: neps-default -neps_decision_tree: true -searcher_args: - eta: 3 - initial_design_type: max_budget - prior_confidence: medium - random_interleave_prob: 0.0 - sample_prior_first: true - sample_prior_at_target: false - prior_weight_type: geometric - inc_sample_type: mutation - inc_mutation_rate: 0.5 - inc_mutation_std: 0.25 - inc_style: dynamic - model_based: false diff --git a/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml b/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml deleted file mode 100644 index c6cbe0eec..000000000 --- a/tests/test_neps_api/solution_yamls/user_yaml_bo.yaml +++ /dev/null @@ -1,8 +0,0 @@ -searcher_name: optimizer_test -searcher_alg: bayesian_optimization -searcher_selection: user-yaml -neps_decision_tree: false -searcher_args: - initial_design_size: 5 - use_priors: true - sample_prior_first: true diff --git a/tests/test_neps_api/test_api.py b/tests/test_neps_api/test_api.py deleted file mode 100644 index ae63b253d..000000000 --- a/tests/test_neps_api/test_api.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations - -import logging -import os -import runpy -from pathlib import Path - -import pytest -import yaml - - -# To change the working directly into the tmp_path when testing function -@pytest.fixture(autouse=True) -def use_tmpdir(tmp_path, request): - os.chdir(tmp_path) - yield - os.chdir(request.config.invocation_dir) - - -# https://stackoverflow.com/a/59745629 -# Fail tests if there is a logging.error -@pytest.fixture(autouse=True) -def no_logs_gte_error(caplog): - yield - errors = [ - record for record in caplog.get_records("call") if record.levelno >= logging.ERROR - ] - assert not errors - - -HERE = Path(__file__).resolve().parent - -testing_scripts = ["default_neps", "baseoptimizer_neps", "user_yaml_neps"] -EXAMPLES_FOLDER = HERE / "testing_scripts" -SOLUTION_FOLDER = HERE / "solution_yamls" -neps_api_example_script = [ - EXAMPLES_FOLDER / f"{example}.py" for example in testing_scripts -] - - -@pytest.mark.neps_api -@pytest.mark.parametrize("example_script", neps_api_example_script) -def test_default_examples(tmp_path: Path, example_script: Path) -> None: - # Running the example files holding multiple neps.run commands. - runpy.run_path(str(example_script), run_name="__main__") - - # Testing each folder with its corresponding expected dictionary - for folder in tmp_path.iterdir(): - info_yaml_path = folder / "optimizer_info.yaml" - - assert info_yaml_path.exists() - loaded_data = yaml.safe_load(info_yaml_path.read_text()) - - solution_yaml_path = SOLUTION_FOLDER / (folder.name + ".yaml") - solution_data = yaml.safe_load(solution_yaml_path.read_text()) - - assert ( - loaded_data == solution_data - ), f"Solution Path: {solution_yaml_path}\nLoaded Path: {info_yaml_path}\n" diff --git a/tests/test_neps_api/testing_scripts/baseoptimizer_neps.py b/tests/test_neps_api/testing_scripts/baseoptimizer_neps.py deleted file mode 100644 index 9a4a4591d..000000000 --- a/tests/test_neps_api/testing_scripts/baseoptimizer_neps.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import logging -from warnings import warn - -import neps -from neps.optimizers.bayesian_optimization.optimizer import BayesianOptimization -from neps.optimizers.multi_fidelity.hyperband import Hyperband -from neps.search_spaces.search_space import SearchSpace - -pipeline_space_fidelity = { - "val1": neps.Float(lower=-10, upper=10), - "val2": neps.Integer(lower=1, upper=5, is_fidelity=True), -} - -pipeline_space = { - "val1": neps.Float(lower=-10, upper=10), - "val2": neps.Integer(lower=1, upper=5), -} - - -def run_pipeline(val1, val2): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline(val1, val2) - -def evaluate_pipeline(val1, val2): - return val1 * val2 - - -def run_pipeline_fidelity(val1, val2): - warn("run_pipeline_fidelity is deprecated, use evaluate_pipeline_fidelity instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline_fidelity(val1, val2) - -def evaluate_pipeline_fidelity(val1, val2): - objective_to_minimize = val1 * val2 - return {"objective_to_minimize": objective_to_minimize, "cost": 1} - - -logging.basicConfig(level=logging.INFO) - -# Case 1: Testing BaseOptimizer as searcher with Bayesian Optimization -search_space = SearchSpace(**pipeline_space) -my_custom_searcher_1 = BayesianOptimization( - pipeline_space=search_space, initial_design_size=5 -) -neps.run( - evaluate_pipeline=evaluate_pipeline, - root_directory="bo_custom_created", - max_evaluations_total=1, - searcher=my_custom_searcher_1, -) - -# Case 2: Testing BaseOptimizer as searcher with Hyperband -search_space_fidelity = SearchSpace(**pipeline_space_fidelity) -my_custom_searcher_2 = Hyperband(pipeline_space=search_space_fidelity, max_cost_total=1) -neps.run( - evaluate_pipeline=evaluate_pipeline_fidelity, - root_directory="hyperband_custom_created", - max_cost_total=1, - searcher=my_custom_searcher_2, -) diff --git a/tests/test_neps_api/testing_scripts/default_neps.py b/tests/test_neps_api/testing_scripts/default_neps.py deleted file mode 100644 index 815adcc9e..000000000 --- a/tests/test_neps_api/testing_scripts/default_neps.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -import logging -from warnings import warn - -import neps - -pipeline_space_fidelity_priors = { - "val1": neps.Float(lower=-10, upper=10, prior=1), - "val2": neps.Integer(lower=1, upper=5, is_fidelity=True), -} - -pipeline_space_not_fidelity_priors = { - "val1": neps.Float(lower=-10, upper=10, prior=1), - "val2": neps.Integer(lower=1, upper=5, prior=1), -} - -pipeline_space_fidelity = { - "val1": neps.Float(lower=-10, upper=10), - "val2": neps.Integer(lower=1, upper=5, is_fidelity=True), -} - -pipeline_space_not_fidelity = { - "val1": neps.Float(lower=-10, upper=10), - "val2": neps.Integer(lower=1, upper=5), -} - - -def run_pipeline(val1, val2): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline(val1, val2) - -def evaluate_pipeline(val1, val2): - return val1 * val2 - - -logging.basicConfig(level=logging.INFO) - -# Testing user input "priorband_bo" with argument changes that should be -# accepted in the run. - -# Case 1: Choosing priorband - -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space_fidelity_priors, - root_directory="priorband_bo_user_decided", - max_evaluations_total=1, - searcher="priorband_bo", - initial_design_size=5, - eta=3, -) - -# Testing neps decision tree on deciding the searcher and rejecting the -# additional arguments. - -# Case 1: Choosing priorband -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space_fidelity_priors, - root_directory="priorband_neps_decided", - max_evaluations_total=1, - initial_design_size=5, - eta=3, -) - -# Case 2: Choosing bayesian_optimization -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space_not_fidelity, - root_directory="bo_neps_decided", - max_evaluations_total=1, -) - -# Case 3: Choosing pibo -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space_not_fidelity_priors, - root_directory="pibo_neps_decided", - max_evaluations_total=1, - initial_design_size=5, -) - -# Case 4: Choosing hyperband -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space_fidelity, - root_directory="hyperband_neps_decided", - max_evaluations_total=1, - eta=2, -) diff --git a/tests/test_neps_api/testing_scripts/user_yaml_neps.py b/tests/test_neps_api/testing_scripts/user_yaml_neps.py deleted file mode 100644 index 2320a90df..000000000 --- a/tests/test_neps_api/testing_scripts/user_yaml_neps.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -import logging -from pathlib import Path -from warnings import warn - -import neps - -pipeline_space = { - "val1": neps.Float(lower=-10, upper=10), - "val2": neps.Integer(lower=1, upper=5), -} - - -def run_pipeline(val1, val2): - warn( - "run_pipeline is deprecated, use evaluate_pipeline instead", - DeprecationWarning, - stacklevel=2, - ) - return evaluate_pipeline(val1, val2) - - -def evaluate_pipeline(val1, val2): - return val1 * val2 - - -logging.basicConfig(level=logging.INFO) - -# Testing using created yaml with api -script_directory = Path(__file__).resolve().parent -parent_directory = script_directory.parent -searcher_path = Path(parent_directory) / "testing_yaml" / "optimizer_test" -neps.run( - evaluate_pipeline=evaluate_pipeline, - pipeline_space=pipeline_space, - root_directory="user_yaml_bo", - max_evaluations_total=1, - searcher=searcher_path, - initial_design_size=5, -) diff --git a/tests/test_neps_api/testing_yaml/optimizer_test.yaml b/tests/test_neps_api/testing_yaml/optimizer_test.yaml deleted file mode 100644 index e6efcf0e7..000000000 --- a/tests/test_neps_api/testing_yaml/optimizer_test.yaml +++ /dev/null @@ -1,5 +0,0 @@ -strategy: bayesian_optimization -# Specific arguments depending on the searcher -initial_design_size: 7 -use_priors: true -sample_prior_first: true diff --git a/tests/test_regression.py b/tests/test_regression.py deleted file mode 100644 index 4f5ea96fe..000000000 --- a/tests/test_regression.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -import logging -import os - -import pytest - -from tests.settings import OPTIMIZERS - -pytest.skip( - "Pretty slow and will be reworked", - allow_module_level=True -) - -@pytest.fixture(autouse=True) -def use_tmpdir(tmp_path, request): - os.chdir(tmp_path) - yield - os.chdir(request.config.invocation_dir) - - -# https://stackoverflow.com/a/59745629 -# Fail tests if there is a logging.error -@pytest.fixture(autouse=True) -def no_logs_gte_error(caplog): - yield - errors = [ - record for record in caplog.get_records("call") if record.levelno >= logging.ERROR - ] - assert not errors - - -@pytest.mark.regression_all -@pytest.mark.parametrize("optimizer", OPTIMIZERS, ids=OPTIMIZERS) -def test_regression_all(optimizer): - from tests.regression_runner import TASK_OBJECTIVE_MAPPING, RegressionRunner - from tests.settings import TASKS - - test_results = {} - test_results["test_agg"] = 0 - test_results["task_agg"] = 0 - for task in TASKS: - ks_test, median_test, median_improvement = RegressionRunner( - objective=TASK_OBJECTIVE_MAPPING[task](optimizer=optimizer, task=task) - ).test() - - test_results[task] = [ks_test, median_test, median_improvement] - - test_results["task_agg"] += ( - 1 if (ks_test + median_test == 2) or median_improvement else 0 - ) - test_results["test_agg"] = ( - test_results["test_agg"] + ks_test + median_test + 2 * median_improvement - ) - - result = ( - 1 - if test_results["task_agg"] >= 1 and test_results["test_agg"] >= len(TASKS) + 1 - else 0 - ) - assert result == 1, f"Test for {optimizer} didn't pass: {test_results}" - logging.info(f"Regression test for {optimizer} passed successfully!") diff --git a/tests/test_runtime/test_default_report_values.py b/tests/test_runtime/test_default_report_values.py index 74c56f466..77b4358fb 100644 --- a/tests/test_runtime/test_default_report_values.py +++ b/tests/test_runtime/test_default_report_values.py @@ -4,15 +4,19 @@ from pytest_cases import fixture -from neps.optimizers.random_search.optimizer import RandomSearch +from neps.optimizers.algorithms import random_search from neps.runtime import DefaultWorker -from neps.search_spaces import Float -from neps.search_spaces.search_space import SearchSpace -from neps.state.neps_state import NePSState -from neps.state.optimizer import OptimizationState, OptimizerInfo -from neps.state.seed_snapshot import SeedSnapshot -from neps.state.settings import DefaultReportValues, OnErrorPossibilities, WorkerSettings -from neps.state.trial import Trial +from neps.space import Float, SearchSpace +from neps.state import ( + DefaultReportValues, + NePSState, + OnErrorPossibilities, + OptimizationState, + OptimizerInfo, + SeedSnapshot, + Trial, + WorkerSettings, +) @fixture @@ -29,11 +33,11 @@ def neps_state(tmp_path: Path) -> NePSState: def test_default_values_on_error( neps_state: NePSState, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( - objective_to_minimize_value_on_error=2.4, # <- Highlight + objective_value_on_error=2.4, # <- Highlight cost_value_on_error=2.4, # <- Highlight learning_curve_on_error=[2.4, 2.5], # <- Highlight ), @@ -56,7 +60,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -82,7 +85,7 @@ def eval_function(*args, **kwargs) -> float: def test_default_values_on_not_specified( neps_state: NePSState, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( @@ -108,7 +111,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -133,7 +135,7 @@ def eval_function(*args, **kwargs) -> float: def test_default_value_objective_to_minimize_curve_take_objective_to_minimize_value( neps_state: NePSState, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues( @@ -160,7 +162,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() diff --git a/tests/test_runtime/test_error_handling_strategies.py b/tests/test_runtime/test_error_handling_strategies.py index 7650cbc2a..7a60887a8 100644 --- a/tests/test_runtime/test_error_handling_strategies.py +++ b/tests/test_runtime/test_error_handling_strategies.py @@ -1,22 +1,26 @@ from __future__ import annotations +import contextlib from dataclasses import dataclass from pathlib import Path import pytest -from pandas.core.common import contextlib from pytest_cases import fixture, parametrize from neps.exceptions import WorkerRaiseError -from neps.optimizers.random_search.optimizer import RandomSearch +from neps.optimizers.algorithms import random_search from neps.runtime import DefaultWorker -from neps.search_spaces import Float -from neps.search_spaces.search_space import SearchSpace -from neps.state.neps_state import NePSState -from neps.state.optimizer import OptimizationState, OptimizerInfo -from neps.state.seed_snapshot import SeedSnapshot -from neps.state.settings import DefaultReportValues, OnErrorPossibilities, WorkerSettings -from neps.state.trial import Trial +from neps.space import Float, SearchSpace +from neps.state import ( + DefaultReportValues, + NePSState, + OnErrorPossibilities, + OptimizationState, + OptimizerInfo, + SeedSnapshot, + Trial, + WorkerSettings, +) @fixture @@ -40,7 +44,7 @@ def test_worker_raises_when_error_in_self( neps_state: NePSState, on_error: OnErrorPossibilities, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), @@ -63,7 +67,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) with pytest.raises(WorkerRaiseError): worker.run() @@ -81,7 +84,7 @@ def eval_function(*args, **kwargs) -> float: def test_worker_raises_when_error_in_other_worker(neps_state: NePSState) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.RAISE_ANY_ERROR, # <- Highlight default_report_values=DefaultReportValues(), @@ -104,14 +107,12 @@ def evaler(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=evaler, settings=settings, - _pre_sample_hooks=None, ) worker2 = DefaultWorker.new( state=neps_state, optimizer=optimizer, evaluation_fn=evaler, settings=settings, - _pre_sample_hooks=None, ) # Worker1 should run 1 and error out @@ -143,9 +144,9 @@ def test_worker_does_not_raise_when_error_in_other_worker( neps_state: NePSState, on_error: OnErrorPossibilities, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( - on_error=OnErrorPossibilities.RAISE_WORKER_ERROR, # <- Highlight + on_error=on_error, # <- Highlight default_report_values=DefaultReportValues(), max_evaluations_total=None, include_in_progress_evaluations_towards_maximum=False, @@ -162,7 +163,7 @@ def test_worker_does_not_raise_when_error_in_other_worker( class _Eval: do_raise: bool - def __call__(self, *args, **kwargs) -> float: + def __call__(self, *args, **kwargs) -> float: # noqa: ARG002 if self.do_raise: raise ValueError("This is an error") return 10 @@ -174,14 +175,12 @@ def __call__(self, *args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=evaler, settings=settings, - _pre_sample_hooks=None, ) worker2 = DefaultWorker.new( state=neps_state, optimizer=optimizer, evaluation_fn=evaler, settings=settings, - _pre_sample_hooks=None, ) # Worker1 should run 1 and error out diff --git a/tests/test_runtime/test_stopping_criterion.py b/tests/test_runtime/test_stopping_criterion.py index e380a9fc0..8a526b553 100644 --- a/tests/test_runtime/test_stopping_criterion.py +++ b/tests/test_runtime/test_stopping_criterion.py @@ -5,15 +5,19 @@ from pytest_cases import fixture -from neps.optimizers.random_search.optimizer import RandomSearch +from neps.optimizers.algorithms import random_search from neps.runtime import DefaultWorker -from neps.search_spaces import Float -from neps.search_spaces.search_space import SearchSpace -from neps.state.neps_state import NePSState -from neps.state.optimizer import OptimizationState, OptimizerInfo -from neps.state.seed_snapshot import SeedSnapshot -from neps.state.settings import DefaultReportValues, OnErrorPossibilities, WorkerSettings -from neps.state.trial import Trial +from neps.space import Float, SearchSpace +from neps.state import ( + DefaultReportValues, + NePSState, + OnErrorPossibilities, + OptimizationState, + OptimizerInfo, + SeedSnapshot, + Trial, + WorkerSettings, +) @fixture @@ -32,7 +36,7 @@ def neps_state(tmp_path: Path) -> NePSState: def test_max_evaluations_total_stopping_criterion( neps_state: NePSState, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -55,7 +59,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -75,7 +78,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) new_worker.run() assert new_worker.worker_cumulative_eval_count == 0 @@ -86,7 +88,7 @@ def eval_function(*args, **kwargs) -> float: def test_worker_evaluations_total_stopping_criterion( neps_state: NePSState, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -109,7 +111,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -130,7 +131,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) new_worker.run() @@ -149,12 +149,12 @@ def eval_function(*args, **kwargs) -> float: def test_include_in_progress_evaluations_towards_maximum_with_work_eval_count( neps_state: NePSState, ) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), max_evaluations_total=2, # <- Highlight, only 2 maximum evaluations allowed - include_in_progress_evaluations_towards_maximum=True, # <- include the inprogress trial + include_in_progress_evaluations_towards_maximum=True, # <- inprogress trial max_cost_total=None, max_evaluations_for_worker=None, max_evaluation_time_total_seconds=None, @@ -177,7 +177,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -203,10 +202,8 @@ def eval_function(*args, **kwargs) -> float: assert the_completed_trial.report.objective_to_minimize == 1.0 -def test_max_cost_total( - neps_state: NePSState, -) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) +def test_max_cost_total(neps_state: NePSState) -> None: + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -229,7 +226,6 @@ def eval_function(*args, **kwargs) -> dict: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -249,16 +245,13 @@ def eval_function(*args, **kwargs) -> dict: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) new_worker.run() assert new_worker.worker_cumulative_eval_count == 0 -def test_worker_cost_total( - neps_state: NePSState, -) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) +def test_worker_cost_total(neps_state: NePSState) -> None: + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -281,7 +274,6 @@ def eval_function(*args, **kwargs) -> dict: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) worker.run() @@ -301,7 +293,6 @@ def eval_function(*args, **kwargs) -> dict: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, ) new_worker.run() assert new_worker.worker_cumulative_eval_count == 2 @@ -315,14 +306,12 @@ def eval_function(*args, **kwargs) -> dict: assert len(trials) == 4 # 2 more trials were ran -def test_worker_wallclock_time( - neps_state: NePSState, -) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) +def test_worker_wallclock_time(neps_state: NePSState) -> None: + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), - max_evaluations_total=1000, # Safety incase it doesn't work that we eventually stop + max_evaluations_total=1000, # Incase it doesn't work that we eventually stop include_in_progress_evaluations_towards_maximum=False, max_cost_total=None, max_evaluations_for_worker=None, @@ -341,7 +330,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, worker_id="dummy", ) worker.run() @@ -360,7 +348,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, worker_id="dummy2", ) new_worker.run() @@ -374,10 +361,8 @@ def eval_function(*args, **kwargs) -> float: assert len_trials_on_second_worker > len_trials_on_first_worker -def test_max_worker_evaluation_time( - neps_state: NePSState, -) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) +def test_max_worker_evaluation_time(neps_state: NePSState) -> None: + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -401,7 +386,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, worker_id="dummy", ) worker.run() @@ -420,7 +404,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, worker_id="dummy2", ) new_worker.run() @@ -434,10 +417,8 @@ def eval_function(*args, **kwargs) -> float: assert len_trials_on_second_worker > len_trials_on_first_worker -def test_max_evaluation_time_global( - neps_state: NePSState, -) -> None: - optimizer = RandomSearch(pipeline_space=SearchSpace(a=Float(0, 1))) +def test_max_evaluation_time_global(neps_state: NePSState) -> None: + optimizer = random_search(pipeline_space=SearchSpace({"a": Float(0, 1)})) settings = WorkerSettings( on_error=OnErrorPossibilities.IGNORE, default_report_values=DefaultReportValues(), @@ -461,7 +442,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, worker_id="dummy", ) worker.run() @@ -480,7 +460,6 @@ def eval_function(*args, **kwargs) -> float: optimizer=optimizer, evaluation_fn=eval_function, settings=settings, - _pre_sample_hooks=None, worker_id="dummy2", ) new_worker.run() diff --git a/tests/test_samplers.py b/tests/test_samplers.py index 8581153e3..57450c4ae 100644 --- a/tests/test_samplers.py +++ b/tests/test_samplers.py @@ -3,9 +3,8 @@ import torch from pytest_cases import parametrize -from neps.sampling.priors import Prior, UniformPrior -from neps.sampling.samplers import BorderSampler, Sampler, Sobol, WeightedSampler -from neps.search_spaces.domain import Domain +from neps.sampling import BorderSampler, Prior, Sampler, Sobol, Uniform, WeightedSampler +from neps.space import Domain def _make_centered_prior(ndim: int) -> Prior: @@ -20,12 +19,12 @@ def _make_centered_prior(ndim: int) -> Prior: [ Sobol(ndim=3), BorderSampler(ndim=3), - UniformPrior(ndim=3), + Uniform(ndim=3), # Convenence method for making a distribution around center points _make_centered_prior(ndim=3), WeightedSampler( - [UniformPrior(ndim=3), _make_centered_prior(3), Sobol(ndim=3)], - weights=torch.tensor([0.5, 0.25, 0.25]), + [Uniform(ndim=3), _make_centered_prior(3), Sobol(ndim=3)], + weights=torch.tensor([0.5, 0.25, 0.25]).tolist(), ), ], ) @@ -57,7 +56,7 @@ def test_sampler_samples_into_domain(sampler: Sampler) -> None: @parametrize( "prior", [ - UniformPrior(ndim=3), + Uniform(ndim=3), # Convenence method for making a distribution around center points _make_centered_prior(ndim=3), ], diff --git a/tests/test_search_space_functions.py b/tests/test_search_space_functions.py deleted file mode 100644 index f7c09f9f8..000000000 --- a/tests/test_search_space_functions.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import torch - -from neps.search_spaces.encoding import ConfigEncoder -from neps.search_spaces.functions import pairwise_dist -from neps.search_spaces.hyperparameters import Categorical, Float, Integer - - -def test_config_encoder_pdist_calculation() -> None: - parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(1, 10), - "c": Float(1, 10), - } - encoder = ConfigEncoder.from_parameters(parameters) - config1 = {"a": "cat", "b": 1, "c": 1.0} - config2 = {"a": "mouse", "b": 10, "c": 10.0} - - # Same config, no distance - x = encoder.encode([config1, config1]) - dist = pairwise_dist(x, encoder=encoder, square_form=False) - assert dist.item() == 0.0 - - # Opposite configs, max distance - x = encoder.encode([config1, config2]) - dist = pairwise_dist(x, encoder=encoder, square_form=False) - - # The first config should have it's p2 euclidean distance as the norm - # of the distances between these two configs, i.e. the distance along the - # diagonal of a unit-square they belong to - _first_config_numerical_encoding = torch.tensor([[0.0, 0.0]], dtype=torch.float64) - _second_config_numerical_encoding = torch.tensor([[1.0, 1.0]], dtype=torch.float64) - _expected_numerical_dist = torch.linalg.norm( - _first_config_numerical_encoding - _second_config_numerical_encoding, - ord=2, - ) - - # The categorical distance should just be one, as they are different - _expected_categorical_dist = 1.0 - - _expected_dist = _expected_numerical_dist + _expected_categorical_dist - assert torch.isclose(dist, _expected_dist) - - -def test_config_encoder_pdist_squareform() -> None: - parameters = { - "a": Categorical(["cat", "mouse", "dog"]), - "b": Integer(1, 10), - "c": Float(1, 10), - } - encoder = ConfigEncoder.from_parameters(parameters) - config1 = {"a": "cat", "b": 1, "c": 1.0} - config2 = {"a": "dog", "b": 5, "c": 5} - config3 = {"a": "mouse", "b": 10, "c": 10.0} - - # Same config, no distance - x = encoder.encode([config1, config2, config3]) - dist = pairwise_dist(x, encoder=encoder, square_form=False) - - # 3 possible distances - assert dist.shape == (3,) - torch.testing.assert_close( - dist, - torch.tensor([1.6285, 2.4142, 1.7857], dtype=torch.float64), - atol=1e-4, - rtol=1e-4, - ) - - dist_sq = pairwise_dist(x, encoder=encoder, square_form=True) - assert dist_sq.shape == (3, 3) - - # Distance to self along diagonal should be 0 - torch.testing.assert_close(dist_sq.diagonal(), torch.zeros(3, dtype=torch.float64)) - - # Should be symmetric - torch.testing.assert_close(dist_sq, dist_sq.T) diff --git a/tests/test_search_space_parsing.py b/tests/test_search_space_parsing.py new file mode 100644 index 000000000..4fd2ea226 --- /dev/null +++ b/tests/test_search_space_parsing.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from neps.space import Categorical, Constant, Float, Integer, Parameter, parsing + + +@pytest.mark.parametrize( + ("config", "expected"), + [ + ( + (0, 1), + Integer(0, 1), + ), + ( + ("1e3", "1e5"), + Integer(1e3, 1e5), + ), + ( + ("1e-3", "1e-1"), + Float(1e-3, 1e-1), + ), + ( + (1e-5, 1e-1), + Float(1e-5, 1e-1), + ), + ( + {"type": "float", "lower": 0.00001, "upper": "1e-1", "log": True}, + Float(0.00001, 0.1, log=True), + ), + ( + {"type": "int", "lower": 3, "upper": 30, "is_fidelity": True}, + Integer(3, 30, is_fidelity=True), + ), + ( + { + "type": "int", + "lower": "1e2", + "upper": "3E4", + "log": True, + "is_fidelity": False, + }, + Integer(100, 30000, log=True, is_fidelity=False), + ), + ( + {"type": "float", "lower": "3.3e-5", "upper": "1.5E-1"}, + Float(3.3e-5, 1.5e-1), + ), + ( + {"type": "cat", "choices": [2, "sgd", "10e-3"]}, + Categorical([2, "sgd", 0.01]), + ), + ( + 0.5, + Constant(0.5), + ), + ( + "1e3", + Constant(1000), + ), + ( + {"type": "cat", "choices": ["adam", "sgd", "rmsprop"]}, + Categorical(["adam", "sgd", "rmsprop"]), + ), + ( + { + "lower": 0.00001, + "upper": 0.1, + "log": True, + "prior": 3.3e-2, + "prior_confidence": "high", + }, + Float(0.00001, 0.1, log=True, prior=3.3e-2, prior_confidence="high"), + ), + ], +) +def test_type_deduction_succeeds(config: Any, expected: Parameter) -> None: + parameter = parsing.as_parameter(config) + assert parameter == expected + + +@pytest.mark.parametrize( + "config", + [ + {"type": int, "lower": 0.00001, "upper": 0.1, "log": True}, # Invalid type + (1, 2.5), # int and float + (1, 2, 3), # too many values + (1,), # too few values + ], +) +def test_parsing_fails(config: dict[str, Any]) -> None: + with pytest.raises(ValueError): + parsing.as_parameter(config) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_settings/overwrite_run_args.yaml b/tests/test_settings/overwrite_run_args.yaml deleted file mode 100644 index e4dd91b4c..000000000 --- a/tests/test_settings/overwrite_run_args.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Full Configuration Template for NePS -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: True - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/full_config" -max_evaluations_total: 20 # Budget -max_cost_total: - -# Debug and Monitoring -overwrite_working_directory: True -post_run_summary: False -development_stage_id: 1 -task_id: 3 - -# Parallelization Setup -max_evaluations_per_run: 6 -continue_until_max_evaluation_completed: False - -# Error Handling -objective_to_minimize_value_on_error: 1.0 -cost_value_on_error: 1.0 -ignore_errors: True - -# Customization Options -searcher: hyperband # Internal key to select a NePS optimizer. - -# Hooks -pre_load_hooks: - hook1: "tests/test_settings/test_settings.py" diff --git a/tests/test_settings/run_args_optimizer_outside.yaml b/tests/test_settings/run_args_optimizer_outside.yaml deleted file mode 100644 index 1d87c32db..000000000 --- a/tests/test_settings/run_args_optimizer_outside.yaml +++ /dev/null @@ -1,16 +0,0 @@ -evaluate_pipeline: - name: evaluate_pipeline - path: "tests/test_settings/test_settings.py" -pipeline_space: - name: pipeline_space - path: "tests/test_settings/test_settings.py" - -root_directory: "path/to/root_directory" -max_evaluations_total: 10 # Budget -searcher: - path: "tests/test_settings/test_settings.py" - name: my_bayesian - # Specific arguments depending on the searcher - initial_design_size: 7 - -overwrite_working_directory: True diff --git a/tests/test_settings/run_args_optimizer_settings.yaml b/tests/test_settings/run_args_optimizer_settings.yaml deleted file mode 100644 index 4c049813f..000000000 --- a/tests/test_settings/run_args_optimizer_settings.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# Full Configuration Template for NePS -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: True - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/full_config" -max_evaluations_total: 20 # Budget -max_cost_total: - -# Debug and Monitoring -overwrite_working_directory: True -post_run_summary: False -development_stage_id: 1 -task_id: 3 - -# Parallelization Setup -max_evaluations_per_run: 6 -continue_until_max_evaluation_completed: False - -# Error Handling -objective_to_minimize_value_on_error: 1.0 -cost_value_on_error: 1.0 -ignore_errors: True - -# Customization Options -searcher: - strategy: "hyperband" # Internal key to select a NePS optimizer. - eta: 3 - initial_design_type: max_budget - use_priors: false - random_interleave_prob: 0.0 - sample_prior_first: false - sample_prior_at_target: false - -# Hooks -pre_load_hooks: - hook1: "tests/test_settings/test_settings.py" diff --git a/tests/test_settings/run_args_optional.yaml b/tests/test_settings/run_args_optional.yaml deleted file mode 100644 index db0ef575a..000000000 --- a/tests/test_settings/run_args_optional.yaml +++ /dev/null @@ -1,14 +0,0 @@ -max_cost_total: -overwrite_working_directory: True -post_run_summary: False -development_stage_id: -task_id: -max_evaluations_per_run: -continue_until_max_evaluation_completed: False -max_evaluations_total: 11 # get ignored -root_directory: "get/ignored" -objective_to_minimize_value_on_error: -cost_value_on_error: -ignore_errors: -searcher: hyperband -pre_load_hooks: diff --git a/tests/test_settings/run_args_required.yaml b/tests/test_settings/run_args_required.yaml deleted file mode 100644 index 47aa6a520..000000000 --- a/tests/test_settings/run_args_required.yaml +++ /dev/null @@ -1,8 +0,0 @@ -evaluate_pipeline: - name: evaluate_pipeline - path: "tests/test_settings/test_settings.py" -pipeline_space: - name: pipeline_space - path: "tests/test_settings/test_settings.py" -max_evaluations_total: 10 -root_directory: "path/to/root_directory" diff --git a/tests/test_settings/test_settings.py b/tests/test_settings/test_settings.py deleted file mode 100644 index ca76445cd..000000000 --- a/tests/test_settings/test_settings.py +++ /dev/null @@ -1,360 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from neps.optimizers.bayesian_optimization.optimizer import BayesianOptimization -from neps.utils.run_args import Default, Settings -from tests.test_yaml_run_args.test_yaml_run_args import ( - evaluate_pipeline, - hook1, - hook2, - pipeline_space, -) - -BASE_PATH = Path("tests") / "test_settings" -evaluate_pipeline = evaluate_pipeline -hook1 = hook1 -hook2 = hook2 -pipeline_space = pipeline_space -my_bayesian = BayesianOptimization - - -@pytest.mark.neps_api -@pytest.mark.parametrize( - ("func_args", "yaml_args", "expected_output"), - [ - ( - { # only essential arguments provided by func_args, no yaml - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "run_args": Default(None), - "overwrite_working_directory": Default(False), - "post_run_summary": Default(True), - "development_stage_id": Default(None), - "task_id": Default(None), - "max_evaluations_total": 10, - "max_evaluations_per_run": Default(None), - "continue_until_max_evaluation_completed": Default(False), - "max_cost_total": Default(None), - "ignore_errors": Default(False), - "objective_to_minimize_value_on_error": Default(None), - "cost_value_on_error": Default(None), - "pre_load_hooks": Default(None), - "searcher": Default("default"), - "searcher_kwargs": {}, - "sample_batch_size": Default(None), - }, - Default(None), - { - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "overwrite_working_directory": False, - "post_run_summary": True, - "development_stage_id": None, - "task_id": None, - "max_evaluations_total": 10, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": "default", - "searcher_kwargs": {}, - "sample_batch_size": None, - }, - ), - ( - { # only required elements of run_args - "evaluate_pipeline": Default(None), - "root_directory": Default(None), - "pipeline_space": Default(None), - "run_args": Default(None), - "overwrite_working_directory": Default(False), - "post_run_summary": Default(True), - "development_stage_id": Default(None), - "task_id": Default(None), - "max_evaluations_total": Default(None), - "max_evaluations_per_run": Default(None), - "continue_until_max_evaluation_completed": Default(False), - "max_cost_total": Default(None), - "ignore_errors": Default(False), - "objective_to_minimize_value_on_error": Default(None), - "cost_value_on_error": Default(None), - "pre_load_hooks": Default(None), - "searcher": Default("default"), - "searcher_kwargs": {}, - "sample_batch_size": Default(None), - }, - "run_args_required.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "overwrite_working_directory": False, - "post_run_summary": True, - "development_stage_id": None, - "task_id": None, - "max_evaluations_total": 10, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": "default", - "searcher_kwargs": {}, - "sample_batch_size": None, - }, - ), - ( - { # required via func_args, optional via yaml - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "run_args": "tests/path/to/run_args", # will be ignored by Settings - "overwrite_working_directory": Default(False), - "post_run_summary": Default(True), - "development_stage_id": Default(None), - "task_id": Default(None), - "max_evaluations_total": 10, - "max_evaluations_per_run": Default(None), - "continue_until_max_evaluation_completed": Default(False), - "max_cost_total": Default(None), - "ignore_errors": Default(False), - "objective_to_minimize_value_on_error": Default(None), - "cost_value_on_error": Default(None), - "pre_load_hooks": Default(None), - "searcher": Default("default"), - "searcher_kwargs": {}, - "sample_batch_size": Default(None), - }, - "run_args_optional.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "overwrite_working_directory": True, - "post_run_summary": False, - "development_stage_id": None, - "task_id": None, - "max_evaluations_total": 10, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": "hyperband", - "searcher_kwargs": {}, - "sample_batch_size": None, - }, - ), - ( - { # overwrite all yaml values - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "run_args": "test", - "overwrite_working_directory": False, - "post_run_summary": True, - "development_stage_id": 5, - "task_id": None, - "max_evaluations_total": 17, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": "default", - "searcher_kwargs": {}, - "sample_batch_size": Default(None), - }, - "overwrite_run_args.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "overwrite_working_directory": False, - "post_run_summary": True, - "development_stage_id": 5, - "task_id": None, - "max_evaluations_total": 17, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": "default", - "searcher_kwargs": {}, - "sample_batch_size": None, - }, - ), - ( - { # optimizer args special case - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "run_args": "test", - "overwrite_working_directory": False, - "post_run_summary": True, - "development_stage_id": 5, - "task_id": None, - "max_evaluations_total": 17, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": Default("default"), - "searcher_kwargs": { - "initial_design_type": "max_budget", - "use_priors": False, - "random_interleave_prob": 0.0, - "sample_prior_first": False, - "sample_prior_at_target": False, - }, - "sample_batch_size": Default(None), - }, - "run_args_optimizer_settings.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "overwrite_working_directory": False, - "post_run_summary": True, - "development_stage_id": 5, - "task_id": None, - "max_evaluations_total": 17, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": { - "strategy": "hyperband", - "eta": 3, - "initial_design_type": "max_budget", - "use_priors": False, - "random_interleave_prob": 0.0, - "sample_prior_first": False, - "sample_prior_at_target": False, - }, - "searcher_kwargs": { - "initial_design_type": "max_budget", - "use_priors": False, - "random_interleave_prob": 0.0, - "sample_prior_first": False, - "sample_prior_at_target": False, - }, - "sample_batch_size": None, - }, - ), - ( - { # load optimizer with args - "evaluate_pipeline": Default(None), - "root_directory": Default(None), - "pipeline_space": Default(None), - "run_args": Default(None), - "overwrite_working_directory": Default(False), - "post_run_summary": Default(True), - "development_stage_id": Default(None), - "task_id": Default(None), - "max_evaluations_total": Default(None), - "max_evaluations_per_run": Default(None), - "continue_until_max_evaluation_completed": Default(False), - "max_cost_total": Default(None), - "ignore_errors": Default(False), - "objective_to_minimize_value_on_error": Default(None), - "cost_value_on_error": Default(None), - "pre_load_hooks": Default(None), - "searcher": Default("default"), - "searcher_kwargs": { - "initial_design_size": 9, - }, - "sample_batch_size": Default(None), - }, - "run_args_optimizer_outside.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "root_directory": "path/to/root_directory", - "pipeline_space": pipeline_space, - "overwrite_working_directory": True, - "post_run_summary": True, - "development_stage_id": None, - "task_id": None, - "max_evaluations_total": 10, - "max_evaluations_per_run": None, - "continue_until_max_evaluation_completed": False, - "max_cost_total": None, - "ignore_errors": False, - "objective_to_minimize_value_on_error": None, - "cost_value_on_error": None, - "pre_load_hooks": None, - "searcher": my_bayesian, - "searcher_kwargs": {"initial_design_size": 9}, - "sample_batch_size": None, - }, - ), - ], -) -def test_check_settings(func_args: dict, yaml_args: str, expected_output: dict) -> None: - """Check if expected settings are set.""" - args = BASE_PATH / yaml_args if isinstance(yaml_args, str) else yaml_args - - settings = Settings(func_args, args) - for key, value in expected_output.items(): - assert getattr(settings, key) == value - - -@pytest.mark.neps_api -@pytest.mark.parametrize( - ("func_args", "yaml_args", "error"), - [ - ( - { - "root_directory": Default(None), - "pipeline_space": Default(None), - "run_args": Default(None), - "overwrite_working_directory": Default(False), - "post_run_summary": Default(True), - "development_stage_id": Default(None), - "task_id": Default(None), - "max_evaluations_total": Default(None), - "max_evaluations_per_run": Default(None), - "continue_until_max_evaluation_completed": Default(False), - "max_cost_total": Default(None), - "ignore_errors": Default(False), - "objective_to_minimize_value_on_error": Default(None), - "cost_value_on_error": Default(None), - "pre_load_hooks": Default(None), - "searcher": Default("default"), - "searcher_kwargs": {}, - "sample_batch_size": Default(None), - }, - Default(None), - ValueError, - ) - ], -) -def test_settings_initialization_error( - func_args: dict, yaml_args: str | Default, error: type[Exception] -) -> None: - """Test if Settings raises Error when essential arguments are missing.""" - with pytest.raises(error): - Settings(func_args, yaml_args) diff --git a/tests/test_state/test_filebased_neps_state.py b/tests/test_state/test_filebased_neps_state.py index ea2751520..3d56d64b8 100644 --- a/tests/test_state/test_filebased_neps_state.py +++ b/tests/test_state/test_filebased_neps_state.py @@ -2,6 +2,7 @@ This could be generalized if we end up with a server based implementation but for now we're just testing the filebased implementation. """ + from __future__ import annotations from pathlib import Path diff --git a/tests/test_state/test_neps_state.py b/tests/test_state/test_neps_state.py index 6d067d896..7605ea50b 100644 --- a/tests/test_state/test_neps_state.py +++ b/tests/test_state/test_neps_state.py @@ -2,6 +2,7 @@ This could be generalized if we end up with a server based implementation but for now we're just testing the filebased implementation. """ + from __future__ import annotations import time @@ -11,59 +12,70 @@ import pytest from pytest_cases import case, fixture, parametrize, parametrize_with_cases -from neps.optimizers import SearcherMapping -from neps.optimizers.base_optimizer import BaseOptimizer -from neps.search_spaces.hyperparameters import ( +from neps.optimizers import AskFunction, PredefinedOptimizers, load_optimizer +from neps.space import ( Categorical, Constant, Float, Integer, + SearchSpace, +) +from neps.state import ( + BudgetInfo, + NePSState, + OptimizationState, + OptimizerInfo, + SeedSnapshot, ) -from neps.search_spaces.search_space import SearchSpace -from neps.state.neps_state import NePSState -from neps.state.optimizer import BudgetInfo, OptimizationState, OptimizerInfo -from neps.state.seed_snapshot import SeedSnapshot @case def case_search_space_no_fid() -> SearchSpace: return SearchSpace( - a=Float(0, 1), - b=Categorical(["a", "b", "c"]), - c=Constant("a"), - d=Integer(0, 10), + { + "a": Float(0, 1), + "b": Categorical(["a", "b", "c"]), + "c": Constant("a"), + "d": Integer(0, 10), + } ) @case def case_search_space_with_fid() -> SearchSpace: return SearchSpace( - a=Float(0, 1), - b=Categorical(["a", "b", "c"]), - c=Constant("a"), - d=Integer(0, 10), - e=Integer(1, 10, is_fidelity=True), + { + "a": Float(0, 1), + "b": Categorical(["a", "b", "c"]), + "c": Constant("a"), + "d": Integer(0, 10), + "e": Integer(1, 10, is_fidelity=True), + } ) @case def case_search_space_no_fid_with_prior() -> SearchSpace: return SearchSpace( - a=Float(0, 1, prior=0.5), - b=Categorical(["a", "b", "c"], prior="a"), - c=Constant("a"), - d=Integer(0, 10, prior=5), + { + "a": Float(0, 1, prior=0.5), + "b": Categorical(["a", "b", "c"], prior="a"), + "c": Constant("a"), + "d": Integer(0, 10, prior=5), + } ) @case def case_search_space_fid_with_prior() -> SearchSpace: return SearchSpace( - a=Float(0, 1, prior=0.5), - b=Categorical(["a", "b", "c"], prior="a"), - c=Constant("a"), - d=Integer(0, 10, prior=5), - e=Integer(1, 10, is_fidelity=True), + { + "a": Float(0, 1, prior=0.5), + "b": Categorical(["a", "b", "c"], prior="a"), + "c": Constant("a"), + "d": Integer(0, 10, prior=5), + "e": Integer(1, 10, is_fidelity=True), + } ) @@ -74,7 +86,9 @@ def case_search_space_fid_with_prior() -> SearchSpace: OPTIMIZER_FAILS_WITH_FIDELITY = [ "random_search", + "bayesian_optimization_cost_aware", "bayesian_optimization", + "bayesian_optimization_prior", "pibo", "cost_cooling_bayesian_optimization", "cost_cooling", @@ -88,26 +102,17 @@ def case_search_space_fid_with_prior() -> SearchSpace: "asha", "asha_prior", "hyperband", - "hyperband_custom_default", + "hyperband_prior", + "async_hb", + "async_hb_prior", "priorband", - "priorband_bo", - "mobster", - "mf_ei_bo", + "priorband_sh", "priorband_asha", - "ifbo", - "priorband_asha_hyperband", -] -OPTIMIZER_REQUIRES_BUDGET = [ - "successive_halving_prior", - "hyperband_custom_default", - "asha", - "priorband", + "priorband_async", "priorband_bo", - "priorband_asha", - "priorband_asha_hyperband", - "hyperband", - "asha_prior", + "bayesian_optimization_cost_aware", "mobster", + "ifbo", ] REQUIRES_PRIOR = { "priorband", @@ -119,13 +124,13 @@ def case_search_space_fid_with_prior() -> SearchSpace: @fixture -@parametrize("key", list(SearcherMapping.keys())) +@parametrize("key", list(PredefinedOptimizers.keys())) @parametrize_with_cases("search_space", cases=".", prefix="case_search_space") -def optimizer_and_key(key: str, search_space: SearchSpace) -> tuple[BaseOptimizer, str]: +def optimizer_and_key(key: str, search_space: SearchSpace) -> tuple[AskFunction, str]: if key in JUST_SKIP: pytest.xfail(f"{key} is not instantiable") - if key in REQUIRES_PRIOR and search_space.hyperparameters["a"].prior is None: + if key in REQUIRES_PRIOR and search_space.searchables["a"].prior is None: pytest.xfail(f"{key} requires a prior") if len(search_space.fidelities) > 0 and key in OPTIMIZER_FAILS_WITH_FIDELITY: @@ -134,15 +139,9 @@ def optimizer_and_key(key: str, search_space: SearchSpace) -> tuple[BaseOptimize if key in OPTIMIZER_REQUIRES_FIDELITY and not len(search_space.fidelities) > 0: pytest.xfail(f"{key} requires a fidelity parameter") - kwargs: dict[str, Any] = { - "pipeline_space": search_space, - } - if key in OPTIMIZER_REQUIRES_BUDGET: - kwargs["max_cost_total"] = 10 - - optimizer_cls = SearcherMapping[key] - - return optimizer_cls(**kwargs), key + kwargs: dict[str, Any] = {} + opt, _ = load_optimizer((key, kwargs), search_space) # type: ignore + return opt, key @parametrize("optimizer_info", [OptimizerInfo({"a": "b"}), OptimizerInfo({})]) @@ -169,7 +168,7 @@ def case_neps_state_filebased( @parametrize_with_cases("neps_state", cases=".", prefix="case_neps_state") def test_sample_trial( neps_state: NePSState, - optimizer_and_key: tuple[BaseOptimizer, str], + optimizer_and_key: tuple[AskFunction, str], ) -> None: optimizer, key = optimizer_and_key if key in REQUIRES_COST and neps_state.lock_and_get_optimizer_state().budget is None: @@ -182,7 +181,6 @@ def test_sample_trial( trial1 = neps_state.lock_and_sample_trial(optimizer=optimizer, worker_id="1") for k, v in trial1.config.items(): - assert k in optimizer.pipeline_space.hyperparameters assert v is not None, f"'{k}' is None in {trial1.config}" # HACK: Unfortunatly due to windows, who's time.time() is not very @@ -196,7 +194,6 @@ def test_sample_trial( trial2 = neps_state.lock_and_sample_trial(optimizer=optimizer, worker_id="1") for k, v in trial1.config.items(): - assert k in optimizer.pipeline_space.hyperparameters assert v is not None, f"'{k}' is None in {trial1.config}" assert trial1 != trial2 diff --git a/tests/test_yaml_run_args/__init__.py b/tests/test_yaml_run_args/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_yaml_run_args/pipeline_space.yaml b/tests/test_yaml_run_args/pipeline_space.yaml deleted file mode 100644 index cd41210f8..000000000 --- a/tests/test_yaml_run_args/pipeline_space.yaml +++ /dev/null @@ -1,9 +0,0 @@ -lr: - lower: 1e-3 - upper: 0.1 -epochs: - lower: 1 - upper: 10 -optimizer: - choices: [adam, sgd, adamw] -batch_size: 64 diff --git a/tests/test_yaml_run_args/run_args_empty.yaml b/tests/test_yaml_run_args/run_args_empty.yaml deleted file mode 100644 index 8f47d1795..000000000 --- a/tests/test_yaml_run_args/run_args_empty.yaml +++ /dev/null @@ -1,23 +0,0 @@ -pipeline_space: -root_directory: - -budget: - evaluations: - max_evaluations_total: - cost: - max_cost_total: - -monitoring: - overwrite_working_directory: - post_run_summary: - development_stage_id: - task_id: - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -search: - searcher: - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/run_args_full.yaml b/tests/test_yaml_run_args/run_args_full.yaml deleted file mode 100644 index 2bf1671fe..000000000 --- a/tests/test_yaml_run_args/run_args_full.yaml +++ /dev/null @@ -1,34 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_yaml_run_args.py" - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" - -budget: - max_evaluations_total: 20 - max_cost_total: 3 - -monitoring: - overwrite_working_directory: true - post_run_summary: true - development_stage_id: "Early_Stage" - task_id: 4 - -parallelization_setup: - max_evaluations_per_run: 5 - continue_until_max_evaluation_completed: true - -error_handling: - objective_to_minimize_value_on_error: 4.2 - cost_value_on_error: 3.7 - ignore_errors: true - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args.py" - hook2: "tests/test_yaml_run_args/test_yaml_run_args.py" diff --git a/tests/test_yaml_run_args/run_args_full_same_level.yaml b/tests/test_yaml_run_args/run_args_full_same_level.yaml deleted file mode 100644 index a7b9948f6..000000000 --- a/tests/test_yaml_run_args/run_args_full_same_level.yaml +++ /dev/null @@ -1,22 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_yaml_run_args" # check if without .py also works - name: "evaluate_pipeline" -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" -max_evaluations_total: 20 -max_cost_total: 4.2 -overwrite_working_directory: true -post_run_summary: false -development_stage_id: 9 -task_id: 2.0 -max_evaluations_per_run: 5 -continue_until_max_evaluation_completed: true -objective_to_minimize_value_on_error: 2.4 -cost_value_on_error: 2.1 -ignore_errors: false -searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args" # check if without .py also works diff --git a/tests/test_yaml_run_args/run_args_invalid_key.yaml b/tests/test_yaml_run_args/run_args_invalid_key.yaml deleted file mode 100644 index 8047f74a8..000000000 --- a/tests/test_yaml_run_args/run_args_invalid_key.yaml +++ /dev/null @@ -1,34 +0,0 @@ -run_pipelin: # typo in key - path: "tests/test_yaml_run_args/test_yaml_run_args.py" - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" - -budget: - max_evaluations_total: 20 - max_cost_total: 3 - -monitoring: - overwrite_working_directory: true - post_run_summary: true - development_stage_id: "Early_Stage" - task_id: 4 - -parallelization_setup: - max_evaluations_per_run: 5 - continue_until_max_evaluation_completed: true - -error_handling: - objective_to_minimize_value_on_error: 4.2 - cost_value_on_error: 3.7 - ignore_errors: true - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args.py" - hook2: "tests/test_yaml_run_args/test_yaml_run_args.py" diff --git a/tests/test_yaml_run_args/run_args_invalid_type.yaml b/tests/test_yaml_run_args/run_args_invalid_type.yaml deleted file mode 100644 index 8e1f1d2d1..000000000 --- a/tests/test_yaml_run_args/run_args_invalid_type.yaml +++ /dev/null @@ -1,29 +0,0 @@ -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" - -budget: - max_evaluations_total: 20 - max_cost_total: - -monitoring: - overwrite_working_directory: true - post_run_summary: Falsee # Error - development_stage_id: "None" - task_id: "None" - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: false - -error_handling: - objective_to_minimize_value_on_error: None - cost_value_on_error: None - ignore_errors: None - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/run_args_key_missing.yaml b/tests/test_yaml_run_args/run_args_key_missing.yaml deleted file mode 100644 index 660349c99..000000000 --- a/tests/test_yaml_run_args/run_args_key_missing.yaml +++ /dev/null @@ -1,22 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_yaml_run_args.py" - # key name is missing -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" -max_evaluations_total: 20 -max_cost_total: 4.2 -overwrite_working_directory: true -post_run_summary: false -development_stage_id: 9 -task_id: 2.0 -max_evaluations_per_run: 5 -continue_until_max_evaluation_completed: true -objective_to_minimize_value_on_error: 2.4 -cost_value_on_error: 2.1 -ignore_errors: false -searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args.py" diff --git a/tests/test_yaml_run_args/run_args_optional_loading_format.yaml b/tests/test_yaml_run_args/run_args_optional_loading_format.yaml deleted file mode 100644 index 640c7d9de..000000000 --- a/tests/test_yaml_run_args/run_args_optional_loading_format.yaml +++ /dev/null @@ -1,24 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_yaml_run_args.py" - name: "evaluate_pipeline" -pipeline_space: # Optional loading - path: "tests/test_yaml_run_args/test_yaml_run_args.py" - name: "pipeline_space" -root_directory: "test_yaml" -max_evaluations_total: 20 -max_cost_total: 4.2 -overwrite_working_directory: true -post_run_summary: false -development_stage_id: 9 -task_id: -max_evaluations_per_run: 5 -continue_until_max_evaluation_completed: true -objective_to_minimize_value_on_error: 2.4 -cost_value_on_error: 2.1 -ignore_errors: false -searcher: # Optional Loading - path: "neps/optimizers/bayesian_optimization/optimizer.py" - name: BayesianOptimization - initial_design_size: 5 -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args.py" diff --git a/tests/test_yaml_run_args/run_args_partial.yaml b/tests/test_yaml_run_args/run_args_partial.yaml deleted file mode 100644 index c5226a64e..000000000 --- a/tests/test_yaml_run_args/run_args_partial.yaml +++ /dev/null @@ -1,27 +0,0 @@ -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" - -budget: - evaluations: - max_evaluations_total: - 20 - cost: - max_cost_total: - -monitoring: - overwrite_working_directory: true - post_run_summary: false - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: false - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/run_args_partial_same_level.yaml b/tests/test_yaml_run_args/run_args_partial_same_level.yaml deleted file mode 100644 index 931986513..000000000 --- a/tests/test_yaml_run_args/run_args_partial_same_level.yaml +++ /dev/null @@ -1,14 +0,0 @@ -pipeline_space: -root_directory: "test_yaml" -max_evaluations_total: 20 -max_cost_total: -overwrite_working_directory: True -post_run_summary: False -development_stage_id: None -task_id: 4 -max_evaluations_per_run: None -continue_until_max_evaluation_completed: True -objective_to_minimize_value_on_error: None -ignore_errors: True -searcher: -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/run_args_wrong_name.yaml b/tests/test_yaml_run_args/run_args_wrong_name.yaml deleted file mode 100644 index c2eb70020..000000000 --- a/tests/test_yaml_run_args/run_args_wrong_name.yaml +++ /dev/null @@ -1,34 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_yaml_run_args.py" - name: run_pipelin # typo in name -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" - -budget: - max_evaluations_total: 20 - max_cost_total: 3 - -monitoring: - overwrite_working_directory: True - post_run_summary: True - development_stage_id: "Early_Stage" - task_id: 4 - -parallelization_setup: - max_evaluations_per_run: 5 - continue_until_max_evaluation_completed: True - -error_handling: - objective_to_minimize_value_on_error: 4.2 - cost_value_on_error: 3.7 - ignore_errors: True - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args.py" - hook2: "tests/test_yaml_run_args/test_yaml_run_args.py" diff --git a/tests/test_yaml_run_args/run_args_wrong_path.yaml b/tests/test_yaml_run_args/run_args_wrong_path.yaml deleted file mode 100644 index b09808cc1..000000000 --- a/tests/test_yaml_run_args/run_args_wrong_path.yaml +++ /dev/null @@ -1,34 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_yaml_ru_args.py" # typo in path - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/pipeline_space.yaml" -root_directory: "test_yaml" - -budget: - max_evaluations_total: 20 - max_cost_total: 3 - -monitoring: - overwrite_working_directory: True - post_run_summary: True - development_stage_id: "Early_Stage" - task_id: 4 - -parallelization_setup: - max_evaluations_per_run: 5 - continue_until_max_evaluation_completed: True - -error_handling: - objective_to_minimize_value_on_error: 4.2 - cost_value_on_error: 3.7 - ignore_errors: True - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: - hook1: "tests/test_yaml_run_args/test_yaml_run_args.py" - hook2: "tests/test_yaml_run_args/test_yaml_run_args.py" diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/__init__.py b/tests/test_yaml_run_args/test_declarative_usage_docs/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml deleted file mode 100644 index afae914e0..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/customizing_neps_optimizer.yaml +++ /dev/null @@ -1,23 +0,0 @@ -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: 20 - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/custominizing_neps_optimizer" -max_evaluations_total: 20 # Budget -searcher: - strategy: bayesian_optimization - name: "my_bayesian" - # Specific arguments depending on the searcher - initial_design_size: 7 - -overwrite_working_directory: True diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml deleted file mode 100644 index f37485f42..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/defining_hooks.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Basic NEPS Configuration Example -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: True - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/hooks" -max_evaluations_total: 20 # Budget - -pre_load_hooks: - hook1: tests/test_yaml_run_args/test_declarative_usage_docs/hooks.py - -overwrite_working_directory: True diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py b/tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py deleted file mode 100644 index adb912682..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from warnings import warn - -import numpy as np - - -def run_pipeline(learning_rate, optimizer, epochs): - """Func for test loading of run_pipeline.""" - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline(learning_rate, optimizer, epochs) - -def evaluate_pipeline(learning_rate, optimizer, epochs): - """Func for test loading of evaluate_pipeline.""" - eval_score = np.random.choice([learning_rate, epochs], 1) if optimizer == "a" else 5.0 - return {"objective_to_minimize": eval_score} - - -def run_pipeline_constant(learning_rate, optimizer, epochs, batch_size): - """Func for test loading of run_pipeline.""" - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline_constant(learning_rate, optimizer, epochs, batch_size) - -def evaluate_pipeline_constant(learning_rate, optimizer, epochs, batch_size): - """Func for test loading of evaluate_pipeline.""" - eval_score = np.random.choice([learning_rate, epochs], 1) if optimizer == "a" else 5.0 - eval_score += batch_size - return {"objective_to_minimize": eval_score} - diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml deleted file mode 100644 index e207a6dfa..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/full_configuration_template.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Full Configuration Template for NePS -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: True - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/full_config" -max_evaluations_total: 20 # Budget -max_cost_total: - -# Debug and Monitoring -overwrite_working_directory: True -post_run_summary: False -development_stage_id: -task_id: - -# Parallelization Setup -max_evaluations_per_run: -continue_until_max_evaluation_completed: False - -# Error Handling -objective_to_minimize_value_on_error: -cost_value_on_error: -ignore_errors: - -# Customization Options -searcher: hyperband # Internal key to select a NePS optimizer. - -# Hooks -pre_load_hooks: diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/hooks.py b/tests/test_yaml_run_args/test_declarative_usage_docs/hooks.py deleted file mode 100644 index 029ac890c..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/hooks.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - - -def hook1(sampler): - """Func to test loading of pre_load_hooks.""" - return sampler - - -def hook2(sampler): - """Func to test loading of pre_load_hooks.""" - return sampler diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml deleted file mode 100644 index c178e8da6..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_own_optimizer.yaml +++ /dev/null @@ -1,22 +0,0 @@ -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: evaluate_pipeline - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - optimizer: - choices: [adam, sgd, adamw] - epochs: 50 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/loading_own_optimizer" -max_evaluations_total: 20 # Budget -searcher: - path: "neps/optimizers/bayesian_optimization/optimizer.py" - name: BayesianOptimization - # Specific arguments depending on your searcher - initial_design_size: 7 - -overwrite_working_directory: True diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml deleted file mode 100644 index 0baad5c10..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/loading_pipeline_space_dict.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Loading pipeline space from a python dict -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - path: tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py - name: pipeline_space # Name of the dict instance - -root_directory: "tests_tmpdir/test_declarative_usage_docs/results_loading_pipeline_space" -max_evaluations_total: 20 # Budget diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/neps_run.py b/tests/test_yaml_run_args/test_declarative_usage_docs/neps_run.py deleted file mode 100644 index fbc2a1bd1..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/neps_run.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -import argparse -from warnings import warn - -import numpy as np - -import neps - - -def run_pipeline_constant(learning_rate, optimizer, epochs, batch_size): - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline_constant(learning_rate, optimizer, epochs, batch_size) - -def evaluate_pipeline_constant(learning_rate, optimizer, epochs, batch_size): - """Func for test loading of evaluate_pipeline.""" - eval_score = np.random.choice([learning_rate, epochs], 1) if optimizer == "a" else 5.0 - eval_score += batch_size - return {"objective_to_minimize": eval_score} - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Run NEPS optimization with run_args.yml." - ) - parser.add_argument("run_args", type=str, help="Path to the YAML configuration file.") - parser.add_argument("--evaluate_pipeline", action="store_true") - args = parser.parse_args() - - if args.evaluate_pipeline: - neps.run(run_args=args.run_args, evaluate_pipeline=evaluate_pipeline_constant) - else: - neps.run(run_args=args.run_args) diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml deleted file mode 100644 index 82882247b..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_optimizer.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Optimizer settings from YAML configuration -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: evaluate_pipeline - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - optimizer: - choices: [adam, sgd, adamw] - epochs: 50 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/outsourcing_optimizer" -max_evaluations_total: 20 # Budget - -searcher: tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml deleted file mode 100644 index 980b8b27b..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/outsourcing_pipeline_space.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Pipeline space settings from YAML -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml - -root_directory: "tests_tmpdir/test_declarative_usage_docs/outsourcing_pipeline_space" -max_evaluations_total: 20 # Budget - diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py b/tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py deleted file mode 100644 index c63c06d29..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -import neps - -pipeline_space = { - "learning_rate": neps.Float(lower=1e-5, upper=1e-1, log=True), - "epochs": neps.Integer(lower=5, upper=20, is_fidelity=True), - "optimizer": neps.Categorical(choices=["adam", "sgd", "adamw"]), - "batch_size": neps.Constant(value=64) -} diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml deleted file mode 100644 index 274d53873..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/pipeline_space.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# pipeline_space including priors and fidelity -learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - prior: 1e-2 - prior_confidence: "medium" -epochs: - lower: 5 - upper: 20 - prior: 10 - is_fidelity: True -optimizer: - choices: [adam, sgd, adamw] - prior: adam - prior_confidence: low -batch_size: 64 diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml deleted file mode 100644 index 94922d78a..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/set_up_optimizer.yaml +++ /dev/null @@ -1,5 +0,0 @@ -strategy: bayesian_optimization -# Specific arguments depending on the searcher -initial_design_size: 7 -use_priors: true -sample_prior_first: false diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml deleted file mode 100644 index 920d15cdf..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Basic NePS Configuration Example -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: True - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/simple_example" -max_evaluations_total: 20 # Budget - - -overwrite_working_directory: True diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml b/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml deleted file mode 100644 index a9abe6a75..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/simple_example_including_run_pipeline.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Simple NePS configuration including evaluate_pipeline -evaluate_pipeline: - path: tests/test_yaml_run_args/test_declarative_usage_docs/evaluate_pipeline.py - name: run_pipeline_constant - -pipeline_space: - learning_rate: - lower: 1e-5 - upper: 1e-1 - log: True # Log scale for learning rate - epochs: - lower: 5 - upper: 20 - is_fidelity: True - optimizer: - choices: [adam, sgd, adamw] - batch_size: 64 - -root_directory: "tests_tmpdir/test_declarative_usage_docs/simple_example_including_evaluate_pipeline" -max_evaluations_total: 20 # Budget - -overwrite_working_directory: True diff --git a/tests/test_yaml_run_args/test_declarative_usage_docs/test_declarative_usage_docs.py b/tests/test_yaml_run_args/test_declarative_usage_docs/test_declarative_usage_docs.py deleted file mode 100644 index 2ec2d0dc0..000000000 --- a/tests/test_yaml_run_args/test_declarative_usage_docs/test_declarative_usage_docs.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path - -import pytest - -BASE_PATH = Path("tests") / "test_yaml_run_args" / "test_declarative_usage_docs" - - -@pytest.mark.neps_api -@pytest.mark.parametrize( - "yaml_file", - [ - "simple_example_including_run_pipeline.yaml", - "full_configuration_template.yaml", - "defining_hooks.yaml", - "customizing_neps_optimizer.yaml", - "loading_own_optimizer.yaml", - "loading_pipeline_space_dict.yaml", - "outsourcing_optimizer.yaml", - "outsourcing_pipeline_space.yaml", - ], -) -def test_run_with_yaml(yaml_file: str) -> None: - """Test 'neps.run' with various run_args.yaml settings to simulate loading options - for variables. - """ - yaml_path = BASE_PATH / yaml_file - assert yaml_path.exists(), f"{yaml_path} does not exist." - - try: - subprocess.check_call([sys.executable, BASE_PATH / "neps_run.py", yaml_path]) - except subprocess.CalledProcessError as e: - pytest.fail(f"NePS run failed for configuration: {yaml_file} with error: {e!s}") - - -@pytest.mark.neps_api -def test_run_with_yaml_and_run_pipeline() -> None: - """Test 'neps.run' with simple_example.yaml as run_args + a run_pipeline that is - provided separately. - """ - yaml_path = BASE_PATH / "simple_example.yaml" - assert yaml_path.exists(), f"{yaml_path} does not exist." - - try: - subprocess.check_call( - [sys.executable, BASE_PATH / "neps_run.py", yaml_path, "--evaluate_pipeline"] - ) - except subprocess.CalledProcessError as e: - pytest.fail( - f"NePS run failed for configuration: simple_example.yaml with error: {e!s}" - ) diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/__init__.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml deleted file mode 100644 index a87e31087..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml" -root_directory: "tests_tmpdir/test_run_args_by_neps_run/results2" - -max_evaluations_total: 5 -max_cost_total: - -monitoring: - overwrite_working_directory: True - post_run_summary: False - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml deleted file mode 100644 index 3ebfcb5d0..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_hyperband_mixed_args.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# args of optimizer from searcher kwargs (neps.run) and from run_args (yaml) - -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml" -root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband" - -max_evaluations_total: 5 -max_cost_total: - -monitoring: - overwrite_working_directory: true - post_run_summary: false - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -searcher: - strategy: hyperband - name: my_hyperband - eta: 8 - initial_design_type: max_budget - - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml deleted file mode 100644 index 2d94740b7..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_priorband_with_args.yaml +++ /dev/null @@ -1,38 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml" -root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband" - -max_evaluations_total: 5 -max_cost_total: - -monitoring: - overwrite_working_directory: true - post_run_summary: false - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -searcher: - strategy: "priorband" - initial_design_type: max_budget - prior_confidence: medium - sample_prior_first: true - sample_prior_at_target: false - prior_weight_type: geometric - inc_sample_type: mutation - inc_mutation_rate: 0.2 - inc_mutation_std: 0.25 - inc_style: dynamic - model_based: true - modelling_type: joint - initial_design_size: 5 - surrogate_model: gp - acquisition: EI - log_prior_weighted: false - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml deleted file mode 100644 index 46913efa0..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/config_select_bo.yaml +++ /dev/null @@ -1,23 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: evaluate_pipeline -pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml" -root_directory: "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo" - -max_evaluations_total: 5 -max_cost_total: - -monitoring: - overwrite_working_directory: true - post_run_summary: false - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -searcher: - strategy: "bayesian_optimization" - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml deleted file mode 100644 index 731dd20bc..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_optimizer.yaml +++ /dev/null @@ -1,27 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: "evaluate_pipeline" -pipeline_space: "tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml" -root_directory: "tests_tmpdir/test_run_args_by_neps_run/results1" - -max_evaluations_total: 5 -max_cost_total: - -monitoring: - overwrite_working_directory: True - post_run_summary: False - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -search: - # Test Case - searcher: - path: "neps/optimizers/bayesian_optimization/optimizer.py" - name: BayesianOptimization - - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml deleted file mode 100644 index eb298ad0c..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/loading_pipeline_space.yaml +++ /dev/null @@ -1,29 +0,0 @@ -evaluate_pipeline: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: evaluate_pipeline -# Test Case -pipeline_space: - path: "tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py" - name: "pipeline_space" -root_directory: "tests/test_yaml_run_args/test_run_args_by_neps_run/results" - -max_evaluations_total: 5 -max_cost_total: - -monitoring: - overwrite_working_directory: True - post_run_summary: False - development_stage_id: None - task_id: None - -parallelization_setup: - max_evaluations_per_run: None - continue_until_max_evaluation_completed: - -search: - searcher: - strategy: "bayesian_optimization" - initial_design_size: 5 - surrogate_model: gp - -pre_load_hooks: None diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py deleted file mode 100644 index 2a2775d4f..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/neps_run.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -import argparse -from warnings import warn - -import numpy as np - -import neps - - -def run_pipeline(learning_rate, epochs, optimizer, batch_size): - """Func for test loading of run_pipeline.""" - warn("run_pipeline is deprecated, use evaluate_pipeline instead", DeprecationWarning, stacklevel=2) - return evaluate_pipeline(learning_rate, epochs, optimizer, batch_size) - -def evaluate_pipeline(learning_rate, epochs, optimizer, batch_size): - """Func for test loading of evaluate_pipeline.""" - eval_score = np.random.choice([learning_rate, epochs], 1) if optimizer == "a" else 5.0 - eval_score += batch_size - return {"objective_to_minimize": eval_score} - - -# For testing the functionality of loading a dictionary from a YAML configuration. -pipeline_space = { - "learning_rate": neps.Float(lower=1e-6, upper=1e-1, log=False), - "epochs": neps.Integer(lower=1, upper=3, is_fidelity=False), - "optimizer": neps.Categorical(choices=["a", "b", "c"]), - "batch_size": neps.Constant(64), -} - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Run NEPS optimization with run_args.yaml.") - parser.add_argument("run_args", type=str, - help="Path to the YAML configuration file.") - parser.add_argument("--kwargs_flag", action="store_true", - help="flag for adding kwargs") - args = parser.parse_args() - - hyperband_args_optimizer = {"random_interleave_prob": 0.9, - "sample_prior_first": False, - "sample_prior_at_target": False, - "eta": 7} - - if args.kwargs_flag: - neps.run(run_args=args.run_args, **hyperband_args_optimizer) - else: - neps.run(run_args=args.run_args) diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml deleted file mode 100644 index cbb031317..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/hyperband_searcher_kwargs_yaml_args.yaml +++ /dev/null @@ -1,11 +0,0 @@ -searcher_name: my_hyperband -searcher_alg: hyperband -searcher_selection: user-run_args-yaml -neps_decision_tree: false -searcher_args: - eta: 7 - initial_design_type: max_budget - use_priors: false - random_interleave_prob: 0.9 - sample_prior_first: false - sample_prior_at_target: false diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml deleted file mode 100644 index 66a4a3dba..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/priorband_args_run_args.yaml +++ /dev/null @@ -1,22 +0,0 @@ -searcher_name: custom_priorband -searcher_alg: priorband -searcher_selection: user-run_args-yaml -neps_decision_tree: false -searcher_args: - eta: 3 - initial_design_type: max_budget - prior_confidence: medium - random_interleave_prob: 0.0 - sample_prior_first: true - sample_prior_at_target: false - prior_weight_type: geometric - inc_sample_type: mutation - inc_mutation_rate: 0.2 - inc_mutation_std: 0.25 - inc_style: dynamic - model_based: true - modelling_type: joint - initial_design_size: 5 - surrogate_model: gp - acquisition: EI - log_prior_weighted: false diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml deleted file mode 100644 index 3bdb09431..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/optimizer_yamls/select_bo_run_args.yaml +++ /dev/null @@ -1,10 +0,0 @@ -searcher_name: custom_bayesian_optimization -searcher_alg: bayesian_optimization -searcher_selection: user-run_args-yaml -neps_decision_tree: false -searcher_args: - initial_design_size: null - use_priors: false - use_cost: false - sample_prior_first: false - device: null diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml deleted file mode 100644 index 9a728d741..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space.yaml +++ /dev/null @@ -1,11 +0,0 @@ -epochs: - lower: 1 - upper: 3 - is_fidelity: False -learning_rate: - lower: 1e-6 - upper: 1e-1 - log: False -optimizer: - choices: ["a", "b", "c"] -batch_size: 64 diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml deleted file mode 100644 index 4a44434d9..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_fidelity.yaml +++ /dev/null @@ -1,11 +0,0 @@ -epochs: - lower: 1 - upper: 3 - is_fidelity: true -learning_rate: - lower: 1e-6 - upper: 1e-1 - log: False -optimizer: - choices: ["a", "b", "c"] -batch_size: 64 diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml b/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml deleted file mode 100644 index 5e8b1d383..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/search_space_with_priors.yaml +++ /dev/null @@ -1,15 +0,0 @@ -epochs: - lower: 1 - upper: 3 - is_fidelity: True -learning_rate: - lower: 1e-6 - upper: 1e-1 - log: False - prior: 1e-3 - prior_confidence: "low" -optimizer: - choices: ["a", "b", "c"] - prior: "b" - prior_confidence: "high" -batch_size: 64 diff --git a/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py b/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py deleted file mode 100644 index 3ac9dc85e..000000000 --- a/tests/test_yaml_run_args/test_run_args_by_neps_run/test_neps_run.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path - -import pytest -import yaml - -BASE_PATH = Path("tests") / "test_yaml_run_args" / "test_run_args_by_neps_run" - - -@pytest.mark.neps_api -@pytest.mark.parametrize( - "config", - [ - {"file_name": "config.yaml"}, - {"file_name": "loading_pipeline_space.yaml"}, - {"file_name": "loading_optimizer.yaml"}, - { - "file_name": "config_select_bo.yaml", - "check_optimizer": True, - "optimizer_path": "select_bo_run_args.yaml", - "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_bo/optimizer_info.yaml", # noqa: E501 - }, - { - "file_name": "config_priorband_with_args.yaml", - "check_optimizer": True, - "optimizer_path": "priorband_args_run_args.yaml", - "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_priorband/optimizer_info.yaml", # noqa: E501 - }, - { - "file_name": "config_hyperband_mixed_args.yaml", - "check_optimizer": True, - "optimizer_path": "hyperband_searcher_kwargs_yaml_args.yaml", - "result_path": "tests_tmpdir/test_run_args_by_neps_run/optimizer_hyperband/optimizer_info.yaml", # noqa: E501 - "args": True, - }, - ], -) -def test_run_with_yaml(config: dict) -> None: - """Test "neps.run" with various run_args.yaml settings to simulate loading options - for variables. - """ - file_name = config["file_name"] - check_optimizer = config.pop("check_optimizer", False) - assert (BASE_PATH / file_name).exists(), f"{file_name} " f"does not exist." - - cmd = [ - sys.executable, - BASE_PATH / "neps_run.py", - BASE_PATH / file_name, - ] - if "args" in config: - cmd.append("--kwargs_flag") - - try: - subprocess.check_call(cmd) # noqa: S603 - except subprocess.CalledProcessError: - pytest.fail(f"NePS run failed for configuration: {file_name}") - - if check_optimizer: - optimizer_path = Path(config.pop("optimizer_path")) - result_path = Path(config.pop("result_path")) - compare_generated_yaml(result_path, optimizer_path) - - -def compare_generated_yaml(result_path: Path, optimizer_path: Path) -> None: - """Compare generated optimizer settings and solution settings.""" - assert result_path.exists(), "Generated YAML file does not exist." - - assert ( - BASE_PATH / "optimizer_yamls" / optimizer_path - ).exists(), "Solution YAML file does not exist." - - with result_path.open("r") as gen_file: - generated_content = yaml.safe_load(gen_file) - - with (BASE_PATH / "optimizer_yamls" / optimizer_path).open("r") as ref_file: - reference_content = yaml.safe_load(ref_file) - - assert ( - generated_content == reference_content - ), "The generated YAML does not match the reference YAML" diff --git a/tests/test_yaml_run_args/test_yaml_run_args.py b/tests/test_yaml_run_args/test_yaml_run_args.py deleted file mode 100644 index b68a07fe9..000000000 --- a/tests/test_yaml_run_args/test_yaml_run_args.py +++ /dev/null @@ -1,245 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from warnings import warn - -import pytest - -import neps -from neps.optimizers.bayesian_optimization.optimizer import BayesianOptimization -from neps.utils.run_args import get_run_args_from_yaml - -BASE_PATH = "tests/test_yaml_run_args/" -pipeline_space = { - "lr": neps.Float(lower=1e-3, upper=0.1), - "optimizer": neps.Categorical(choices=["adam", "sgd", "adamw"]), - "epochs": neps.Integer(lower=1, upper=10), - "batch_size": neps.Constant(value=64), -} - - -def run_pipeline(): - """Func to test loading of run_pipeline.""" - warn( - "run_pipeline is deprecated, use evaluate_pipeline instead", - DeprecationWarning, - stacklevel=2, - ) - return evaluate_pipeline() - - -def evaluate_pipeline(): - """Func to test loading of evaluate_pipeline.""" - return - - -def hook1(sampler): - """Func to test loading of pre_load_hooks.""" - return sampler - - -def hook2(sampler): - """Func to test loading of pre_load_hooks.""" - return sampler - - -def check_run_args(yaml_path_run_args: str, expected_output: dict) -> None: - """Validates the loaded NEPS configuration against expected settings. - - Loads NEPS configuration settings from a specified YAML file and verifies - against expected settings, including function objects, dict and classes. Special - handling is applied to compare functions. - - Args: - yaml_path_run_args (str): The path to the YAML configuration file. - expected_output (dict): The expected NePS configuration settings. - - Raises: - AssertionError: If any configuration setting does not match the expected value. - """ - output = get_run_args_from_yaml(BASE_PATH + yaml_path_run_args) - - def are_functions_equivalent( - f1: Callable | list[Callable], f2: Callable | list[Callable] - ) -> bool: - """Compares functions or lists of functions for equivalence by their bytecode, - useful when identical functions have different memory addresses. This method - identifies if functions, despite being distinct instances, perform identical - operations. - - Parameters: - - func1: Function or list of functions to compare. - - func2: Function or list of functions to compare against func1. - - Returns: - bool: True if the functions or all functions in the lists are equivalent, - False otherwise. - """ - if isinstance(f1, list) and isinstance(f2, list): - if len(f1) != len(f2): - return False - return all( - f1_item.__code__.co_code == f2_item.__code__.co_code - for f1_item, f2_item in zip(f1, f2, strict=False) - ) - return f1.__code__.co_code == f2.__code__.co_code - - # Compare keys with a function/list of functions as their values - # Special because they include a module loading procedure by a path and the name of - # the function - for special_key in ["evaluate_pipeline", "pre_load_hooks"]: - if special_key in expected_output: - func_expected = expected_output.pop(special_key) - func_output = output.pop(special_key) - assert are_functions_equivalent(func_expected, func_output), ( - f"Mismatch in {special_key} " f"function(s)" - ) - # Compare instances of a subclass of BaseOptimizer - if "searcher" in expected_output and not isinstance(expected_output["searcher"], str): - # 'searcher': BaseOptimizer() - optimizer_expected = expected_output.pop("searcher") - optimizer_output = output.pop("searcher", None) - assert isinstance(optimizer_output, type(optimizer_expected)) - - # Assert that the rest of the output dict matches the expected output dict - assert output == expected_output, f"Expected {expected_output}, but got {output}" - - -@pytest.mark.neps_api -@pytest.mark.parametrize( - ("yaml_path", "expected_output"), - [ - ( - "run_args_full.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "pipeline_space": pipeline_space, - "root_directory": "test_yaml", - "max_evaluations_total": 20, - "max_cost_total": 3, - "overwrite_working_directory": True, - "post_run_summary": True, - "development_stage_id": "Early_Stage", - "task_id": 4, - "max_evaluations_per_run": 5, - "continue_until_max_evaluation_completed": True, - "objective_to_minimize_value_on_error": 4.2, - "cost_value_on_error": 3.7, - "ignore_errors": True, - "searcher": { - "strategy": "bayesian_optimization", - "initial_design_size": 5, - }, - "pre_load_hooks": [hook1, hook2], - }, - ), - ( - "run_args_full_same_level.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "pipeline_space": pipeline_space, - "root_directory": "test_yaml", - "max_evaluations_total": 20, - "max_cost_total": 4.2, - "overwrite_working_directory": True, - "post_run_summary": False, - "development_stage_id": 9, - "task_id": 2.0, - "max_evaluations_per_run": 5, - "continue_until_max_evaluation_completed": True, - "objective_to_minimize_value_on_error": 2.4, - "cost_value_on_error": 2.1, - "ignore_errors": False, - "searcher": { - "strategy": "bayesian_optimization", - "initial_design_size": 5, - }, - "pre_load_hooks": [hook1], - }, - ), - ( - "run_args_partial.yaml", - { - "pipeline_space": pipeline_space, - "root_directory": "test_yaml", - "max_evaluations_total": 20, - "overwrite_working_directory": True, - "post_run_summary": False, - "continue_until_max_evaluation_completed": False, - "searcher": { - "strategy": "bayesian_optimization", - "initial_design_size": 5, - }, - }, - ), - ( - "run_args_partial_same_level.yaml", - { - "root_directory": "test_yaml", - "max_evaluations_total": 20, - "overwrite_working_directory": True, - "post_run_summary": False, - "task_id": 4, - "continue_until_max_evaluation_completed": True, - "ignore_errors": True, - }, - ), - ("run_args_empty.yaml", {}), - ( - "run_args_optional_loading_format.yaml", - { - "evaluate_pipeline": evaluate_pipeline, - "pipeline_space": pipeline_space, - "root_directory": "test_yaml", - "max_evaluations_total": 20, - "max_cost_total": 4.2, - "overwrite_working_directory": True, - "post_run_summary": False, - "development_stage_id": 9, - "max_evaluations_per_run": 5, - "continue_until_max_evaluation_completed": True, - "objective_to_minimize_value_on_error": 2.4, - "cost_value_on_error": 2.1, - "ignore_errors": False, - "searcher": BayesianOptimization, - "searcher_kwargs": {"initial_design_size": 5}, - "pre_load_hooks": [hook1], - }, - ), - ], -) -def test_yaml_config(yaml_path: str, expected_output: dict) -> None: - """Tests NePS configuration loading from run_args=YAML, comparing expected settings - against loaded ones. Covers hierarchical levels and partial/full of yaml - dict definitions. - - Args: - yaml_path (str): Path to the YAML file. - expected_output (dict): Expected configuration settings. - """ - check_run_args(yaml_path, expected_output) - - -@pytest.mark.neps_api -@pytest.mark.parametrize( - ("yaml_path", "expected_exception"), - [ - ("run_args_invalid_type.yaml", TypeError), - ("run_args_wrong_path.yaml", ImportError), - ("run_args_invalid_key.yaml", KeyError), - ("run_args_wrong_name.yaml", ImportError), - ("run_args_key_missing.yaml", KeyError), - ], -) -def test_yaml_failure_cases(yaml_path: str, expected_exception: type[Exception]) -> None: - """Tests for expected exceptions when loading erroneous NePS configurations from YAML. - - Each case checks if `get_run_args_from_yaml` raises the correct exception for errors - like invalid types, missing keys, and incorrect paths in YAML configurations. - - Args: - yaml_path (str): Path to the error-containing YAML file. - expected_exception (Exception): Expected exception type. - """ - with pytest.raises(expected_exception): - get_run_args_from_yaml(BASE_PATH + yaml_path) diff --git a/tests/test_yaml_search_space/__init__.py b/tests/test_yaml_search_space/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_yaml_search_space/config_including_unknown_types.yaml b/tests/test_yaml_search_space/config_including_unknown_types.yaml deleted file mode 100644 index 778fa2f12..000000000 --- a/tests/test_yaml_search_space/config_including_unknown_types.yaml +++ /dev/null @@ -1,17 +0,0 @@ -learning_rate: - type: numerical - lower: 0.00001 - upper: 0.1 - log: true - -num_epochs: - type: numerical - lower: 3 - upper: 30 - is_fidelity: True - -optimizer: - type: numerical - choices: ["adam", "sgd", "rmsprop"] - -dropout_rate: 0.5 diff --git a/tests/test_yaml_search_space/config_including_wrong_types.yaml b/tests/test_yaml_search_space/config_including_wrong_types.yaml deleted file mode 100644 index 62a852c7b..000000000 --- a/tests/test_yaml_search_space/config_including_wrong_types.yaml +++ /dev/null @@ -1,17 +0,0 @@ -learning_rate: - type: int # wrong type - lower: 0.00001 - upper: 0.1 - log: true - -num_epochs: - type: int - lower: 3 - upper: 30 - is_fidelity: True - -optimizer: - type: cat - choices: ["adam", "sgd", "rmsprop"] - -dropout_rate: 0.5 diff --git a/tests/test_yaml_search_space/correct_config.yaml b/tests/test_yaml_search_space/correct_config.yaml deleted file mode 100644 index b8c348663..000000000 --- a/tests/test_yaml_search_space/correct_config.yaml +++ /dev/null @@ -1,28 +0,0 @@ -param_float1: - lower: 0.00001 - upper: 0.1 - log: TRUE - is_fidelity: off - -param_int1: - lower: 3 - upper: 30 - log: false - is_fidelity: on -param_int2: - type: int - lower: 1E2 - upper: 3e4 - log: ON - is_fidelity: FALSE - -param_float2: - lower: 3.3e-5 - upper: 1.5E-1 - -param_cat: - choices: [2, "sgd", 10e-3] - -param_const1: 0.5 - -param_const2: 1e3 diff --git a/tests/test_yaml_search_space/correct_config_including_priors.yml b/tests/test_yaml_search_space/correct_config_including_priors.yml deleted file mode 100644 index 305cb3aac..000000000 --- a/tests/test_yaml_search_space/correct_config_including_priors.yml +++ /dev/null @@ -1,18 +0,0 @@ -learning_rate: - lower: 0.00001 - upper: 0.1 - log: true - prior: 3.3E-2 - prior_confidence: high - -num_epochs: - lower: 3 - upper: 30 - is_fidelity: True - -optimizer: - choices: [adam, 90E-3, rmsprop] - prior: 0.09 - prior_confidence: "medium" - -dropout_rate: 1E3 diff --git a/tests/test_yaml_search_space/correct_config_including_types.yaml b/tests/test_yaml_search_space/correct_config_including_types.yaml deleted file mode 100644 index 4ae8cb9c7..000000000 --- a/tests/test_yaml_search_space/correct_config_including_types.yaml +++ /dev/null @@ -1,31 +0,0 @@ -param_float1: - type: float - lower: 0.00001 - upper: 1e-1 - log: true - -param_int1: - type: integer - lower: 3 - upper: 30 - is_fidelity: True - -param_int2: - type: "int" - lower: 1e2 - upper: 3E4 - log: true - is_fidelity: false - -param_float2: - type: "float" - lower: 3.3e-5 - upper: 1.5E-1 - -param_cat: - type: cat - choices: [2, "sgd", 10E-3] - -param_const1: 0.5 - -param_const2: 1e3 diff --git a/tests/test_yaml_search_space/default_not_in_range_config.yaml b/tests/test_yaml_search_space/default_not_in_range_config.yaml deleted file mode 100644 index e0ec7d584..000000000 --- a/tests/test_yaml_search_space/default_not_in_range_config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -param_float1: - lower: 0.00001 - upper: 0.1 -prior: 0.0000000001 - log: false - is_fidelity: true diff --git a/tests/test_yaml_search_space/default_value_not_in_choices_config.yaml b/tests/test_yaml_search_space/default_value_not_in_choices_config.yaml deleted file mode 100644 index bc2e76d56..000000000 --- a/tests/test_yaml_search_space/default_value_not_in_choices_config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -cat1: - choices: ["a", "b", "c"] - prior: "d" diff --git a/tests/test_yaml_search_space/inconsistent_types_config.yml b/tests/test_yaml_search_space/inconsistent_types_config.yml deleted file mode 100644 index 509dcb9ea..000000000 --- a/tests/test_yaml_search_space/inconsistent_types_config.yml +++ /dev/null @@ -1,14 +0,0 @@ -learning_rate: - lower: "string" # Lower is now a string - upper: 1e3 - log: true - -num_epochs: - lower: 3 - upper: 30 - is_fidelity: True - -optimizer: - choices: ["adam", "sgd", "rmsprop"] - -dropout_rate: 0.5 diff --git a/tests/test_yaml_search_space/inconsistent_types_config2.yml b/tests/test_yaml_search_space/inconsistent_types_config2.yml deleted file mode 100644 index cd4517f33..000000000 --- a/tests/test_yaml_search_space/inconsistent_types_config2.yml +++ /dev/null @@ -1,15 +0,0 @@ -learning_rate: - type: int - lower: 2.3 # float - upper: 1e3 - log: true - -num_epochs: - lower: 3 - upper: 30 - is_fidelity: True - -optimizer: - choices: ["adam", "sgd", "rmsprop"] - -dropout_rate: 0.5 diff --git a/tests/test_yaml_search_space/incorrect_config.txt b/tests/test_yaml_search_space/incorrect_config.txt deleted file mode 100644 index 4f468d64f..000000000 --- a/tests/test_yaml_search_space/incorrect_config.txt +++ /dev/null @@ -1,4 +0,0 @@ -learning_rate # : is missing - lower: 0.00001 - upper: 0.1 - log: true diff --git a/tests/test_yaml_search_space/incorrect_fidelity_bounds_config.yaml b/tests/test_yaml_search_space/incorrect_fidelity_bounds_config.yaml deleted file mode 100644 index 552c775d2..000000000 --- a/tests/test_yaml_search_space/incorrect_fidelity_bounds_config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -param_float1: - lower: 0.00001 - upper: 0.1 - log: TRUE - is_fidelity: off - -param_int1: - lower: -3 # negative fidelity range - upper: 30 - log: false - is_fidelity: on - -param_int2: - type: int - lower: 1E2 - upper: 3e4 - log: ON - is_fidelity: FALSE - -param_float2: - lower: 3.3e-5 - upper: 1.5E-1 diff --git a/tests/test_yaml_search_space/missing_key_config.yml b/tests/test_yaml_search_space/missing_key_config.yml deleted file mode 100644 index 94e56c9cd..000000000 --- a/tests/test_yaml_search_space/missing_key_config.yml +++ /dev/null @@ -1,13 +0,0 @@ -learning_rate: - lower: 0.00001 - log: true - -num_epochs: - lower: 3 - upper: 30 - is_fidelity: True - -optimizer: - choices: ["adam", "sgd", "rmsprop"] - -dropout_rate: 0.5 diff --git a/tests/test_yaml_search_space/not_allowed_key_config.yml b/tests/test_yaml_search_space/not_allowed_key_config.yml deleted file mode 100644 index 12d76862b..000000000 --- a/tests/test_yaml_search_space/not_allowed_key_config.yml +++ /dev/null @@ -1,25 +0,0 @@ -float_name1: - lower: 3e-5 - upper: 0.1 - -float_name2: - type: "float" # Optional, as neps infers type from 'lower' and 'upper' - lower: 1.7 - upper: 42.0 - log: true - -categorical_name1: - choices: [0, 1] - -categorical_name2: - type: cat - choices: ["a", "b", "c"] - -integer_name1: - lower: 32 - upper: 128 - fidelity: True # error, fidelity instead of is_fidelity - -integer_name2: - lower: -5 - upper: 5 diff --git a/tests/test_yaml_search_space/not_boolean_type_is_fidelity_cat_config.yaml b/tests/test_yaml_search_space/not_boolean_type_is_fidelity_cat_config.yaml deleted file mode 100644 index 344270858..000000000 --- a/tests/test_yaml_search_space/not_boolean_type_is_fidelity_cat_config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -cat1: - choices: ["a", "b", "c"] - is_fidelity: fals - prior: "c" diff --git a/tests/test_yaml_search_space/not_boolean_type_is_fidelity_float_config.yaml b/tests/test_yaml_search_space/not_boolean_type_is_fidelity_float_config.yaml deleted file mode 100644 index 729f61919..000000000 --- a/tests/test_yaml_search_space/not_boolean_type_is_fidelity_float_config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -param_float1: - lower: 0.00001 - upper: 0.1 - prior: 0.001 - log: false - is_fidelity: truee diff --git a/tests/test_yaml_search_space/not_boolean_type_log_config.yaml b/tests/test_yaml_search_space/not_boolean_type_log_config.yaml deleted file mode 100644 index c8ad4c47e..000000000 --- a/tests/test_yaml_search_space/not_boolean_type_log_config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -param_float1: - lower: 0.00001 - upper: 0.1 - prior: 0.001 - log: falsee - is_fidelity: true diff --git a/tests/test_yaml_search_space/test_search_space.py b/tests/test_yaml_search_space/test_search_space.py deleted file mode 100644 index ac1a4ed36..000000000 --- a/tests/test_yaml_search_space/test_search_space.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from neps import Categorical, Constant, Float, Integer -from neps.search_spaces.search_space import ( - SearchSpaceFromYamlFileError, - pipeline_space_from_yaml, -) - -BASE_PATH = "tests/test_yaml_search_space/" - - -@pytest.mark.neps_api -def test_correct_yaml_files(): - def test_correct_yaml_file(path): - """Test the function with a correctly formatted YAML file.""" - pipeline_space = pipeline_space_from_yaml(path) - assert isinstance(pipeline_space, dict) - float1 = Float(0.00001, 0.1, log=True, is_fidelity=False) - assert float1.__eq__(pipeline_space["param_float1"]) is True - int1 = Integer(3, 30, log=False, is_fidelity=True) - assert int1.__eq__(pipeline_space["param_int1"]) is True - int2 = Integer(100, 30000, log=True, is_fidelity=False) - assert int2.__eq__(pipeline_space["param_int2"]) is True - float2 = Float(3.3e-5, 0.15, log=False) - assert float2.__eq__(pipeline_space["param_float2"]) is True - cat1 = Categorical([2, "sgd", 10e-3]) - assert cat1.__eq__(pipeline_space["param_cat"]) is True - const1 = Constant(0.5) - assert const1.__eq__(pipeline_space["param_const1"]) is True - const2 = Constant(1e3) - assert const2.__eq__(pipeline_space["param_const2"]) is True - - test_correct_yaml_file(BASE_PATH + "correct_config.yaml") - test_correct_yaml_file(BASE_PATH + "correct_config_including_types.yaml") - - -@pytest.mark.neps_api -def test_correct_including_priors_yaml_file(): - """Test the function with a correctly formatted YAML file.""" - pipeline_space = pipeline_space_from_yaml( - BASE_PATH + "correct_config_including_priors.yml" - ) - assert isinstance(pipeline_space, dict) - float1 = Float( - 0.00001, 0.1, log=True, is_fidelity=False, prior=3.3e-2, prior_confidence="high" - ) - assert float1.__eq__(pipeline_space["learning_rate"]) is True - int1 = Integer(3, 30, log=False, is_fidelity=True) - assert int1.__eq__(pipeline_space["num_epochs"]) is True - cat1 = Categorical(["adam", 90e-3, "rmsprop"], prior=90e-3, prior_confidence="medium") - assert cat1.__eq__(pipeline_space["optimizer"]) is True - const1 = Constant(1e3) - assert const1.__eq__(pipeline_space["dropout_rate"]) is True - - -@pytest.mark.neps_api -def test_incorrect_yaml_file(): - """Test the function with an incorrectly formatted YAML file.""" - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(Path(BASE_PATH + "incorrect_config.txt")) - assert excinfo.value.exception_type == "ValueError" - - -@pytest.mark.neps_api -def test_yaml_file_with_missing_key(): - """Test the function with a YAML file missing a required key.""" - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "missing_key_config.yml") - assert excinfo.value.exception_type == "KeyError" - - -@pytest.mark.neps_api -def test_yaml_file_with_inconsistent_types(): - """Test the function with a YAML file having inconsistent types for - 'lower' and 'upper'. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "inconsistent_types_config.yml") - assert str(excinfo.value.exception_type == "TypeError") - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(Path(BASE_PATH + "inconsistent_types_config2.yml")) - assert excinfo.value.exception_type == "TypeError" - - -@pytest.mark.neps_api -def test_yaml_file_including_wrong_types(): - """Test the function with a YAML file that defines the wrong but existing type - int to float as an optional argument. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(Path(BASE_PATH + "inconsistent_types_config2.yml")) - assert excinfo.value.exception_type == "TypeError" - - -@pytest.mark.neps_api -def test_yaml_file_including_unkown_types(): - """Test the function with a YAML file that defines an unknown type as an optional - argument. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "config_including_unknown_types.yaml") - assert excinfo.value.exception_type == "TypeError" - - -@pytest.mark.neps_api -def test_yaml_file_including_not_allowed_parameter_keys(): - """Test the function with a YAML file that defines an unknown type as an optional - argument. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "not_allowed_key_config.yml") - assert excinfo.value.exception_type == "TypeError" - - -@pytest.mark.neps_api -def test_yaml_file_default_parameter_not_in_range(): - """Test if the default value outside the specified range is - correctly identified and handled. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "default_not_in_range_config.yaml") - assert excinfo.value.exception_type == "ValueError" - - -@pytest.mark.neps_api -def test_float_log_not_boolean(): - """Test if an exception is raised when the 'log' attribute is not a boolean.""" - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "not_boolean_type_log_config.yaml") - assert excinfo.value.exception_type == "TypeError" - - -@pytest.mark.neps_api -def test_float_is_fidelity_not_boolean(): - """Test if an exception is raised when for Float the 'is_fidelity' - attribute is not a boolean. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml( - BASE_PATH + "not_boolean_type_is_fidelity_float_config.yaml" - ) - assert excinfo.value.exception_type == "TypeError" - - -@pytest.mark.neps_api -def test_categorical_default_value_not_in_choices(): - """Test if a ValueError is raised when the default value is not in the choices - for a Categorical. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "default_value_not_in_choices_config.yaml") - assert excinfo.value.exception_type == "ValueError" - - -@pytest.mark.neps_api -def test_incorrect_fidelity_parameter_bounds(): - """Test if a ValueError is raised when the bounds of a fidelity parameter are - not correctly specified. - """ - with pytest.raises(SearchSpaceFromYamlFileError) as excinfo: - pipeline_space_from_yaml(BASE_PATH + "incorrect_fidelity_bounds_config.yaml") - assert excinfo.value.exception_type == "ValueError"