-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
various storage and config modifications
- Loading branch information
Showing
37 changed files
with
714 additions
and
532 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,120 +1,4 @@ | ||
| import sys | ||
| import typer | ||
| import logging | ||
| import logging.handlers | ||
| from medperf.init import initialize | ||
|
|
||
| from medperf import __version__ | ||
| import medperf.config as config | ||
| from medperf.ui.factory import UIFactory | ||
| from medperf.decorators import clean_except, configurable | ||
| from medperf.comms.factory import CommsFactory | ||
| import medperf.comms.auth as auth_module | ||
| import medperf.commands.result.result as result | ||
| from medperf.commands.result.create import BenchmarkExecution | ||
| from medperf.commands.result.submit import ResultSubmission | ||
| import medperf.commands.mlcube.mlcube as mlcube | ||
| import medperf.commands.dataset.dataset as dataset | ||
| import medperf.commands.auth.auth as auth | ||
| import medperf.commands.benchmark.benchmark as benchmark | ||
| import medperf.commands.profile as profile | ||
| from medperf.utils import ( | ||
| set_custom_config, | ||
| set_unique_tmp_config, | ||
| init_storage, | ||
| setup_logging, | ||
| ) | ||
| import medperf.commands.association.association as association | ||
| import medperf.commands.compatibility_test.compatibility_test as compatibility_test | ||
|
|
||
|
|
||
| app = typer.Typer() | ||
| app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes") | ||
| app.add_typer(result.app, name="result", help="Manage results") | ||
| app.add_typer(dataset.app, name="dataset", help="Manage datasets") | ||
| app.add_typer(benchmark.app, name="benchmark", help="Manage benchmarks") | ||
| app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes") | ||
| app.add_typer(result.app, name="result", help="Manage results") | ||
| app.add_typer(association.app, name="association", help="Manage associations") | ||
| app.add_typer(profile.app, name="profile", help="Manage profiles") | ||
| app.add_typer(compatibility_test.app, name="test", help="Manage compatibility tests") | ||
| app.add_typer(auth.app, name="auth", help="Authentication") | ||
|
|
||
|
|
||
| @app.command("run") | ||
| @clean_except | ||
| def execute( | ||
| benchmark_uid: int = typer.Option( | ||
| ..., "--benchmark", "-b", help="UID of the desired benchmark" | ||
| ), | ||
| data_uid: int = typer.Option( | ||
| ..., "--data_uid", "-d", help="Registered Dataset UID" | ||
| ), | ||
| model_uid: int = typer.Option( | ||
| ..., "--model_uid", "-m", help="UID of model to execute" | ||
| ), | ||
| approval: bool = typer.Option(False, "-y", help="Skip approval step"), | ||
| ignore_model_errors: bool = typer.Option( | ||
| False, | ||
| "--ignore-model-errors", | ||
| help="Ignore failing model cubes, allowing for possibly submitting partial results", | ||
| ), | ||
| no_cache: bool = typer.Option( | ||
| False, | ||
| "--no-cache", | ||
| help="Ignore existing results. The experiment then will be rerun", | ||
| ), | ||
| ): | ||
| """Runs the benchmark execution step for a given benchmark, prepared dataset and model""" | ||
| result = BenchmarkExecution.run( | ||
| benchmark_uid, | ||
| data_uid, | ||
| [model_uid], | ||
| ignore_model_errors=ignore_model_errors, | ||
| no_cache=no_cache, | ||
| )[0] | ||
| if result.id: # TODO: use result.is_registered once PR #338 is merged | ||
| config.ui.print( # TODO: msg should be colored yellow | ||
| """An existing registered result for the requested execution has been\n | ||
| found. If you wish to submit a new result for the same execution,\n | ||
| please run the command again with the --no-cache option.\n""" | ||
| ) | ||
| else: | ||
| ResultSubmission.run(result.generated_uid, approved=approval) | ||
| config.ui.print("✅ Done!") | ||
|
|
||
|
|
||
| def version_callback(value: bool): | ||
| if value: | ||
| print(f"MedPerf version {__version__}") | ||
| raise typer.Exit() | ||
|
|
||
|
|
||
| @app.callback() | ||
| @configurable | ||
| def main( | ||
| ctx: typer.Context, | ||
| version: bool = typer.Option( | ||
| None, "--version", callback=version_callback, is_eager=True | ||
| ), | ||
| ): | ||
| # Set inline parameters | ||
| inline_args = ctx.params | ||
| set_custom_config(inline_args) | ||
| set_unique_tmp_config() | ||
|
|
||
| init_storage() | ||
| log = config.loglevel.upper() | ||
| log_lvl = getattr(logging, log) | ||
| setup_logging(log_lvl) | ||
| logging.info(f"Running MedPerf v{__version__} on {log_lvl} logging level") | ||
| logging.info(f"Executed command: {' '.join(sys.argv[1:])}") | ||
|
|
||
| config.ui = UIFactory.create_ui(config.ui) | ||
| config.comms = CommsFactory.create_comms(config.comms, config.server) | ||
| auth_class = getattr(auth_module, config.auth_class) | ||
| config.auth = auth_class() | ||
| config.ui.print(f"MedPerf {__version__}") | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| app() | ||
| initialize() | ||
| from medperf.cli import app # noqa |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| import sys | ||
| import typer | ||
| import logging | ||
| import logging.handlers | ||
|
|
||
| from medperf import __version__ | ||
| import medperf.config as config | ||
| from medperf.decorators import clean_except, add_inline_parameters | ||
| import medperf.commands.result.result as result | ||
| from medperf.commands.result.create import BenchmarkExecution | ||
| from medperf.commands.result.submit import ResultSubmission | ||
| import medperf.commands.mlcube.mlcube as mlcube | ||
| import medperf.commands.dataset.dataset as dataset | ||
| import medperf.commands.auth.auth as auth | ||
| import medperf.commands.benchmark.benchmark as benchmark | ||
| import medperf.commands.profile as profile | ||
| import medperf.commands.association.association as association | ||
| import medperf.commands.compatibility_test.compatibility_test as compatibility_test | ||
| import medperf.commands.storage as storage | ||
|
|
||
| app = typer.Typer() | ||
| app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes") | ||
| app.add_typer(result.app, name="result", help="Manage results") | ||
| app.add_typer(dataset.app, name="dataset", help="Manage datasets") | ||
| app.add_typer(benchmark.app, name="benchmark", help="Manage benchmarks") | ||
| app.add_typer(mlcube.app, name="mlcube", help="Manage mlcubes") | ||
| app.add_typer(result.app, name="result", help="Manage results") | ||
| app.add_typer(association.app, name="association", help="Manage associations") | ||
| app.add_typer(profile.app, name="profile", help="Manage profiles") | ||
| app.add_typer(compatibility_test.app, name="test", help="Manage compatibility tests") | ||
| app.add_typer(auth.app, name="auth", help="Authentication") | ||
| app.add_typer(storage.app, name="storage", help="Storage management") | ||
|
|
||
|
|
||
| @app.command("run") | ||
| @clean_except | ||
| def execute( | ||
| benchmark_uid: int = typer.Option( | ||
| ..., "--benchmark", "-b", help="UID of the desired benchmark" | ||
| ), | ||
| data_uid: int = typer.Option( | ||
| ..., "--data_uid", "-d", help="Registered Dataset UID" | ||
| ), | ||
| model_uid: int = typer.Option( | ||
| ..., "--model_uid", "-m", help="UID of model to execute" | ||
| ), | ||
| approval: bool = typer.Option(False, "-y", help="Skip approval step"), | ||
| ignore_model_errors: bool = typer.Option( | ||
| False, | ||
| "--ignore-model-errors", | ||
| help="Ignore failing model cubes, allowing for possibly submitting partial results", | ||
| ), | ||
| no_cache: bool = typer.Option( | ||
| False, | ||
| "--no-cache", | ||
| help="Ignore existing results. The experiment then will be rerun", | ||
| ), | ||
| ): | ||
| """Runs the benchmark execution step for a given benchmark, prepared dataset and model""" | ||
| result = BenchmarkExecution.run( | ||
| benchmark_uid, | ||
| data_uid, | ||
| [model_uid], | ||
| ignore_model_errors=ignore_model_errors, | ||
| no_cache=no_cache, | ||
| )[0] | ||
| if result.id: # TODO: use result.is_registered once PR #338 is merged | ||
| config.ui.print( # TODO: msg should be colored yellow | ||
| """An existing registered result for the requested execution has been\n | ||
| found. If you wish to submit a new result for the same execution,\n | ||
| please run the command again with the --no-cache option.\n""" | ||
| ) | ||
| else: | ||
| ResultSubmission.run(result.generated_uid, approved=approval) | ||
| config.ui.print("✅ Done!") | ||
|
|
||
|
|
||
| def version_callback(value: bool): | ||
| if value: | ||
| print(f"MedPerf version {__version__}") | ||
| raise typer.Exit() | ||
|
|
||
|
|
||
| @app.callback() | ||
| @add_inline_parameters | ||
| def main( | ||
| ctx: typer.Context, | ||
| version: bool = typer.Option( | ||
| None, "--version", callback=version_callback, is_eager=True | ||
| ), | ||
| ): | ||
| # Set inline parameters | ||
| inline_args = ctx.params | ||
| for param in inline_args: | ||
| setattr(config, param, inline_args[param]) | ||
|
|
||
| # Update logging level according to the passed inline params | ||
| loglevel = config.loglevel.upper() | ||
| logging.getLogger().setLevel(loglevel) | ||
| logging.getLogger("requests").setLevel(loglevel) | ||
|
|
||
| logging.info(f"Running MedPerf v{__version__} on {loglevel} logging level") | ||
| logging.info(f"Executed command: {' '.join(sys.argv[1:])}") | ||
|
|
||
| config.ui.print(f"MedPerf {__version__}") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| import typer | ||
|
|
||
| from medperf import config | ||
| from medperf.decorators import clean_except | ||
| from medperf.utils import cleanup | ||
| from medperf.storage.utils import move_storage | ||
|
|
||
| app = typer.Typer() | ||
|
|
||
|
|
||
| @app.command("move") | ||
| @clean_except | ||
| def move(path: str = typer.Option(..., "--target", "-t", help="Target path")): | ||
| """Moves all storage folders to a target path. Folders include: | ||
| Benchmarks, datasets, mlcubes, results, tests, ... | ||
| Args: | ||
| path (str): target path | ||
| """ | ||
| move_storage(path) | ||
|
|
||
|
|
||
| @app.command("cleanup") | ||
| def clean(): | ||
| """Cleans up clutter paths""" | ||
|
|
||
| # Force cleanup to be true | ||
| config.cleanup = True | ||
| cleanup() |
Oops, something went wrong.