Skip to content

Commit

Permalink
refactor: set env vars before mlflow import
Browse files Browse the repository at this point in the history
  • Loading branch information
gmertes committed Nov 5, 2024
1 parent 77c26e6 commit a1d970f
Showing 1 changed file with 10 additions and 10 deletions.
20 changes: 10 additions & 10 deletions src/anemoi/training/diagnostics/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ def get_mlflow_logger(config: DictConfig) -> None:
LOGGER.debug("MLFlow logging is disabled.")
return None

# 35 retries allow for 1 hour of server downtime
http_max_retries = config.diagnostics.log.mlflow.get("http_max_retries", 35)

os.environ["MLFLOW_HTTP_REQUEST_MAX_RETRIES"] = str(http_max_retries)
os.environ["_MLFLOW_HTTP_REQUEST_MAX_RETRIES_LIMIT"] = str(http_max_retries + 1)

# these are the default values, but set them explicitly in case they change
os.environ["MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR"] = "2"
os.environ["MLFLOW_HTTP_REQUEST_BACKOFF_JITTER"] = "1"

from anemoi.training.diagnostics.mlflow.logger import AnemoiMLflowLogger

resumed = config.training.run_id is not None
Expand Down Expand Up @@ -55,16 +65,6 @@ def get_mlflow_logger(config: DictConfig) -> None:
)
log_hyperparams = False

# 35 retries allow for 1 hour of server downtime
http_max_retries = config.diagnostics.log.mlflow.get("http_max_retries", 35)

os.environ["MLFLOW_HTTP_REQUEST_MAX_RETRIES"] = str(http_max_retries)
os.environ["_MLFLOW_HTTP_REQUEST_MAX_RETRIES_LIMIT"] = str(http_max_retries + 1)

# these are the default values, but set them explicitly in case they change
os.environ["MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR"] = "2"
os.environ["MLFLOW_HTTP_REQUEST_BACKOFF_JITTER"] = "1"

LOGGER.info("AnemoiMLFlow logging to %s", tracking_uri)
logger = AnemoiMLflowLogger(
experiment_name=config.diagnostics.log.mlflow.experiment_name,
Expand Down

0 comments on commit a1d970f

Please sign in to comment.