From 5c0e27ce3bab674588753c20de08933dc0cdbccb Mon Sep 17 00:00:00 2001 From: notoraptor Date: Fri, 6 Oct 2023 10:43:05 -0400 Subject: [PATCH 1/2] Add a specific improved function _jobs_are_old(cluster_name) to tell if cluster jobs are old. Much faster to use than get_jobs(). --- clockwork_web/core/users_helper.py | 45 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/clockwork_web/core/users_helper.py b/clockwork_web/core/users_helper.py index 86862e64..f829734a 100644 --- a/clockwork_web/core/users_helper.py +++ b/clockwork_web/core/users_helper.py @@ -592,19 +592,32 @@ def render_template_with_user_settings(template_name_or_list, **context): # Get cluster status (if jobs are old and cluster has error). for cluster_name in context["clusters"]: - # Default status values. - jobs_are_old = False + # Cluster error cannot yet be checked, so + # cluster_has_error is always False for now. cluster_has_error = False + context["clusters"][cluster_name]["status"] = { + "jobs_are_old": _jobs_are_old(cluster_name), + "cluster_has_error": cluster_has_error, + } + + return render_template(template_name_or_list, **context) + + +def _jobs_are_old(cluster_name): + """Return True if last slurm update in given cluster is older than 30 days.""" + + jobs_are_old = False - # Check if jobs are old. - jobs, _ = get_jobs(cluster_names=[cluster_name]) - job_dates = [ - job["cw"]["last_slurm_update"] - for job in jobs - if "last_slurm_update" in job["cw"] - ] - if job_dates: - most_recent_job_edition = max(job_dates) + mongodb_filter = {"slurm.cluster_name": cluster_name} + mc = get_db() + job_with_max_cw_last_slurm_update = list( + mc["jobs"].find(mongodb_filter).sort([("cw.last_slurm_update", -1)]).limit(1) + ) + + if job_with_max_cw_last_slurm_update: + (job,) = job_with_max_cw_last_slurm_update + if "last_slurm_update" in job["cw"]: + most_recent_job_edition = job["cw"]["last_slurm_update"] current_timestamp = datetime.now().timestamp() elapsed_time = timedelta( seconds=current_timestamp - most_recent_job_edition @@ -613,12 +626,4 @@ def render_template_with_user_settings(template_name_or_list, **context): max_delay = timedelta(days=30) jobs_are_old = elapsed_time > max_delay - # Cluster error cannot yet be checked, so - # cluster_has_error is always False for now. - - context["clusters"][cluster_name]["status"] = { - "jobs_are_old": jobs_are_old, - "cluster_has_error": cluster_has_error, - } - - return render_template(template_name_or_list, **context) + return jobs_are_old From 32be0752254c1b87208998e1a9f3246f892d6005 Mon Sep 17 00:00:00 2001 From: notoraptor Date: Thu, 21 Mar 2024 16:37:12 -0400 Subject: [PATCH 2/2] Set delay to 2 days. --- clockwork_web/core/users_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clockwork_web/core/users_helper.py b/clockwork_web/core/users_helper.py index f829734a..47b5fd74 100644 --- a/clockwork_web/core/users_helper.py +++ b/clockwork_web/core/users_helper.py @@ -604,7 +604,7 @@ def render_template_with_user_settings(template_name_or_list, **context): def _jobs_are_old(cluster_name): - """Return True if last slurm update in given cluster is older than 30 days.""" + """Return True if last slurm update in given cluster is older than 2 days.""" jobs_are_old = False @@ -622,8 +622,8 @@ def _jobs_are_old(cluster_name): elapsed_time = timedelta( seconds=current_timestamp - most_recent_job_edition ) - # Let's say the latest jobs edition must not be older than 30 days ago. - max_delay = timedelta(days=30) + # Let's say the latest jobs edition must not be older than max_delay. + max_delay = timedelta(days=2) jobs_are_old = elapsed_time > max_delay return jobs_are_old