Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Accélérer la récupération du statut "jobs are old" d'un cluster #184

Merged
merged 2 commits into from
Mar 26, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 27 additions & 22 deletions clockwork_web/core/users_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -592,33 +592,38 @@ def render_template_with_user_settings(template_name_or_list, **context):

# Get cluster status (if jobs are old and cluster has error).
for cluster_name in context["clusters"]:
# Default status values.
jobs_are_old = False
# Cluster error cannot yet be checked, so
# cluster_has_error is always False for now.
cluster_has_error = False
context["clusters"][cluster_name]["status"] = {
"jobs_are_old": _jobs_are_old(cluster_name),
"cluster_has_error": cluster_has_error,
}

return render_template(template_name_or_list, **context)


def _jobs_are_old(cluster_name):
"""Return True if last slurm update in given cluster is older than 2 days."""

jobs_are_old = False

# Check if jobs are old.
jobs, _ = get_jobs(cluster_names=[cluster_name])
job_dates = [
job["cw"]["last_slurm_update"]
for job in jobs
if "last_slurm_update" in job["cw"]
]
if job_dates:
most_recent_job_edition = max(job_dates)
mongodb_filter = {"slurm.cluster_name": cluster_name}
mc = get_db()
job_with_max_cw_last_slurm_update = list(
mc["jobs"].find(mongodb_filter).sort([("cw.last_slurm_update", -1)]).limit(1)
)

if job_with_max_cw_last_slurm_update:
(job,) = job_with_max_cw_last_slurm_update
if "last_slurm_update" in job["cw"]:
most_recent_job_edition = job["cw"]["last_slurm_update"]
current_timestamp = datetime.now().timestamp()
elapsed_time = timedelta(
seconds=current_timestamp - most_recent_job_edition
)
# Let's say the latest jobs edition must not be older than 30 days ago.
max_delay = timedelta(days=30)
# Let's say the latest jobs edition must not be older than max_delay.
max_delay = timedelta(days=2)
jobs_are_old = elapsed_time > max_delay

# Cluster error cannot yet be checked, so
# cluster_has_error is always False for now.

context["clusters"][cluster_name]["status"] = {
"jobs_are_old": jobs_are_old,
"cluster_has_error": cluster_has_error,
}

return render_template(template_name_or_list, **context)
return jobs_are_old
Loading