Allow to display job-user dicts

mila-iqia · Feb 16, 2024 · 1f44065 · 1f44065
1 parent 28e4c18
commit 1f44065
Show file tree

Hide file tree

Showing 7 changed files with 68 additions and 40 deletions.
diff --git a/clockwork_web/browser_routes/jobs.py b/clockwork_web/browser_routes/jobs.py
@@ -101,7 +101,8 @@ def route_search():
     - "sort_asc" is an optional integer and used to specify if sorting is
       ascending (1) or descending (-1). Default is 1.
     - "job_array" is optional and used to specify the job array in which we are looking for jobs
-    - "job_label" is optional and used to specify the label associated to jobs we are looking for
+    - "job_label_name" is optional and used to specify the label name associated to jobs we are looking for
+    - "job_label_content" is optional and used to specify the label value associated to jobs we are looking for
 
     .. :quickref: list all Slurm job as formatted html
     """
@@ -165,7 +166,8 @@ def route_search():
                 "sort_by": query.sort_by,
                 "sort_asc": query.sort_asc,
                 "job_array": query.job_array,
-                "job_label": query.job_label,
+                "job_label_name": query.job_label_name,
+                "job_label_content": query.job_label_content,
             },
         )
 

diff --git a/clockwork_web/core/jobs_helper.py b/clockwork_web/core/jobs_helper.py
@@ -161,7 +161,7 @@ def get_filtered_and_paginated_jobs(
     if LD_jobs:
         label_map = {}
         # Collect all labels related to found jobs,
-        # and store them in a dict with keys (user ID, job ID)
+        # and store them in a dict with keys (user ID, job ID, cluster_name)
         for label in list(
             mc["labels"].find(
                 combine_all_mongodb_filters(
@@ -175,10 +175,13 @@ def get_filtered_and_paginated_jobs(
         ):
             # Remove MongoDB identifier, as we won't use it.
             label.pop("_id")
-            label_map.setdefault((label["user_id"], label["job_id"]), []).append(label)
-        # Populate jobs with labels using job's user email and job ID to find related labels in labels dict.
+            key = (label["user_id"], label["job_id"], label["cluster_name"])
+            assert key not in label_map
+            label_map[key] = label["labels"]
+        # Populate jobs with labels using job's user email,  job ID and cluster name
+        # to find related labels in labels dict.
         for job in LD_jobs:
-            key = (job["cw"]["mila_email_username"], int(job["slurm"]["job_id"]))
+            key = (job["cw"]["mila_email_username"], int(job["slurm"]["job_id"]), job["slurm"]["cluster_name"])
             if key in label_map:
                 job["job_labels"] = label_map[key]
 
@@ -260,7 +263,8 @@ def get_jobs(
     sort_by="submit_time",
     sort_asc=-1,
     job_array=None,
-    job_label=None,
+    job_label_name=None,
+    job_label_content=None,
 ):
     """
     Set up the filters according to the parameters and retrieve the requested jobs from the database.
@@ -278,7 +282,8 @@ def get_jobs(
         sort_asc                Whether or not to sort in ascending order (1)
                                 or descending order (-1).
         job_array               ID of job array in which we look for jobs.
-        job_label               label (string) we must find in jobs to look for.
+        job_label_name          name of label (string) we must find in jobs to look for.
+        job_label_content       content of label (string) we must find in jobs to look for.
 
     Returns:
         A tuple containing:
@@ -288,12 +293,12 @@ def get_jobs(
     """
     # If job label is specified,
     # get job indices from jobs associated to this label.
-    if job_label is not None:
+    if job_label_name is not None and job_label_content is not None:
         mc = get_db()
         label_job_ids = [
             str(label["job_id"])
             for label in mc["labels"].find(
-                combine_all_mongodb_filters({"name": job_label})
+                combine_all_mongodb_filters({f"labels.{job_label_name}": job_label_content})
             )
         ]
         if job_ids:

diff --git a/clockwork_web/core/search_helper.py b/clockwork_web/core/search_helper.py
@@ -21,7 +21,8 @@ def parse_search_request(user, args, force_pagination=True):
     want_count = to_boolean(want_count)
 
     job_array = args.get("job_array", type=int, default=None)
-    job_label = args.get("job_label", type=str, default=None) or None
+    job_label_name = args.get("job_label_name", type=str, default=None) or None
+    job_label_content = args.get("job_label_content", type=str, default=None) or None
 
     default_page_number = "1" if force_pagination else None
 
@@ -72,7 +73,8 @@ def parse_search_request(user, args, force_pagination=True):
         sort_asc=sort_asc,
         want_count=want_count,
         job_array=job_array,
-        job_label=job_label,
+        job_label_name=job_label_name,
+        job_label_content=job_label_content,
     )
 
     #########################
@@ -117,6 +119,7 @@ def search_request(user, args, force_pagination=True):
         sort_by=query.sort_by,
         sort_asc=query.sort_asc,
         job_array=query.job_array,
-        job_label=query.job_label,
+        job_label_name=query.job_label_name,
+        job_label_content=query.job_label_content,
     )
     return (query, jobs, nbr_total_jobs)
diff --git a/clockwork_web/templates/base.html b/clockwork_web/templates/base.html
@@ -324,8 +324,11 @@ <h1><a data-bs-toggle="collapse" data-bs-target=".formCollapse" aria-expanded="f
                                 {% if previous_request_args['job_array'] is not none %}
                                 <input type="hidden" name="job_array" value="{{ previous_request_args['job_array'] }}"/>
                                 {% endif %}
-                                {% if previous_request_args['job_label'] is not none %}
-                                <input type="hidden" name="job_label" value="{{ previous_request_args['job_label'] }}"/>
+                                {% if previous_request_args['job_label_name'] is not none %}
+                                <input type="hidden" name="job_label_name" value="{{ previous_request_args['job_label_name'] }}"/>
+                                {% endif %}
+                                {% if previous_request_args['job_label_content'] is not none %}
+                                <input type="hidden" name="job_label_content" value="{{ previous_request_args['job_label_content'] }}"/>
                                 {% endif %}
 
 				                 <div class="row align-items-center">
@@ -338,9 +341,9 @@ <h1><a data-bs-toggle="collapse" data-bs-target=".formCollapse" aria-expanded="f
 										 </a>
 										 {% endif %}
 
-										 {% if previous_request_args['job_label'] is not none %}
-										 <a href="{{ modify_query(job_label='') }}" title="Reset filter by job label" class="px-3 py-2">
-											 Label "{{ previous_request_args['job_label'] }}"&nbsp;&nbsp;&nbsp;&nbsp;
+										 {% if previous_request_args['job_label_name'] is not none and previous_request_args['job_label_content'] is not none %}
+										 <a href="{{ modify_query(job_label_name='', job_label_content='') }}" title="Reset filter by job label" class="px-3 py-2">
+											 Label <strong>{{ previous_request_args['job_label_name'] }}</strong>: "{{ previous_request_args['job_label_content'] }}"&nbsp;&nbsp;&nbsp;&nbsp;
 											 <i class="fa-solid fa-circle-xmark" style="color: #888a85;"></i>
 										 </a>
 										 {% endif %}

diff --git a/clockwork_web/templates/jobs_search.html b/clockwork_web/templates/jobs_search.html
@@ -200,10 +200,11 @@ <h1>JOBS</h1>
                             <!-- Job labels -->
                             {% if (web_settings | check_web_settings_column_display(page_name, "job_labels")) %}
                                 <td>
-                                    {% for D_label in D_job.get('job_labels', []) %}
+                                    {% for D_label_name, D_label_content in D_job.get('job_labels', {}).items() %}
                                     <p>
-                                    <a href="{{ modify_query(job_label=D_label['name'], page_num=1) }}" title="Filter by job label &quot;{{D_label['name']}}&quot;">
-                                        {{ D_label['name'] }}
+                                    <a href="{{ modify_query(job_label_name=D_label_name, job_label_content=D_label_content, page_num=1) }}" title="Filter by job label &quot;{{D_label_name}}&quot;: &quot;{{D_label_content}}&quot;">
+                                        <strong>{{ D_label_name }}</strong><br/>
+                                        {{ D_label_content }}
                                     </a>
                                     </p>
                                     {% endfor %}

diff --git a/test_common/fake_data.json b/test_common/fake_data.json
@@ -5965,34 +5965,45 @@
   ],
   "labels": [
     {
-      "name": "je suis un label 1",
       "user_id": "[email protected]",
-      "job_id": 795002
+      "job_id": 795002,
+      "cluster_name": "mila",
+      "labels": {
+        "name": "je suis un label 1"
+      }
     },
     {
-      "name": "je suis un label 2",
       "user_id": "[email protected]",
-      "job_id": 606872
-    },
-    {
-      "name": "je suis un label 3",
-      "user_id": "[email protected]",
-      "job_id": 834395
+      "job_id": 606872,
+      "cluster_name": "mila",
+      "labels": {
+        "name": "je suis un label 2"
+      }
     },
     {
-      "name": "je suis un label 3",
       "user_id": "[email protected]",
-      "job_id": 154325
+      "job_id": 834395,
+      "cluster_name": "graham",
+      "labels": {
+        "name": "je suis un label 3"
+      }
     },
     {
-      "name": "je suis un label 4",
       "user_id": "[email protected]",
-      "job_id": 154325
+      "job_id": 154325,
+      "cluster_name": "graham",
+      "labels": {
+        "name": "je suis un label 3",
+        "name2": "je suis un label 4"
+      }
     },
     {
-      "name": "je suis un label 1",
       "user_id": "[email protected]",
-      "job_id": 613024
+      "job_id": 613024,
+      "cluster_name": "graham",
+      "labels": {
+        "name": "je suis un label 1"
+      }
     }
   ]
 }
diff --git a/test_common/fake_data.py b/test_common/fake_data.py
@@ -25,9 +25,10 @@ def fake_data():
     for job in E["jobs"]:
         job_id = int(job["slurm"]["job_id"])
         user_id = job["cw"]["mila_email_username"]
+        cluster_name = job["slurm"]["cluster_name"]
         for label in E["labels"]:
-            if label["job_id"] == job_id and label["user_id"] == user_id:
-                job.setdefault("job_labels", []).append(label)
+            if label["job_id"] == job_id and label["user_id"] == user_id and label["cluster_name"] == cluster_name:
+                job["job_labels"] = label["labels"]
 
     mutate_some_job_status(E)
     return E
@@ -80,7 +81,7 @@ def populate_fake_data(db_insertion_point, json_file=None, mutate=False):
     )
     db_insertion_point["gpu"].create_index([("name", 1)], name="gpu_name")
     db_insertion_point["labels"].create_index(
-        [("user_id", 1), ("job_id", 1), ("name", 1)], name="job_label_index"
+        [("user_id", 1), ("job_id", 1), ("cluster_name", 1), ("labels", 1)], name="job_label_index"
     )
 
     for k in ["users", "jobs", "nodes", "gpu", "labels"]:
@@ -109,7 +110,9 @@ def cleanup_function():
             db_insertion_point["gpu"].delete_many({"name": e["name"]})
 
         for e in E["labels"]:
-            db_insertion_point["labels"].delete_many({"name": e["name"]})
+            copy_e = e
+            copy_e.pop("labels")
+            db_insertion_point["labels"].delete_many(copy_e)
 
         for (k, sub, id_field) in [
             ("jobs", "slurm", "job_id"),