Merge pull request #185 from notoraptor/web-user-prop-dicts-in

Dictionnaires job-utilisateur visibles dans l'interface web

clockwork_frontend_test/test_jobs_search.py

@@ -640,6 +640,59 @@ def test_filter_by_job_array(page: Page):
     _check_jobs_table(page, JOBS_SEARCH_DEFAULT_TABLE)
 
 
+def test_filter_by_job_user_props(page: Page):
+    # Login
+    page.goto(f"{BASE_URL}/login/[email protected]")
+    # Go to settings.
+    page.goto(f"{BASE_URL}/settings/")
+    radio_job_user_props = page.locator("input#jobs_list_job_user_props_toggle")
+    expect(radio_job_user_props).to_be_checked(checked=False)
+    # Check column job_user_props.
+    radio_job_user_props.click()
+    expect(radio_job_user_props).to_be_checked(checked=True)
+    # Back to jobs/search.
+    page.goto(f"{BASE_URL}/jobs/search")
+
+    job_id = page.get_by_text("795002")
+    expect(job_id).to_have_count(1)
+    parent_row = page.locator("table#search_table tbody tr").filter(has=job_id)
+    expect(parent_row).to_have_count(1)
+    cols = parent_row.locator("td")
+    link_job_user_prop = cols.nth(4).locator("a")
+    expect(link_job_user_prop).to_have_count(1)
+    expect(link_job_user_prop).to_contain_text("name je suis une user prop 1")
+    link_job_user_prop.click()
+    expect(page).to_have_url(
+        f"{BASE_URL}/jobs/search?"
+        f"user_prop_name=name"
+        f"&user_prop_content=je+suis+une+user+prop+1"
+        f"&page_num=1"
+    )
+    _check_jobs_table(
+        page,
+        [
+            ["mila", "student06 @mila.quebec", "795002"],
+            ["graham", "student12 @mila.quebec", "613024"],
+        ],
+    )
+
+    filter_reset = page.get_by_title("Reset filter by job user prop")
+    expect(filter_reset).to_contain_text('User prop name: "je suis une user prop 1"')
+    filter_reset.click()
+
+    expect(page).to_have_url(
+        f"{BASE_URL}/jobs/search?user_prop_name=&user_prop_content=&page_num=1"
+    )
+    _check_jobs_table(page, JOBS_SEARCH_DEFAULT_TABLE)
+
+    # Back to default settings.
+    page.goto(f"{BASE_URL}/settings/")
+    radio_job_user_props = page.locator("input#jobs_list_job_user_props_toggle")
+    expect(radio_job_user_props).to_be_checked(checked=True)
+    radio_job_user_props.click()
+    expect(radio_job_user_props).to_be_checked(checked=False)
+
+
 def test_jobs_table_sorting_by_cluster(page: Page):
     _load_jobs_search_page(page)
     expected_content = [

clockwork_frontend_test/test_settings.py

@@ -223,3 +223,66 @@ def test_jobs_search_columns(page: Page):
     expect(headers.nth(7)).to_contain_text("Start time")
     expect(headers.nth(8)).to_contain_text("End time")
     expect(headers.nth(9)).to_contain_text("Links")
+
+
+def test_jobs_search_column_job_user_props(page: Page):
+    # Login
+    page.goto(f"{BASE_URL}/login/[email protected]")
+    # Check default jobs search columns.
+    page.goto(f"{BASE_URL}/jobs/search")
+    headers = page.locator("table#search_table thead tr th")
+    expect(headers).to_have_count(10)
+    expect(headers.nth(0)).to_contain_text("Cluster")
+    expect(headers.nth(1)).to_contain_text("User (@mila.quebec)")
+    expect(headers.nth(2)).to_contain_text("Job ID")
+    expect(headers.nth(3)).to_contain_text("Job array")
+    expect(headers.nth(4)).to_contain_text("Job name [:20]")
+    expect(headers.nth(5)).to_contain_text("Job state")
+    expect(headers.nth(6)).to_contain_text("Submit time")
+    expect(headers.nth(7)).to_contain_text("Start time")
+    expect(headers.nth(8)).to_contain_text("End time")
+    expect(headers.nth(9)).to_contain_text("Links")
+
+    # Go to settings.
+    page.goto(f"{BASE_URL}/settings/")
+    radio_job_user_props = page.locator("input#jobs_list_job_user_props_toggle")
+    expect(radio_job_user_props).to_be_checked(checked=False)
+    # Check column job_user_props.
+    radio_job_user_props.click()
+    expect(radio_job_user_props).to_be_checked(checked=True)
+
+    # Check column job_user_props is indeed now displayed in jobs/search.
+    page.goto(f"{BASE_URL}/jobs/search")
+    headers = page.locator("table#search_table thead tr th")
+    expect(headers).to_have_count(11)
+    expect(headers.nth(0)).to_contain_text("Cluster")
+    expect(headers.nth(1)).to_contain_text("User (@mila.quebec)")
+    expect(headers.nth(2)).to_contain_text("Job ID")
+    expect(headers.nth(3)).to_contain_text("Job array")
+    expect(headers.nth(4)).to_contain_text("Job-user props")
+    expect(headers.nth(5)).to_contain_text("Job name [:20]")
+    expect(headers.nth(6)).to_contain_text("Job state")
+    expect(headers.nth(7)).to_contain_text("Submit time")
+    expect(headers.nth(8)).to_contain_text("Start time")
+    expect(headers.nth(9)).to_contain_text("End time")
+    expect(headers.nth(10)).to_contain_text("Links")
+
+    # Back to default settings.
+    page.goto(f"{BASE_URL}/settings/")
+    radio_job_user_props = page.locator("input#jobs_list_job_user_props_toggle")
+    expect(radio_job_user_props).to_be_checked(checked=True)
+    radio_job_user_props.click()
+    expect(radio_job_user_props).to_be_checked(checked=False)
+    page.goto(f"{BASE_URL}/jobs/search")
+    headers = page.locator("table#search_table thead tr th")
+    expect(headers).to_have_count(10)
+    expect(headers.nth(0)).to_contain_text("Cluster")
+    expect(headers.nth(1)).to_contain_text("User (@mila.quebec)")
+    expect(headers.nth(2)).to_contain_text("Job ID")
+    expect(headers.nth(3)).to_contain_text("Job array")
+    expect(headers.nth(4)).to_contain_text("Job name [:20]")
+    expect(headers.nth(5)).to_contain_text("Job state")
+    expect(headers.nth(6)).to_contain_text("Submit time")
+    expect(headers.nth(7)).to_contain_text("Start time")
+    expect(headers.nth(8)).to_contain_text("End time")
+    expect(headers.nth(9)).to_contain_text("Links")

clockwork_web/browser_routes/jobs.py

@@ -101,6 +101,8 @@ def route_search():
     - "sort_asc" is an optional integer and used to specify if sorting is
       ascending (1) or descending (-1). Default is 1.
     - "job_array" is optional and used to specify the job array in which we are looking for jobs
+    - "user_prop_name" is optional and used to specify the user prop name associated to jobs we are looking for
+    - "user_prop_content" is optional and used to specify the user prop value associated to jobs we are looking for
 
     .. :quickref: list all Slurm job as formatted html
     """
@@ -164,6 +166,8 @@ def route_search():
                 "sort_by": query.sort_by,
                 "sort_asc": query.sort_asc,
                 "job_array": query.job_array,
+                "user_prop_name": query.user_prop_name,
+                "user_prop_content": query.user_prop_content,
             },
         )
 

clockwork_web/core/jobs_helper.py

@@ -7,6 +7,7 @@
 import time
 
 from flask.globals import current_app
+from flask_login import current_user
 from ..db import get_db
 
 
@@ -157,6 +158,44 @@ def get_filtered_and_paginated_jobs(
         # on the server because not enough memory was allocated to perform the sorting.
         LD_jobs = list(mc["jobs"].find(mongodb_filter))
 
+    # Get job user props
+    if LD_jobs and current_user:
+        user_props_map = {}
+        # Collect all job user props related to found jobs,
+        # and store them in a dict with keys (mila email username, job ID, cluster_name)
+        for user_props in list(
+            mc["job_user_props"].find(
+                combine_all_mongodb_filters(
+                    {
+                        "job_id": {
+                            "$in": [int(job["slurm"]["job_id"]) for job in LD_jobs]
+                        },
+                        "mila_email_username": current_user.mila_email_username,
+                    }
+                )
+            )
+        ):
+            key = (
+                user_props["mila_email_username"],
+                user_props["job_id"],
+                user_props["cluster_name"],
+            )
+            assert key not in user_props_map
+            user_props_map[key] = user_props["props"]
+
+        if user_props_map:
+            # Populate jobs with user props using
+            # current user email, job ID and job cluster name
+            # to find related user props in props map.
+            for job in LD_jobs:
+                key = (
+                    current_user.mila_email_username,
+                    int(job["slurm"]["job_id"]),
+                    job["slurm"]["cluster_name"],
+                )
+                if key in user_props_map:
+                    job["job_user_props"] = user_props_map[key]
+
     # Set nbr_total_jobs
     if want_count:
         # Get the number of filtered jobs (not paginated)
@@ -235,6 +274,8 @@ def get_jobs(
     sort_by="submit_time",
     sort_asc=-1,
     job_array=None,
+    user_prop_name=None,
+    user_prop_content=None,
 ):
     """
     Set up the filters according to the parameters and retrieve the requested jobs from the database.
@@ -252,13 +293,33 @@ def get_jobs(
         sort_asc                Whether or not to sort in ascending order (1)
                                 or descending order (-1).
         job_array               ID of job array in which we look for jobs.
+        user_prop_name          name of user prop (string) we must find in jobs to look for.
+        user_prop_content       content of user prop (string) we must find in jobs to look for.
 
     Returns:
         A tuple containing:
             - the list of jobs as first entity
             - the total number of jobs corresponding of the filters in the databse, if want_count has been set to
             True, None otherwise, as second element
     """
+    # If job user prop is specified,
+    # get job indices from jobs associated to this prop.
+    if user_prop_name is not None and user_prop_content is not None:
+        mc = get_db()
+        props_job_ids = [
+            str(user_props["job_id"])
+            for user_props in mc["job_user_props"].find(
+                combine_all_mongodb_filters(
+                    {f"props.{user_prop_name}": user_prop_content}
+                )
+            )
+        ]
+        if job_ids:
+            # If job ids where provided, make intersection between given job ids and props job ids.
+            job_ids = list(set(props_job_ids) & set(job_ids))
+        else:
+            # Otherwise, just use props job ids.
+            job_ids = props_job_ids
 
     # Set up and combine filters
     filter = get_global_filter(
@@ -405,6 +466,7 @@ def get_jobs_properties_list_per_page():
             "user",
             "job_id",
             "job_array",
+            "job_user_props",
             "job_name",
             "job_state",
             "start_time",

clockwork_web/core/search_helper.py

@@ -21,6 +21,8 @@ def parse_search_request(user, args, force_pagination=True):
     want_count = to_boolean(want_count)
 
     job_array = args.get("job_array", type=int, default=None)
+    user_prop_name = args.get("user_prop_name", type=str, default=None) or None
+    user_prop_content = args.get("user_prop_content", type=str, default=None) or None
 
     default_page_number = "1" if force_pagination else None
 
@@ -71,6 +73,8 @@ def parse_search_request(user, args, force_pagination=True):
         sort_asc=sort_asc,
         want_count=want_count,
         job_array=job_array,
+        user_prop_name=user_prop_name,
+        user_prop_content=user_prop_content,
     )
 
     #########################
@@ -115,5 +119,7 @@ def search_request(user, args, force_pagination=True):
         sort_by=query.sort_by,
         sort_asc=query.sort_asc,
         job_array=query.job_array,
+        user_prop_name=query.user_prop_name,
+        user_prop_content=query.user_prop_content,
     )
     return (query, jobs, nbr_total_jobs)

clockwork_web/core/users_helper.py

@@ -605,7 +605,6 @@ def render_template_with_user_settings(template_name_or_list, **context):
 
 def _jobs_are_old(cluster_name):
     """Return True if last slurm update in given cluster is older than 2 days."""
-
     jobs_are_old = False
 
     mongodb_filter = {"slurm.cluster_name": cluster_name}

clockwork_web/templates/base.html

@@ -24,7 +24,7 @@
 			<script src="https://cdnjs.cloudflare.com/ajax/libs/jqueryui/1.13.2/jquery-ui.min.js" integrity="sha512-57oZ/vW8ANMjR/KQ6Be9v/+/h6bq9/l3f0Oc7vn6qMqyhvPd1cvKBRWWpzu0QoneImqr2SkmO4MSqU+RpHom3Q==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 
 			<script type="text/javascript" src="{{ url_for('static', filename='js/sortable.min.js') }}"></script>
-			
+
 			<script type="text/javascript" src="{{ url_for('static', filename='js/moment-with-locales.min.js') }}"></script>
 			<script type="text/javascript" src="{{ url_for('static', filename='js/livestamp.min.js') }}"></script>
 
@@ -323,6 +323,12 @@ <h1><a data-bs-toggle="collapse" data-bs-target=".formCollapse" aria-expanded="f
                                 <input type="hidden" name="sort_asc" value="{{ previous_request_args['sort_asc'] }}"/>
                                 {% if previous_request_args['job_array'] is not none %}
                                 <input type="hidden" name="job_array" value="{{ previous_request_args['job_array'] }}"/>
+                                {% endif %}
+                                {% if previous_request_args['user_prop_name'] is not none %}
+                                <input type="hidden" name="user_prop_name" value="{{ previous_request_args['user_prop_name'] }}"/>
+                                {% endif %}
+                                {% if previous_request_args['user_prop_content'] is not none %}
+                                <input type="hidden" name="user_prop_content" value="{{ previous_request_args['user_prop_content'] }}"/>
                                 {% endif %}
 
 				                 <div class="row align-items-center">
@@ -334,6 +340,13 @@ <h1><a data-bs-toggle="collapse" data-bs-target=".formCollapse" aria-expanded="f
 											 <i class="fa-solid fa-circle-xmark" style="color: #888a85;"></i>
 										 </a>
 										 {% endif %}
+
+										 {% if previous_request_args['user_prop_name'] is not none and previous_request_args['user_prop_content'] is not none %}
+										 <a href="{{ modify_query(user_prop_name='', user_prop_content='') }}" title="Reset filter by job user prop" class="px-3 py-2">
+											 User prop <strong>{{ previous_request_args['user_prop_name'] }}</strong>: "{{ previous_request_args['user_prop_content'] }}"&nbsp;&nbsp;&nbsp;&nbsp;
+											 <i class="fa-solid fa-circle-xmark" style="color: #888a85;"></i>
+										 </a>
+										 {% endif %}
 									 </div>
 				                    <!-- button -->
 				                    <div class="col-sm-12 col-md-4 offset-md-8">

clockwork_web/templates/jobs_search.html

@@ -101,6 +101,10 @@ <h1>JOBS</h1>
                             {% if (web_settings | check_web_settings_column_display(page_name, "job_array")) %}
                                 <th>Job array</th>
                             {% endif %}
+                            <!-- Job user props header -->
+                            {% if (web_settings | check_web_settings_column_display(page_name, "job_user_props")) %}
+                                <th>Job-user props</th>
+                            {% endif %}
                             <!-- Job name header -->
                             {% if (web_settings | check_web_settings_column_display(page_name, "job_name")) %}
                                 {% set sort_by = "name" %}
@@ -193,6 +197,20 @@ <h1>JOBS</h1>
                                 </td>
                             {% endif %}
 
+                            <!-- Job user props -->
+                            {% if (web_settings | check_web_settings_column_display(page_name, "job_user_props")) %}
+                                <td>
+                                    {% for D_user_prop_name, D_user_prop_content in D_job.get('job_user_props', {}).items() %}
+                                    <p>
+                                    <a href="{{ modify_query(user_prop_name=D_user_prop_name, user_prop_content=D_user_prop_content, page_num=1) }}" title="Filter by job-user prop &quot;{{D_user_prop_name}}&quot;: &quot;{{D_user_prop_content}}&quot;">
+                                        <strong>{{ D_user_prop_name }}</strong><br/>
+                                        {{ D_user_prop_content }}
+                                    </a>
+                                    </p>
+                                    {% endfor %}
+                                </td>
+                            {% endif %}
+
                             <!-- Job name -->
                             {% if (web_settings | check_web_settings_column_display(page_name, "job_name")) %}
                                 <td>{{D_job['slurm'].get("name", "")[0:20]}}</td>

clockwork_web/templates/settings.html

@@ -279,6 +279,7 @@ <h1>{{ gettext("User settings %(mila_email_username)s", mila_email_username=curr
 		                            <th>{{ gettext("User (@mila.quebec)") }}</th>
 		                            <th>{{ gettext("Job ID") }}</th>
 		                            <th>{{ gettext("Job array") }}</th>
+		                            <th>{{ gettext("Job-user props") }}</th>
 		                            <th>{{ gettext("Job name [:20]") }}</th>
 		                            <th>{{ gettext("Job state") }}</th>
 		                            <th>{{ gettext("Submit time") }}</th>
@@ -291,7 +292,7 @@ <h1>{{ gettext("User settings %(mila_email_username)s", mila_email_username=curr
 		                    <tbody>
 		                    	<tr>
 		                    		{% set page_name = "jobs_list" %}
-									{% for column_name in ["clusters", "user","job_id", "job_array", "job_name", "job_state", "submit_time", "start_time", "end_time", "links"] %}
+									{% for column_name in ["clusters", "user","job_id", "job_array", "job_user_props", "job_name", "job_state", "submit_time", "start_time", "end_time", "links"] %}
 		                    		<td><div class="form-check form-switch">
 										{% if (web_settings | check_web_settings_column_display(page_name, column_name)) %}
 											<input name="{{page_name}}_{{column_name}}_toggle" id="{{page_name}}_{{column_name}}_toggle" type="checkbox" class="form-check-input" onclick="switch_column_setting('{{page_name}}', '{{column_name}}')" checked />

clockwork_web/user.py

@@ -93,6 +93,10 @@ def boolean(value):
         self.web_settings.setdefault("column_display", {}).setdefault("jobs_list", {})[
             "actions"
         ] = False
+        # By default, do not display column "job_user_props" in job tables.
+        self.web_settings.setdefault("column_display", {}).setdefault(
+            "jobs_list", {}
+        ).setdefault("job_user_props", False)
 
     def get_id(self):
         return self.mila_email_username