Remove slurm_openstack_tools collection

ansible/.gitignore

@@ -76,3 +76,7 @@ roles/*
 !roles/pulp_site/**
 !roles/doca/
 !roles/doca/**
+!roles/slurm_stats/
+!roles/slurm_stats/**
+!roles/pytools/
+!roles/pytools/**

ansible/monitoring.yml

@@ -17,14 +17,10 @@
 - name: Setup slurm stats
   hosts: slurm_stats
   tags: slurm_stats
-  collections:
-    - stackhpc.slurm_openstack_tools
   tasks:
     - include_role:
         name: slurm-stats
-        apply:
-          # Collection currently requires root for all tasks.
-          become: true
+        become: true
 
 - name: Deploy filebeat
   hosts: filebeat

ansible/roles/pytools/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

ansible/roles/pytools/README.md

@@ -0,0 +1,25 @@
+pytools
+=========
+
+Install a python package into a venv.
+
+Role Variables
+--------------
+- `pytools_python_package`: Optional. Name of `dnf` python package to use. Default `python3`.
+- `pytools_venv_path`: Required. Path to install at, e.g. `/opt/mytool`.
+- `pytools_venv_command`: Optional. Command to run to create venv. Default uses the `pytools_python_package` python's `venv` module.
+- `pytools_pip_name`: Required. Name or URL of python package to install, passed to [ansible.builtin.pip:name](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/pip_module.html#parameter-name).
+- `pytools_editable`: Optional. Whether to install the package using `pip`'s editable mode (installing source to `{{ pytools_venv_path/src }}`). Bool, default `false`.
+- `pytools_user`: Optional. User to install as. Default `root`.
+
+
+Example Playbook
+----------------
+
+    - hosts: compute
+      tasks:
+        - import_role:
+            name: pytools
+          vars:
+            pytools_venv_path: /opt/slurm-tools
+            pytools_pip_name: "git+https://github.com/stackhpc/slurm-openstack-tools.git@main#egg=slurm_openstack_tools"

ansible/roles/pytools/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+# defaults file for pytools
+pytools_python_package: python3
+#pytools_venv_path:
+pytools_venv_command: "{{ pytools_python_package }} -m venv"
+#pytools_pip_name:
+pytools_editable: false
+pytools_user: root

ansible/roles/pytools/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+- name: Install packages
+  dnf:
+    name: "['git'] + {{ pytools_python_package }}"
+  become: true
+
+- name: Create virtualenv directory
+  file:
+    path: "{{ pytools_venv_path }}"
+    owner: "{{ pytools_user }}"
+    group: "{{ pytools_user }}"
+    state: directory
+  become: true
+
+- block:
+  - name: Upgrade pip
+    # This needs to a separate step so that we use the updated version
+    # to install the packages below.
+    pip:
+      name: pip
+
+  - name: Create virtualenv
+    pip:
+      name: "{{ pytools_pip_name }}"
+      editable: "{{ pytools_editable }}"
+
+  module_defaults:
+    ansible.builtin.pip:
+      virtualenv: "{{ pytools_venv_path }}"
+      virtualenv_command: "{{ pytools_venv_command }}"
+      state: latest
+  become: true
+  become_user: "{{ pytools_user }}"

ansible/roles/slurm_stats/README.md

@@ -0,0 +1,33 @@
+stackhpc.slurm_openstack_tools.slurm-stats
+==========================================
+
+Configures slurm-stats from https://github.com/stackhpc/slurm-openstack-tools.git which
+transforms sacct output into a form that is more amenable for importing into elasticsearch/loki.
+
+Requirements
+------------
+
+Role Variables
+--------------
+
+See `defaults/main.yml`.
+
+Dependencies
+------------
+
+Example Playbook
+----------------
+
+    - hosts: compute
+      tasks:
+        - import_role:
+            name: stackhpc.slurm_openstack_tools.slurm-stats
+
+
+License
+-------
+
+Apache-2.0
+
+Author Information
+------------------

ansible/roles/slurm_stats/defaults/main.yml

@@ -0,0 +1,16 @@
+---
+####################
+# log rotate options
+####################
+
+# These options affect the contents of the log-rotate file.
+# See: man logrotate
+
+# Log files are rotated count times before being removed
+slurm_stats_log_rotate_content_rotate: 7
+
+# How frequently are the log files rotated. Can be one of daily, monthly, ...
+slurm_stats_log_rotate_content_frequency: daily
+
+# Which branch/tag etc to install from
+slurm_stats_gitref: main

ansible/roles/slurm_stats/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+- name: Setup slurm tools
+  include_role:
+    name: pytools
+  vars:
+    pytools_venv_path: /opt/slurm-tools
+    pytools_pip_name: "git+https://github.com/stackhpc/slurm-openstack-tools.git@{{ slurm_stats_gitref }}#egg=slurm_openstack_tools"
+
+- name: Create a directory to house the log files
+  file:
+    state: directory
+    path: /var/log/slurm-stats
+  become: true
+
+- name: Create cron job
+  cron:
+    name: Generate slurm stats
+    minute: "*/5"
+    user: root
+    # NOTE: lasttimestamp is stored at /root/lasttimestamp
+    job: "TZ=UTC /opt/slurm-tools/bin/slurm-stats >> /var/log/slurm-stats/finished_jobs.json"
+    cron_file: slurm-stats
+  become: true
+
+- name: Setup log rotate
+  copy:
+    content: |
+      # WARNING: This file is managed by ansible, do not modify.
+      /var/log/slurm-stats/finished_jobs.json {
+              {{ slurm_stats_log_rotate_content_frequency }}
+              rotate {{ slurm_stats_log_rotate_content_rotate }}
+              compress
+              delaycompress
+      }
+    dest: /etc/logrotate.d/slurm-stats
+  become: true

ansible/slurm.yml

@@ -9,16 +9,6 @@
     - include_role:
         name:  mysql
 
-- name: Setup slurm-driven reimage
-  hosts: rebuild
-  become: yes
-  tags:
-    - rebuild
-    - openhpc
-  tasks:
-    - import_role:
-        name: stackhpc.slurm_openstack_tools.rebuild
-
 - name: Setup slurm
   hosts: openhpc
   become: yes

ansible/validate.yml

@@ -40,16 +40,6 @@
         tasks_from: validate.yml
       tags: validate
 
-- name: Validate rebuild configuration
-  hosts: rebuild
-  gather_facts: false
-  tags: rebuild
-  tasks:
-    - import_role:
-        name: stackhpc.slurm_openstack_tools.rebuild
-        tasks_from: validate.yml
-      tags: validate
-
 - name: Validate openondemand configuration
   hosts:
     - openondemand

environments/.stackhpc/terraform/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250114-1627-bccc88b5",
-        "RL9": "openhpc-RL9-250114-1626-bccc88b5"
+        "RL8": "openhpc-RL8-250115-1510-99f67c6d",
+        "RL9": "openhpc-RL9-250115-1510-99f67c6d"
     }
 }

requirements.yml

@@ -28,9 +28,6 @@ collections:
     version: 1.16.2
   - name: community.grafana
     version: 1.5.4
-  - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
-    type: git
-    version: v0.2.0
   - name: ansible.posix
     version: 1.5.4
   - name: ansible.netcommon