diff --git a/ansible/.gitignore b/ansible/.gitignore index 1cabb8ad..c6e7887c 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -76,3 +76,7 @@ roles/* !roles/pulp_site/** !roles/doca/ !roles/doca/** +!roles/slurm_stats/ +!roles/slurm_stats/** +!roles/pytools/ +!roles/pytools/** diff --git a/ansible/monitoring.yml b/ansible/monitoring.yml index 84f31968..3ec86aca 100644 --- a/ansible/monitoring.yml +++ b/ansible/monitoring.yml @@ -17,14 +17,10 @@ - name: Setup slurm stats hosts: slurm_stats tags: slurm_stats - collections: - - stackhpc.slurm_openstack_tools tasks: - include_role: name: slurm-stats - apply: - # Collection currently requires root for all tasks. - become: true + become: true - name: Deploy filebeat hosts: filebeat diff --git a/ansible/roles/slurm_stats/README.md b/ansible/roles/slurm_stats/README.md new file mode 100644 index 00000000..69032c13 --- /dev/null +++ b/ansible/roles/slurm_stats/README.md @@ -0,0 +1,33 @@ +stackhpc.slurm_openstack_tools.slurm-stats +========================================== + +Configures slurm-stats from https://github.com/stackhpc/slurm-openstack-tools.git which +transforms sacct output into a form that is more amenable for importing into elasticsearch/loki. + +Requirements +------------ + +Role Variables +-------------- + +See `defaults/main.yml`. + +Dependencies +------------ + +Example Playbook +---------------- + + - hosts: compute + tasks: + - import_role: + name: stackhpc.slurm_openstack_tools.slurm-stats + + +License +------- + +Apache-2.0 + +Author Information +------------------ diff --git a/ansible/roles/slurm_stats/defaults/main.yml b/ansible/roles/slurm_stats/defaults/main.yml new file mode 100644 index 00000000..4aaa7578 --- /dev/null +++ b/ansible/roles/slurm_stats/defaults/main.yml @@ -0,0 +1,13 @@ +--- +#################### +# log rotate options +#################### + +# These options affect the contents of the log-rotate file. +# See: man logrotate + +# Log files are rotated count times before being removed +slurm_stats_log_rotate_content_rotate: 7 + +# How frequently are the log files rotated. Can be one of daily, monthly, ... +slurm_stats_log_rotate_content_frequency: daily diff --git a/ansible/roles/slurm_stats/tasks/main.yml b/ansible/roles/slurm_stats/tasks/main.yml new file mode 100644 index 00000000..d681b7f3 --- /dev/null +++ b/ansible/roles/slurm_stats/tasks/main.yml @@ -0,0 +1,34 @@ +--- + +- name: Setup slurm tools + include_role: + name: pytools + +- name: Create a directory to house the log files + file: + state: directory + path: /var/log/slurm-stats + become: true + +- name: Create cron job + cron: + name: Generate slurm stats + minute: "*/5" + user: root + # NOTE: lasttimestamp is stored at /root/lasttimestamp + job: "TZ=UTC /opt/slurm-tools/bin/slurm-stats >> /var/log/slurm-stats/finished_jobs.json" + cron_file: slurm-stats + become: true + +- name: Setup log rotate + copy: + content: | + # WARNING: This file is managed by ansible, do not modify. + /var/log/slurm-stats/finished_jobs.json { + {{ slurm_stats_log_rotate_content_frequency }} + rotate {{ slurm_stats_log_rotate_content_rotate }} + compress + delaycompress + } + dest: /etc/logrotate.d/slurm-stats + become: true diff --git a/ansible/roles/slurm_tools/.travis.yml b/ansible/roles/slurm_tools/.travis.yml new file mode 100644 index 00000000..36bbf620 --- /dev/null +++ b/ansible/roles/slurm_tools/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/ansible/roles/slurm_tools/README.md b/ansible/roles/slurm_tools/README.md new file mode 100644 index 00000000..9724c446 --- /dev/null +++ b/ansible/roles/slurm_tools/README.md @@ -0,0 +1,12 @@ +slurm_tools +========= + +Install python-based tools from https://github.com/stackhpc/slurm-openstack-tools.git into `/opt/slurm-tools/bin/`. + +Role Variables +-------------- + +- `pytools_editable`: Optional bool. Whether to install the package using `pip`'s + editable mode (installing source to `/opt/slurm-tools/src`). Default `false`. +- `pytools_gitref`: Optional. Git branch/tag/commit etc to install. Default `master`. +- `pytools_user`: Optional user to install as. Default `root`. diff --git a/ansible/roles/slurm_tools/defaults/main.yml b/ansible/roles/slurm_tools/defaults/main.yml new file mode 100644 index 00000000..39070255 --- /dev/null +++ b/ansible/roles/slurm_tools/defaults/main.yml @@ -0,0 +1,4 @@ +--- +pytools_editable: false +pytools_gitref: master +pytools_user: root diff --git a/ansible/roles/slurm_tools/tasks/main.yml b/ansible/roles/slurm_tools/tasks/main.yml new file mode 100644 index 00000000..e1eb3e51 --- /dev/null +++ b/ansible/roles/slurm_tools/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: install python3 + package: + name: python3,git + become: true + +- name: Create virtualenv directory + file: + path: /opt/slurm-tools + owner: "{{ pytools_user }}" + group: "{{ pytools_user }}" + state: directory + become: true + +- block: + - name: Upgrade pip + # This needs to a separate step so that we use the updated version + # to install the packages below. + pip: + name: pip + + - name: Create virtualenv + pip: + name: "git+https://github.com/stackhpc/slurm-openstack-tools.git@{{ pytools_gitref }}#egg=slurm_openstack_tools" + editable: "{{ pytools_editable }}" + + module_defaults: + ansible.builtin.pip: + virtualenv: /opt/slurm-tools + virtualenv_command: python3 -m venv + state: latest + become: true + become_user: "{{ pytools_user }}" diff --git a/ansible/slurm.yml b/ansible/slurm.yml index cf282f78..71e2ab9c 100644 --- a/ansible/slurm.yml +++ b/ansible/slurm.yml @@ -9,16 +9,6 @@ - include_role: name: mysql -- name: Setup slurm-driven reimage - hosts: rebuild - become: yes - tags: - - rebuild - - openhpc - tasks: - - import_role: - name: stackhpc.slurm_openstack_tools.rebuild - - name: Setup slurm hosts: openhpc become: yes diff --git a/ansible/validate.yml b/ansible/validate.yml index d02caac6..ca0a9585 100644 --- a/ansible/validate.yml +++ b/ansible/validate.yml @@ -40,16 +40,6 @@ tasks_from: validate.yml tags: validate -- name: Validate rebuild configuration - hosts: rebuild - gather_facts: false - tags: rebuild - tasks: - - import_role: - name: stackhpc.slurm_openstack_tools.rebuild - tasks_from: validate.yml - tags: validate - - name: Validate openondemand configuration hosts: - openondemand diff --git a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json index 37bd8c3d..de5e8e19 100644 --- a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json +++ b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json @@ -1,6 +1,6 @@ { "cluster_image": { - "RL8": "openhpc-RL8-250114-1627-bccc88b5", - "RL9": "openhpc-RL9-250114-1626-bccc88b5" + "RL8": "openhpc-RL8-250115-1510-99f67c6d", + "RL9": "openhpc-RL9-250115-1510-99f67c6d" } } diff --git a/requirements.yml b/requirements.yml index 71adbc6e..3eff9168 100644 --- a/requirements.yml +++ b/requirements.yml @@ -28,9 +28,6 @@ collections: version: 1.16.2 - name: community.grafana version: 1.5.4 - - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools - type: git - version: v0.2.0 - name: ansible.posix version: 1.5.4 - name: ansible.netcommon