From fc22af4cb2c63dee23e8f29b73924d4cc4b96697 Mon Sep 17 00:00:00 2001 From: krisfield Date: Mon, 25 Mar 2019 14:18:33 -0500 Subject: [PATCH 001/280] Bump to WatchAnalytics 3.1.2 --- config/core/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index ff8dab685..c06b9aa07 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -253,7 +253,7 @@ list: version: tags/0.3.0 - name: WatchAnalytics repo: https://github.com/enterprisemediawiki/WatchAnalytics.git - version: tags/3.1.1 + version: tags/3.1.2 config: | $egPendingReviewsEmphasizeDays = 10; // makes Pending Reviews shake after X days - name: Variables From f34a3cd6acc0cf1808bd94d190c5e31569858c88 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 25 Mar 2019 17:32:24 -0500 Subject: [PATCH 002/280] 31.3.0 release (#1129) --- RELEASE-NOTES.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index cb3bb5bf1..f620cd4a4 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,28 @@ Release Notes ============= +## Meza 31.3.0 + +WatchAnalytics to 3.1.2 for diff in PendingReviews, improved on-page banners + +### Commits since 31.2.5 + +* fc22af4 Bump to WatchAnalytics 3.1.2 +* 70b4e2b Bump to WatchAnalytics 3.1.1 +* ce403a5 Bump WatchAnalytics to 3.1.0 + +### Contributors + +* 4 krisfield +* 1 James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.3.0 +sudo meza deploy +``` + ## Meza 31.2.4 ### Commits since 31.2.3 From 186298a6cab7585e160f927cf92909dd14b1c2f8 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 25 Apr 2019 10:43:21 -0400 Subject: [PATCH 003/280] The SFTP server is in a different location on Debian It's important to have the path correct. This affects, among other things, the ability to use SSHFS with your nodes. --- src/roles/base/templates/sshd_config.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/roles/base/templates/sshd_config.j2 b/src/roles/base/templates/sshd_config.j2 index a2ac12d5a..bf9ce8a56 100644 --- a/src/roles/base/templates/sshd_config.j2 +++ b/src/roles/base/templates/sshd_config.j2 @@ -143,7 +143,11 @@ AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE AcceptEnv XMODIFIERS # override default of no subsystems +{% if ansible_os_family == 'Debian' %} +Subsystem sftp /usr/lib/openssh/sftp-server +{% else %} Subsystem sftp /usr/libexec/openssh/sftp-server +{% endif %} # Example of overriding settings on a per-user basis #Match User anoncvs From 6ae982b8f7f14c8085aaad17154da1a5b8028626 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 25 Apr 2019 16:13:43 -0400 Subject: [PATCH 004/280] Use 'remote_src' When the parsoid host is not the controller, the src file is on the remote. Therefore, use remote_src: yes so that Ansible finds the file to copy. Otherwise, this fails when the Parsoid host in inventory is not the same host as the controller. In my example setup, I have two application servers and both are designated as Parsoid servers. One is the controller, the other app server is just an app server. --- src/roles/nodejs/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roles/nodejs/tasks/main.yml b/src/roles/nodejs/tasks/main.yml index 60371eb5e..a24642302 100644 --- a/src/roles/nodejs/tasks/main.yml +++ b/src/roles/nodejs/tasks/main.yml @@ -72,6 +72,7 @@ unarchive: src: /tmp/node.tar.xz dest: /usr/local/lib/node + remote_src: yes extra_opts: [--strip-components=1] when: "node_version.stdout != 'v8.11.1'" From 4127c780d406a3c32a6b5f142e57773504d6545e Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 7 May 2019 20:22:53 -0500 Subject: [PATCH 005/280] Use base ImageMagick rather than Meza's own RPM --- src/roles/imagemagick/tasks/main.yml | 19 +++++++++++++++---- .../mediawiki/templates/LocalSettings.php.j2 | 3 --- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/roles/imagemagick/tasks/main.yml b/src/roles/imagemagick/tasks/main.yml index f6dc7302a..0cc001ddd 100644 --- a/src/roles/imagemagick/tasks/main.yml +++ b/src/roles/imagemagick/tasks/main.yml @@ -1,8 +1,19 @@ --- -- name: Ensure ghostscript installed - yum: name=ghostscript state=installed -- name: Install Imagemagick from meza repo - yum: name=https://raw.github.com/enterprisemediawiki/meza-packages/master/RPMs/imagemagick_7.0.3_x86_64.rpm +- name: Ensure old ImageMagick installed from Meza RPM + yum: + name: imagemagick-7.0.3-1.x86_64 + state: absent + +- name: Ensure ImageMagick at latest version + yum: + name: + - ghostscript + - ImageMagick + - ImageMagick-devel + state: latest + tags: + - latest + - name: Copy xpdf bin64 files to /usr/local/bin copy: src: xpdf-3.04-bin64/ diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index a959e1096..922f899c3 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -601,9 +601,6 @@ $wgMaxImageArea = 1.25e10; // Images on [[Snorkel]] fail without this ini_set( 'pcre.backtrack_limit', 1000000000 ); //1 billion -$wgUseImageMagick = true; -$wgImageMagickConvertCommand = '/usr/local/bin/convert'; - // Allowed file types $wgFileExtensions = array( 'aac', From 43691c4c205327d29868c12f3d7321d99b38f782 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 9 May 2019 09:53:52 -0500 Subject: [PATCH 006/280] Vagrantfile set mount_options: ["dmode=755,fmode=755"] Windows only Doing this on Windows has no effect on the /opt/meza git repository, but doing it on Linux causes a change to all files adding +x, thus making many uncommitted changes. --- Vagrantfile | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 7966bf3dd..42e8bb837 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -14,6 +14,28 @@ else configuration = YAML::load(File.read("#{File.dirname(__FILE__)}/vagrantconf.default.yml")) end +# Source: +# https://stackoverflow.com/questions/26811089/vagrant-how-to-have-host-platform-specific-provisioning-steps +module OS + def OS.windows? + (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil + end + + def OS.mac? + (/darwin/ =~ RUBY_PLATFORM) != nil + end + + def OS.unix? + !OS.windows? + end + + # Not ideal. BSD is Unix but is not Mac, but would return true for Linux. + def OS.linux? + OS.unix? and not OS.mac? + end +end + + # All Vagrant configuration is done below. The "2" in Vagrant.configure # configures the configuration version (we support older styles for # backwards compatibility). Please don't change it unless you know what @@ -145,7 +167,13 @@ Vagrant.configure("2") do |config| # Disable default synced folder at /vagrant, instead put at /opt/meza app1.vm.synced_folder ".", "/vagrant", disabled: true - app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: "vagrant", group: "vagrant", mount_options: ["dmode=755,fmode=755"] + + if OS.windows? + # puts "Vagrant host: Windows" + app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: "vagrant", group: "vagrant", mount_options: ["dmode=755,fmode=755"] + else + app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: "vagrant", group: "vagrant" + end # app1.vm.synced_folder ".", "/opt/meza", type: "smb" # app1.vm.synced_folder ".", "/opt/meza", type: "rsync", From 46f7ac66debc81c51fad0ce669dc30d9b4e48c19 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 10 May 2019 10:31:08 -0500 Subject: [PATCH 007/280] Make net adapter select statement break on newlines --- src/scripts/create-vm.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scripts/create-vm.sh b/src/scripts/create-vm.sh index c2694cb0d..8b253c694 100644 --- a/src/scripts/create-vm.sh +++ b/src/scripts/create-vm.sh @@ -129,12 +129,16 @@ if (( $(grep -c . <<<"$hostonlyadapter") > 1 )); then echo "You have multiple Host-Only adapters. Their info is above. Choose which to use below." echo + OLD_IFS=$IFS + IFS=$'\n' select adapter in $hostonlyadapter; do echo "You chose: $adapter" hostonlyadapter=$adapter break done + IFS=${OLD_IFS} + # # If Host-Only adapter is blank, create one From bd6b1038423331dea8d31febb4d076a8ae772756 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 9 Feb 2018 00:19:30 -0500 Subject: [PATCH 008/280] Minor spelling mistakes --- src/roles/firewalld/tasks/main.yml | 2 +- src/roles/haproxy/templates/errors/500.http.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/firewalld/tasks/main.yml b/src/roles/firewalld/tasks/main.yml index 2d61779f6..e4819dd5d 100644 --- a/src/roles/firewalld/tasks/main.yml +++ b/src/roles/firewalld/tasks/main.yml @@ -1,6 +1,6 @@ --- # These are handled within the base role now such that they aren't re-run -# for each play that makes a firewall modification. Instead this role is to bue +# for each play that makes a firewall modification. Instead this role is to be # used sort of like a library of functions. Plays will use the uncommented-out # tasks below as needed. # diff --git a/src/roles/haproxy/templates/errors/500.http.j2 b/src/roles/haproxy/templates/errors/500.http.j2 index d3ab29e86..43ea3d86c 100644 --- a/src/roles/haproxy/templates/errors/500.http.j2 +++ b/src/roles/haproxy/templates/errors/500.http.j2 @@ -62,7 +62,7 @@

If the problem persists feel free to contact us.

From af9eb52065b04b2de33439123d810f4602c1c8f4 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 10 May 2019 13:04:54 -0500 Subject: [PATCH 009/280] Move vault pass file from meza-ansible home to /opt/conf-meza/vault --- config/core/defaults.yml | 1 + src/playbooks/site.yml | 3 ++- src/roles/haproxy/tasks/main.yml | 6 ++--- src/scripts/meza.py | 26 ++++++++++++++++---- tests/docker/backup-to-remote.setup.sh | 2 +- tests/docker/import-from-alt-remote.setup.sh | 2 +- 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index af4282d84..836de008c 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -14,6 +14,7 @@ m_use_production_settings: True m_config_core: /opt/meza/config/core m_local_secret: /opt/conf-meza/secret m_local_public: /opt/conf-meza/public +m_config_vault: /opt/conf-meza/vault m_home: /opt/conf-meza/users # Config files written by Ansible which need a place to live on non-controller diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index c14de06a3..db3092469 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -8,6 +8,7 @@ become: yes vars: m_home: "/opt/conf-meza/users" + m_config_vault: "/opt/conf-meza/vault" tasks: - name: Ensure no password on meza-ansible user on controller shell: passwd --delete meza-ansible @@ -58,7 +59,7 @@ shell: > ansible-vault encrypt /opt/conf-meza/secret/{{ env }}/secret.yml - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + --vault-password-file {{ m_config_vault }}/vault-pass-{{ env }}.txt failed_when: False # Note: without this, the encryption above changes mode to 0600 and ownership diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 541e5d3a5..ffd736f99 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -55,7 +55,7 @@ shell: > ansible-vault encrypt {{ item }} - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + --vault-password-file {{ m_config_vault }}/vault-pass-{{ env }}.txt failed_when: False delegate_to: localhost run_once: True @@ -67,7 +67,7 @@ shell: > ansible-vault view /opt/conf-meza/secret/{{ env }}/ssl/meza.key - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + --vault-password-file {{ m_config_vault }}/vault-pass-{{ env }}.txt register: decrypted_key delegate_to: localhost run_once: True @@ -76,7 +76,7 @@ shell: > ansible-vault view /opt/conf-meza/secret/{{ env }}/ssl/meza.crt - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + --vault-password-file {{ m_config_vault }}/vault-pass-{{ env }}.txt register: decrypted_cert delegate_to: localhost run_once: True diff --git a/src/scripts/meza.py b/src/scripts/meza.py index f2cb6ffbe..c5d9823c1 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -325,7 +325,7 @@ def meza_command_setup_env (argv, return_not_exit=False): print "Please review your host file. Run command:" print " sudo vi /opt/conf-meza/secret/{}/hosts".format(env) print "Please review your secret config. It is encrypted, so edit by running:" - print " sudo ansible-vault edit /opt/conf-meza/secret/{}/secret.yml --vault-password-file /opt/conf-meza/users/meza-ansible/.vault-pass-{}.txt".format(env,env) + print " sudo ansible-vault edit /opt/conf-meza/secret/{}/secret.yml --vault-password-file {}".format(env,vault_pass_file) if return_not_exit: return rc else: @@ -680,12 +680,28 @@ def meza_shell_exec_exit( return_code=0 ): def get_vault_pass_file ( env ): import pwd import grp + home_dir = defaults['m_home'] - vault_pass_file = '{}/meza-ansible/.vault-pass-{}.txt'.format(home_dir,env) + legacy_file = '{}/meza-ansible/.vault-pass-{}.txt'.format(home_dir,env) + + vault_dir = defaults['m_config_vault'] + vault_pass_file = '{}/vault-pass-{}.txt'.format(vault_dir, env) + if not os.path.isfile( vault_pass_file ): - with open( vault_pass_file, 'w' ) as f: - f.write( random_string( num_chars=64 ) ) - f.close() + if not os.path.exists( vault_dir ): + os.mkdir( vault_dir ) + meza_chown( vault_dir, 'meza-ansible', 'wheel' ) + os.chmod( vault_dir, 0o700 ) + + # If legacy vault password file exists copy that into new location. + # Otherwise, create one in the new location + if os.path.isfile( legacy_file ): + from shutil import copyfile + copyfile(legacy_file, vault_pass_file) + else: + with open( vault_pass_file, 'w' ) as f: + f.write( random_string( num_chars=64 ) ) + f.close() # Run this everytime, since it should be fast and if meza-ansible can't # read this then you're stuck! diff --git a/tests/docker/backup-to-remote.setup.sh b/tests/docker/backup-to-remote.setup.sh index a9be9b14d..efe8e3fdc 100644 --- a/tests/docker/backup-to-remote.setup.sh +++ b/tests/docker/backup-to-remote.setup.sh @@ -47,7 +47,7 @@ ${docker_exec_1[@]} bash -c "echo -e 'sshd_config_PasswordAuthentication: \"yes\ # secret.yml is encrypted. decrypt first, make edits, re-encrypt. # secret_yml="/opt/conf-meza/secret/$env_name/secret.yml" -# vault_pass="/opt/conf-meza/users/meza-ansible/.vault-pass-$env_name.txt" +# vault_pass="/opt/conf-meza/vault/vault-pass-$env_name.txt" # ${docker_exec_1[@]} bash -c "ansible-vault decrypt $secret_yml --vault-password-file $vault_pass" # ${docker_exec_1[@]} bash -c "echo -e '\n' >> $secret_yml" # ${docker_exec_1[@]} bash -c "echo 'mysql_root_password_update: yes' >> $secret_yml" diff --git a/tests/docker/import-from-alt-remote.setup.sh b/tests/docker/import-from-alt-remote.setup.sh index d22f2f035..a3ae0e845 100644 --- a/tests/docker/import-from-alt-remote.setup.sh +++ b/tests/docker/import-from-alt-remote.setup.sh @@ -28,7 +28,7 @@ docker_exec_2=( "${docker_exec[@]}" ) # Location of secret.yml file, hosts file, and vault pass file secret_yml="/opt/conf-meza/secret/$env_name/secret.yml" hosts_file="/opt/conf-meza/secret/$env_name/hosts" -vault_pass="/opt/conf-meza/users/meza-ansible/.vault-pass-$env_name.txt" +vault_pass="/opt/conf-meza/vault/vault-pass-$env_name.txt" # CONTAINER 1 # (1) Get local secret config from repo From 51f5e34fefa71ea7f41a2d18170e2d644e9ef82f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 11 May 2019 10:43:08 -0500 Subject: [PATCH 010/280] Re-enable PEAR package; not used by default, but used by MS SQL --- src/roles/apache-php/tasks/php.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/roles/apache-php/tasks/php.yml b/src/roles/apache-php/tasks/php.yml index 9509324f6..ef1b4efc9 100644 --- a/src/roles/apache-php/tasks/php.yml +++ b/src/roles/apache-php/tasks/php.yml @@ -118,7 +118,9 @@ # Post 7.0, use the pear1u package for all versions of PHP # PEAR is no longer a requirement for Meza. Mail and Net_SMTP installed with # Composer via MW core (MW 1.32+) or composer.local.json (MW 1.31 and lower) - # - pear1u + # However, other packages may require it, for example installing sqlsrv and + # pdo_sqlsrv PECL packages for MS SQL use. + - pear1u # Not available for PHP 7, due to being built into PHP 7 # - php56u-pecl-jsonc From 527852f59a97150f698fc8577fc764b755a83688 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 11 May 2019 10:44:23 -0500 Subject: [PATCH 011/280] Get public config repo as meza-ansible, not root --- src/roles/init-controller-config/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index d3cb2f017..856ee5297 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -28,6 +28,8 @@ # If a git repo is defined use that for config - name: Get local config repo if set + become: yes + become_user: "meza-ansible" git: repo: "{{ local_config_repo.repo }}" dest: "{{ m_local_public }}" From 6ecc855e9a5af4e49a0eab7e09a7fdf669eabf41 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 11 May 2019 11:20:46 -0500 Subject: [PATCH 012/280] Create role remote-dir-check to verify if remote uploads dir exists --- src/roles/remote-dir-check/tasks/main.yml | 61 +++++++++++++++++++++++ src/roles/verify-wiki/tasks/main.yml | 25 ++++++---- 2 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 src/roles/remote-dir-check/tasks/main.yml diff --git a/src/roles/remote-dir-check/tasks/main.yml b/src/roles/remote-dir-check/tasks/main.yml new file mode 100644 index 000000000..63f408a54 --- /dev/null +++ b/src/roles/remote-dir-check/tasks/main.yml @@ -0,0 +1,61 @@ +--- +# Check if a directory exists on a remote server +# +# Inputs: +# remote_server: +# directory_path: +# remote_server_user: + +- name: "Initially assume directory does not exist" + set_fact: + remote_dir_exists: False + +# +# Put meza-ansible's private key and known_hosts on server within /root +# +- name: "Grant keys to {{ inventory_hostname }}" + include_role: + name: key-transfer + tasks_from: grant-keys + vars: + granted_server: "{{ inventory_hostname }}" + when: + remote_server != inventory_hostname + +- name: "Check for existence of {{ directory_path }} on {{ remote_server }}" + shell: > + ssh + -o StrictHostKeyChecking=no + -i /root/meza-ansible-id_rsa + -o UserKnownHostsFile=/root/meza-ansible-known_hosts + {{ remote_server_user }}@{{ remote_server }} + '[ -d {{ uploads_backup_dir_path }} ]' + failed_when: False + register: remote_dir_check + run_once: true + when: remote_server != inventory_hostname + +# If not actually a remote server +- name: "Check for existence of {{ directory_path }} on {{ inventory_hostname }}" + shell: '[ -d {{ uploads_backup_dir_path }} ]' + failed_when: False + register: local_dir_check + run_once: true + when: remote_server == inventory_hostname + +- debug: { var: remote_dir_check } +- debug: { var: local_dir_check } + +- name: "Set remote_dir_exists to true" + set_fact: + remote_dir_exists: True + when: (not local_dir_check|skipped and local_dir_check.rc == 0) or (not remote_dir_check|skipped and remote_dir_check.rc == 0) + +- debug: { var: remote_dir_exists } + +- name: "Revoke keys from {{ inventory_hostname }}" + include_role: + name: key-transfer + tasks_from: revoke-keys + vars: + granted_server: "{{ inventory_hostname }}" diff --git a/src/roles/verify-wiki/tasks/main.yml b/src/roles/verify-wiki/tasks/main.yml index 4a6155618..775b13e9c 100644 --- a/src/roles/verify-wiki/tasks/main.yml +++ b/src/roles/verify-wiki/tasks/main.yml @@ -178,6 +178,9 @@ run_once: true delegate_to: "{{ groups['db-master'][0] }}" +- debug: { var: do_sql_dump } +- debug: { var: wiki_exists } +- debug: { var: intend_overwrite_from_backup } # # SECTION: DUMP SQL to DB master @@ -299,13 +302,15 @@ # # SECTION: Does backup server have uploads? # -- name: "{{ wiki_id }} - Check if wiki's uploads backup dir exists on backups.0" - stat: - path: "{{ uploads_backup_dir_path }}" - register: images_backup_dir - delegate_to: "{{ uploads_backup_server }}" - remote_user: "{{ uploads_backup_server_remote_user }}" - run_once: true +- name: + include_role: + name: remote-dir-check + vars: + remote_server: "{{ uploads_backup_server }}" + directory_path: "{{ uploads_backup_dir_path }}" + remote_server_user: "{{ uploads_backup_server_remote_user }}" + +- debug: { var: remote_dir_exists } # @@ -314,12 +319,12 @@ - name: "{{ wiki_id }} - Set fact if SHOULD overwrite uploads data (only possible if backup exists)" set_fact: do_overwrite_uploads_from_backup: True - when: intend_overwrite_from_backup and images_backup_dir.stat.exists + when: intend_overwrite_from_backup and remote_dir_exists - name: "{{ wiki_id }} - Set fact if SHOULD NOT overwrite uploads data" set_fact: do_overwrite_uploads_from_backup: False - when: not intend_overwrite_from_backup or not images_backup_dir.stat.exists + when: not intend_overwrite_from_backup or not remote_dir_exists # @@ -336,7 +341,7 @@ pulling_from_user: "{{ uploads_backup_server_remote_user }}" run_once: true when: - images_backup_dir.stat.exists + remote_dir_exists and (not wiki_has_uploads or do_overwrite_uploads_from_backup) From 4d56386fd3f83f7f27bb4d6545fdff009a62d29f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 11 May 2019 15:35:42 -0500 Subject: [PATCH 013/280] Set permissions for /opt/conf-meza and /opt/conf-meza/public Make these dirs suitable for meza-ansible to manage them without using sudo. --- config/core/defaults.yml | 5 ++++ src/playbooks/site.yml | 8 ++++++ .../init-controller-config/tasks/main.yml | 26 +++++++++---------- src/scripts/getmeza.sh | 1 + 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 836de008c..a07597ed8 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -215,6 +215,11 @@ m_backups_owner: root m_backups_group: root +m_config_public_mode: "0755" +m_config_public_owner: meza-ansible +m_config_public_group: wheel + + # # PHP config # diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index db3092469..cb25420e4 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -71,6 +71,14 @@ group: wheel mode: "0600" + - name: Ensure /opt/conf-meza owned by meza-ansible + file: + path: "/opt/conf-meza" + owner: meza-ansible + group: wheel + mode: "0755" + + # FIXME 800: Run against localhost - hosts: app-servers become: yes diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index 856ee5297..1e07fdc22 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -57,9 +57,9 @@ file: path: "{{ m_local_public }}" state: directory - owner: root - group: root - mode: 0755 + owner: "{{ m_config_public_owner }}" + group: "{{ m_config_public_group }}" + mode: "{{ m_config_public_mode }}" recurse: true delegate_to: localhost run_once: true @@ -71,20 +71,20 @@ file: path: "{{ m_local_public }}/wikis" state: directory - owner: root - group: root - mode: 0755 + owner: "{{ m_config_public_owner }}" + group: "{{ m_config_public_group }}" + mode: "{{ m_config_public_mode }}" delegate_to: localhost run_once: true - name: Ensure pre/post settings directories exists in config file: - path: "/opt/conf-meza/public/{{ item }}" + path: "{{ m_local_public }}/{{ item }}" state: directory - owner: root - group: root - mode: 0755 + owner: "{{ m_config_public_owner }}" + group: "{{ m_config_public_group }}" + mode: "{{ m_config_public_mode }}" delegate_to: localhost run_once: true with_items: @@ -96,9 +96,9 @@ template: src: "templates/{{ item }}.j2" dest: "{{ m_local_public }}/{{ item }}" - owner: root - group: root - mode: 0755 + owner: "{{ m_config_public_owner }}" + group: "{{ m_config_public_group }}" + mode: "{{ m_config_public_mode }}" force: no delegate_to: localhost run_once: true diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index c0e0e38de..07cf9d952 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -80,6 +80,7 @@ if $ret; then fi fi +chown meza-ansible:wheel /opt/conf-meza echo echo "Add ansible master user" From 68ac393023573311d119b6d58341189b0648f8fe Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 12 May 2019 12:45:32 -0500 Subject: [PATCH 014/280] Add tags for rsync-uploads and better debug --- src/roles/verify-wiki/tasks/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/roles/verify-wiki/tasks/main.yml b/src/roles/verify-wiki/tasks/main.yml index 775b13e9c..02b3a696b 100644 --- a/src/roles/verify-wiki/tasks/main.yml +++ b/src/roles/verify-wiki/tasks/main.yml @@ -182,6 +182,20 @@ - debug: { var: wiki_exists } - debug: { var: intend_overwrite_from_backup } +- debug: + msg: | + do_sql_dump: "{{ do_sql_dump }}" + wiki_exists: "{{ wiki_exists }}" + intend_overwrite_from_backup: "{{ intend_overwrite_from_backup }}" + wiki_id: "{{ wiki_id }}" + remote_server: "{{ sql_backup_server }}" + remote_server_ssh_user: "{{ db_backup_server_remote_user }}" + remote_server_mysql_user: "{{ backups_server_db_dump.mysql_user }}" + remote_server_mysql_pass: "hidden" + dump_database: "wiki_{{ wiki_id }}" + target_server: "{{ groups['db-master'][0] }}" + target_server_path: "{{ m_tmp }}/wiki.sql" + # # SECTION: DUMP SQL to DB master # @@ -309,8 +323,12 @@ remote_server: "{{ uploads_backup_server }}" directory_path: "{{ uploads_backup_dir_path }}" remote_server_user: "{{ uploads_backup_server_remote_user }}" + tags: + - verify-wiki-uploads - debug: { var: remote_dir_exists } + tags: + - verify-wiki-uploads # @@ -320,11 +338,15 @@ set_fact: do_overwrite_uploads_from_backup: True when: intend_overwrite_from_backup and remote_dir_exists + tags: + - verify-wiki-uploads - name: "{{ wiki_id }} - Set fact if SHOULD NOT overwrite uploads data" set_fact: do_overwrite_uploads_from_backup: False when: not intend_overwrite_from_backup or not remote_dir_exists + tags: + - verify-wiki-uploads # @@ -343,6 +365,8 @@ when: remote_dir_exists and (not wiki_has_uploads or do_overwrite_uploads_from_backup) + tags: + - verify-wiki-uploads # From cd79230d2e70e285d7b38dd24983c7859d369fa7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 11 Jan 2019 18:06:00 -0600 Subject: [PATCH 015/280] Make checks for wiki existence during backup go to right server Previously when a server backing up from another server checks to see if a wiki exists on the remote server, it was actually checking to see if the wiki existed locally. This made no sesnse and caused errors when a wiki did not exist on the remote server but a backup file was generated anyway. --- src/roles/remote-mysqldump/tasks/main.yml | 37 +++++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/roles/remote-mysqldump/tasks/main.yml b/src/roles/remote-mysqldump/tasks/main.yml index 9a96a2594..8b199b567 100644 --- a/src/roles/remote-mysqldump/tasks/main.yml +++ b/src/roles/remote-mysqldump/tasks/main.yml @@ -62,16 +62,47 @@ # # Verify wiki exists on source server before attempting to dump from it # + +# +# If remote_server != target_server, use SSH to run mysqlshow. Else, doit locally +# +# FIXME #818: Remove StrictHostKeyChecking=no when tests properly add host keys (users should do so, too, of course) +- name: remote_server ({{ remote_server }}) != target_server ({{ target_server }}); run mysqlshow over SSH + set_fact: + mysqlshow_command: > + ssh + -o StrictHostKeyChecking=no + -i /root/meza-ansible-id_rsa + -o UserKnownHostsFile=/root/meza-ansible-known_hosts + {{ remote_server_ssh_user }}@{{ remote_server }} + "mysqlshow + {{ user_option }} + {{ password_option }} + {{ dump_database }} + | grep -v Wildcard + | grep -o {{ dump_database }}" + when: remote_server != target_server +- name: remote_server == target_server ({{ target_server }}); run mysqlshow locally + set_fact: + mysqlshow_command: > + mysqlshow + {{ user_option }} + {{ password_option }} + {{ dump_database }} + | grep -v Wildcard + | grep -o {{ dump_database }} + when: remote_server == target_server +- debug: { msg: "{{ mysqlshow_command }}" } + - name: "{{ wiki_id }} - check if wiki database exists ON SOURCE SERVER" - shell: 'mysqlshow "wiki_{{ wiki_id }}" | grep -v Wildcard | grep -o wiki_{{ wiki_id }}' + shell: "{{ mysqlshow_command }}" register: source_wiki_exists_check - delegate_to: "{{ target_server }}" failed_when: False run_once: true - - debug: var: source_wiki_exists_check + - name: "{{ wiki_id }} - Set fact if database wiki_{{ wiki_id }} DOES exist ON SOURCE SERVER" set_fact: source_wiki_exists: True From db6f80589f6421ba939eecd5fe89f96948103b76 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 12 May 2019 14:36:06 -0500 Subject: [PATCH 016/280] Give undefined debug vars print vals --- src/roles/verify-wiki/tasks/main.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/roles/verify-wiki/tasks/main.yml b/src/roles/verify-wiki/tasks/main.yml index 02b3a696b..401338a4e 100644 --- a/src/roles/verify-wiki/tasks/main.yml +++ b/src/roles/verify-wiki/tasks/main.yml @@ -184,17 +184,17 @@ - debug: msg: | - do_sql_dump: "{{ do_sql_dump }}" - wiki_exists: "{{ wiki_exists }}" - intend_overwrite_from_backup: "{{ intend_overwrite_from_backup }}" - wiki_id: "{{ wiki_id }}" - remote_server: "{{ sql_backup_server }}" - remote_server_ssh_user: "{{ db_backup_server_remote_user }}" - remote_server_mysql_user: "{{ backups_server_db_dump.mysql_user }}" - remote_server_mysql_pass: "hidden" - dump_database: "wiki_{{ wiki_id }}" - target_server: "{{ groups['db-master'][0] }}" - target_server_path: "{{ m_tmp }}/wiki.sql" + do_sql_dump: {{ do_sql_dump | default('') }} + wiki_exists: {{ wiki_exists | default('') }} + intend_overwrite_from_backup: {{ intend_overwrite_from_backup | default('') }} + wiki_id: {{ wiki_id | default('') }} + remote_server: {{ sql_backup_server | default('') }} + remote_server_ssh_user: {{ db_backup_server_remote_user | default('') }} + remote_server_mysql_user: {{ backups_server_db_dump.mysql_user | default('') }} + remote_server_mysql_pass: + dump_database: wiki_{{ wiki_id | default('') }} + target_server: {{ groups['db-master'][0] }} + target_server_path: {{ m_tmp }}/wiki.sql # # SECTION: DUMP SQL to DB master From 112317c5af41cc02d9cb91225777bfea18e0e752 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 12 May 2019 15:27:35 -0500 Subject: [PATCH 017/280] Handle undefined backups_server_db_dump --- src/roles/verify-wiki/tasks/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/roles/verify-wiki/tasks/main.yml b/src/roles/verify-wiki/tasks/main.yml index 401338a4e..8154e3b0d 100644 --- a/src/roles/verify-wiki/tasks/main.yml +++ b/src/roles/verify-wiki/tasks/main.yml @@ -138,6 +138,7 @@ do_sql_dump: True sql_file_match: "wiki_{{ wiki_id }}.sql" db_backup_server_remote_user: "{{ backups_server_db_dump.remote_user }}" + db_backup_server_mysql_user: "{{ backups_server_db_dump.mysql_user }}" sql_backup_server_set: True when: "backups_server_db_dump is defined" @@ -190,7 +191,7 @@ wiki_id: {{ wiki_id | default('') }} remote_server: {{ sql_backup_server | default('') }} remote_server_ssh_user: {{ db_backup_server_remote_user | default('') }} - remote_server_mysql_user: {{ backups_server_db_dump.mysql_user | default('') }} + remote_server_mysql_user: {{ db_backup_server_mysql_user | default('') }} remote_server_mysql_pass: dump_database: wiki_{{ wiki_id | default('') }} target_server: {{ groups['db-master'][0] }} @@ -205,7 +206,7 @@ vars: remote_server: "{{ sql_backup_server }}" remote_server_ssh_user: "{{ db_backup_server_remote_user }}" - remote_server_mysql_user: "{{ backups_server_db_dump.mysql_user }}" + remote_server_mysql_user: "{{ db_backup_server_mysql_user }}" remote_server_mysql_pass: "{{ backups_server_db_dump.mysql_pass }}" dump_database: "wiki_{{ wiki_id }}" target_server: "{{ groups['db-master'][0] }}" From 84bc0a0eab2266950382eefeb5f92d98f8227589 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 12 May 2019 17:17:40 -0500 Subject: [PATCH 018/280] Install mysql client on backup servers for direct mysqldump --- src/playbooks/site.yml | 12 ++++++++++++ src/roles/remote-mysqldump/tasks/main.yml | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index cb25420e4..a485d23b5 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -284,6 +284,18 @@ mysql_replication_master: "{{ groups['db-master'][0] }}" mysql_replication_role: slave +- hosts: backup-servers + become: yes + tags: backup-servers + roles: + - set-vars + tasks: + - name: Ensure MariaDB client installed on backup servers + yum: + name: mariadb + state: present + tags: + - latest - hosts: elastic-servers become: yes diff --git a/src/roles/remote-mysqldump/tasks/main.yml b/src/roles/remote-mysqldump/tasks/main.yml index 8b199b567..edfb3154c 100644 --- a/src/roles/remote-mysqldump/tasks/main.yml +++ b/src/roles/remote-mysqldump/tasks/main.yml @@ -99,6 +99,8 @@ register: source_wiki_exists_check failed_when: False run_once: true + delegate_to: "{{ target_server }}" + - debug: var: source_wiki_exists_check From 7ea1f982862e23032477e26a8dfaf44b6acc4113 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 12 May 2019 19:17:11 -0500 Subject: [PATCH 019/280] Use known user, not no user, when mysqldump user unspecified --- src/roles/remote-mysqldump/tasks/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/roles/remote-mysqldump/tasks/main.yml b/src/roles/remote-mysqldump/tasks/main.yml index edfb3154c..ee73d3a33 100644 --- a/src/roles/remote-mysqldump/tasks/main.yml +++ b/src/roles/remote-mysqldump/tasks/main.yml @@ -32,9 +32,9 @@ set_fact: user_option: "-u {{ remote_server_mysql_user }}" when: remote_server_mysql_user is defined -- name: Set mysql user option to empty string if not defined +- name: Set mysql user option to use wiki application user if not defined set_fact: - user_option: "" + user_option: "-u {{ wiki_app_db_user.name }}" when: remote_server_mysql_user is not defined # @@ -44,9 +44,9 @@ set_fact: password_option: "-p{{ remote_server_mysql_pass }}" when: remote_server_mysql_pass is defined -- name: Set mysql password option to empty string if not defined +- name: Set mysql password option use wiki application user if not defined set_fact: - password_option: "" + password_option: "-p{{ wiki_app_db_user.password }}" when: remote_server_mysql_pass is not defined # From 02cb384d71e3621714e00e797df298c8dc1c6757 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 23 Mar 2019 12:11:55 -0500 Subject: [PATCH 020/280] WIP: release notes script --- .gitignore | 3 ++ src/scripts/do-release.sh | 109 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 src/scripts/do-release.sh diff --git a/.gitignore b/.gitignore index 6baaaa61c..f72899d43 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ .vagrant/* vagrantconf.yml + +# file used by src/scripts/do-release.sh +.release-notes.tmp diff --git a/src/scripts/do-release.sh b/src/scripts/do-release.sh new file mode 100644 index 000000000..f14ab7f57 --- /dev/null +++ b/src/scripts/do-release.sh @@ -0,0 +1,109 @@ +#!/bin/sh +# +# Generate release notes for Meza + +# +# SET VARIABLES FOR COLORIZING BASH OUTPUT +# +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# +# SETUP KNOWN VARS PRIOR TO USER INPUT +# +PREVIOUS_RELEASES=$(git tag -l | sed '/^v0/ d' | sed '/^v1/ d') +LATEST="${PREVIOUS_RELEASES##*$'\n'}" +GIT_HASH=$(git rev-parse HEAD | cut -c1-8) + +# +# READ IN USER INPUTS +# +read -p "Add optional single line of overview text: " OVERVIEW + +echo -e "${BLUE}" +echo "${PREVIOUS_RELEASES}" +echo -e "${NC}" + +while [ -z "$OLD_VERSION" ]; do + read -p "Enter previous release number (options in blue above): " -i "$LATEST" -e OLD_VERSION +done; + +while [ -z "$NEW_VERSION" ]; do + read -p "Enter new version number in form X.Y.Z: " NEW_VERSION +done; + +# +# SETUP VARS BASED UPON USER INPUT +# +MAJOR_VERSION=$(echo "$OLD_VERSION" | cut -f1 -d".") +RELEASE_BRANCH="${MAJOR_VERSION}.x" +COMMITS=$(git log --oneline --no-merges "${OLD_VERSION}..HEAD" | while read line; do echo "* $line"; done) +CONTRIBUTORS=$(git shortlog -sn "${OLD_VERSION}..HEAD" | while read line; do echo "* $line"; done) + +# +# GENERATE RELEASE NOTES INTO TEMP FILE +# +RELEASE_NOTES_FILE=./.release-notes.tmp +cat > ${RELEASE_NOTES_FILE} <<- EOM +${OVERVIEW} + +### Commits since $OLD_VERSION + +${COMMITS} + +### Contributors + +${CONTRIBUTORS} + +### Mediawiki.org pages updated + +* TBD + +### What still isn't documented? + +* TBD +* See [list of issues and pull requests indicating missing docs](https://github.com/enterprisemediawiki/meza/pulls?utf8=%E2%9C%93&q=label%3A%22open+post-merge+actions%22+) + +# How to upgrade + +\`\`\`bash +sudo meza update ${NEW_VERSION} +sudo meza deploy +\`\`\` +EOM + +# +# OUTPUT RELEASE NOTES IN GREEN ON COMMAND LINE +# +echo -e "${GREEN}" +cat "${RELEASE_NOTES_FILE}" +echo -e "${NC}" + +# +# TO-DO: Automate edit of release notes +# +# sed -e '1,/=============/r.release-notes.tmp' ./RELEASE-NOTES.md + +# +# OUTPUT DIRECTIONS FOR COMPLETING RELEASE +# +echo +echo "1. Edit RELEASE-NOTES.md" +echo -e " * Copy the ${GREEN}green text${NC} from above and add it under the title ${GREEN}## Meza $NEW_VERSION${NC}" +echo " * Edit the text as required" +echo "2. Commit your changes and submit a pull request" +echo "3. After the PR is merged create a new release of Meza with these details:" +echo " * Tag: $NEW_VERSION" +echo " * Title: Meza $NEW_VERSION" +echo -e " * Description: ${GREEN}green text${NC} from above (edits as required)" +echo "4. Bump the release branch $RELEASE_BRANCH to this release:" +echo -e " ${RED}git checkout $RELEASE_BRANCH" +echo " git merge $GIT_HASH --ff-only" +echo -e " git push origin $RELEASE_BRANCH${NC}" +echo -e "5. Update ${BLUE}https://www.mediawiki.org/wiki/Meza/Version_history${NC}" +echo -e "6. Announce on ${BLUE}https://riot.im/app/#/room/#mwstake-MEZA:matrix.org${NC}" +echo + +rm ${RELEASE_NOTES_FILE} From 7fe7b2ebd8fe77b0d9f138dd256986c70dca80d0 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 24 Mar 2019 09:44:05 -0500 Subject: [PATCH 021/280] Make release script edit RELEASE-NOTES.md --- src/scripts/do-release.sh | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) mode change 100644 => 100755 src/scripts/do-release.sh diff --git a/src/scripts/do-release.sh b/src/scripts/do-release.sh old mode 100644 new mode 100755 index f14ab7f57..13f3c41be --- a/src/scripts/do-release.sh +++ b/src/scripts/do-release.sh @@ -47,6 +47,7 @@ CONTRIBUTORS=$(git shortlog -sn "${OLD_VERSION}..HEAD" | while read line; do ech # RELEASE_NOTES_FILE=./.release-notes.tmp cat > ${RELEASE_NOTES_FILE} <<- EOM + ${OVERVIEW} ### Commits since $OLD_VERSION @@ -77,33 +78,37 @@ EOM # # OUTPUT RELEASE NOTES IN GREEN ON COMMAND LINE # -echo -e "${GREEN}" -cat "${RELEASE_NOTES_FILE}" -echo -e "${NC}" +# I think preferable not to output this here +# echo -e "${GREEN}" +# cat "${RELEASE_NOTES_FILE}" +# echo -e "${NC}" + # # TO-DO: Automate edit of release notes # -# sed -e '1,/=============/r.release-notes.tmp' ./RELEASE-NOTES.md +sed -i -e '/=============/r.release-notes.tmp' ./RELEASE-NOTES.md +sed -i "s/=============/\0\n\n## Meza $NEW_VERSION/" ./RELEASE-NOTES.md + # # OUTPUT DIRECTIONS FOR COMPLETING RELEASE # echo -echo "1. Edit RELEASE-NOTES.md" -echo -e " * Copy the ${GREEN}green text${NC} from above and add it under the title ${GREEN}## Meza $NEW_VERSION${NC}" -echo " * Edit the text as required" -echo "2. Commit your changes and submit a pull request" +echo "Release notes generated. To complete the release do the following:" +echo +echo -e "1. Check changes to RELEASE-NOTES.md with ${RED}git diff${NC}" +echo "2. Commit changes and submit a pull request" echo "3. After the PR is merged create a new release of Meza with these details:" echo " * Tag: $NEW_VERSION" echo " * Title: Meza $NEW_VERSION" -echo -e " * Description: ${GREEN}green text${NC} from above (edits as required)" -echo "4. Bump the release branch $RELEASE_BRANCH to this release:" +echo -e " * Description: the ${GREEN}Meza $NEW_VERSION${NC} section from RELEASE-NOTES.md" +echo -e "4. Move the ${GREEN}$RELEASE_BRANCH${NC} branch to the same point as the ${GREEN}${NEW_VERSION}${NC} release:" echo -e " ${RED}git checkout $RELEASE_BRANCH" echo " git merge $GIT_HASH --ff-only" echo -e " git push origin $RELEASE_BRANCH${NC}" -echo -e "5. Update ${BLUE}https://www.mediawiki.org/wiki/Meza/Version_history${NC}" -echo -e "6. Announce on ${BLUE}https://riot.im/app/#/room/#mwstake-MEZA:matrix.org${NC}" +echo -e "5. Update ${GREEN}https://www.mediawiki.org/wiki/Meza/Version_history${NC}" +echo -e "6. Announce on ${GREEN}https://riot.im/app/#/room/#mwstake-MEZA:matrix.org${NC}" echo rm ${RELEASE_NOTES_FILE} From 47f65182796f67c20822938e914076ac1bc58941 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 24 Mar 2019 10:07:59 -0500 Subject: [PATCH 022/280] Improvements to rel notes script order --- src/scripts/do-release.sh | 44 +++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/scripts/do-release.sh b/src/scripts/do-release.sh index 13f3c41be..0fc2227e7 100755 --- a/src/scripts/do-release.sh +++ b/src/scripts/do-release.sh @@ -18,28 +18,54 @@ LATEST="${PREVIOUS_RELEASES##*$'\n'}" GIT_HASH=$(git rev-parse HEAD | cut -c1-8) # -# READ IN USER INPUTS +# WELCOME MESSAGE # -read -p "Add optional single line of overview text: " OVERVIEW +echo +echo "* * * * * * * * * * * * * * * * * * * * * * * *" +echo "* *" +echo "* Meza Release Generator *" +echo "* *" +echo "* * * * * * * * * * * * * * * * * * * * * * * *" -echo -e "${BLUE}" +# +# USER INPUT: CHOOSE OLD VERSION NUMBER TO BASE FROM +# +echo -e "${GREEN}" echo "${PREVIOUS_RELEASES}" echo -e "${NC}" while [ -z "$OLD_VERSION" ]; do - read -p "Enter previous release number (options in blue above): " -i "$LATEST" -e OLD_VERSION + read -p "Enter previous release number (options in green above): " -i "$LATEST" -e OLD_VERSION done; +# +# SETUP LIST OF COMMITS FOR DISPLAY NOW AND INCLUSION IN RELEASE-NOTES.MD +# +COMMITS=$(git log --oneline --no-merges "${OLD_VERSION}..HEAD" | while read line; do echo "* $line"; done) + +echo +echo -e "From ${GREEN}${OLD_VERSION}${NC} to ${GREEN}HEAD${NC}, these are the non-merge commits:" +echo -e "${GREEN}" +echo "${COMMITS}" +echo -e "${NC}" + +# +# USER INPUT: CHOOSE NEW VERSION NUMBER +# while [ -z "$NEW_VERSION" ]; do read -p "Enter new version number in form X.Y.Z: " NEW_VERSION done; +# +# USER INPUT: OVERVIEW TEXT +# +read -p "Based upon commits above, choose optional 1-line overview: " OVERVIEW + # # SETUP VARS BASED UPON USER INPUT # -MAJOR_VERSION=$(echo "$OLD_VERSION" | cut -f1 -d".") +MAJOR_VERSION=$(echo "$NEW_VERSION" | cut -f1 -d".") RELEASE_BRANCH="${MAJOR_VERSION}.x" -COMMITS=$(git log --oneline --no-merges "${OLD_VERSION}..HEAD" | while read line; do echo "* $line"; done) CONTRIBUTORS=$(git shortlog -sn "${OLD_VERSION}..HEAD" | while read line; do echo "* $line"; done) # @@ -95,6 +121,12 @@ sed -i "s/=============/\0\n\n## Meza $NEW_VERSION/" ./RELEASE-NOTES.md # OUTPUT DIRECTIONS FOR COMPLETING RELEASE # echo +echo "* * * * * * * * * * * * * * * * * * * * * * * *" +echo "* *" +echo "* Release process started *" +echo "* *" +echo "* * * * * * * * * * * * * * * * * * * * * * * *" +echo echo "Release notes generated. To complete the release do the following:" echo echo -e "1. Check changes to RELEASE-NOTES.md with ${RED}git diff${NC}" From 7ce4ddbf60fcfa0252b19cd455cc52a0d5360c0b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 25 Mar 2019 15:43:40 -0500 Subject: [PATCH 023/280] Make script actually do release commit. Beware. --- src/scripts/do-release.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/scripts/do-release.sh b/src/scripts/do-release.sh index 0fc2227e7..bfb9169ab 100755 --- a/src/scripts/do-release.sh +++ b/src/scripts/do-release.sh @@ -84,15 +84,6 @@ ${COMMITS} ${CONTRIBUTORS} -### Mediawiki.org pages updated - -* TBD - -### What still isn't documented? - -* TBD -* See [list of issues and pull requests indicating missing docs](https://github.com/enterprisemediawiki/meza/pulls?utf8=%E2%9C%93&q=label%3A%22open+post-merge+actions%22+) - # How to upgrade \`\`\`bash @@ -116,6 +107,16 @@ EOM sed -i -e '/=============/r.release-notes.tmp' ./RELEASE-NOTES.md sed -i "s/=============/\0\n\n## Meza $NEW_VERSION/" ./RELEASE-NOTES.md +# +# COMMIT CHANGE +# +git add RELEASE-NOTES.md +# Set current branch as base branch +BASE_BRANCH=$(git branch | grep \* | cut -d ' ' -f2) +RELEASE_BRANCH="${NEW_VERSION}-release" +git checkout -b "${RELEASE_BRANCH}" +git commit -m "${NEW_VERSION} release" +git push origin "$BASE_BRANCH" # # OUTPUT DIRECTIONS FOR COMPLETING RELEASE @@ -127,10 +128,10 @@ echo "* Release process started *" echo "* *" echo "* * * * * * * * * * * * * * * * * * * * * * * *" echo -echo "Release notes generated. To complete the release do the following:" +echo "Release notes generated, committed, and pushed. " echo -echo -e "1. Check changes to RELEASE-NOTES.md with ${RED}git diff${NC}" -echo "2. Commit changes and submit a pull request" +echo -e "1. Check what you pushed with ${RED}git diff HEAD~1..HEAD${NC}" +echo -e "2. Open a pull request at ${GREEN}https://github.com/enterprisemediawiki/meza/compare/${BASE_BRANCH}...${RELEASE_BRANCH}?expand=1${NC}" echo "3. After the PR is merged create a new release of Meza with these details:" echo " * Tag: $NEW_VERSION" echo " * Title: Meza $NEW_VERSION" @@ -141,6 +142,8 @@ echo " git merge $GIT_HASH --ff-only" echo -e " git push origin $RELEASE_BRANCH${NC}" echo -e "5. Update ${GREEN}https://www.mediawiki.org/wiki/Meza/Version_history${NC}" echo -e "6. Announce on ${GREEN}https://riot.im/app/#/room/#mwstake-MEZA:matrix.org${NC}" +echo -e "7. Update pages on ${GREEN}https://mediawiki.org/wiki/Meza${NC}" echo rm ${RELEASE_NOTES_FILE} + From 0d23cf7a140171fed05b872aadaeff5138553690 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 13 May 2019 08:21:36 -0500 Subject: [PATCH 024/280] Don't push --- src/scripts/do-release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripts/do-release.sh b/src/scripts/do-release.sh index bfb9169ab..e1430ccef 100755 --- a/src/scripts/do-release.sh +++ b/src/scripts/do-release.sh @@ -116,7 +116,7 @@ BASE_BRANCH=$(git branch | grep \* | cut -d ' ' -f2) RELEASE_BRANCH="${NEW_VERSION}-release" git checkout -b "${RELEASE_BRANCH}" git commit -m "${NEW_VERSION} release" -git push origin "$BASE_BRANCH" +# git push origin "$BASE_BRANCH" # # OUTPUT DIRECTIONS FOR COMPLETING RELEASE @@ -130,7 +130,7 @@ echo "* * * * * * * * * * * * * * * * * * * * * * * *" echo echo "Release notes generated, committed, and pushed. " echo -echo -e "1. Check what you pushed with ${RED}git diff HEAD~1..HEAD${NC}" +echo -e "1. Check what you committed with ${RED}git diff HEAD~1..HEAD${NC}, then push" echo -e "2. Open a pull request at ${GREEN}https://github.com/enterprisemediawiki/meza/compare/${BASE_BRANCH}...${RELEASE_BRANCH}?expand=1${NC}" echo "3. After the PR is merged create a new release of Meza with these details:" echo " * Tag: $NEW_VERSION" From 4687f14b6e6e6b68565d3148834ccedc68dfb30c Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 13 May 2019 09:56:22 -0500 Subject: [PATCH 025/280] 31.4.0 release --- RELEASE-NOTES.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index f620cd4a4..d75f17f20 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,46 @@ Release Notes ============= +## Meza 31.4.0 + +Make importing from a live server simpler and more secure by not requiring sudo on the remote. Also use a more stable version of ImageMagick. + +### Commits since 31.3.0 + +* 0d23cf7 Don't push +* 7ce4ddb Make script actually do release commit. Beware. +* 47f6518 Improvements to rel notes script order +* 7fe7b2e Make release script edit RELEASE-NOTES.md +* 02cb384 WIP: release notes script +* 7ea1f98 Use known user, not no user, when mysqldump user unspecified +* 84bc0a0 Install mysql client on backup servers for direct mysqldump +* 112317c Handle undefined backups_server_db_dump +* db6f805 Give undefined debug vars print vals +* cd79230 Make checks for wiki existence during backup go to right server +* 68ac393 Add tags for rsync-uploads and better debug +* 4d56386 Set permissions for /opt/conf-meza and /opt/conf-meza/public +* 6ecc855 Create role remote-dir-check to verify if remote uploads dir exists +* 527852f Get public config repo as meza-ansible, not root +* 51f5e34 Re-enable PEAR package; not used by default, but used by MS SQL +* af9eb52 Move vault pass file from meza-ansible home to /opt/conf-meza/vault +* bd6b103 Minor spelling mistakes +* 43691c4 Vagrantfile set mount_options: ["dmode=755,fmode=755"] Windows only +* 4127c78 Use base ImageMagick rather than Meza's own RPM +* 6ae982b Use 'remote_src' +* 186298a The SFTP server is in a different location on Debian + +### Contributors + +* 25 James Montalvo +* 3 Greg Rundlett + +# How to upgrade + +```bash +sudo meza update 31.4.0 +sudo meza deploy +``` + ## Meza 31.3.0 WatchAnalytics to 3.1.2 for diff in PendingReviews, improved on-page banners From 6789e7dca28bc68784067100f7c7056c3b51dcce Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 13 May 2019 17:09:43 -0500 Subject: [PATCH 026/280] Ansible Git module fails with /tmp mounted with noexec; set TMPDIR as workaround --- src/roles/apache-php/tasks/profiling.yml | 3 +++ .../init-controller-config/tasks/main.yml | 3 +++ src/roles/mediawiki/tasks/main.yml | 18 ++++++++++++++++++ src/roles/parsoid/tasks/main.yml | 3 +++ src/roles/saml/tasks/main.yml | 6 ++++++ tests/deploys/setup-alt-source-backup.yml | 4 ++++ 6 files changed, 37 insertions(+) diff --git a/src/roles/apache-php/tasks/profiling.yml b/src/roles/apache-php/tasks/profiling.yml index b767b92bc..c7235a6a4 100644 --- a/src/roles/apache-php/tasks/profiling.yml +++ b/src/roles/apache-php/tasks/profiling.yml @@ -38,6 +38,9 @@ - pecl/mongodb - name: Ensure XHGui present + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: https://github.com/perftools/xhgui.git dest: "{{ m_profiling_xhgui_directory }}" diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index 1e07fdc22..ee8dfc75d 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -30,6 +30,9 @@ - name: Get local config repo if set become: yes become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: "{{ local_config_repo.repo }}" dest: "{{ m_local_public }}" diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 5e9dad419..722beff72 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -72,6 +72,9 @@ - name: Ensure proper MediaWiki git version installed become: yes become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: https://github.com/wikimedia/mediawiki.git dest: "{{ m_mediawiki }}" @@ -118,6 +121,9 @@ - name: Ensure core meza extensions installed (non-Composer) become: yes become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/extensions/{{ item.name }}" @@ -138,6 +144,9 @@ - always - name: Ensure core meza skins installed (non-Composer) + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/skins/{{ item.name }}" @@ -151,6 +160,9 @@ - name: Ensure local meza extensions installed (non-Composer) become: yes become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/extensions/{{ item.name }}" @@ -165,6 +177,9 @@ - latest - name: Ensure local meza skins installed (non-Composer) + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/skins/{{ item.name }}" @@ -287,6 +302,9 @@ # LANDING PAGE # - name: Ensure WikiBlender installed + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: https://github.com/jamesmontalvo3/WikiBlender.git dest: "{{ m_htdocs }}/WikiBlender" diff --git a/src/roles/parsoid/tasks/main.yml b/src/roles/parsoid/tasks/main.yml index c1ea521e6..69f062da5 100644 --- a/src/roles/parsoid/tasks/main.yml +++ b/src/roles/parsoid/tasks/main.yml @@ -7,6 +7,9 @@ # working directory is wiped out on each run. Then We can immediately patch # the repo in the following step (optionally, if we want tags) - name: Get Parsoid repository + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: https://github.com/wikimedia/parsoid.git dest: "{{ m_parsoid_path }}" diff --git a/src/roles/saml/tasks/main.yml b/src/roles/saml/tasks/main.yml index 44518dcd4..e1192f0f8 100644 --- a/src/roles/saml/tasks/main.yml +++ b/src/roles/saml/tasks/main.yml @@ -1,6 +1,9 @@ --- - name: Ensure SimpleSamlPhp (PHP SAML library) installed + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: https://github.com/simplesamlphp/simplesamlphp.git dest: "{{ m_simplesamlphp_path }}" @@ -10,6 +13,9 @@ - latest - name: Ensure SimpleSamlAuth (MediaWiki extension) installed + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: # Main version of this extension, jornane/mwSimpleSamlAuth, is slow to # update. Use this fork instead, which should stay current with jornane diff --git a/tests/deploys/setup-alt-source-backup.yml b/tests/deploys/setup-alt-source-backup.yml index 08bd19716..67e5a1880 100644 --- a/tests/deploys/setup-alt-source-backup.yml +++ b/tests/deploys/setup-alt-source-backup.yml @@ -9,6 +9,7 @@ vars: alt_source_backups_dir: /opt/alt/backups m_home: /opt/conf-meza/users + m_tmp: /opt/data-meza/tmp tasks: - name: Ensure packages installed @@ -23,6 +24,9 @@ - perl-DBD-MySQL - name: Ensure backups repo in place + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" git: repo: https://github.com/jamesmontalvo3/meza-test-backups.git dest: "{{ alt_source_backups_dir }}" From e919123f0896d997ebd6b2443ba39a65fec15b42 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 16 May 2019 12:15:43 -0500 Subject: [PATCH 027/280] Don't use services during docker image building --- src/roles/elasticsearch/tasks/main.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index 256f6afa0..e218b5303 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -110,18 +110,29 @@ register: version_found retries: 10 delay: 10 + when: docker_skip_tasks is not defined or not docker_skip_tasks - name: Display current Elasticsearch full version number debug: var: version_found.json.version.number + when: docker_skip_tasks is not defined or not docker_skip_tasks - name: Display desired Elasticsearch version debug: var: elasticsearch_major_version -- name: Set the Elasticsearch major version numbers +- name: Set the Elasticsearch major version found set_fact: es_version_found: "{{ version_found.json.version.number | list | first }}" + when: docker_skip_tasks is not defined or not docker_skip_tasks + +- name: "Docker image building only: set found version to desired version" + set_fact: + es_version_found: "{{ elasticsearch_major_version | list | first }}" + when: docker_skip_tasks is defined and docker_skip_tasks + +- name: Set the Elasticsearch major version desired + set_fact: es_version_desired: "{{ elasticsearch_major_version | list | first }}" # Do false first. Then force_do_elasticsearch_upgrade can override From 36f965fbc2b703f0d0422764587461e25baed73c Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 16 May 2019 13:28:34 -0500 Subject: [PATCH 028/280] Reorder AND statement since initial_wikis_dir_check undefined in docker build --- src/roles/mediawiki/tasks/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 722beff72..661bdea56 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -371,12 +371,16 @@ vars: wiki_id: demo wiki_name: Demo Wiki - when: "initial_wikis_dirs_check.files|length == 0 and (docker_skip_tasks is not defined or not docker_skip_tasks)" + when: + - (docker_skip_tasks is not defined or not docker_skip_tasks) + - initial_wikis_dirs_check.files|length == 0 - name: Re-sync configuration between controller and app servers if Demo just configured include_role: name: sync-configs - when: "initial_wikis_dirs_check.files|length == 0 and (docker_skip_tasks is not defined or not docker_skip_tasks)" + when: + - (docker_skip_tasks is not defined or not docker_skip_tasks) + - initial_wikis_dirs_check.files|length == 0 From 0795348149ddb71dae2baad093677431698cfbd4 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 16 May 2019 14:17:19 -0500 Subject: [PATCH 029/280] More docker skip tasks --- src/roles/mediawiki/tasks/main.yml | 1 + src/roles/parsoid/tasks/main.yml | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 661bdea56..aaeb4e721 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -482,6 +482,7 @@ - name: Verify metastore index upgraded shell: WIKI={{ list_of_wikis[0] }} php /opt/htdocs/mediawiki/extensions/CirrusSearch/maintenance/metastore.php --upgrade run_once: true + when: docker_skip_tasks is not defined or not docker_skip_tasks # Wikis are totally built at this point, but SMW and search need rebuilding # FIXME #811: Will this work when controller is not an app server? diff --git a/src/roles/parsoid/tasks/main.yml b/src/roles/parsoid/tasks/main.yml index 69f062da5..58648ce8e 100644 --- a/src/roles/parsoid/tasks/main.yml +++ b/src/roles/parsoid/tasks/main.yml @@ -62,7 +62,10 @@ service: name: parsoid state: stopped - when: modify_parsoid_user and parsoid_service_status.stat.exists + when: + - docker_skip_tasks is not defined or not docker_skip_tasks + - modify_parsoid_user + - parsoid_service_status.stat.exists - name: Ensure parsoid user exists user: From 40e43885785151ec5424a19fc84a4aef5fb38d22 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 17 May 2019 08:35:51 -0500 Subject: [PATCH 030/280] Don't recreate meza-ansible if user already exists (Revert #965) PR #965 did some work to fix issues with restrictive umask settings. It also stated that it "made getmeza.sh more idempotent" but one of the changes was to make it so each run of the script attempted to create user meza-ansible, and in the process recreated SSH keys for the user. This is the opposite of idempotent, and must be undone. This is currently breaking docker builds. #965 does not indicate if there was a specific reason for making this change. --- src/scripts/getmeza.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index 07cf9d952..af67ab811 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -78,15 +78,14 @@ if $ret; then else echo "meza-ansible home-dir in correct location" fi +else + echo + echo "Add ansible master user" + source "/opt/meza/src/scripts/ssh-users/setup-master-user.sh" fi chown meza-ansible:wheel /opt/conf-meza -echo -echo "Add ansible master user" -source "/opt/meza/src/scripts/ssh-users/setup-master-user.sh" - - # Don't require TTY or visible password for sudo. Ref #769 sed -r -i "s/^Defaults\\s+requiretty/#Defaults requiretty/g;" /etc/sudoers sed -r -i "s/^Defaults\\s+\!visiblepw/#Defaults \\!visiblepw/g;" /etc/sudoers From 87a401236d76d3bbbffa8811a3a047f5a105e025 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 14 Dec 2018 16:07:53 -0600 Subject: [PATCH 031/280] Make autodeployer, overwrite-deploy, and backups-cleanup configurable --- config/core/defaults.yml | 4 ++ src/playbooks/site.yml | 16 +++++ src/roles/autodeployer/tasks/main.yml | 58 +++++++++++++++++++ .../templates/check-for-changes.sh.j2} | 0 .../autodeployer/templates/do-deploy.sh.j2} | 0 .../autodeployer/templates/git-fetch.sh.j2} | 0 .../templates/meza-autodeployer-cron.j2 | 27 +++++++++ .../autodeployer/templates/no-notify.sh.j2} | 0 .../templates/slack-notify.sh.j2} | 0 src/roles/backups-cleanup/tasks/main.yml | 17 ++++++ .../templates/backups-cleanup.sh.j2 | 14 +++++ .../templates/meza-cron-backups-cleanup.j2 | 6 ++ 12 files changed, 142 insertions(+) create mode 100644 src/roles/autodeployer/tasks/main.yml rename src/{scripts/autodeployer/check-for-changes.sh => roles/autodeployer/templates/check-for-changes.sh.j2} (100%) mode change 100755 => 100644 rename src/{scripts/autodeployer/do-deploy.sh => roles/autodeployer/templates/do-deploy.sh.j2} (100%) mode change 100755 => 100644 rename src/{scripts/autodeployer/git-fetch.sh => roles/autodeployer/templates/git-fetch.sh.j2} (100%) mode change 100755 => 100644 create mode 100644 src/roles/autodeployer/templates/meza-autodeployer-cron.j2 rename src/{scripts/autodeployer/no-notify.sh => roles/autodeployer/templates/no-notify.sh.j2} (100%) mode change 100755 => 100644 rename src/{scripts/autodeployer/slack-notify.sh => roles/autodeployer/templates/slack-notify.sh.j2} (100%) mode change 100755 => 100644 create mode 100644 src/roles/backups-cleanup/tasks/main.yml create mode 100644 src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 create mode 100644 src/roles/backups-cleanup/templates/meza-cron-backups-cleanup.j2 diff --git a/config/core/defaults.yml b/config/core/defaults.yml index a07597ed8..001c41958 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -214,6 +214,10 @@ m_backups_mode: "0755" m_backups_owner: root m_backups_group: root +# mode/ownership for anything in /etc/cron.d +m_crond_mode: "0644" +m_crond_owner: root +m_crond_group: root m_config_public_mode: "0755" m_config_public_owner: meza-ansible diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index a485d23b5..380083097 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -87,6 +87,13 @@ - umask-set - init-controller-config +- hosts: localhost + become: yes + roles: + - set-vars + - role: autodeployer + when: force_deploy is defined or autodeployer is defined + # Ensure proper base setup on all servers in inventory, with the exception of # servers in "exclude-all" group. At present, the intent of this group is to # allow servers which serve as sources for database and user-uploaded files, @@ -370,6 +377,15 @@ - role: meza-log when: docker_skip_tasks is not defined or not docker_skip_tasks +- hosts: backup-servers + become: yes + tags: + - backup-cleanup + roles: + - set-vars + - role: backups-cleanup + when: backups_cleanup is defined and backups_cleanup.crontime is defined + - hosts: all:!exclude-all:!load-balancers-unmanaged become: yes tags: diff --git a/src/roles/autodeployer/tasks/main.yml b/src/roles/autodeployer/tasks/main.yml new file mode 100644 index 000000000..2ce35b8ec --- /dev/null +++ b/src/roles/autodeployer/tasks/main.yml @@ -0,0 +1,58 @@ +--- + +# Enable force_deploy by putting something like the following in secret.yml +# +# force_deploy: +# crontime: "0 18 * * 0" +# overwrite: true +# extra_options: "--skip-tags smw-data,search-index" + + +- name: + set_fact: + _force_deploy_notify_prefix: "Backup" + _force_deploy_options: "--overwrite {{ force_deploy.extra_options|default('') }}" + when: + - force_deploy is defined + - force_deploy.overwrite is defined + - force_deploy.overwrite + +- name: + set_fact: + _force_deploy_notify_prefix: "Deploy" + _force_deploy_options: "{{ force_deploy.extra_options|default('') }}" + when: + - force_deploy is defined + - > + force_deploy.overwrite is not defined + or (force_deploy.overwrite is defined and not force_deploy.overwrite) + +- name: "Ensure {{ m_deploy }}/autodeployer exists" + file: + path: "{{ m_deploy }}/autodeployer" + owner: root + group: root + mode: 0644 + state: directory + +- name: Ensure autodeployer scripts in place + template: + src: "{{ item }}.j2" + dest: "{{ m_deploy }}/autodeployer/{{ item }}" + owner: root + group: root + mode: "0744" + with_items: + - "check-for-changes.sh" + - "do-deploy.sh" + - "git-fetch.sh" + - "no-notify.sh" + - "slack-notify.sh" + +- name: Ensure autodeployer cron in place + template: + src: "meza-autodeployer-cron.j2" + dest: "/etc/cron.d/meza-autodeployer-cron-{{ env }}" + owner: "{{ m_crond_owner }}" + group: "{{ m_crond_group }}" + mode: "{{ m_crond_mode }}" diff --git a/src/scripts/autodeployer/check-for-changes.sh b/src/roles/autodeployer/templates/check-for-changes.sh.j2 old mode 100755 new mode 100644 similarity index 100% rename from src/scripts/autodeployer/check-for-changes.sh rename to src/roles/autodeployer/templates/check-for-changes.sh.j2 diff --git a/src/scripts/autodeployer/do-deploy.sh b/src/roles/autodeployer/templates/do-deploy.sh.j2 old mode 100755 new mode 100644 similarity index 100% rename from src/scripts/autodeployer/do-deploy.sh rename to src/roles/autodeployer/templates/do-deploy.sh.j2 diff --git a/src/scripts/autodeployer/git-fetch.sh b/src/roles/autodeployer/templates/git-fetch.sh.j2 old mode 100755 new mode 100644 similarity index 100% rename from src/scripts/autodeployer/git-fetch.sh rename to src/roles/autodeployer/templates/git-fetch.sh.j2 diff --git a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 new file mode 100644 index 000000000..0188ae2dd --- /dev/null +++ b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 @@ -0,0 +1,27 @@ +# Do deploys for environment "{{ env }}" +SHELL=/bin/bash +PATH=/sbin:/bin:/usr/sbin:/usr/bin +MAILTO=root + +{% if force_deploy is defined and force_deploy.crontime is defined %} +# +# Force deploy +# FIXME: At present this extracts environment from /opt/.deploy-meza/config.sh. +# This works fine if the controller is used for only one environment, but +# will get confused if multiple environments are used. Environment should +# be explicitly passed into this script. +# +{{ force_deploy.crontime }} root {{ m_scripts }}/autodeployer/do-deploy.sh "{{ _force_deploy_notify_prefix }}" "{{ _force_deploy_options }}" "deploy-" +{% endif %} + + +{% if autodeployer is defined and autodeployer.crontime is defined %} +# +# Auto-deploy on config and Meza changes +# FIXME: At present this extracts environment from /opt/.deploy-meza/config.sh. +# This works fine if the controller is used for only one environment, but +# will get confused if multiple environments are used. Environment should +# be explicitly passed into this script. +# +{{ autodeployer.crontime }} root {{ m_scripts }}/autodeployer/check-for-changes.sh > {{ m_logs }}/deploy/check-for-changes.log 2>&1 +{% endif %} diff --git a/src/scripts/autodeployer/no-notify.sh b/src/roles/autodeployer/templates/no-notify.sh.j2 old mode 100755 new mode 100644 similarity index 100% rename from src/scripts/autodeployer/no-notify.sh rename to src/roles/autodeployer/templates/no-notify.sh.j2 diff --git a/src/scripts/autodeployer/slack-notify.sh b/src/roles/autodeployer/templates/slack-notify.sh.j2 old mode 100755 new mode 100644 similarity index 100% rename from src/scripts/autodeployer/slack-notify.sh rename to src/roles/autodeployer/templates/slack-notify.sh.j2 diff --git a/src/roles/backups-cleanup/tasks/main.yml b/src/roles/backups-cleanup/tasks/main.yml new file mode 100644 index 000000000..eeb9d03ef --- /dev/null +++ b/src/roles/backups-cleanup/tasks/main.yml @@ -0,0 +1,17 @@ +--- + +- name: Ensure backups cleanup script in place + template: + src: "backups-cleanup.sh.j2" + dest: "{{ m_deploy }}/backups-cleanup.sh" + owner: root + group: root + mode: "0744" + +- name: Ensure backups cleanup cron in place + template: + src: "meza-cron-backups-cleanup.j2" + dest: "{{ m_deploy }}/meza-cron-backups-cleanup" + owner: "{{ m_crond_owner }}" + group: "{{ m_crond_group }}" + mode: "{{ m_crond_mode }}" diff --git a/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 b/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 new file mode 100644 index 000000000..260415359 --- /dev/null +++ b/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/sh + +# change directory to backups location +cd {{ m_backups }}/{{ env }} + +# remove all odd-numbered days (keep only even-numbered days) +{% for removal_rule in backups_cleanup.removal_rules %} +rm {{ removal_rule }} -f +{% endfor %} + +# compress all SQL files +echo "gzip start time: $(date +\"%T\")" +gzip -9 ./*/*.sql +echo "gzip end time: $(date +\"%T\")" diff --git a/src/roles/backups-cleanup/templates/meza-cron-backups-cleanup.j2 b/src/roles/backups-cleanup/templates/meza-cron-backups-cleanup.j2 new file mode 100644 index 000000000..7291ab3f6 --- /dev/null +++ b/src/roles/backups-cleanup/templates/meza-cron-backups-cleanup.j2 @@ -0,0 +1,6 @@ +# Cleanup backup files for environment "{{ env }}" +SHELL=/bin/bash +PATH=/sbin:/bin:/usr/sbin:/usr/bin +MAILTO=root + +{{ backups_cleanup.crontime }} root {{ m_deploy }}/backups-cleanup.sh > {{ m_logs }}/cleanup/backups-cleanup-`date "+\%Y\%m\%d"`.log 2>&1 From a5396d8f2ce7d3e4893c5185bc10dbe316761a69 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 15 Dec 2018 14:19:58 -0600 Subject: [PATCH 032/280] Fix location of backups-cleanup cron --- src/roles/backups-cleanup/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/backups-cleanup/tasks/main.yml b/src/roles/backups-cleanup/tasks/main.yml index eeb9d03ef..934c4c317 100644 --- a/src/roles/backups-cleanup/tasks/main.yml +++ b/src/roles/backups-cleanup/tasks/main.yml @@ -11,7 +11,7 @@ - name: Ensure backups cleanup cron in place template: src: "meza-cron-backups-cleanup.j2" - dest: "{{ m_deploy }}/meza-cron-backups-cleanup" + dest: "/etc/cron.d/meza-cron-backups-cleanup-{{ env }}" owner: "{{ m_crond_owner }}" group: "{{ m_crond_group }}" mode: "{{ m_crond_mode }}" From 28df0422f3cf6456ad645b945a7cca72d9a9a1f2 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 17 Dec 2018 22:25:55 -0600 Subject: [PATCH 033/280] Fix autodeployer crons --- src/roles/autodeployer/templates/meza-autodeployer-cron.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 index 0188ae2dd..15a547b5d 100644 --- a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 +++ b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 @@ -11,7 +11,7 @@ MAILTO=root # will get confused if multiple environments are used. Environment should # be explicitly passed into this script. # -{{ force_deploy.crontime }} root {{ m_scripts }}/autodeployer/do-deploy.sh "{{ _force_deploy_notify_prefix }}" "{{ _force_deploy_options }}" "deploy-" +{{ force_deploy.crontime }} root {{ m_deploy }}/autodeployer/do-deploy.sh "{{ _force_deploy_notify_prefix }}" "{{ _force_deploy_options }}" "deploy-" {% endif %} @@ -23,5 +23,5 @@ MAILTO=root # will get confused if multiple environments are used. Environment should # be explicitly passed into this script. # -{{ autodeployer.crontime }} root {{ m_scripts }}/autodeployer/check-for-changes.sh > {{ m_logs }}/deploy/check-for-changes.log 2>&1 +{{ autodeployer.crontime }} root {{ m_deploy }}/autodeployer/check-for-changes.sh > {{ m_logs }}/deploy/check-for-changes.log 2>&1 {% endif %} From 36104ea9f24ead464da456b69071c803a51d24a6 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 15 May 2019 15:51:48 -0500 Subject: [PATCH 034/280] Have meza-ansible do autodeployer git-fetch --- src/roles/autodeployer/templates/git-fetch.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/autodeployer/templates/git-fetch.sh.j2 b/src/roles/autodeployer/templates/git-fetch.sh.j2 index abbab7769..659facd71 100644 --- a/src/roles/autodeployer/templates/git-fetch.sh.j2 +++ b/src/roles/autodeployer/templates/git-fetch.sh.j2 @@ -12,6 +12,6 @@ REPO="$1" DEST="$2" VERSION="$3" -ansible localhost -m git -a "repo=$REPO dest=$DEST version=$VERSION" +sudo -u meza-ansible ansible localhost -m git -a "repo=$REPO dest=$DEST version=$VERSION" cd "$CWD" From 220df48974b094373a5ddf5febd45c86e3c600ef Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 15 May 2019 16:09:42 -0500 Subject: [PATCH 035/280] Add autodeployer tag --- src/playbooks/site.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index 380083097..ab95d443b 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -93,6 +93,8 @@ - set-vars - role: autodeployer when: force_deploy is defined or autodeployer is defined + tags: + - autodeployer # Ensure proper base setup on all servers in inventory, with the exception of # servers in "exclude-all" group. At present, the intent of this group is to From 3854c6c4ec210284d0a2a084eda13236f1bc4fbf Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 16 May 2019 10:55:06 -0500 Subject: [PATCH 036/280] Make sure to use TMPDIR when doing Ansible Git operations --- config/core/adhoc/ansible.cfg | 3 +++ src/roles/autodeployer/templates/git-fetch.sh.j2 | 4 +++- src/roles/base-config-scripts/templates/config.php.j2 | 6 +----- src/roles/base-config-scripts/templates/config.sh.j2 | 6 +----- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/config/core/adhoc/ansible.cfg b/config/core/adhoc/ansible.cfg index e0c9ee54e..da11c4c9f 100644 --- a/config/core/adhoc/ansible.cfg +++ b/config/core/adhoc/ansible.cfg @@ -10,3 +10,6 @@ stdout_callback = json ; use callbacks on ad hoc commands bin_ansible_callbacks=True + +ansible_user = meza-ansible + diff --git a/src/roles/autodeployer/templates/git-fetch.sh.j2 b/src/roles/autodeployer/templates/git-fetch.sh.j2 index 659facd71..b0ffb018d 100644 --- a/src/roles/autodeployer/templates/git-fetch.sh.j2 +++ b/src/roles/autodeployer/templates/git-fetch.sh.j2 @@ -12,6 +12,8 @@ REPO="$1" DEST="$2" VERSION="$3" -sudo -u meza-ansible ansible localhost -m git -a "repo=$REPO dest=$DEST version=$VERSION" +source /opt/.deploy-meza/config.sh + +sudo TMPDIR="$m_tmp" -u meza-ansible ansible localhost -m git -a "repo=$REPO dest=$DEST version=$VERSION" cd "$CWD" diff --git a/src/roles/base-config-scripts/templates/config.php.j2 b/src/roles/base-config-scripts/templates/config.php.j2 index bc3050912..7ec83b17d 100644 --- a/src/roles/base-config-scripts/templates/config.php.j2 +++ b/src/roles/base-config-scripts/templates/config.php.j2 @@ -12,14 +12,10 @@ $wiki_app_fqdn = '{{ wiki_app_fqdn }}'; $m_install = "{{ m_install }}"; $m_meza = "{{ m_meza }}"; -# config dir $m_config_core = "{{ m_config_core }}"; - -# scripts dir $m_scripts = "{{ m_scripts }}"; - -# test dir $m_test = "{{ m_test }}"; +$m_tmp = "{{ m_tmp }}"; # data dir $m_meza_data = "{{ m_meza_data }}"; diff --git a/src/roles/base-config-scripts/templates/config.sh.j2 b/src/roles/base-config-scripts/templates/config.sh.j2 index fa874fafe..968b60484 100644 --- a/src/roles/base-config-scripts/templates/config.sh.j2 +++ b/src/roles/base-config-scripts/templates/config.sh.j2 @@ -11,14 +11,10 @@ m_environment="{{ env }}" m_install="{{ m_install }}" m_meza="{{ m_meza }}" -# config dir m_config_core="{{ m_config_core }}" - -# scripts dir m_scripts="{{ m_scripts }}" - -# test dir m_test="{{ m_test }}" +m_tmp="{{ m_tmp }}" # data dir m_meza_data="{{ m_meza_data }}" From bddb79738a9d7deb98496e8fbaedf42bdd1b57cd Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 19 May 2019 15:31:52 -0500 Subject: [PATCH 037/280] Unique VM names, /opt/meza owned by UID/GID 10000 in Vagrant --- Vagrantfile | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 42e8bb837..75d7efb33 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,6 +1,7 @@ # -*- mode: ruby -*- # vi: set ft=ruby : require 'yaml' +require 'digest/sha1' if not File.file?("#{File.dirname(__FILE__)}/.vagrant/id_rsa") system(" @@ -14,6 +15,9 @@ else configuration = YAML::load(File.read("#{File.dirname(__FILE__)}/vagrantconf.default.yml")) end +# Note: In 32.x+ this will be configurable +box_os = "centos" + # Source: # https://stackoverflow.com/questions/26811089/vagrant-how-to-have-host-platform-specific-provisioning-steps module OS @@ -35,6 +39,8 @@ module OS end end +mezaDirName = File.dirname(__FILE__).rpartition("/").last +mezaInstallUnique = Digest::SHA1.hexdigest File.dirname(__FILE__) # All Vagrant configuration is done below. The "2" in Vagrant.configure # configures the configuration version (we support older styles for @@ -51,7 +57,8 @@ Vagrant.configure("2") do |config| config.vm.define "app2" do |app2| app2.vm.box = "bento/centos-7.4" - app2.vm.hostname = 'app2' + hostname = 'app2-' + box_os + app2.vm.hostname = hostname app2.vm.network :private_network, ip: "192.168.56.57" @@ -60,7 +67,7 @@ Vagrant.configure("2") do |config| v.customize ['modifyvm', :id, '--cableconnected1', 'on'] v.customize ["modifyvm", :id, "--memory", configuration["app2"]["memory"] ] v.customize ["modifyvm", :id, "--cpus", configuration["app2"]["cpus"] ] - v.customize ["modifyvm", :id, "--name", "app2"] + v.customize ["modifyvm", :id, "--name", mezaDirName + '-' + hostname + '-' + mezaInstallUnique] end # Non-controlling server should not have meza @@ -102,7 +109,8 @@ Vagrant.configure("2") do |config| config.vm.define "db2" do |db2| db2.vm.box = "bento/centos-7.4" - db2.vm.hostname = 'db2' + hostname = 'db2-' + box_os + db2.vm.hostname = hostname db2.vm.network :private_network, ip: "192.168.56.58" @@ -111,7 +119,7 @@ Vagrant.configure("2") do |config| v.customize ['modifyvm', :id, '--cableconnected1', 'on'] v.customize ["modifyvm", :id, "--memory", configuration["db2"]["memory"] ] v.customize ["modifyvm", :id, "--cpus", configuration["db2"]["cpus"] ] - v.customize ["modifyvm", :id, "--name", "db2"] + v.customize ["modifyvm", :id, "--name", mezaDirName + '-' + hostname + '-' + mezaInstallUnique] end # Non-controlling server should not have meza @@ -149,10 +157,11 @@ Vagrant.configure("2") do |config| config.vm.define "app1", primary: true do |app1| - # app1.vm.box = "centos/7" + # app1.vm.box = "centos/7"x app1.vm.box = "bento/centos-7.4" # app1.vm.box = "geerlingguy/centos7" - app1.vm.hostname = 'app1' + hostname = 'app1-' + box_os + app1.vm.hostname = hostname # app1.vm.box_url = "ubuntu/precise64" app1.vm.network :private_network, ip: "192.168.56.56" @@ -162,17 +171,21 @@ Vagrant.configure("2") do |config| v.customize ['modifyvm', :id, '--cableconnected1', 'on'] v.customize ["modifyvm", :id, "--memory", configuration["app1"]["memory"] ] v.customize ["modifyvm", :id, "--cpus", configuration["app1"]["cpus"] ] - v.customize ["modifyvm", :id, "--name", "app1"] + v.customize ["modifyvm", :id, "--name", mezaDirName + '-' + hostname + '-' + mezaInstallUnique] end # Disable default synced folder at /vagrant, instead put at /opt/meza app1.vm.synced_folder ".", "/vagrant", disabled: true if OS.windows? - # puts "Vagrant host: Windows" - app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: "vagrant", group: "vagrant", mount_options: ["dmode=755,fmode=755"] + # Vagrant provisioning happens after mounts, so since meza-ansible doesn't + # exist yet at the time of mounting cannot specify owner appropriately. + # Also, at least on Windows it's not possible to change the owner/group + # after it is mounted, so instead we pick a UID and GID and meza-ansible + # and wheel are changed to these IDs after they are created. + app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: 10000, group: 10000, mount_options: ["dmode=755,fmode=755"] else - app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: "vagrant", group: "vagrant" + app1.vm.synced_folder ".", "/opt/meza", type: "virtualbox", owner: 10000, group: 10000 end # app1.vm.synced_folder ".", "/opt/meza", type: "smb" @@ -201,6 +214,9 @@ Vagrant.configure("2") do |config| chown meza-ansible:meza-ansible /opt/conf-meza/users/meza-ansible/.ssh/id_rsa.pub cat /opt/conf-meza/users/meza-ansible/.ssh/id_rsa.pub >> /opt/conf-meza/users/meza-ansible/.ssh/authorized_keys + + usermod -u 10000 meza-ansible + groupmod -g 10000 wheel SHELL # From 04e8ddbb8e5e260762cb9120aed1d64796f67832 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 20 May 2019 14:48:50 -0500 Subject: [PATCH 038/280] Prevent simultaneous deploys (#1157) Make Meza not allow deploys when one is already running --- src/scripts/getmeza.sh | 3 +++ src/scripts/meza.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index af67ab811..5d703e9f1 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -61,6 +61,9 @@ mkdir -p /opt/conf-meza/secret chmod 755 /opt/conf-meza chmod 755 /opt/conf-meza/secret +# Required initially for creating lock files +mkdir -p /opt/data-meza + # If user meza-ansible already exists, make sure home directory is correct # (update from old meza versions) ret=false diff --git a/src/scripts/meza.py b/src/scripts/meza.py index c5d9823c1..b4f375420 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -72,6 +72,12 @@ def meza_command_deploy (argv): rc = check_environment(env) + lock_success = request_lock_for_deploy(env) + + if not lock_success: + print "Deploy for environment {} in progress. Exiting".format(env) + sys.exit(1) + # return code != 0 means failure if rc != 0: if env == "monolith": @@ -107,10 +113,33 @@ def meza_command_deploy (argv): return_code = meza_shell_exec( shell_cmd ) + unlock_deploy(env) + meza_shell_exec_exit( return_code ) +def request_lock_for_deploy (env): + import os + lock_file = get_lock_file_path(env) + if os.path.isfile( lock_file ): + print "Deploy lock file already exists at {}".format(lock_file) + return False + else: + print "Create deploy lock file at {}".format(lock_file) + with open( lock_file, 'w' ) as f: + f.write( "deploying" ) + f.close() + return True +def unlock_deploy(env): + import os + lock_file = get_lock_file_path(env) + if os.path.exists( lock_file ): + os.remove( lock_file ) +def get_lock_file_path(env): + import os + lock_file = os.path.join( defaults['m_meza_data'], "env-{}-deploy.lock".format(env) ) + return lock_file # env # dev From dd60b91d2bf1e06650a98660dde72588ebca596e Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 20 May 2019 19:20:52 -0500 Subject: [PATCH 039/280] Add meza subcommand to check if deploy underway --- src/scripts/meza.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index b4f375420..d4791f6fc 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -55,7 +55,7 @@ def main (argv): command = argv[0] - command_fn = "meza_command_{}".format( argv[0] ) + command_fn = "meza_command_{}".format( argv[0] ).replace("-","_") # if command_fn is a valid Python function, pass it all remaining args if command_fn in globals() and callable( globals()[command_fn] ): @@ -141,6 +141,18 @@ def get_lock_file_path(env): lock_file = os.path.join( defaults['m_meza_data'], "env-{}-deploy.lock".format(env) ) return lock_file +# "meza deploy-check " to return 0 on no deploy, 1 on deploy is active +def meza_command_deploy_check (argv): + import os + env = argv[0] + lock_file = get_lock_file_path(env) + if os.path.isfile( lock_file ): + print "Meza environment {} deploying; {} exists".format(env,lock_file) + sys.exit(1) + else: + print "Meza environment {} not deploying".format(env) + sys.exit(0) + # env # dev # dev-networking --> vbox-networking ?? From a8f1aca21f035b39d952dce3a3817d3e5cbc8c60 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 20 May 2019 19:40:59 -0500 Subject: [PATCH 040/280] Add 'meza deploy-(un)lock commands; Autodeployer use them to avoid conflicts --- .../templates/check-for-changes.sh.j2 | 18 +++++++++++++++ src/scripts/meza.py | 23 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/roles/autodeployer/templates/check-for-changes.sh.j2 b/src/roles/autodeployer/templates/check-for-changes.sh.j2 index d5fc37b33..8cf0376e6 100644 --- a/src/roles/autodeployer/templates/check-for-changes.sh.j2 +++ b/src/roles/autodeployer/templates/check-for-changes.sh.j2 @@ -58,6 +58,15 @@ if [ ! -z "$autodeployer_slack_username" ]; then SLACK_USERNAME="$autodeployer_s if [ ! -z "$autodeployer_slack_channel" ]; then SLACK_CHANNEL="$autodeployer_slack_channel"; fi if [ ! -z "$autodeployer_slack_icon_url" ]; then SLACK_ICON_URL="$autodeployer_slack_icon_url"; fi +meza deploy-check "$m_environment" +if [ $? -eq 0 ]; then + echo "Checked if deploy underway. Is not. Placing deploy lock while checking for changes." + meza deploy-lock "$m_environment" +else + echo "Deploy is underway. Exiting." + exit 1 +fi + # If SLACK_TOKEN is set, send notification via slack. Else, use no-notify script if [ ! -z "$SLACK_TOKEN" ]; then NOTIFY="$DIR/slack-notify.sh" @@ -181,6 +190,7 @@ set -e # end FIXME from above. # Neither Meza mor config changed? Exit. # if [ -z "$PUBLIC_CONFIG_AFTER_HASH$MEZA_AFTER_HASH" ]; then + meza deploy-unlock "$m_environment" echo "Nothing to deploy" exit 0; fi @@ -251,5 +261,13 @@ echo "Deploying" if [ -z "$DEPLOY_TYPE" ]; then DEPLOY_TYPE="Deploy"; fi if [ -z "$DEPLOY_ARGS" ]; then DEPLOY_ARGS=""; fi if [ -z "$DEPLOY_LOG_PREFIX" ]; then DEPLOY_LOG_PREFIX="deploy-after-config-change-"; fi + +# This isn't perfect, as there is still a little that will happen in do-deploy +# below before the actual deploy starts, and a separate deploy _could_ start +# in between now and then, but the likelihood is low, and the impact is only +# this deploy wouldn't happen since it's actual deploy would check for the lock +# file and would fail/exit. +meza deploy-unlock "$m_environment" + source "$DIR/do-deploy.sh" echo "Done" diff --git a/src/scripts/meza.py b/src/scripts/meza.py index d4791f6fc..76ac11b7b 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -135,6 +135,9 @@ def unlock_deploy(env): lock_file = get_lock_file_path(env) if os.path.exists( lock_file ): os.remove( lock_file ) + return True + else: + return False def get_lock_file_path(env): import os @@ -153,6 +156,26 @@ def meza_command_deploy_check (argv): print "Meza environment {} not deploying".format(env) sys.exit(0) +def meza_command_deploy_lock (argv): + env = argv[0] + success = request_lock_for_deploy(env) + if success: + print "Environment {} locked for deploy".format(env) + sys.exit(0) + else: + print "Environment {} could not be locked".format(env) + sys.exit(1) + +def meza_command_deploy_unlock (argv): + env = argv[0] + success = unlock_deploy(env) + if success: + print "Environment {} deploy lock removed".format(env) + sys.exit(0) + else: + print "Environment {} is not deploying".format(env) + sys.exit(1) + # env # dev # dev-networking --> vbox-networking ?? From 9438f7c4cfbe96a078ac01b9f77d9d561ae88e81 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 21 May 2019 17:27:04 -0500 Subject: [PATCH 041/280] Handle sigint; also better info in lock file --- src/scripts/meza.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 76ac11b7b..c43825cb4 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -6,6 +6,19 @@ import sys, getopt, os +# Handle pressing of ctrl-c. Make sure to remove lock file when deploying. +deploy_lock_environment = False +def sigint_handler(sig, frame): + print('Cancelling...') + if deploy_lock_environment: + print('Deploy underway...removing lock file') + unlock_deploy(deploy_lock_environment) + sys.exit(1) + +import signal +signal.signal(signal.SIGINT, sigint_handler) + + def load_yaml ( filepath ): import yaml with open(filepath, 'r') as stream: @@ -118,17 +131,26 @@ def meza_command_deploy (argv): meza_shell_exec_exit( return_code ) def request_lock_for_deploy (env): - import os + import os, datetime lock_file = get_lock_file_path(env) if os.path.isfile( lock_file ): print "Deploy lock file already exists at {}".format(lock_file) return False else: print "Create deploy lock file at {}".format(lock_file) + pid = str( os.getpid() ) + timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S") + + # Before creating lock file, this global must be set in order for ctrl-c + # interrupts (SIGINT) to be properly managed (SIGINT will call + # sigint_handler function) + global deploy_lock_environment + deploy_lock_environment = env + with open( lock_file, 'w' ) as f: - f.write( "deploying" ) + f.write( "{}\n{}".format(pid,timestamp) ) f.close() - return True + return { "pid": pid, "timestamp": timestamp } def unlock_deploy(env): import os From c5ef0e55ebb856db3513be863e5f5dae8740f1ce Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 21 May 2019 17:30:13 -0500 Subject: [PATCH 042/280] Add meza deploy-kill, deploy-log, deploy-tail functions --- src/scripts/meza.py | 53 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index c43825cb4..04170cdfd 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -172,32 +172,73 @@ def meza_command_deploy_check (argv): env = argv[0] lock_file = get_lock_file_path(env) if os.path.isfile( lock_file ): - print "Meza environment {} deploying; {} exists".format(env,lock_file) + print "Meza environment '{}' deploying; {} exists".format(env,lock_file) sys.exit(1) else: - print "Meza environment {} not deploying".format(env) + print "Meza environment '{}' not deploying".format(env) sys.exit(0) def meza_command_deploy_lock (argv): env = argv[0] success = request_lock_for_deploy(env) if success: - print "Environment {} locked for deploy".format(env) + print "Environment '{}' locked for deploy".format(env) sys.exit(0) else: - print "Environment {} could not be locked".format(env) + print "Environment '{}' could not be locked".format(env) sys.exit(1) def meza_command_deploy_unlock (argv): env = argv[0] success = unlock_deploy(env) if success: - print "Environment {} deploy lock removed".format(env) + print "Environment '{}' deploy lock removed".format(env) sys.exit(0) else: - print "Environment {} is not deploying".format(env) + print "Environment '{}' is not deploying".format(env) sys.exit(1) +def meza_command_deploy_kill (argv): + env = argv[0] + lock_file = get_lock_file_path(env) + if os.path.isfile( lock_file ): + print "Meza environment {} deploying; killing...".format(env) + di = get_deploy_info(env) + os.system( "kill $(ps -o pid= --ppid {})".format(di['pid']) ) + import time + time.sleep(2) + os.system( 'wall "Meza deploy terminated using \'meza deploy-kill\' command."' ) + sys.exit(0) + else: + print "Meza environment '{}' not deploying".format(env) + sys.exit(1) + +def get_deploy_info (env): + import os + lock_file = get_lock_file_path(env) + if not os.path.isfile( lock_file ): + print "Environment '{}' not deploying".format(env) + return False + with open( lock_file, 'r' ) as f: + pid = f.readline() + timestamp = f.readline() + f.close() + return { "pid": pid, "timestamp": timestamp } + +def get_deploy_log_path (env): + timestamp = get_deploy_info(env)["timestamp"] + filename = "{}-{}.log".format( env,timestamp ) + log_path = os.path.join( defaults['m_logs'], 'deploy-output', filename ) + return log_path + +def meza_command_deploy_log (argv): + env = argv[0] + print get_deploy_log_path(env) + +def meza_command_deploy_tail (argv): + env = argv[0] + os.system( " ".join(["tail", "-f", get_deploy_log_path(env)]) ) + # env # dev # dev-networking --> vbox-networking ?? From a120ac329d5ab4cf1c52d0ed425f1d16626e559d Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 21 May 2019 17:32:11 -0500 Subject: [PATCH 043/280] Make meza_shell_exec use subprocess; optionally write to log file --- src/scripts/meza.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 04170cdfd..c4aff6d07 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -124,7 +124,9 @@ def meza_command_deploy (argv): if len(argv) > 0: shell_cmd = shell_cmd + argv - return_code = meza_shell_exec( shell_cmd ) + deploy_log = get_deploy_log_path(env) + + return_code = meza_shell_exec( shell_cmd, deploy_log ) unlock_deploy(env) @@ -754,7 +756,7 @@ def playbook_cmd ( playbook, env=False, more_extra_vars=False ): # FIXME install --> setup dev-networking, setup docker, deploy monolith (special case) -def meza_shell_exec ( shell_cmd ): +def meza_shell_exec ( shell_cmd, log_file=False ): # Get errors with user meza-ansible trying to write to the calling-user's # home directory if don't cd to a neutral location. By cd'ing to this @@ -762,16 +764,6 @@ def meza_shell_exec ( shell_cmd ): starting_wd = os.getcwd() os.chdir( "/opt/meza/config/core" ) - # import subprocess - # # child = subprocess.Popen(shell_cmd, stdout=subprocess.PIPE) - # child = subprocess.Popen(shell_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - # if return_output: - # output = child.communicate()[0] - # else: - # print child.communicate()[0] - # rc = child.returncode - - # # FIXME #874: For some reason `sudo -u meza-ansible ...` started failing in # fall 2017. Using `su meza-ansible -c "..."` works. It is not @@ -786,7 +778,18 @@ def meza_shell_exec ( shell_cmd ): cmd = ' '.join(shell_cmd) print cmd - rc = os.system(cmd) + + import subprocess + + if log_file: + log = open(log_file,'a') + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + for line in iter(proc.stdout.readline, b''): + print( line.rstrip() ) + if log_file: + log.write( line ) + + rc = proc.returncode # Move back to original working directory os.chdir( starting_wd ) From 500997425cb91103c35099f295acb3d01080700c Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 21 May 2019 17:32:34 -0500 Subject: [PATCH 044/280] Always have ansible show colors --- config/core/ansible.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/core/ansible.cfg b/config/core/ansible.cfg index 535aa60a4..19f039439 100644 --- a/config/core/ansible.cfg +++ b/config/core/ansible.cfg @@ -26,3 +26,8 @@ stdout_callback = debug ; forks = 1 remote_tmp = /tmp/${USER}/ansible + +# Makes deploys write to logs with color info. Puts extra characters into logs +# which is ugly when viewed in an editor (Vim) but looks better when you do +# `less -r /path/to/file` or similar commands. +force_color = 1 From e7b8ad36fe81fa494d8c650febd7eed2dfde5f02 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 21 May 2019 20:20:44 -0500 Subject: [PATCH 045/280] Make sure deploy log directory exists --- src/scripts/meza.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index c4aff6d07..30097b9eb 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -230,7 +230,13 @@ def get_deploy_info (env): def get_deploy_log_path (env): timestamp = get_deploy_info(env)["timestamp"] filename = "{}-{}.log".format( env,timestamp ) - log_path = os.path.join( defaults['m_logs'], 'deploy-output', filename ) + + log_dir = os.path.join( defaults['m_logs'], 'deploy-output' ) + log_path = os.path.join( log_dir, filename ) + + if not os.path.isdir( log_dir ): + os.makedirs( log_dir ) + return log_path def meza_command_deploy_log (argv): From 60a680c02a5bb80609e61d3f31e40936e34021ac Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 21 May 2019 21:41:42 -0500 Subject: [PATCH 046/280] Add wait() to capture return code Fix for deploy exiting with: Traceback (most recent call last): File "/usr/bin/meza", line 966, in main(sys.argv[1:]) File "/usr/bin/meza", line 75, in main globals()[command_fn]( argv[1:] ) File "/usr/bin/meza", line 133, in meza_command_deploy meza_shell_exec_exit( return_code ) File "/usr/bin/meza", line 810, in meza_shell_exec_exit if int(return_code) > 0: TypeError: int() argument must be a string or a number, not 'NoneType' --- src/scripts/meza.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 30097b9eb..dd3e06a7f 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -794,6 +794,7 @@ def meza_shell_exec ( shell_cmd, log_file=False ): print( line.rstrip() ) if log_file: log.write( line ) + proc.wait() rc = proc.returncode From 2513a36c0bc2bebbdcffcda76b80fe42b64bd3ff Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 22 May 2019 15:38:26 -0500 Subject: [PATCH 047/280] Set ownership of meza and config; fix role:init-controller-config init-controller-config was being run on app-servers, but then had delegate:localhost on all tasks. This didn't make sense. Instead just run the play on localhost. --- config/core/defaults.yml | 10 +++++ src/playbooks/site.yml | 11 +++--- .../init-controller-config/tasks/main.yml | 38 +++++++++++-------- src/scripts/getmeza.sh | 1 + 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 001c41958..a2ad6a7dd 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -188,6 +188,13 @@ clean_upload_stash_crontime: "0 18 * * *" # FILE MODES, OWNERS, GROUPS # +m_meza_owner: meza-ansible +m_meza_group: wheel +# Don't set mode for /opt/meza for now. Don't want to impact execute bit which +# is managed by Git +# FIXME: Later specify read and write permissions only if Ansible supports + + # uploads directory. Note: user meza-ansible is in group "apache" m_uploads_dir_mode: "0775" m_uploads_dir_owner: apache @@ -223,6 +230,9 @@ m_config_public_mode: "0755" m_config_public_owner: meza-ansible m_config_public_group: wheel +m_config_secret_mode: "0750" +m_config_secret_owner: meza-ansible +m_config_secret_group: wheel # # PHP config diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index ab95d443b..dbc00347f 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -78,23 +78,23 @@ group: wheel mode: "0755" - -# FIXME 800: Run against localhost -- hosts: app-servers +# Set umask on all servers. Perhaps this should move above localhost steps since +# it may impact them, and role "umask-set" has no config requirements +- hosts: all:!exclude-all:!load-balancers-unmanaged become: yes roles: - - set-vars - umask-set - - init-controller-config - hosts: localhost become: yes roles: - set-vars + - init-controller-config - role: autodeployer when: force_deploy is defined or autodeployer is defined tags: - autodeployer + - controller # Ensure proper base setup on all servers in inventory, with the exception of # servers in "exclude-all" group. At present, the intent of this group is to @@ -104,7 +104,6 @@ become: yes roles: - set-vars - - umask-set - base - base-config-scripts tags: base diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index ee8dfc75d..706cd4beb 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -23,8 +23,6 @@ stat: path: "{{ m_local_public }}" register: controller_local_config - delegate_to: localhost - run_once: true # If a git repo is defined use that for config - name: Get local config repo if set @@ -39,8 +37,6 @@ version: "{{ local_config_repo.version | default('master') }}" force: "{{ local_config_repo.force | default(false) | bool }}" umask: "0002" - delegate_to: localhost - run_once: true when: not controller_local_config.stat.exists and local_config_repo.repo is defined @@ -51,8 +47,6 @@ stat: path: "{{ m_local_public }}" register: recheck_controller - delegate_to: localhost - run_once: true # At this point, whether a m_local directory exists on the controller or not, # ensure the directory exists and is configured properly @@ -64,8 +58,8 @@ group: "{{ m_config_public_group }}" mode: "{{ m_config_public_mode }}" recurse: true - delegate_to: localhost - run_once: true + tags: + - file-perms # Still no config for controller? This must be a new installation. Copy from @@ -77,9 +71,6 @@ owner: "{{ m_config_public_owner }}" group: "{{ m_config_public_group }}" mode: "{{ m_config_public_mode }}" - delegate_to: localhost - run_once: true - - name: Ensure pre/post settings directories exists in config file: @@ -88,8 +79,6 @@ owner: "{{ m_config_public_owner }}" group: "{{ m_config_public_group }}" mode: "{{ m_config_public_mode }}" - delegate_to: localhost - run_once: true with_items: - preLocalSettings.d - postLocalSettings.d @@ -103,9 +92,28 @@ group: "{{ m_config_public_group }}" mode: "{{ m_config_public_mode }}" force: no - delegate_to: localhost - run_once: true with_items: - MezaLocalExtensions.yml - MezaLocalSkins.yml - public.yml + +- name: "Ensure {{ m_meza }} properly owned" + file: + path: "{{ m_meza }}" + owner: "{{ m_meza_owner }}" + group: "{{ m_meza_group }}" + state: directory + recurse: Yes + tags: + - file-perms + +- name: "Ensure {{ m_local_secret }} properly owned" + file: + path: "{{ m_local_secret }}" + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_mode }}" + state: directory + recurse: Yes + tags: + - file-perms diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index 5d703e9f1..a5d780505 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -88,6 +88,7 @@ else fi chown meza-ansible:wheel /opt/conf-meza +chown meza-ansible:wheel /opt/meza # Don't require TTY or visible password for sudo. Ref #769 sed -r -i "s/^Defaults\\s+requiretty/#Defaults requiretty/g;" /etc/sudoers From 0502a41cc2de11c53d89e65d9a49684d301d8188 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 27 May 2019 04:26:53 -0500 Subject: [PATCH 048/280] Ansible 2.8 fixes (#1162) * Fix for ansible upgrade * WIP * WIP * WIP * WIP * WIP * [skip ci] * Cleanup --- src/roles/backup-db-wikis/tasks/main.yml | 18 +++++++++--------- src/roles/backup-uploads/tasks/main.yml | 21 +++++++++------------ src/roles/nodejs/tasks/main.yml | 10 +++++++++- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/roles/backup-db-wikis/tasks/main.yml b/src/roles/backup-db-wikis/tasks/main.yml index 0fdf763bd..eff966c34 100644 --- a/src/roles/backup-db-wikis/tasks/main.yml +++ b/src/roles/backup-db-wikis/tasks/main.yml @@ -25,15 +25,15 @@ group: "{{ m_backups_group }}" with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" -# copy from server A (db-master) to server B (backups) -- name: Copy SQL files to backups - synchronize: - # copy from server A - src: "{{ m_tmp }}/{{ env }}_{{ item }}.sql" - # copy to server B - dest: "{{ m_backups }}/{{ env }}/{{ item }}/{{ backup_timestamp }}_wiki.sql" - # server A - delegate_to: "{{ groups['db-master'][0] }}" +- name: "Run role:rsync - Copy SQL files to backups" + include_role: + name: rsync + vars: + pulling_to_server: "{{ inventory_hostname }}" + pulling_to_path: "{{ m_backups }}/{{ env }}/{{ item }}/{{ backup_timestamp }}_wiki.sql" + pulling_from_server: "{{ groups['db-master'][0] }}" + pulling_from_path: "{{ m_tmp }}/{{ env }}_{{ item }}.sql" + pulling_from_user: "meza-ansible" with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" # Remove temp SQL files, only needs to be done on first backup server diff --git a/src/roles/backup-uploads/tasks/main.yml b/src/roles/backup-uploads/tasks/main.yml index 2d863c244..fe49022f4 100644 --- a/src/roles/backup-uploads/tasks/main.yml +++ b/src/roles/backup-uploads/tasks/main.yml @@ -36,16 +36,13 @@ group: root with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" -# copy from server A (app.0) to server B (backups) -- name: Copy uploads directories to backups - synchronize: - # copy from server A - src: "{{ m_uploads_dir }}/{{ item }}/" - # copy to server B - dest: "{{ m_backups }}/{{ env }}/{{ item }}/uploads" - recursive: yes - # server A - delegate_to: "{{ groups['app-servers'][0] }}" +- name: "Run role:rsync - Copy uploads directory to backups" + include_role: + name: rsync + vars: + pulling_to_server: "{{ inventory_hostname }}" + pulling_to_path: "{{ m_backups }}/{{ env }}/{{ item }}/uploads" + pulling_from_server: "{{ groups['app-servers'][0] }}" + pulling_from_path: "{{ m_uploads_dir }}/{{ item }}/" + pulling_from_user: "meza-ansible" with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" - - diff --git a/src/roles/nodejs/tasks/main.yml b/src/roles/nodejs/tasks/main.yml index a24642302..82096a257 100644 --- a/src/roles/nodejs/tasks/main.yml +++ b/src/roles/nodejs/tasks/main.yml @@ -15,11 +15,19 @@ # https://bugs.centos.org/view.php?id=13669&nbn=8 # https://bugzilla.redhat.com/show_bug.cgi?id=1481008 # https://bugzilla.redhat.com/show_bug.cgi?id=1481470 +- name: Get OS minor version (EL-only) + shell: cat /etc/redhat-release | grep -Eo '[0-9]+' | sed -n 2p + register: rhel_os_minor_version + when: ansible_os_family == 'RedHat' +- debug: + var: rhel_os_minor_version - name: Ensure http-parser installed from RPM for {{ ansible_distribution_version }} yum: name: https://kojipkgs.fedoraproject.org//packages/http-parser/2.7.1/3.el7/x86_64/http-parser-2.7.1-3.el7.x86_64.rpm state: present - when: ansible_distribution_version.split('.')[1] | int <= 3 + when: + - ansible_os_family == 'RedHat' + - rhel_os_minor_version.stdout | int <= 3 # - name: Ensure Node.js and npm are installed. # yum: From 0f879f8f950179c892e7e33a8daa68a15489fb9a Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 16 Apr 2019 17:29:57 -0500 Subject: [PATCH 049/280] Make pip3 symlink for RHEL --- src/roles/mediawiki/tasks/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index aaeb4e721..be2ed980a 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -547,3 +547,15 @@ owner: root group: root mode: 0755 + when: ansible_os_family == 'RedHat' + +- name: "Ensure pip3 symlink in place" + file: + # dest = symlink, src = dir linked to + src: "/usr/bin/pip3.5" + dest: "/usr/bin/pip3" + state: link + owner: root + group: root + mode: 0755 + when: ansible_os_family == 'RedHat' From 0155726ed877a3f1450b007196d7e0605f5537a7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 27 May 2019 10:54:20 -0500 Subject: [PATCH 050/280] Add pip3 (31.x didn't have it yet) --- src/roles/mediawiki/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index be2ed980a..16988adce 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -533,7 +533,9 @@ # MediaWiki 1.31+. - name: Ensure Python3 present yum: - name: python35u + name: + - python35u + - python35u-pip state: latest tags: - latest From 3db517a69c1af3ab8ecfb80ec0d74c8c1a153cb1 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 27 May 2019 11:34:14 -0500 Subject: [PATCH 051/280] Add pip for Python 2.7 --- src/roles/base/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index ed3ea66a8..376238dbd 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -130,6 +130,7 @@ - tree - cronie - rsync + - python-pip tags: - latest From a621460fef7547512940cf8d75b187d5a3d4e61d Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 27 May 2019 12:19:06 -0500 Subject: [PATCH 052/280] Remove yum:PackageKit to remove error Error was "yum lockfile is held by another process" --- src/roles/base/tasks/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index ed3ea66a8..38d342edb 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -49,6 +49,13 @@ line: 'alt-meza-ansible ALL=(ALL) NOPASSWD: ALL' validate: 'visudo -cf %s' +# Without this get error "yum lockfile is held by another process" occasionally +- name: Ensure PackageKit is removed so it doesn't try to upgrade packages on its own + yum: + name: PackageKit + state: absent + when: ansible_os_family == 'RedHat' + - name: ensure deltarpm is installed and latest yum: name=deltarpm state=latest tags: From 6e54a869c8aa175e885bf423f4b1f0fa0ca87c1f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 28 May 2019 23:51:35 -0500 Subject: [PATCH 053/280] Enable rsync push backups (#1166) --- config/core/defaults.yml | 12 +++- src/playbooks/push-backup.yml | 25 +++++++ src/playbooks/site.yml | 8 +-- src/roles/backup-config/tasks/main.yml | 55 ++++++++++++++ src/roles/backup-db-wikis-push/tasks/main.yml | 59 +++++++++++++++ src/roles/backup-uploads-push/tasks/main.yml | 48 +++++++++++++ src/roles/base/tasks/main.yml | 8 +++ src/roles/rsync-push/tasks/main.yml | 71 +++++++++++++++++++ src/scripts/meza.py | 13 ++++ 9 files changed, 289 insertions(+), 10 deletions(-) create mode 100644 src/playbooks/push-backup.yml create mode 100644 src/roles/backup-config/tasks/main.yml create mode 100644 src/roles/backup-db-wikis-push/tasks/main.yml create mode 100644 src/roles/backup-uploads-push/tasks/main.yml create mode 100644 src/roles/rsync-push/tasks/main.yml diff --git a/config/core/defaults.yml b/config/core/defaults.yml index a2ad6a7dd..ce233bed2 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -195,6 +195,12 @@ m_meza_group: wheel # FIXME: Later specify read and write permissions only if Ansible supports +m_data_mode: "0755" +m_data_owner: meza-ansible +m_data_group: wheel + + + # uploads directory. Note: user meza-ansible is in group "apache" m_uploads_dir_mode: "0775" m_uploads_dir_owner: apache @@ -217,9 +223,9 @@ m_logs_owner: meza-ansible m_logs_group: wheel -m_backups_mode: "0755" -m_backups_owner: root -m_backups_group: root +m_backups_mode: "0775" +m_backups_owner: meza-ansible +m_backups_group: meza-backups # mode/ownership for anything in /etc/cron.d m_crond_mode: "0644" diff --git a/src/playbooks/push-backup.yml b/src/playbooks/push-backup.yml new file mode 100644 index 000000000..7293c5454 --- /dev/null +++ b/src/playbooks/push-backup.yml @@ -0,0 +1,25 @@ +--- + +# Define a timestamp fact to persist throughout this playbook +- hosts: all:!exclude-all:!load-balancers-nonmeza:!load-balancers-nonmeza-external:!load-balancers-nonmeza-internal + tasks: + - set_fact: + backup_timestamp: "{{lookup('pipe','date +%Y%m%d%H%M%S')}}" + +# FIXME #803: If a slave is available, maybe pull from there +- hosts: db-master + become: yes + roles: + - set-vars + - dump-db-wikis + - backup-db-wikis-push + tags: db + +- hosts: backup-servers + become: yes + roles: + - set-vars + - backup-uploads-push + tags: + - uploads + - upload diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index dbc00347f..67f7539c6 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -297,13 +297,7 @@ tags: backup-servers roles: - set-vars - tasks: - - name: Ensure MariaDB client installed on backup servers - yum: - name: mariadb - state: present - tags: - - latest + - backup-config - hosts: elastic-servers become: yes diff --git a/src/roles/backup-config/tasks/main.yml b/src/roles/backup-config/tasks/main.yml new file mode 100644 index 000000000..c20b6b6ee --- /dev/null +++ b/src/roles/backup-config/tasks/main.yml @@ -0,0 +1,55 @@ +--- + +- name: Ensure MariaDB client installed on backup servers + yum: + name: mariadb + state: present + tags: + - latest + +- name: Ensure meza-backups group exists + group: + name: meza-backups + state: present + +- name: Ensure backups directory exists + file: + path: "{{ m_backups }}" + state: directory + mode: "{{ m_backups_mode }}" + owner: "{{ m_backups_owner }}" + group: "{{ m_backups_group }}" + +- name: Ensure backups directory exists for environment + file: + path: "{{ m_backups }}/{{ env }}" + state: directory + mode: "{{ m_backups_mode }}" + owner: "{{ m_backups_owner }}" + group: "{{ m_backups_group }}" + +- name: Get individual wikis from controller config + find: + paths: "{{ m_local_public }}/wikis/" + file_type: directory + register: wiki_dirs + delegate_to: localhost + +- name: Ensure backups directory exists for each wiki + file: + path: "{{ m_backups }}/{{ env }}/{{ item }}" + state: directory + mode: "{{ m_backups_mode }}" + owner: "{{ m_backups_owner }}" + group: "{{ m_backups_group }}" + with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" + +- name: Ensure backups uploads directory exists for each wiki + file: + path: "{{ m_backups }}/{{ env }}/{{ item }}/uploads" + state: directory + mode: "{{ m_backups_mode }}" + owner: "{{ m_backups_owner }}" + group: "{{ m_backups_group }}" + with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" + diff --git a/src/roles/backup-db-wikis-push/tasks/main.yml b/src/roles/backup-db-wikis-push/tasks/main.yml new file mode 100644 index 000000000..b692ea507 --- /dev/null +++ b/src/roles/backup-db-wikis-push/tasks/main.yml @@ -0,0 +1,59 @@ +--- + +- name: Get individual wikis from controller config + find: + paths: "{{ m_local_public }}/wikis/" + file_type: directory + register: wiki_dirs + delegate_to: localhost + +# NOTE: no validation of proper directory setup on pushing-to server. Non-push +# role "backup-db-wikis" has these tasks to handle directory setup: +# - name: Ensure backups directory exists for environment +# - name: Ensure backups directory exists for each wiki + +- name: Set remote_server_base_path if set in configuration + set_fact: + remote_server_base_path: "{{ backups_server_db_push.sql_files_path }}" + when: + - backups_server_db_push.sql_files_path is defined + +- name: Set remote_server_base_path if NOT set in configuration + set_fact: + remote_server_base_path: "{{ m_backups }}/{{ env }}//" + when: + - backups_server_db_push.sql_files_path is not defined + +- name: Output value of remote_server_base_path ( will be replaced by each wiki_id) + debug: { var: remote_server_base_path } + +- name: "Run role:rsync-push - Copy SQL files to {{ backups_server_db_push.addr }}" + include_role: + name: rsync-push + vars: + pushing_from_server: "{{ inventory_hostname }}" + pushing_from_path: "{{ m_tmp }}/{{ env }}_{{ item }}.sql" + pushing_to_server: "{{ backups_server_db_push.addr }}" + # remote_server_base_path + backup_timestamp + _wiki.sql, but replace with wiki_id (item) + pushing_to_path: "{{ remote_server_base_path | regex_replace('', item) }}{{ backup_timestamp }}_wiki.sql" + pushing_to_user: "{{ backups_server_db_push.remote_user }}" + with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" + +# Remove temp SQL files, only needs to be done on first backup server +- name: Remove SQL files from DB master {{ m_tmp }} + file: + path: "{{ m_tmp }}/{{ env }}_{{ item }}.sql" + state: absent + with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" + run_once: true + +- name: "Clean out all but the latest SQL backup file for {{ item }} IF DESIRED" + include_role: + name: sql-backup-cleanup + vars: + cleanup_wiki: "{{ item }}" + run_once: true + tags: + - sql-backup-cleanup + with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" + when: do_cleanup_sql_backup diff --git a/src/roles/backup-uploads-push/tasks/main.yml b/src/roles/backup-uploads-push/tasks/main.yml new file mode 100644 index 000000000..df1ab126d --- /dev/null +++ b/src/roles/backup-uploads-push/tasks/main.yml @@ -0,0 +1,48 @@ +--- + + +- name: Get individual wikis from controller config + find: + paths: "{{ m_local_public }}/wikis/" + file_type: directory + register: wiki_dirs + delegate_to: localhost + +# FIXME #805: option to delete existing uploads backups initially (clean slate) + +# NOTE: no validation of proper directory setup on pushing-to server. Non-push +# role "backup-uploads" has these tasks to handle directory setup: +# - name: Ensure backups directory exists for environment +# - name: Ensure backups directory exists for each wiki +# - name: Ensure backups uploads directory exists for each wiki + +- name: Set remote_server_base_path if set in configuration + set_fact: + # Likely path if pushing to a live Meza uploads directory: + # /opt/data-meza/uploads// + remote_server_base_path: "{{ backups_server_uploads_push.uploads_dir_path }}" + when: + - backups_server_uploads_push.uploads_dir_path is defined + +- name: Set remote_server_base_path if NOT set in configuration + set_fact: + # Likely path if pushing to a Meza backups directory: + # /opt/data-meza/backups///uploads/ + remote_server_base_path: "{{ m_backups }}/{{ env }}//uploads/" + when: + - backups_server_uploads_push.uploads_dir_path is not defined + +- name: Output value of remote_server_base_path ( will be replaced by each wiki_id) + debug: { var: remote_server_base_path } + +- name: "Run role:rsync-push - Copy uploads directory to {{ backups_server_uploads_push.addr }}" + include_role: + name: rsync-push + vars: + pushing_from_server: "{{ inventory_hostname }}" + pushing_from_path: "{{ m_uploads_dir }}/{{ item }}/" + pushing_to_server: "{{ backups_server_uploads_push.addr }}" + # remote_server_base_path + backup_timestamp, but replace with wiki_id (item) + pushing_to_path: "{{ remote_server_base_path | regex_replace('', item) }}" + pushing_to_user: "{{ backups_server_uploads_push.remote_user }}" + with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index cc3fd448e..c0b79eefa 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -210,6 +210,14 @@ # pip: # name: cryptography +- name: Ensure data directory configured + file: + path: "{{ m_meza_data }}" + state: directory + owner: "{{ m_data_mode }}" + group: "{{ m_data_mode }}" + mode: "{{ m_data_mode }}" + - name: Ensure deploy directory in place file: path: "{{ m_deploy }}" diff --git a/src/roles/rsync-push/tasks/main.yml b/src/roles/rsync-push/tasks/main.yml new file mode 100644 index 000000000..6a4fe6a51 --- /dev/null +++ b/src/roles/rsync-push/tasks/main.yml @@ -0,0 +1,71 @@ +--- +# Rsync push role +# +# Inputs: +# pushing_from_server: +# pushing_from_path: +# pushing_to_server: +# pushing_to_path: +# pushing_to_user: + + +# +# Put meza-ansible's private key and known_hosts on server within /root +# +- name: "Grant keys to {{ pushing_from_server }}" + include_role: + name: key-transfer + tasks_from: grant-keys + vars: + granted_server: "{{ pushing_from_server }}" + when: + pushing_from_server != pushing_to_server + + +# FIXME #818: Remove StrictHostKeyChecking=no when tests properly add host keys (users should do so, too, of course) +# Note on --omit-dir-times: This will not preserve modification times on directories. +# It may be necessary for the remote user to be the owner of a directory, not just +# within the group that owns it, in order to be able to maintain these times. +- name: Set rsync command fact for pushing-from server {{ pushing_from_server }} NOT pushing-to server + set_fact: + rsync_command: > + rsync + --delay-updates + -F + --compress + --copy-links + --archive + --omit-dir-times + "{{ pushing_from_path }}" + --rsh="/usr/bin/ssh -S none -o StrictHostKeyChecking=no + -l {{ pushing_to_user }} -i /root/meza-ansible-id_rsa + -o UserKnownHostsFile=/root/meza-ansible-known_hosts" + "{{ pushing_to_server }}:{{ pushing_to_path }}" + when: + pushing_to_server != pushing_from_server + +- name: Set rsync command fact for pushing-from and pushing-to server IS {{ pushing_from_server }} + set_fact: + rsync_command: > + rsync + --delay-updates + -F + --compress + --copy-links + --archive + "{{ pushing_from_path }}" + "{{ pushing_to_path }}" + when: + pushing_to_server == pushing_from_server + +- name: Run rsync + shell: "{{ rsync_command }}" + delegate_to: "{{ pushing_from_server }}" + run_once: true + +- name: "Revoke keys from {{ pushing_from_server }}" + include_role: + name: key-transfer + tasks_from: revoke-keys + vars: + granted_server: "{{ pushing_from_server }}" diff --git a/src/scripts/meza.py b/src/scripts/meza.py index dd3e06a7f..fbb5990a6 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -728,6 +728,19 @@ def meza_command_docker (argv): sys.exit(1) +def meza_command_push_backup (argv): + + env = argv[0] + + rc = check_environment(env) + if rc != 0: + meza_shell_exec_exit(rc) + + shell_cmd = playbook_cmd( 'push-backup', env ) + argv[1:] + rc = meza_shell_exec( shell_cmd ) + + meza_shell_exec_exit(rc) + def playbook_cmd ( playbook, env=False, more_extra_vars=False ): command = ['sudo', '-u', 'meza-ansible', 'ansible-playbook', From bce32d315beb37055d7bea3ce7d9f81805028255 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 29 May 2019 07:58:55 -0500 Subject: [PATCH 054/280] Autodeployer check for changes to secret config --- .../templates/check-for-changes.sh.j2 | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/src/roles/autodeployer/templates/check-for-changes.sh.j2 b/src/roles/autodeployer/templates/check-for-changes.sh.j2 index 8cf0376e6..bb09ad5a4 100644 --- a/src/roles/autodeployer/templates/check-for-changes.sh.j2 +++ b/src/roles/autodeployer/templates/check-for-changes.sh.j2 @@ -76,11 +76,77 @@ fi GIT_FETCH="$DIR/git-fetch.sh" + +# Set SECRET config version +# +# FIXME: For now can only get repo from existing (add option: from var) +# FIXME: For now only track master branch (add options: existing and from var) +SECRET_CONFIG_DEST="/opt/conf-meza/secret" +cd "$SECRET_CONFIG_DEST" +SECRET_CONFIG_REPO="SECRET_CONFIG_REPO=$(git remote get-url origin)" +SECRET_CONFIG_VERSION="master" +SECRET_CONFIG_CHANGE=$($GIT_FETCH "$SECRET_CONFIG_REPO" "$SECRET_CONFIG_DEST" "$SECRET_CONFIG_VERSION") + +# +# Check if attempt to git-pull PUBLIC CONFIG failed +# +# FIXME: For some reason the jq command below was not working if it was within +# the conditional, so it has to be out here, where it forces us to +# temporarily allow errors. +set +e +echo "Did git fetch fail on secret config?" +echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e +if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$SECRET_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating secret config failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; +fi + +# +# Check if changes were made to SECRET CONFIG +# +echo "Were there changes to secret config?" +echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e +if [ $? -eq 0 ]; then + SECRET_CONFIG_BEFORE_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + SECRET_CONFIG_AFTER_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $SECRET_CONFIG_BEFORE_HASH" + echo "After hash: $SECRET_CONFIG_BEFORE_HASH" + + pushd "$SECRET_CONFIG_DEST" + SECRET_CONFIG_DIFF=$(git diff "$SECRET_CONFIG_BEFORE_HASH" "$SECRET_CONFIG_AFTER_HASH" 2>&1) + SECRET_CONFIG_COMMITS=$(git log --oneline "$SECRET_CONFIG_BEFORE_HASH...$SECRET_CONFIG_AFTER_HASH" 2>&1) + pushd +else + SECRET_CONFIG_DIFF="" + SECRET_CONFIG_AFTER_HASH="" +fi + +# Make sure config.sh is up-to-date after secret config change above, since it +# may impact what version of public config and Meza are checked out below +meza setbaseconfig "$m_environment" +# FIXME: sourcing config.sh here will overwrite previously set vars with any new +# values, and will add any newly defined vars, but if config changed to +# remove an existing var then that value will not be unset. +source /opt/.deploy-meza/config.sh + + # Set PUBLIC config version # # Could optionally set public config's repo in secret config, but since that is # not done universally, not going to enforce it here. Just use whatever repo is # currently being used as origin. +# +# FIXME: I believe the above comment is wrong. I believe it is pulling the repo +# URL from config. Perhaps it should downmode to using the existing repo +# if not set in config (public or secret). PUBLIC_CONFIG_DEST="/opt/conf-meza/public" PUBLIC_CONFIG_REPO="$local_config_repo_repo" PUBLIC_CONFIG_VERSION="$local_config_repo_version" @@ -129,7 +195,7 @@ else fi # Make sure config.sh is up-to-date after public config change above, since it -# impacts what version of +# potentially impacts what version of Meza is checked out meza setbaseconfig "$m_environment" source /opt/.deploy-meza/config.sh @@ -187,9 +253,9 @@ set -e # end FIXME from above. # -# Neither Meza mor config changed? Exit. +# Meza, public, and secret config all unchanged? Exit. # -if [ -z "$PUBLIC_CONFIG_AFTER_HASH$MEZA_AFTER_HASH" ]; then +if [ -z "$SECRET_CONFIG_AFTER_HASH$PUBLIC_CONFIG_AFTER_HASH$MEZA_AFTER_HASH" ]; then meza deploy-unlock "$m_environment" echo "Nothing to deploy" exit 0; From 1dc989486b0beea9bd43d386742c37d558939f92 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 29 May 2019 08:40:22 -0500 Subject: [PATCH 055/280] Use secret_config_repo to define secret config --- .../templates/check-for-changes.sh.j2 | 155 +++++++++--------- .../templates/config.php.j2 | 7 + .../templates/config.sh.j2 | 6 + 3 files changed, 91 insertions(+), 77 deletions(-) diff --git a/src/roles/autodeployer/templates/check-for-changes.sh.j2 b/src/roles/autodeployer/templates/check-for-changes.sh.j2 index bb09ad5a4..3a6da5a3a 100644 --- a/src/roles/autodeployer/templates/check-for-changes.sh.j2 +++ b/src/roles/autodeployer/templates/check-for-changes.sh.j2 @@ -26,29 +26,32 @@ if [ "$(whoami)" != "root" ]; then exit 1 fi -# Check if a deploy is happening. -# -# This will cause the script to exit if a deploy is currently underway, thus -# preventing two deploys from happening at once. -# -# FIXME: not implemented yet -# source "$DIR/check-deploy.sh" - -# Gets info about public config +# Gets initial config defined by secret and public config (and Meza defaults). +# Really all this does is grabs $m_environment, but instead whatever is calling +# this script should pass that in as an argument. source /opt/.deploy-meza/config.sh -# -# FIXME: For now, don't touch secret config. At some point find a way to -# configure it's repo and version. - -# Make sure config.sh is up-to-date in case there has been a secret config -# change since the last deploy, which could impact local_config_repo var. +# Now that $m_environment is available, force update of config.sh using the +# `meza setbaseconfig` command. This will rebuild config.sh based upon current +# state of secret config, public config, and Meza. meza setbaseconfig "$m_environment" source /opt/.deploy-meza/config.sh +# +# Check if deploy is underway +# +meza deploy-check "$m_environment" +if [ $? -eq 0 ]; then + echo "Checked if deploy underway. Is not. Placing deploy lock while checking for changes." + meza deploy-lock "$m_environment" +else + echo "Deploy is underway. Exiting." + exit 1 +fi + -if [ -z "$local_config_repo_repo" ]; then - >&2 echo "Auto-deploy requires 'local_config_repo' set in secret or public config" +if [ -z "$local_config_repo_repo$secret_config_repo_repo$enforce_meza_version" ]; then + >&2 echo "Auto-deploy requires a public config, secret config, or Meza version to be set." exit 1; fi @@ -58,14 +61,6 @@ if [ ! -z "$autodeployer_slack_username" ]; then SLACK_USERNAME="$autodeployer_s if [ ! -z "$autodeployer_slack_channel" ]; then SLACK_CHANNEL="$autodeployer_slack_channel"; fi if [ ! -z "$autodeployer_slack_icon_url" ]; then SLACK_ICON_URL="$autodeployer_slack_icon_url"; fi -meza deploy-check "$m_environment" -if [ $? -eq 0 ]; then - echo "Checked if deploy underway. Is not. Placing deploy lock while checking for changes." - meza deploy-lock "$m_environment" -else - echo "Deploy is underway. Exiting." - exit 1 -fi # If SLACK_TOKEN is set, send notification via slack. Else, use no-notify script if [ ! -z "$SLACK_TOKEN" ]; then @@ -77,66 +72,72 @@ fi GIT_FETCH="$DIR/git-fetch.sh" -# Set SECRET config version -# -# FIXME: For now can only get repo from existing (add option: from var) -# FIXME: For now only track master branch (add options: existing and from var) -SECRET_CONFIG_DEST="/opt/conf-meza/secret" -cd "$SECRET_CONFIG_DEST" -SECRET_CONFIG_REPO="SECRET_CONFIG_REPO=$(git remote get-url origin)" -SECRET_CONFIG_VERSION="master" -SECRET_CONFIG_CHANGE=$($GIT_FETCH "$SECRET_CONFIG_REPO" "$SECRET_CONFIG_DEST" "$SECRET_CONFIG_VERSION") - -# -# Check if attempt to git-pull PUBLIC CONFIG failed -# -# FIXME: For some reason the jq command below was not working if it was within -# the conditional, so it has to be out here, where it forces us to -# temporarily allow errors. -set +e -echo "Did git fetch fail on secret config?" -echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e -if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$SECRET_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating secret config failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" +if [ ! -z "$secret_config_repo_repo" ]; then + + # Set SECRET config version + # + # FIXME: For now can only get repo from existing (add option: from var) + # FIXME: For now only track master branch (add options: existing and from var) + SECRET_CONFIG_DEST="/opt/conf-meza/secret" + SECRET_CONFIG_REPO="$secret_config_repo_repo" + SECRET_CONFIG_VERSION="$secret_config_repo_version" + SECRET_CONFIG_CHANGE=$($GIT_FETCH "$SECRET_CONFIG_REPO" "$SECRET_CONFIG_DEST" "$SECRET_CONFIG_VERSION") + + # + # Check if attempt to git-pull SECRET CONFIG failed + # + # FIXME: For some reason the jq command below was not working if it was within + # the conditional, so it has to be out here, where it forces us to + # temporarily allow errors. + set +e + echo "Did git fetch fail on secret config?" + echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e + if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$SECRET_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating secret config failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; + fi - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY + # + # Check if changes were made to SECRET CONFIG + # + echo "Were there changes to secret config?" + echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e + if [ $? -eq 0 ]; then + SECRET_CONFIG_BEFORE_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + SECRET_CONFIG_AFTER_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $SECRET_CONFIG_BEFORE_HASH" + echo "After hash: $SECRET_CONFIG_BEFORE_HASH" + + pushd "$SECRET_CONFIG_DEST" + SECRET_CONFIG_DIFF=$(git diff "$SECRET_CONFIG_BEFORE_HASH" "$SECRET_CONFIG_AFTER_HASH" 2>&1) + SECRET_CONFIG_COMMITS=$(git log --oneline "$SECRET_CONFIG_BEFORE_HASH...$SECRET_CONFIG_AFTER_HASH" 2>&1) + pushd + else + SECRET_CONFIG_DIFF="" + SECRET_CONFIG_AFTER_HASH="" fi - exit 1; -fi -# -# Check if changes were made to SECRET CONFIG -# -echo "Were there changes to secret config?" -echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e -if [ $? -eq 0 ]; then - SECRET_CONFIG_BEFORE_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - SECRET_CONFIG_AFTER_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $SECRET_CONFIG_BEFORE_HASH" - echo "After hash: $SECRET_CONFIG_BEFORE_HASH" - - pushd "$SECRET_CONFIG_DEST" - SECRET_CONFIG_DIFF=$(git diff "$SECRET_CONFIG_BEFORE_HASH" "$SECRET_CONFIG_AFTER_HASH" 2>&1) - SECRET_CONFIG_COMMITS=$(git log --oneline "$SECRET_CONFIG_BEFORE_HASH...$SECRET_CONFIG_AFTER_HASH" 2>&1) - pushd + # Make sure config.sh is up-to-date after secret config change above, since it + # may impact what version of public config and Meza are checked out below + meza setbaseconfig "$m_environment" + # FIXME: sourcing config.sh here will overwrite previously set vars with any new + # values, and will add any newly defined vars, but if config changed to + # remove an existing var then that value will not be unset. + source /opt/.deploy-meza/config.sh + else SECRET_CONFIG_DIFF="" SECRET_CONFIG_AFTER_HASH="" fi -# Make sure config.sh is up-to-date after secret config change above, since it -# may impact what version of public config and Meza are checked out below -meza setbaseconfig "$m_environment" -# FIXME: sourcing config.sh here will overwrite previously set vars with any new -# values, and will add any newly defined vars, but if config changed to -# remove an existing var then that value will not be unset. -source /opt/.deploy-meza/config.sh - # Set PUBLIC config version # diff --git a/src/roles/base-config-scripts/templates/config.php.j2 b/src/roles/base-config-scripts/templates/config.php.j2 index 7ec83b17d..3f2721ad0 100644 --- a/src/roles/base-config-scripts/templates/config.php.j2 +++ b/src/roles/base-config-scripts/templates/config.php.j2 @@ -349,6 +349,13 @@ $slack_username_disk_usage="{{ slack_username_disk_usage }}"; # # Auto-deployer # +{% if secret_config_repo is defined and secret_config_repo.repo is defined %} +# Info about local public config +$secret_config_repo_repo="{{ secret_config_repo.repo }}"; +$secret_config_repo_version="{{ secret_config_repo.version | default('master') }}"; +$secret_config_repo_force="{{ secret_config_repo.force | default(false) | bool }}"; +{% endif %} + {% if local_config_repo is defined and local_config_repo.repo is defined %} # Info about local public config $local_config_repo_repo="{{ local_config_repo.repo }}"; diff --git a/src/roles/base-config-scripts/templates/config.sh.j2 b/src/roles/base-config-scripts/templates/config.sh.j2 index 968b60484..c3e515425 100644 --- a/src/roles/base-config-scripts/templates/config.sh.j2 +++ b/src/roles/base-config-scripts/templates/config.sh.j2 @@ -178,6 +178,12 @@ slack_username_disk_usage="{{ slack_username_disk_usage }}" # # Auto-deployer # +{% if secret_config_repo is defined and secret_config_repo.repo is defined %} +# Info about local public config +local_config_repo_repo="{{ secret_config_repo.repo }}" +local_config_repo_version="{{ secret_config_repo.version | default('master') }}" +local_config_repo_force="{{ secret_config_repo.force | default(false) | bool }}" +{% endif %} {% if local_config_repo is defined and local_config_repo.repo is defined %} # Info about local public config From 901f2b915533eb6ce53aeac58fdeb12a6582bda2 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 29 May 2019 08:46:34 -0500 Subject: [PATCH 056/280] Make public config and Meza management by autodeployer optional --- .../templates/check-for-changes.sh.j2 | 221 ++++++++++-------- 1 file changed, 130 insertions(+), 91 deletions(-) diff --git a/src/roles/autodeployer/templates/check-for-changes.sh.j2 b/src/roles/autodeployer/templates/check-for-changes.sh.j2 index 3a6da5a3a..945195052 100644 --- a/src/roles/autodeployer/templates/check-for-changes.sh.j2 +++ b/src/roles/autodeployer/templates/check-for-changes.sh.j2 @@ -71,7 +71,9 @@ fi GIT_FETCH="$DIR/git-fetch.sh" - +# +# If management of SECRET config desired +# if [ ! -z "$secret_config_repo_repo" ]; then # Set SECRET config version @@ -139,118 +141,130 @@ else fi -# Set PUBLIC config version # -# Could optionally set public config's repo in secret config, but since that is -# not done universally, not going to enforce it here. Just use whatever repo is -# currently being used as origin. +# If management of PUBLIC config desired # -# FIXME: I believe the above comment is wrong. I believe it is pulling the repo -# URL from config. Perhaps it should downmode to using the existing repo -# if not set in config (public or secret). -PUBLIC_CONFIG_DEST="/opt/conf-meza/public" -PUBLIC_CONFIG_REPO="$local_config_repo_repo" -PUBLIC_CONFIG_VERSION="$local_config_repo_version" -PUBLIC_CONFIG_CHANGE=$($GIT_FETCH "$PUBLIC_CONFIG_REPO" "$PUBLIC_CONFIG_DEST" "$PUBLIC_CONFIG_VERSION") +if [ ! -z "$local_config_repo_repo" ]; then -# -# Check if attempt to git-pull PUBLIC CONFIG failed -# -# FIXME: For some reason the jq command below was not working if it was within -# the conditional, so it has to be out here, where it forces us to -# temporarily allow errors. -set +e -echo "Did git fetch fail on public config?" -echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e -if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$PUBLIC_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating public config failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" + # Set PUBLIC config version + # + # Could optionally set public config's repo in secret config, but since that is + # not done universally, not going to enforce it here. Just use whatever repo is + # currently being used as origin. + # + # FIXME: I believe the above comment is wrong. I believe it is pulling the repo + # URL from config. Perhaps it should downmode to using the existing repo + # if not set in config (public or secret). + PUBLIC_CONFIG_DEST="/opt/conf-meza/public" + PUBLIC_CONFIG_REPO="$local_config_repo_repo" + PUBLIC_CONFIG_VERSION="$local_config_repo_version" + PUBLIC_CONFIG_CHANGE=$($GIT_FETCH "$PUBLIC_CONFIG_REPO" "$PUBLIC_CONFIG_DEST" "$PUBLIC_CONFIG_VERSION") - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY + # + # Check if attempt to git-pull PUBLIC CONFIG failed + # + # FIXME: For some reason the jq command below was not working if it was within + # the conditional, so it has to be out here, where it forces us to + # temporarily allow errors. + set +e + echo "Did git fetch fail on public config?" + echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e + if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$PUBLIC_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating public config failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; fi - exit 1; -fi -# -# Check if changes were made to PUBLIC CONFIG -# -echo "Were there changes to public config?" -echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e -if [ $? -eq 0 ]; then - PUBLIC_CONFIG_BEFORE_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - PUBLIC_CONFIG_AFTER_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $PUBLIC_CONFIG_BEFORE_HASH" - echo "After hash: $PUBLIC_CONFIG_BEFORE_HASH" - - pushd "$PUBLIC_CONFIG_DEST" - PUBLIC_CONFIG_DIFF=$(git diff "$PUBLIC_CONFIG_BEFORE_HASH" "$PUBLIC_CONFIG_AFTER_HASH" 2>&1) - PUBLIC_CONFIG_COMMITS=$(git log --oneline "$PUBLIC_CONFIG_BEFORE_HASH...$PUBLIC_CONFIG_AFTER_HASH" 2>&1) - pushd + # + # Check if changes were made to PUBLIC CONFIG + # + echo "Were there changes to public config?" + echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e + if [ $? -eq 0 ]; then + PUBLIC_CONFIG_BEFORE_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + PUBLIC_CONFIG_AFTER_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $PUBLIC_CONFIG_BEFORE_HASH" + echo "After hash: $PUBLIC_CONFIG_BEFORE_HASH" + + pushd "$PUBLIC_CONFIG_DEST" + PUBLIC_CONFIG_DIFF=$(git diff "$PUBLIC_CONFIG_BEFORE_HASH" "$PUBLIC_CONFIG_AFTER_HASH" 2>&1) + PUBLIC_CONFIG_COMMITS=$(git log --oneline "$PUBLIC_CONFIG_BEFORE_HASH...$PUBLIC_CONFIG_AFTER_HASH" 2>&1) + pushd + else + PUBLIC_CONFIG_DIFF="" + PUBLIC_CONFIG_AFTER_HASH="" + fi + + # Make sure config.sh is up-to-date after public config change above, since it + # potentially impacts what version of Meza is checked out + meza setbaseconfig "$m_environment" + source /opt/.deploy-meza/config.sh + else PUBLIC_CONFIG_DIFF="" PUBLIC_CONFIG_AFTER_HASH="" fi -# Make sure config.sh is up-to-date after public config change above, since it -# potentially impacts what version of Meza is checked out -meza setbaseconfig "$m_environment" -source /opt/.deploy-meza/config.sh - - -# This could change based upon changes to public config, so only check for it at -# this point, not earlier. -if [ -z "$enforce_meza_version" ]; then - >&2 echo "Auto-deploy requires 'enforce_meza_version' var set in public or secret config" - exit 1; -fi - -# Set MEZA version -MEZA_REPO="https://github.com/enterprisemediawiki/meza" -MEZA_DEST="/opt/meza" -MEZA_VERSION="$enforce_meza_version" -MEZA_CHANGE=$($GIT_FETCH "$MEZA_REPO" "$MEZA_DEST" "$MEZA_VERSION") # -# Check if attempt to git-pull MEZA failed +# If management of Meza version desired # -echo "Did git fetch fail on Meza?" -echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e -if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$MEZA_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating Meza failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" +if [ ! -z "$enforce_meza_version" ]; then - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY + # Set MEZA version + MEZA_REPO="https://github.com/enterprisemediawiki/meza" + MEZA_DEST="/opt/meza" + MEZA_VERSION="$enforce_meza_version" + MEZA_CHANGE=$($GIT_FETCH "$MEZA_REPO" "$MEZA_DEST" "$MEZA_VERSION") + + # + # Check if attempt to git-pull MEZA failed + # + echo "Did git fetch fail on Meza?" + echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e + if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$MEZA_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating Meza failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; fi - exit 1; -fi -# -# Check if changes were made to MEZA -# -echo "Were there changes to Meza?" -echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e -if [ $? -eq 0 ]; then - MEZA_BEFORE_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - MEZA_AFTER_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $MEZA_BEFORE_HASH" - echo "After hash: $MEZA_AFTER_HASH" + # + # Check if changes were made to MEZA + # + echo "Were there changes to Meza?" + echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e + if [ $? -eq 0 ]; then + MEZA_BEFORE_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + MEZA_AFTER_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $MEZA_BEFORE_HASH" + echo "After hash: $MEZA_AFTER_HASH" + + pushd "$MEZA_DEST" + MEZA_COMMITS=$(git log --oneline "$MEZA_BEFORE_HASH...$MEZA_AFTER_HASH" 2>&1) + pushd - pushd "$MEZA_DEST" - MEZA_COMMITS=$(git log --oneline "$MEZA_BEFORE_HASH...$MEZA_AFTER_HASH" 2>&1) - pushd + else + MEZA_AFTER_HASH="" + fi + set -e # end FIXME from above. else MEZA_AFTER_HASH="" fi -set -e # end FIXME from above. # @@ -262,6 +276,31 @@ if [ -z "$SECRET_CONFIG_AFTER_HASH$PUBLIC_CONFIG_AFTER_HASH$MEZA_AFTER_HASH" ]; exit 0; fi +# +# Notify if SECRET CONFIG changed +# +if [ ! -z "$SECRET_CONFIG_AFTER_HASH" ]; then + + MESSAGE=$(cat <<-END + Secret config changed versions: + FROM: \`$SECRET_CONFIG_BEFORE_HASH\` + TO: \`$SECRET_CONFIG_AFTER_HASH\` + + Tracking version: \`$SECRET_CONFIG_VERSION\` + + Further information about secret config withheld +END +) + + echo -e "$MESSAGE" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$MESSAGE" + SLACK_COLOR="good" + source $NOTIFY + fi +fi + # # Notify if PUBLIC CONFIG changed # From 006d5edbad121988620849c3f7696c201eb9297e Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 29 May 2019 22:17:15 -0500 Subject: [PATCH 057/280] Reduce duplication in check-for-changes.sh --- src/roles/autodeployer/tasks/main.yml | 2 + .../templates/check-for-changes.sh.j2 | 245 ++---------------- .../autodeployer/templates/diff-notify.sh.j2 | 55 ++++ .../autodeployer/templates/repo-pull.sh.j2 | 67 +++++ .../templates/config.sh.j2 | 6 +- 5 files changed, 149 insertions(+), 226 deletions(-) create mode 100644 src/roles/autodeployer/templates/diff-notify.sh.j2 create mode 100644 src/roles/autodeployer/templates/repo-pull.sh.j2 diff --git a/src/roles/autodeployer/tasks/main.yml b/src/roles/autodeployer/tasks/main.yml index 2ce35b8ec..e13de6b12 100644 --- a/src/roles/autodeployer/tasks/main.yml +++ b/src/roles/autodeployer/tasks/main.yml @@ -48,6 +48,8 @@ - "git-fetch.sh" - "no-notify.sh" - "slack-notify.sh" + - "repo-pull.sh" + - "diff-notify.sh" - name: Ensure autodeployer cron in place template: diff --git a/src/roles/autodeployer/templates/check-for-changes.sh.j2 b/src/roles/autodeployer/templates/check-for-changes.sh.j2 index 945195052..1e6543739 100644 --- a/src/roles/autodeployer/templates/check-for-changes.sh.j2 +++ b/src/roles/autodeployer/templates/check-for-changes.sh.j2 @@ -76,68 +76,15 @@ GIT_FETCH="$DIR/git-fetch.sh" # if [ ! -z "$secret_config_repo_repo" ]; then - # Set SECRET config version - # - # FIXME: For now can only get repo from existing (add option: from var) - # FIXME: For now only track master branch (add options: existing and from var) SECRET_CONFIG_DEST="/opt/conf-meza/secret" SECRET_CONFIG_REPO="$secret_config_repo_repo" SECRET_CONFIG_VERSION="$secret_config_repo_version" - SECRET_CONFIG_CHANGE=$($GIT_FETCH "$SECRET_CONFIG_REPO" "$SECRET_CONFIG_DEST" "$SECRET_CONFIG_VERSION") - - # - # Check if attempt to git-pull SECRET CONFIG failed - # - # FIXME: For some reason the jq command below was not working if it was within - # the conditional, so it has to be out here, where it forces us to - # temporarily allow errors. - set +e - echo "Did git fetch fail on secret config?" - echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e - if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$SECRET_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating secret config failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY - fi - exit 1; - fi - - # - # Check if changes were made to SECRET CONFIG - # - echo "Were there changes to secret config?" - echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e - if [ $? -eq 0 ]; then - SECRET_CONFIG_BEFORE_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - SECRET_CONFIG_AFTER_HASH=$(echo "$SECRET_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $SECRET_CONFIG_BEFORE_HASH" - echo "After hash: $SECRET_CONFIG_BEFORE_HASH" - - pushd "$SECRET_CONFIG_DEST" - SECRET_CONFIG_DIFF=$(git diff "$SECRET_CONFIG_BEFORE_HASH" "$SECRET_CONFIG_AFTER_HASH" 2>&1) - SECRET_CONFIG_COMMITS=$(git log --oneline "$SECRET_CONFIG_BEFORE_HASH...$SECRET_CONFIG_AFTER_HASH" 2>&1) - pushd - else - SECRET_CONFIG_DIFF="" - SECRET_CONFIG_AFTER_HASH="" - fi - - # Make sure config.sh is up-to-date after secret config change above, since it - # may impact what version of public config and Meza are checked out below - meza setbaseconfig "$m_environment" - # FIXME: sourcing config.sh here will overwrite previously set vars with any new - # values, and will add any newly defined vars, but if config changed to - # remove an existing var then that value will not be unset. - source /opt/.deploy-meza/config.sh -else - SECRET_CONFIG_DIFF="" - SECRET_CONFIG_AFTER_HASH="" + source "$DIR/repo-pull.sh" "secret config" "$SECRET_CONFIG_DEST" "$SECRET_CONFIG_REPO" "$SECRET_CONFIG_VERSION" + SECRET_CONFIG_DIFF="$DIFF" + SECRET_CONFIG_BEFORE_HASH="$BEFORE_HASH" + SECRET_CONFIG_AFTER_HASH="$AFTER_HASH" + SECRET_CONFIG_COMMITS="$COMMITS" fi @@ -146,70 +93,16 @@ fi # if [ ! -z "$local_config_repo_repo" ]; then - # Set PUBLIC config version - # - # Could optionally set public config's repo in secret config, but since that is - # not done universally, not going to enforce it here. Just use whatever repo is - # currently being used as origin. - # - # FIXME: I believe the above comment is wrong. I believe it is pulling the repo - # URL from config. Perhaps it should downmode to using the existing repo - # if not set in config (public or secret). PUBLIC_CONFIG_DEST="/opt/conf-meza/public" PUBLIC_CONFIG_REPO="$local_config_repo_repo" PUBLIC_CONFIG_VERSION="$local_config_repo_version" - PUBLIC_CONFIG_CHANGE=$($GIT_FETCH "$PUBLIC_CONFIG_REPO" "$PUBLIC_CONFIG_DEST" "$PUBLIC_CONFIG_VERSION") - - # - # Check if attempt to git-pull PUBLIC CONFIG failed - # - # FIXME: For some reason the jq command below was not working if it was within - # the conditional, so it has to be out here, where it forces us to - # temporarily allow errors. - set +e - echo "Did git fetch fail on public config?" - echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e - if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$PUBLIC_CONFIG_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating public config failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY - fi - exit 1; - fi - - # - # Check if changes were made to PUBLIC CONFIG - # - echo "Were there changes to public config?" - echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e - if [ $? -eq 0 ]; then - PUBLIC_CONFIG_BEFORE_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - PUBLIC_CONFIG_AFTER_HASH=$(echo "$PUBLIC_CONFIG_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $PUBLIC_CONFIG_BEFORE_HASH" - echo "After hash: $PUBLIC_CONFIG_BEFORE_HASH" - - pushd "$PUBLIC_CONFIG_DEST" - PUBLIC_CONFIG_DIFF=$(git diff "$PUBLIC_CONFIG_BEFORE_HASH" "$PUBLIC_CONFIG_AFTER_HASH" 2>&1) - PUBLIC_CONFIG_COMMITS=$(git log --oneline "$PUBLIC_CONFIG_BEFORE_HASH...$PUBLIC_CONFIG_AFTER_HASH" 2>&1) - pushd - else - PUBLIC_CONFIG_DIFF="" - PUBLIC_CONFIG_AFTER_HASH="" - fi - - # Make sure config.sh is up-to-date after public config change above, since it - # potentially impacts what version of Meza is checked out - meza setbaseconfig "$m_environment" - source /opt/.deploy-meza/config.sh -else - PUBLIC_CONFIG_DIFF="" - PUBLIC_CONFIG_AFTER_HASH="" + source "$DIR/repo-pull.sh" "public config" "$PUBLIC_CONFIG_DEST" "$PUBLIC_CONFIG_REPO" "$PUBLIC_CONFIG_VERSION" + PUBLIC_CONFIG_DIFF="$DIFF" + PUBLIC_CONFIG_BEFORE_HASH="$BEFORE_HASH" + PUBLIC_CONFIG_AFTER_HASH="$AFTER_HASH" + PUBLIC_CONFIG_COMMITS="$COMMITS" + fi @@ -222,48 +115,13 @@ if [ ! -z "$enforce_meza_version" ]; then MEZA_REPO="https://github.com/enterprisemediawiki/meza" MEZA_DEST="/opt/meza" MEZA_VERSION="$enforce_meza_version" - MEZA_CHANGE=$($GIT_FETCH "$MEZA_REPO" "$MEZA_DEST" "$MEZA_VERSION") - - # - # Check if attempt to git-pull MEZA failed - # - echo "Did git fetch fail on Meza?" - echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e - if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$MEZA_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating Meza failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY - fi - exit 1; - fi - - # - # Check if changes were made to MEZA - # - echo "Were there changes to Meza?" - echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e - if [ $? -eq 0 ]; then - MEZA_BEFORE_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - MEZA_AFTER_HASH=$(echo "$MEZA_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $MEZA_BEFORE_HASH" - echo "After hash: $MEZA_AFTER_HASH" - - pushd "$MEZA_DEST" - MEZA_COMMITS=$(git log --oneline "$MEZA_BEFORE_HASH...$MEZA_AFTER_HASH" 2>&1) - pushd - - else - MEZA_AFTER_HASH="" - fi - set -e # end FIXME from above. -else - MEZA_AFTER_HASH="" + source "$DIR/repo-pull.sh" "public config" "$MEZA_DEST" "$MEZA_REPO" "$MEZA_VERSION" + MEZA_DIFF="$DIFF" + MEZA_BEFORE_HASH="$BEFORE_HASH" + MEZA_AFTER_HASH="$AFTER_HASH" + MEZA_COMMITS="$COMMITS" + fi @@ -280,82 +138,23 @@ fi # Notify if SECRET CONFIG changed # if [ ! -z "$SECRET_CONFIG_AFTER_HASH" ]; then - - MESSAGE=$(cat <<-END - Secret config changed versions: - FROM: \`$SECRET_CONFIG_BEFORE_HASH\` - TO: \`$SECRET_CONFIG_AFTER_HASH\` - - Tracking version: \`$SECRET_CONFIG_VERSION\` - - Further information about secret config withheld -END -) - - echo -e "$MESSAGE" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$MESSAGE" - SLACK_COLOR="good" - source $NOTIFY - fi + # arg 5 = commits, 6 = diff. Don't show for secret config + source "$DIR/diff-notify.sh" "Secret config" "$SECRET_CONFIG_BEFORE_HASH" "$SECRET_CONFIG_AFTER_HASH" "$SECRET_CONFIG_VERSION" "" "" fi # # Notify if PUBLIC CONFIG changed # if [ ! -z "$PUBLIC_CONFIG_AFTER_HASH" ]; then - - MESSAGE=$(cat <<-END - Public config changed versions: - FROM: \`$PUBLIC_CONFIG_BEFORE_HASH\` - TO: \`$PUBLIC_CONFIG_AFTER_HASH\` - - Tracking version: \`$PUBLIC_CONFIG_VERSION\` - - Commits: - $PUBLIC_CONFIG_COMMITS - - Diff: - \`\`\` - $PUBLIC_CONFIG_DIFF - \`\`\` -END -) - - echo -e "$MESSAGE" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$MESSAGE" - SLACK_COLOR="good" - source $NOTIFY - fi + source "$DIR/diff-notify.sh" "Public config" "$PUBLIC_CONFIG_BEFORE_HASH" "$PUBLIC_CONFIG_AFTER_HASH" "$PUBLIC_CONFIG_VERSION" "$PUBLIC_CONFIG_COMMITS" "$PUBLIC_CONFIG_DIFF" fi # # Notify if MEZA changed # if [ ! -z "$MEZA_AFTER_HASH" ]; then - - MESSAGE=$(cat <<-END - Meza application changed versions: - FROM: \`$MEZA_BEFORE_HASH\` - TO: \`$MEZA_AFTER_HASH\` - - Tracking version: \`$MEZA_VERSION\` - - Commits: - $MEZA_COMMITS -END -) - - echo -e "$MESSAGE" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$MESSAGE" - SLACK_COLOR="good" - source $NOTIFY - fi + # Don't show arg 6 (diff) for Meza (often too long) + source "$DIR/diff-notify.sh" "Meza" "$MEZA_BEFORE_HASH" "$MEZA_AFTER_HASH" "$MEZA_VERSION" "$MEZA_COMMITS" "" fi diff --git a/src/roles/autodeployer/templates/diff-notify.sh.j2 b/src/roles/autodeployer/templates/diff-notify.sh.j2 new file mode 100644 index 000000000..45ea01579 --- /dev/null +++ b/src/roles/autodeployer/templates/diff-notify.sh.j2 @@ -0,0 +1,55 @@ +#!/bin/sh +# +# + +DESC="$1" +BEFORE_HASH="$2" +AFTER_HASH="$3" +VERSION="$4" +COMMITS="$5" +DIFF="$6" + +# This unsets positional parameters (e.g. $1, $2, etc), necessary for the +# `source $NOTIFY` below not to have $1 and $2 injected +set -- + +# Show commits? +if [ ! -z "$COMMITS" ]; then +COMMITS_TEXT=$(cat <<-END + + +Commits: +$COMMITS +END) +else + COMMITS_TEXT="" +fi + +# Show diff? +if [ ! -z "$DIFF" ]; then +DIFF_TEXT=$(cat <<-END + + +Diff: +\`\`\` +$DIFF +\`\`\` +END) +fi + +MESSAGE=$(cat <<-END +$DESC changed versions: + FROM: \`$BEFORE_HASH\` + TO: \`$AFTER_HASH\` + +Tracking version: \`$VERSION\` $COMMITS_TEXT $DIFF_TEXT +END +) + +echo -e "$MESSAGE" + +if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$MESSAGE" + SLACK_COLOR="good" + source $NOTIFY +fi diff --git a/src/roles/autodeployer/templates/repo-pull.sh.j2 b/src/roles/autodeployer/templates/repo-pull.sh.j2 new file mode 100644 index 000000000..1f40b6c51 --- /dev/null +++ b/src/roles/autodeployer/templates/repo-pull.sh.j2 @@ -0,0 +1,67 @@ +#!/bin/sh +# +# Attempt update of repo +# +# Inputs: +# $1 = repo description. Ex: secret config, public config, Meza +# $2 = destination. Ex: /opt/conf-meza/secret +# $3 = repo. Ex: https://github.com/enterprisemediawiki/meza.git +# $4 = version. Ex: master, tags/31.1.1 + +DESC="$1" +DEST="$2" +REPO="$3" +VERSION="$4" + +GIT_CHANGE=$($GIT_FETCH "$REPO" "$DEST" "$VERSION") + +# +# Check if attempt to git-pull SECRET CONFIG failed +# +# FIXME: For some reason the jq command below was not working if it was within +# the conditional, so it has to be out here, where it forces us to +# temporarily allow errors. +set +e +echo "Did git fetch fail on $DESC?" +echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e +if [ $? -eq 0 ]; then + FAILED_MSG=$(echo "$GIT_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) + FULL_MSG="Updating $DESC failed with following message:\n $FAILED_MSG" + >&2 echo -e "$FULL_MSG" + + if [ ! -z "SLACK_TOKEN" ]; then + SLACK_MESSAGE="$FULL_MSG" + SLACK_COLOR="danger" + source $NOTIFY + fi + exit 1; +fi + +# +# Check if changes were made to SECRET CONFIG +# +echo "Were there changes to $DESC?" +echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e +if [ $? -eq 0 ]; then + BEFORE_HASH=$(echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) + AFTER_HASH=$(echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) + echo "Before hash: $BEFORE_HASH" + echo "After hash: $AFTER_HASH" + + pushd "$DEST" + DIFF=$(git diff "$BEFORE_HASH" "$AFTER_HASH" 2>&1) + COMMITS=$(git log --oneline "$BEFORE_HASH...$AFTER_HASH" 2>&1) + pushd +else + DIFF="" + AFTER_HASH="" + COMMITS="" +fi + +# Make sure config.sh is up-to-date after secret config change above, since it +# may impact what version of public config and Meza are checked out below +meza setbaseconfig "$m_environment" +# FIXME: sourcing config.sh here will overwrite previously set vars with any new +# values, and will add any newly defined vars, but if config changed to +# remove an existing var then that value will not be unset. +source /opt/.deploy-meza/config.sh diff --git a/src/roles/base-config-scripts/templates/config.sh.j2 b/src/roles/base-config-scripts/templates/config.sh.j2 index c3e515425..4fe3e2e77 100644 --- a/src/roles/base-config-scripts/templates/config.sh.j2 +++ b/src/roles/base-config-scripts/templates/config.sh.j2 @@ -180,9 +180,9 @@ slack_username_disk_usage="{{ slack_username_disk_usage }}" # {% if secret_config_repo is defined and secret_config_repo.repo is defined %} # Info about local public config -local_config_repo_repo="{{ secret_config_repo.repo }}" -local_config_repo_version="{{ secret_config_repo.version | default('master') }}" -local_config_repo_force="{{ secret_config_repo.force | default(false) | bool }}" +secret_config_repo_repo="{{ secret_config_repo.repo }}" +secret_config_repo_version="{{ secret_config_repo.version | default('master') }}" +secret_config_repo_force="{{ secret_config_repo.force | default(false) | bool }}" {% endif %} {% if local_config_repo is defined and local_config_repo.repo is defined %} From 4dfb030d2dd73aa009eaabbc30f89b9aee05d7c8 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 31 May 2019 17:03:24 -0500 Subject: [PATCH 058/280] Belt and suspenders for ensuring deploy unlocks --- src/roles/autodeployer/templates/do-deploy.sh.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/roles/autodeployer/templates/do-deploy.sh.j2 b/src/roles/autodeployer/templates/do-deploy.sh.j2 index c113a0b42..e38c5f160 100644 --- a/src/roles/autodeployer/templates/do-deploy.sh.j2 +++ b/src/roles/autodeployer/templates/do-deploy.sh.j2 @@ -83,6 +83,9 @@ set +e meza deploy "$m_environment" $DEPLOY_ARGS \ > /opt/data-meza/logs/${LOG_PREFIX}`date "+%Y%m%d%H%M%S"`.log 2>&1 +# This is overkill but just to be sure in case the above deploy errors +meza deploy-unlock "$m_environment" + # If deploy success, notify. Else retry once. if [ $? -eq 0 ]; then source $NOTIFY "$DEPLOY_TYPE complete" "good" @@ -94,6 +97,9 @@ else meza deploy "$m_environment" $DEPLOY_ARGS \ > /opt/data-meza/logs/${LOG_PREFIX}`date "+%Y%m%d%H%M%S"`.log 2>&1 + # This is overkill but just to be sure in case the above deploy errors + meza deploy-unlock "$m_environment" + if [ $? -eq 0 ]; then source $NOTIFY "$DEPLOY_TYPE complete" "good" else From 64896a5c10b9d557a58fa921b44107a9c1fd9fd1 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 31 May 2019 20:38:50 -0500 Subject: [PATCH 059/280] Replace autodeployer scripts with Ansible --- src/playbooks/check-for-changes.yml | 399 ++++++++++++++++++ src/playbooks/deploy-notify.yml | 18 + src/roles/autodeployer/tasks/do-deploy.yml | 58 +++ src/roles/autodeployer/tasks/main.yml | 24 -- .../templates/check-for-changes.sh.j2 | 178 -------- .../autodeployer/templates/diff-notify.sh.j2 | 55 --- .../autodeployer/templates/do-deploy.sh.j2 | 113 ----- .../autodeployer/templates/git-fetch.sh.j2 | 19 - .../templates/meza-autodeployer-cron.j2 | 4 +- .../autodeployer/templates/no-notify.sh.j2 | 5 - .../autodeployer/templates/repo-pull.sh.j2 | 67 --- .../autodeployer/templates/slack-notify.sh.j2 | 111 ----- src/scripts/meza.py | 90 ++++ 13 files changed, 567 insertions(+), 574 deletions(-) create mode 100644 src/playbooks/check-for-changes.yml create mode 100644 src/playbooks/deploy-notify.yml create mode 100644 src/roles/autodeployer/tasks/do-deploy.yml delete mode 100644 src/roles/autodeployer/templates/check-for-changes.sh.j2 delete mode 100644 src/roles/autodeployer/templates/diff-notify.sh.j2 delete mode 100644 src/roles/autodeployer/templates/do-deploy.sh.j2 delete mode 100644 src/roles/autodeployer/templates/git-fetch.sh.j2 delete mode 100644 src/roles/autodeployer/templates/no-notify.sh.j2 delete mode 100644 src/roles/autodeployer/templates/repo-pull.sh.j2 delete mode 100644 src/roles/autodeployer/templates/slack-notify.sh.j2 diff --git a/src/playbooks/check-for-changes.yml b/src/playbooks/check-for-changes.yml new file mode 100644 index 000000000..b6c7d8f9a --- /dev/null +++ b/src/playbooks/check-for-changes.yml @@ -0,0 +1,399 @@ +--- + +- hosts: localhost + become: yes + roles: + - set-vars + - umask-set + tasks: + + - name: "Exit if one of the following not setup: secret, public, or Meza versions" + debug: + msg: "local_config_repo, secret_config_repo, or enforce_meza_version must be configured" + failed_when: True + when: > + ( + local_config_repo is not defined + and local_config_repo.repo is not defined + and local_config_repo.version is not defined + ) + or + ( + secret_config_repo is not defined + and secret_config_repo.repo is not defined + and secret_config_repo.version is not defined + ) + or + ( + enforce_meza_version is not defined + ) + + - name: + set_fact: + autodeployer_slack_icon_url: "{{ autodeployer.slack_icon_url | default('https://github.com/enterprisemediawiki/meza/raw/master/src/roles/configure-wiki/files/logo.png') }}" + when: autodeployer is defined + + # + # SECRET CONFIG + # + - name: Do secret config + block: + - name: Ensure proper secret config + become: yes + become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" + git: + repo: "{{ secret_config_repo.repo }}" + dest: "{{ m_local_secret }}" + version: "{{ secret_config_repo.version }}" + track_submodules: no + recursive: no + umask: "0002" + register: secret_config_git_operation + + # temporary + - debug: + var: secret_config_git_operation + + - name: Reset vars if secret config changed + include_role: + name: set-vars + when: secret_config_git_operation.changed + + # + # Send notification for Meza changes + # + - name: Send notification via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: | + Secret config changed versions: + FROM: `{{ secret_config_git_operation.before }}` + TO: `{{ secret_config_git_operation.after }}` + + Tracking version: `{{ secret_config_repo.version }}` + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + register: secret_config_notify + failed_when: False + when: + - secret_config_git_operation.changed + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + - name: Send notification via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Secret config slack notification error" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + failed_when: False + when: + - secret_config_git_operation.changed + - secret_config_notify.failed + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + rescue: + - name: Send secret config warning via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Error with secret config git operations" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "danger" + link_names: 1 + + - name: "End autodeployer: Secret config Git failure" + debug: + msg: "Secret config Git failure" + + - meta: end_play + + + when: + - secret_config_repo is defined + - secret_config_repo.repo is defined + - secret_config_repo.version is defined + + + + + + + # + # PUBLIC CONFIG + # + - name: Do public config + block: + + - name: Ensure proper public config + become: yes + become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" + git: + repo: "{{ local_config_repo.repo }}" + dest: "{{ m_local_public }}" + version: "{{ local_config_repo.version }}" + track_submodules: no + recursive: no + umask: "0002" + register: public_config_git_operation + + # temporary + - debug: + var: public_config_git_operation + + - name: Reset vars if public config changed + include_role: + name: set-vars + when: public_config_git_operation.changed + + # + # Send notification for public config changes + # + - name: Get diff of public config change + shell: > + cd {{ m_local_public }} && + git diff "{{ public_config_git_operation.before }}" "{{ public_config_git_operation.after }}" 2>&1 + register: public_config_git_diff + when: public_config_git_operation.changed + + - name: Get commits in public config change + shell: > + cd {{ m_local_public }} && + git log --oneline "{{ public_config_git_operation.before }}...{{ public_config_git_operation.after }}" 2>&1 + register: public_config_git_commits + when: public_config_git_operation.changed + + - name: Send notification via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: | + Public config changed versions: + FROM: `{{ public_config_git_operation.before }}` + TO: `{{ public_config_git_operation.after }}` + + Tracking version: `{{ local_config_repo.version }}` + + Commits: + {{ public_config_git_commits.stdout }} + + Diff: + ``` + {{ public_config_git_diff.stdout }} + ``` + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + register: public_config_notify + failed_when: False + when: + - public_config_git_operation.changed + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + - name: Send notification via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Public config slack notification error" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + failed_when: False + when: + - public_config_git_operation.changed + - public_config_notify.failed + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + rescue: + - name: Send public config warning via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Error with public config git operations" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "danger" + link_names: 1 + + - name: "End autodeployer: Public config Git failure" + debug: + msg: "Public config Git failure" + + - meta: end_play + + when: + - local_config_repo is defined + - local_config_repo.repo is defined + - local_config_repo.version is defined + + + + + + + # + # MEZA + # + - name: Do Meza + block: + + + - name: Ensure proper Meza version + become: yes + become_user: "meza-ansible" + # Ref #1149 for TMPDIR environment var + environment: + TMPDIR: "{{ m_tmp }}" + git: + repo: "https://github.com/enterprisemediawiki/meza.git" + dest: "{{ m_meza }}" + version: "{{ enforce_meza_version }}" + track_submodules: no + recursive: no + umask: "0002" + register: meza_git_operation + + # temporary + - debug: + var: meza_git_operation + + - name: Reset vars if Meza changed + include_role: + name: set-vars + when: meza_git_operation.changed + + # + # Send notification for Meza changes + # + - name: Get commits in Meza change + shell: > + cd {{ m_meza }} && + git log --oneline "{{ meza_git_operation.before }}...{{ meza_git_operation.after }}" 2>&1 + register: meza_git_commits + when: meza_git_operation.changed + + - name: Send notification via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: | + Meza changed versions: + FROM: `{{ meza_git_operation.before }}` + TO: `{{ meza_git_operation.after }}` + + Tracking version: `{{ local_config_repo.version }}` + + Commits: + {{ meza_git_commits.stdout }} + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + register: meza_notify + failed_when: False + when: + - meza_git_operation.changed + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + - name: Send notification via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Meza application slack notification error" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + failed_when: False + when: + - meza_git_operation.changed + - meza_notify.failed + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + rescue: + - name: Send Meza application warning via Slack + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Error with Meza application git operations" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "danger" + link_names: 1 + + - name: "End autodeployer: Meza Git failure" + debug: + msg: "Meza Git failure" + + - meta: end_play + + when: enforce_meza_version is defined + + + + + # + # Not doing deploy? End play + # + - block: + - name: "End autodeployer if no changes made" + debug: + msg: "Nothing new to deploy" + + - meta: end_play + when: + - not secret_config_git_operation.changed + - not public_config_git_operation.changed + - not meza_git_operation.changed + + + # + # Do the deploy + # + - name: Do deploy + include_role: + name: autodeployer + tasks_from: do-deploy + + + - name: Remove umask setting + include_role: + name: umask-unset diff --git a/src/playbooks/deploy-notify.yml b/src/playbooks/deploy-notify.yml new file mode 100644 index 000000000..996bbcdf7 --- /dev/null +++ b/src/playbooks/deploy-notify.yml @@ -0,0 +1,18 @@ +--- + +- hosts: localhost + become: yes + roles: + - set-vars + - umask-set + tasks: + + - name: Do deploy + include_role: + name: autodeployer + tasks_from: do-deploy + + + - name: Remove umask setting + include_role: + name: umask-unset diff --git a/src/roles/autodeployer/tasks/do-deploy.yml b/src/roles/autodeployer/tasks/do-deploy.yml new file mode 100644 index 000000000..51c9833ad --- /dev/null +++ b/src/roles/autodeployer/tasks/do-deploy.yml @@ -0,0 +1,58 @@ +--- + +# this task is duplicated from check-for-changes.yml +- name: + set_fact: + autodeployer_slack_icon_url: "{{ autodeployer.slack_icon_url | default('https://github.com/enterprisemediawiki/meza/raw/master/src/roles/configure-wiki/files/logo.png') }}" + when: autodeployer is defined + +- name: Notify deploy is starting + slack: + token: "{{ autodeployer.slack_token }}" + msg: "{{ deploy_type | default('Deploy') }} starting" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + when: + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + +- name: Deploy + shell: > + meza deploy-unlock {{ env }} + && meza deploy {{ env }} {{ deploy_args }} + failed_when: False + register: first_deploy + +- name: Declare success + set_fact: + deploy_status: "complete" + deploy_notification_color: "good" + when: first_deploy.rc == 0 + +- name: Declare failure + set_fact: + deploy_status: "failed" + deploy_notification_color: "danger" + when: first_deploy.rc != 0 + +- name: "Notify deploy {{ deploy_status }}" + slack: + token: "{{ autodeployer.slack_token }}" + msg: "{{ deploy_type | default('Deploy') }} {{ deploy_status }}" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "{{ deploy_notification_color }}" + link_names: 1 + when: + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined diff --git a/src/roles/autodeployer/tasks/main.yml b/src/roles/autodeployer/tasks/main.yml index e13de6b12..a8d97e628 100644 --- a/src/roles/autodeployer/tasks/main.yml +++ b/src/roles/autodeployer/tasks/main.yml @@ -27,30 +27,6 @@ force_deploy.overwrite is not defined or (force_deploy.overwrite is defined and not force_deploy.overwrite) -- name: "Ensure {{ m_deploy }}/autodeployer exists" - file: - path: "{{ m_deploy }}/autodeployer" - owner: root - group: root - mode: 0644 - state: directory - -- name: Ensure autodeployer scripts in place - template: - src: "{{ item }}.j2" - dest: "{{ m_deploy }}/autodeployer/{{ item }}" - owner: root - group: root - mode: "0744" - with_items: - - "check-for-changes.sh" - - "do-deploy.sh" - - "git-fetch.sh" - - "no-notify.sh" - - "slack-notify.sh" - - "repo-pull.sh" - - "diff-notify.sh" - - name: Ensure autodeployer cron in place template: src: "meza-autodeployer-cron.j2" diff --git a/src/roles/autodeployer/templates/check-for-changes.sh.j2 b/src/roles/autodeployer/templates/check-for-changes.sh.j2 deleted file mode 100644 index 1e6543739..000000000 --- a/src/roles/autodeployer/templates/check-for-changes.sh.j2 +++ /dev/null @@ -1,178 +0,0 @@ -#!/bin/sh -# -# Check for changes to Meza and public config repository, then deploy as needed -# -# Run this command without args: -# -# sudo ./check-for-changes.sh -# -# Or in a passwordless sudoer's (e.g. root) crontab like: -# -# 22 * * * * /opt/meza/src/scripts/autodeployer/check-for-changes.sh >> /opt/data-meza/logs/autodeploy-`date "+\%Y-\%m-\%d"`.log 2>&1 -# - -# Don't allow errors -set -e - -echo "Start meza auto-deployer" -echo $(date "+%Y-%m-%d %H:%M:%S") - -# Path to this file's directory -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# must be root or sudoer -if [ "$(whoami)" != "root" ]; then - echo "Try running this script with sudo: \"sudo bash unite-the-wikis.sh\"" - exit 1 -fi - -# Gets initial config defined by secret and public config (and Meza defaults). -# Really all this does is grabs $m_environment, but instead whatever is calling -# this script should pass that in as an argument. -source /opt/.deploy-meza/config.sh - -# Now that $m_environment is available, force update of config.sh using the -# `meza setbaseconfig` command. This will rebuild config.sh based upon current -# state of secret config, public config, and Meza. -meza setbaseconfig "$m_environment" -source /opt/.deploy-meza/config.sh - -# -# Check if deploy is underway -# -meza deploy-check "$m_environment" -if [ $? -eq 0 ]; then - echo "Checked if deploy underway. Is not. Placing deploy lock while checking for changes." - meza deploy-lock "$m_environment" -else - echo "Deploy is underway. Exiting." - exit 1 -fi - - -if [ -z "$local_config_repo_repo$secret_config_repo_repo$enforce_meza_version" ]; then - >&2 echo "Auto-deploy requires a public config, secret config, or Meza version to be set." - exit 1; -fi - -# Set Slack notify variables that are the same for all notifications -if [ ! -z "$autodeployer_slack_token" ]; then SLACK_TOKEN="$autodeployer_slack_token"; fi -if [ ! -z "$autodeployer_slack_username" ]; then SLACK_USERNAME="$autodeployer_slack_username"; fi -if [ ! -z "$autodeployer_slack_channel" ]; then SLACK_CHANNEL="$autodeployer_slack_channel"; fi -if [ ! -z "$autodeployer_slack_icon_url" ]; then SLACK_ICON_URL="$autodeployer_slack_icon_url"; fi - - -# If SLACK_TOKEN is set, send notification via slack. Else, use no-notify script -if [ ! -z "$SLACK_TOKEN" ]; then - NOTIFY="$DIR/slack-notify.sh" -else - NOTIFY="$DIR/no-notify.sh" -fi - -GIT_FETCH="$DIR/git-fetch.sh" - -# -# If management of SECRET config desired -# -if [ ! -z "$secret_config_repo_repo" ]; then - - SECRET_CONFIG_DEST="/opt/conf-meza/secret" - SECRET_CONFIG_REPO="$secret_config_repo_repo" - SECRET_CONFIG_VERSION="$secret_config_repo_version" - - source "$DIR/repo-pull.sh" "secret config" "$SECRET_CONFIG_DEST" "$SECRET_CONFIG_REPO" "$SECRET_CONFIG_VERSION" - SECRET_CONFIG_DIFF="$DIFF" - SECRET_CONFIG_BEFORE_HASH="$BEFORE_HASH" - SECRET_CONFIG_AFTER_HASH="$AFTER_HASH" - SECRET_CONFIG_COMMITS="$COMMITS" -fi - - -# -# If management of PUBLIC config desired -# -if [ ! -z "$local_config_repo_repo" ]; then - - PUBLIC_CONFIG_DEST="/opt/conf-meza/public" - PUBLIC_CONFIG_REPO="$local_config_repo_repo" - PUBLIC_CONFIG_VERSION="$local_config_repo_version" - - source "$DIR/repo-pull.sh" "public config" "$PUBLIC_CONFIG_DEST" "$PUBLIC_CONFIG_REPO" "$PUBLIC_CONFIG_VERSION" - PUBLIC_CONFIG_DIFF="$DIFF" - PUBLIC_CONFIG_BEFORE_HASH="$BEFORE_HASH" - PUBLIC_CONFIG_AFTER_HASH="$AFTER_HASH" - PUBLIC_CONFIG_COMMITS="$COMMITS" - -fi - - -# -# If management of Meza version desired -# -if [ ! -z "$enforce_meza_version" ]; then - - # Set MEZA version - MEZA_REPO="https://github.com/enterprisemediawiki/meza" - MEZA_DEST="/opt/meza" - MEZA_VERSION="$enforce_meza_version" - - source "$DIR/repo-pull.sh" "public config" "$MEZA_DEST" "$MEZA_REPO" "$MEZA_VERSION" - MEZA_DIFF="$DIFF" - MEZA_BEFORE_HASH="$BEFORE_HASH" - MEZA_AFTER_HASH="$AFTER_HASH" - MEZA_COMMITS="$COMMITS" - -fi - - -# -# Meza, public, and secret config all unchanged? Exit. -# -if [ -z "$SECRET_CONFIG_AFTER_HASH$PUBLIC_CONFIG_AFTER_HASH$MEZA_AFTER_HASH" ]; then - meza deploy-unlock "$m_environment" - echo "Nothing to deploy" - exit 0; -fi - -# -# Notify if SECRET CONFIG changed -# -if [ ! -z "$SECRET_CONFIG_AFTER_HASH" ]; then - # arg 5 = commits, 6 = diff. Don't show for secret config - source "$DIR/diff-notify.sh" "Secret config" "$SECRET_CONFIG_BEFORE_HASH" "$SECRET_CONFIG_AFTER_HASH" "$SECRET_CONFIG_VERSION" "" "" -fi - -# -# Notify if PUBLIC CONFIG changed -# -if [ ! -z "$PUBLIC_CONFIG_AFTER_HASH" ]; then - source "$DIR/diff-notify.sh" "Public config" "$PUBLIC_CONFIG_BEFORE_HASH" "$PUBLIC_CONFIG_AFTER_HASH" "$PUBLIC_CONFIG_VERSION" "$PUBLIC_CONFIG_COMMITS" "$PUBLIC_CONFIG_DIFF" -fi - -# -# Notify if MEZA changed -# -if [ ! -z "$MEZA_AFTER_HASH" ]; then - # Don't show arg 6 (diff) for Meza (often too long) - source "$DIR/diff-notify.sh" "Meza" "$MEZA_BEFORE_HASH" "$MEZA_AFTER_HASH" "$MEZA_VERSION" "$MEZA_COMMITS" "" -fi - - -# -# Do deploy -# -echo "Deploying" -# Allow overriding variables by only setting them if they're empty -if [ -z "$DEPLOY_TYPE" ]; then DEPLOY_TYPE="Deploy"; fi -if [ -z "$DEPLOY_ARGS" ]; then DEPLOY_ARGS=""; fi -if [ -z "$DEPLOY_LOG_PREFIX" ]; then DEPLOY_LOG_PREFIX="deploy-after-config-change-"; fi - -# This isn't perfect, as there is still a little that will happen in do-deploy -# below before the actual deploy starts, and a separate deploy _could_ start -# in between now and then, but the likelihood is low, and the impact is only -# this deploy wouldn't happen since it's actual deploy would check for the lock -# file and would fail/exit. -meza deploy-unlock "$m_environment" - -source "$DIR/do-deploy.sh" -echo "Done" diff --git a/src/roles/autodeployer/templates/diff-notify.sh.j2 b/src/roles/autodeployer/templates/diff-notify.sh.j2 deleted file mode 100644 index 45ea01579..000000000 --- a/src/roles/autodeployer/templates/diff-notify.sh.j2 +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# - -DESC="$1" -BEFORE_HASH="$2" -AFTER_HASH="$3" -VERSION="$4" -COMMITS="$5" -DIFF="$6" - -# This unsets positional parameters (e.g. $1, $2, etc), necessary for the -# `source $NOTIFY` below not to have $1 and $2 injected -set -- - -# Show commits? -if [ ! -z "$COMMITS" ]; then -COMMITS_TEXT=$(cat <<-END - - -Commits: -$COMMITS -END) -else - COMMITS_TEXT="" -fi - -# Show diff? -if [ ! -z "$DIFF" ]; then -DIFF_TEXT=$(cat <<-END - - -Diff: -\`\`\` -$DIFF -\`\`\` -END) -fi - -MESSAGE=$(cat <<-END -$DESC changed versions: - FROM: \`$BEFORE_HASH\` - TO: \`$AFTER_HASH\` - -Tracking version: \`$VERSION\` $COMMITS_TEXT $DIFF_TEXT -END -) - -echo -e "$MESSAGE" - -if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$MESSAGE" - SLACK_COLOR="good" - source $NOTIFY -fi diff --git a/src/roles/autodeployer/templates/do-deploy.sh.j2 b/src/roles/autodeployer/templates/do-deploy.sh.j2 deleted file mode 100644 index e38c5f160..000000000 --- a/src/roles/autodeployer/templates/do-deploy.sh.j2 +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/sh -# -# Do deploy. Notify on success. Notify and retry on fail. -# -# To use this script, either pass in DEPLOY_TYPE, DEPLOY_ARGS, AND LOG_PREFIX, -# or set them ahead of time and `source` the script: -# -# 1. ./do-deploy.sh "Backup" "--skip-tags search-index" "test-deploy-" -# -# 2. DEPLOY_TYPE="Backup" -# DEPLOY_ARGS="--skip-tags search-index" -# LOG_PREFIX="test-deploy-" -# -# Args: -# DEPLOY_TYPE: Just used in notifications for type of deploy, e.g. "Backup -# starting" or "Deploy starting" where "Backup" and "Deploy" are -# the DEPLOY_TYPE -# DEPLOY_ARGS: Any arguments that are going to get added to the deploy -# command. So if you want to do: -# `meza deploy dev --tags mediawiki --skip-tags latest` -# The DEPLOY_ARGS would be "--tags mediawiki --skip-tags latest" -# LOG_PREFIX: Logs are written to /opt/data-meza/logs to a file ending in -# the date/time and ".log". Prefix it with something like -# "nightly-backup-" to make "nightly-backup-$DATETIME.log" - -# Path to this file's directory -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - - -# Setting configuration for DEPLOY_TYPE, DEPLOY_ARGS, and LOG_PREFIX -# -# If any of them is being set via script argument $1, $2, or $3, respectively, -# then use that value. Otherwise check if they already have a value set earlier -# within the enviroment, and set a default if not. -if [ ! -z "$1" ]; then - DEPLOY_TYPE="$1" -elif [ -z "$DEPLOY_TYPE" ]; then - DEPLOY_TYPE="Deploy" -fi - -if [ ! -z "$2" ]; then - DEPLOY_ARGS="$2" -elif [ -z "$DEPLOY_ARGS" ]; then - DEPLOY_ARGS="" -fi - -if [ ! -z "$3" ]; then - LOG_PREFIX="$3" -elif [ -z "$LOG_PREFIX" ]; then - LOG_PREFIX="deploy-" -fi - -# Gets info about public config -source /opt/.deploy-meza/config.sh - -# If SLACK_TOKEN not set from outside this script, grab from config.sh -# Same goes for other slack vars -if [ -z "$SLACK_TOKEN" ] && [ ! -z "$autodeployer_slack_token" ]; then - SLACK_TOKEN="$autodeployer_slack_token" -fi -if [ -z "$SLACK_USERNAME" ] && [ ! -z "$autodeployer_slack_username" ]; then - SLACK_USERNAME="$autodeployer_slack_username" -fi -if [ -z "$SLACK_CHANNEL" ] && [ ! -z "$autodeployer_slack_channel" ]; then - SLACK_CHANNEL="$autodeployer_slack_channel" -fi -if [ -z "$SLACK_ICON_URL" ] && [ ! -z "$autodeployer_slack_icon_url" ]; then - SLACK_ICON_URL="$autodeployer_slack_icon_url" -fi - - -# If SLACK_TOKEN is set, send notification via slack. Else, use no-notify script -if [ ! -z "$SLACK_TOKEN" ]; then - NOTIFY="$DIR/slack-notify.sh" -else - NOTIFY="$DIR/no-notify.sh" -fi - -source $NOTIFY "$DEPLOY_TYPE starting" "good" - -# First try at deploy. Allow failures so we can capture them later -set +e -meza deploy "$m_environment" $DEPLOY_ARGS \ - > /opt/data-meza/logs/${LOG_PREFIX}`date "+%Y%m%d%H%M%S"`.log 2>&1 - -# This is overkill but just to be sure in case the above deploy errors -meza deploy-unlock "$m_environment" - -# If deploy success, notify. Else retry once. -if [ $? -eq 0 ]; then - source $NOTIFY "$DEPLOY_TYPE complete" "good" -else - source $NOTIFY "$DEPLOY_TYPE attempt failed. Retrying..." "warning" - - # sourcing scripts above may reset -e. Explicitly set prior to meza deploy - set +e - meza deploy "$m_environment" $DEPLOY_ARGS \ - > /opt/data-meza/logs/${LOG_PREFIX}`date "+%Y%m%d%H%M%S"`.log 2>&1 - - # This is overkill but just to be sure in case the above deploy errors - meza deploy-unlock "$m_environment" - - if [ $? -eq 0 ]; then - source $NOTIFY "$DEPLOY_TYPE complete" "good" - else - source $NOTIFY "$DEPLOY_TYPE failed" "danger" - fi -fi - -set -e - - - diff --git a/src/roles/autodeployer/templates/git-fetch.sh.j2 b/src/roles/autodeployer/templates/git-fetch.sh.j2 deleted file mode 100644 index b0ffb018d..000000000 --- a/src/roles/autodeployer/templates/git-fetch.sh.j2 +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -# -# Use Ansible's Git module plus jq to easily set repository versions - -# Get directory to return to at end -CWD=$(pwd) - -# change to directory holding appropriate ansible.cfg file for this operation -cd /opt/meza/config/core/adhoc - -REPO="$1" -DEST="$2" -VERSION="$3" - -source /opt/.deploy-meza/config.sh - -sudo TMPDIR="$m_tmp" -u meza-ansible ansible localhost -m git -a "repo=$REPO dest=$DEST version=$VERSION" - -cd "$CWD" diff --git a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 index 15a547b5d..25b7d07e9 100644 --- a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 +++ b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 @@ -11,7 +11,7 @@ MAILTO=root # will get confused if multiple environments are used. Environment should # be explicitly passed into this script. # -{{ force_deploy.crontime }} root {{ m_deploy }}/autodeployer/do-deploy.sh "{{ _force_deploy_notify_prefix }}" "{{ _force_deploy_options }}" "deploy-" +{{ force_deploy.crontime }} root meza deploy-notify "{{ env }}" "{{ _force_deploy_notify_prefix }}" "{{ _force_deploy_options }}" {% endif %} @@ -23,5 +23,5 @@ MAILTO=root # will get confused if multiple environments are used. Environment should # be explicitly passed into this script. # -{{ autodeployer.crontime }} root {{ m_deploy }}/autodeployer/check-for-changes.sh > {{ m_logs }}/deploy/check-for-changes.log 2>&1 +{{ autodeployer.crontime }} root meza autodeploy "{{ env }}" "Deploy" "" >> {{ m_logs }}/deploy/check-for-changes-`date "+\%Y\%m\%d"`.log 2>&1 {% endif %} diff --git a/src/roles/autodeployer/templates/no-notify.sh.j2 b/src/roles/autodeployer/templates/no-notify.sh.j2 deleted file mode 100644 index f73d123bc..000000000 --- a/src/roles/autodeployer/templates/no-notify.sh.j2 +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -# -# Script used to handle when no notification system is set - -echo -e "$(date "+\%Y-\%m-\%d"): No notification method set.\n Color: $2\n Message: $1" >> /opt/data-meza/logs/missing-notifier.log diff --git a/src/roles/autodeployer/templates/repo-pull.sh.j2 b/src/roles/autodeployer/templates/repo-pull.sh.j2 deleted file mode 100644 index 1f40b6c51..000000000 --- a/src/roles/autodeployer/templates/repo-pull.sh.j2 +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/sh -# -# Attempt update of repo -# -# Inputs: -# $1 = repo description. Ex: secret config, public config, Meza -# $2 = destination. Ex: /opt/conf-meza/secret -# $3 = repo. Ex: https://github.com/enterprisemediawiki/meza.git -# $4 = version. Ex: master, tags/31.1.1 - -DESC="$1" -DEST="$2" -REPO="$3" -VERSION="$4" - -GIT_CHANGE=$($GIT_FETCH "$REPO" "$DEST" "$VERSION") - -# -# Check if attempt to git-pull SECRET CONFIG failed -# -# FIXME: For some reason the jq command below was not working if it was within -# the conditional, so it has to be out here, where it forces us to -# temporarily allow errors. -set +e -echo "Did git fetch fail on $DESC?" -echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.failed' -e -if [ $? -eq 0 ]; then - FAILED_MSG=$(echo "$GIT_CHANGE" | jq .plays[0].tasks[0].hosts.localhost.msg -r) - FULL_MSG="Updating $DESC failed with following message:\n $FAILED_MSG" - >&2 echo -e "$FULL_MSG" - - if [ ! -z "SLACK_TOKEN" ]; then - SLACK_MESSAGE="$FULL_MSG" - SLACK_COLOR="danger" - source $NOTIFY - fi - exit 1; -fi - -# -# Check if changes were made to SECRET CONFIG -# -echo "Were there changes to $DESC?" -echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.changed' -e -if [ $? -eq 0 ]; then - BEFORE_HASH=$(echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.before' -r) - AFTER_HASH=$(echo "$GIT_CHANGE" | jq '.plays[0].tasks[0].hosts.localhost.after' -r) - echo "Before hash: $BEFORE_HASH" - echo "After hash: $AFTER_HASH" - - pushd "$DEST" - DIFF=$(git diff "$BEFORE_HASH" "$AFTER_HASH" 2>&1) - COMMITS=$(git log --oneline "$BEFORE_HASH...$AFTER_HASH" 2>&1) - pushd -else - DIFF="" - AFTER_HASH="" - COMMITS="" -fi - -# Make sure config.sh is up-to-date after secret config change above, since it -# may impact what version of public config and Meza are checked out below -meza setbaseconfig "$m_environment" -# FIXME: sourcing config.sh here will overwrite previously set vars with any new -# values, and will add any newly defined vars, but if config changed to -# remove an existing var then that value will not be unset. -source /opt/.deploy-meza/config.sh diff --git a/src/roles/autodeployer/templates/slack-notify.sh.j2 b/src/roles/autodeployer/templates/slack-notify.sh.j2 deleted file mode 100644 index 9f51b0ea8..000000000 --- a/src/roles/autodeployer/templates/slack-notify.sh.j2 +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh -# -# Use Ansible's Slack module to send messages. -# -# Required variables -# SLACK_TOKEN="" # Exits script if you don't set this. -# -# Recommended variables -# SLACK_MESSAGE="" # Defaults to a pointless message. Set this one. -# SLACK_COLOR="" # Defaults to "good" which means "green". Also available -# are "warning" (orange) and "danger" (red) or using hex -# code (e.g. #439FE0 = light blue) -# -# Optional variables -# SLACK_USERNAME="" # Will use your Slack token's default username by default -# SLACK_CHANNEL="" # Will use your Slack token's default channel by default -# SLACK_ICON_URL="" # will use Meza logo by default - -# Get directory to return to at end -CWD=$(pwd) - -# change to directory holding appropriate ansible.cfg file for this operation -cd /opt/meza/config/core/adhoc - - - -if [ -z "$SLACK_TOKEN" ]; then - >&2 echo "You need to set a SLACK_TOKEN variable" - exit 1; -fi - -# if first param not empty, use it for SLACK_MESSAGE -if [ ! -z "$1" ]; then - SLACK_MESSAGE="$1" -elif [ -z "$SLACK_MESSAGE" ]; then - SLACK_MESSAGE="Empty message." -fi - -# if second param not empty, use it for SLACK_COLOR -if [ ! -z "$2" ]; then - SLACK_COLOR="$2" -elif [ -z "$SLACK_COLOR" ]; then - SLACK_COLOR="good" # assume all is well -fi - -if [ ! -z "$SLACK_CHANNEL" ]; then - SLACK_CHANNEL_WITH_PARAM="channel='$SLACK_CHANNEL'" -else - SLACK_CHANNEL_WITH_PARAM="" # use default for token -fi - -if [ ! -z "$SLACK_USERNAME" ]; then - SLACK_USERNAME_WITH_PARAM="username='$SLACK_USERNAME'" -else - SLACK_USERNAME_WITH_PARAM="" # use default for token -fi - -if [ -z "$SLACK_ICON_URL" ]; then - SLACK_ICON_URL="https://github.com/enterprisemediawiki/meza/raw/master/src/roles/configure-wiki/files/logo.png" -fi - - -# Escape chars: ' -SLACK_MESSAGE=$(echo "$SLACK_MESSAGE" | sed "s/'/\\\'/g") -# FIXME: also handle ``` inside diffs, which are wrapped in ```...so only escape the inner ones. - -# Turn on allowing failures -set +e - -# for debug -echo "DEBUG OUTPUT OF SLACK NOTIFY COMMAND" -echo \ -"token='$SLACK_TOKEN' \ -$SLACK_CHANNEL_WITH_PARAM \ -msg='$SLACK_MESSAGE' \ -$SLACK_USERNAME_WITH_PARAM \ -icon_url=$SLACK_ICON_URL \ -link_names=1 \ -color=$SLACK_COLOR" - -# Attempt to send message -ansible localhost -m slack -a \ -"token='$SLACK_TOKEN' \ -$SLACK_CHANNEL_WITH_PARAM \ -msg='$SLACK_MESSAGE' \ -$SLACK_USERNAME_WITH_PARAM \ -icon_url=$SLACK_ICON_URL \ -link_names=1 \ -color=$SLACK_COLOR" - -# If message fails, send a generic message -if [ $? -eq 0 ]; then - echo "Slack notify success" -else - echo "Slack notify fail. Attempted message was:" - echo "$SLACK_MESSAGE" - SLACK_MESSAGE="Slack message failed. See logs for attempted message." - ansible localhost -m slack -a \ -"token='$SLACK_TOKEN' \ -$SLACK_CHANNEL_WITH_PARAM \ -msg='$SLACK_MESSAGE' \ -$SLACK_USERNAME_WITH_PARAM \ -icon_url=$SLACK_ICON_URL \ -link_names=1 \ -color=$SLACK_COLOR" -fi - -# Turn off allowing errors -set -e - -cd "$CWD" diff --git a/src/scripts/meza.py b/src/scripts/meza.py index fbb5990a6..e0f2d08c1 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -132,6 +132,96 @@ def meza_command_deploy (argv): meza_shell_exec_exit( return_code ) +# +# Intended to be used by cron job to check for changes to config and meza. Can +# also be called with `meza autodeploy ` +# +def meza_command_autodeploy (argv): + + env = argv[0] + + rc = check_environment(env) + + lock_success = request_lock_for_deploy(env) + + if not lock_success: + print "Deploy for environment {} in progress. Exiting".format(env) + sys.exit(1) + + # return code != 0 means failure + if rc != 0: + sys.exit(rc) + + more_extra_vars = False + + # strip environment off of it + argv = argv[1:] + + if len( argv ) > 0: + more_extra_vars = { + 'deploy_type': argv[0] + } + argv = argv[1:] # strip deploy type off + + if len( argv ) > 0: + more_extra_vars['deploy_args'] = argv[0] + argv = argv[1:] # strip deploy args off + + shell_cmd = playbook_cmd( 'check-for-changes', env, more_extra_vars ) + if len(argv) > 0: + shell_cmd = shell_cmd + argv + + return_code = meza_shell_exec( shell_cmd ) + + unlock_deploy(env) # double check + + meza_shell_exec_exit( return_code ) + +# Just a wrapper on deploy that does some notifications. This needs some +# improvement. FIXME. Lots of duplication between this and meza_command_deploy +# and meza_command_autodeploy +def meza_command_deploy_notify (argv): + + env = argv[0] + + rc = check_environment(env) + + lock_success = request_lock_for_deploy(env) + + if not lock_success: + print "Deploy for environment {} in progress. Exiting".format(env) + sys.exit(1) + + # return code != 0 means failure + if rc != 0: + sys.exit(rc) + + more_extra_vars = False + + # strip environment off of it + argv = argv[1:] + + if len( argv ) > 0: + more_extra_vars = { + 'deploy_type': argv[0] + } + argv = argv[1:] # strip deploy type off + + if len( argv ) > 0: + more_extra_vars['deploy_args'] = argv[0] + argv = argv[1:] # strip deploy args off + + shell_cmd = playbook_cmd( 'deploy-notify', env, more_extra_vars ) + if len(argv) > 0: + shell_cmd = shell_cmd + argv + + return_code = meza_shell_exec( shell_cmd ) + + unlock_deploy(env) # double check + + meza_shell_exec_exit( return_code ) + + def request_lock_for_deploy (env): import os, datetime lock_file = get_lock_file_path(env) From 2784f81bf62277bc5e1be69a164b725a57d8b2b1 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 31 May 2019 11:44:27 -0500 Subject: [PATCH 060/280] Add option to recursively set perms on uploads dir; always run on overwrite --- src/roles/verify-wiki/tasks/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/roles/verify-wiki/tasks/main.yml b/src/roles/verify-wiki/tasks/main.yml index 8154e3b0d..d8dbf0aa8 100644 --- a/src/roles/verify-wiki/tasks/main.yml +++ b/src/roles/verify-wiki/tasks/main.yml @@ -370,6 +370,11 @@ - verify-wiki-uploads +- name: "{{ wiki_id }} - Force m_recursively_set_uploads_permissions = True" + set_fact: + m_recursively_set_uploads_permissions: True + when: intend_overwrite_from_backup + # # SECTION: Ensure backups still in good config # @@ -380,4 +385,4 @@ mode: "{{ m_uploads_dir_mode }}" owner: "{{ m_uploads_dir_owner }}" group: "{{ m_uploads_dir_group }}" - # recursive? + recurse: "{{ m_recursively_set_uploads_permissions | default(False) }}" From 195cbead14b0b67e23709b66a6349e984a39dfd3 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 1 Jun 2019 13:54:00 -0500 Subject: [PATCH 061/280] Fixes for autodeployer logic and misplaced variables --- src/playbooks/check-for-changes.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/playbooks/check-for-changes.yml b/src/playbooks/check-for-changes.yml index b6c7d8f9a..afcac4d83 100644 --- a/src/playbooks/check-for-changes.yml +++ b/src/playbooks/check-for-changes.yml @@ -7,7 +7,7 @@ - umask-set tasks: - - name: "Exit if one of the following not setup: secret, public, or Meza versions" + - name: "Exit if at least one of the following not setup: secret, public, or Meza versions" debug: msg: "local_config_repo, secret_config_repo, or enforce_meza_version must be configured" failed_when: True @@ -17,13 +17,13 @@ and local_config_repo.repo is not defined and local_config_repo.version is not defined ) - or + and ( secret_config_repo is not defined and secret_config_repo.repo is not defined and secret_config_repo.version is not defined ) - or + and ( enforce_meza_version is not defined ) @@ -310,7 +310,7 @@ FROM: `{{ meza_git_operation.before }}` TO: `{{ meza_git_operation.after }}` - Tracking version: `{{ local_config_repo.version }}` + Tracking version: `{{ enforce_meza_version }}` Commits: {{ meza_git_commits.stdout }} @@ -380,9 +380,9 @@ - meta: end_play when: - - not secret_config_git_operation.changed - - not public_config_git_operation.changed - - not meza_git_operation.changed + - ( secret_config_git_operation is not defined or not secret_config_git_operation.changed ) + - ( public_config_git_operation is not defined or not public_config_git_operation.changed ) + - ( meza_git_operation is not defined or not meza_git_operation.changed ) # From f1666ab5dc72a90734454343c25b43ebdaa75e64 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 1 Jun 2019 19:31:25 -0500 Subject: [PATCH 062/280] 31.5.0 release --- RELEASE-NOTES.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index d75f17f20..9f30da39a 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,100 @@ Release Notes ============= +## Meza 31.5.0 + +Major deploy and autodeploy improvements; Push backups to remote server; Security and general improvements; bug fixes + +### Commits since 31.4.0 + +#### Autodeploy on changes to secret config and use Ansible for autodeployer + +Autodeployer has previously just tracked public config and the Meza application. Now it will check secret config, too. Additionally, autodeployer was rewritten in Ansible. Shell scripting got too cumbersome. + +* 195cbea Fixes for autodeployer logic and misplaced variables +* 64896a5 Replace autodeployer scripts with Ansible +* 4dfb030 Belt and suspenders for ensuring deploy unlocks +* 006d5ed Reduce duplication in check-for-changes.sh +* 901f2b9 Make public config and Meza management by autodeployer optional +* 1dc9894 Use secret_config_repo to define secret config +* bce32d3 Autodeployer check for changes to secret config + +#### Prevent simultaneous deploys and improve logging + +Starting a deploy now creates a lock file. Other deploys cannot start until the locking deploy is complete. Additionally, all deploys automatically write to a log file and print to stdout. In the future this will be used to display deploy logs via the web interface. + +* 60a680c Add wait() to capture return code +* e7b8ad3 Make sure deploy log directory exists +* 5009974 Always have ansible show colors +* a120ac3 Make meza_shell_exec use subprocess; optionally write to log file +* c5ef0e5 Add meza deploy-kill, deploy-log, deploy-tail functions +* 9438f7c Handle sigint; also better info in lock file +* a8f1aca Add 'meza deploy-(un)lock commands; Autodeployer use them to avoid conflicts +* dd60b91 Add meza subcommand to check if deploy underway +* 04e8ddb Prevent simultaneous deploys (#1157) + +#### Make autodeployer configurable + +In Meza 31.x prior to this release autodeployer, overwrite-deploys, and backups-cleanup had to be configured manually via crontab. 32.x has had the ability to configure these things in public/secret config for a while. This release pulls that functionality into 31.x. + +* 220df48 Add autodeployer tag +* 87a4012 Make autodeployer, overwrite-deploy, and backups-cleanup configurable +* a5396d8 Fix location of backups-cleanup cron +* 28df042 Fix autodeployer crons + +#### Push backups to an alternate server + +Required if for security reasons dev/int servers cannot SSH into production to grab backups. Instead production can push backups directly to other servers. This was essentially possible before by making the other servers in the `backup-servers` group, but that (a) made it so production managed software configuration on the remote servers (as Meza does for all its server groups) and (b) it put file uploads in the `/opt/data-meza/backups` directory rather than in `/opt/data-meza/uploads`. So you'd have to do some symlink or have duplicated data. With pushed backups the production server (or whatever server is pushing) just needs to be setup so user `meza-ansible` can SSH into the server with a lesser-privileged account. The user must be in group `apache` and `meza-backups`. + +* 6e54a86 Enable rsync push backups (#1166) +* 2784f81 Add option to recursively set perms on uploads dir; always run on overwrite + +#### Security improvements + +Steadily trying to reduce where `root` is required + +* 36104ea Have meza-ansible do autodeployer git-fetch +* 2513a36 Set ownership of meza and config; fix role:init-controller-config + +#### General improvements + +* Vagrant improvements + * bddb797 Unique VM names, /opt/meza owned by UID/GID 10000 in Vagrant + * Unique VM names allows you to boot multiple Meza's on one host + * UID/GID hack required to support using less `root`. Ref #1155 +* Add `pip` and `pip3` + * 3db517a Add pip for Python 2.7 + * 0155726 Add pip3 (31.x didn't have it yet) + * 0f879f8 Make pip3 symlink for RHEL + +#### Fix issues with creating Docker images for testing + +Rebuilding Docker images for testing is not required often. It really only needs to be done when major changes are made or when a very long time has passed between generating images and new images will make test jobs run faster. Since a long time had passed, certain things had been added to Meza that unexpectedly caused issues with Docker builds. + +* 36f965f Reorder AND statement since initial_wikis_dir_check undefined in docker build +* e919123 Don't use services during docker image building +* 0795348 More docker skip tasks + +#### Bug fixes + +* 46f7ac6 Make net adapter select statement break on newlines +* 40e4388 Don't recreate meza-ansible if user already exists (Revert #965) +* a621460 Remove yum:PackageKit to remove error +* 0502a41 Ansible 2.8 fixes (#1162) (`ansible_distribution_version` no longer present and `synchronize` module keeps getting harder to use) +* 6789e7d Ansible Git module fails with /tmp mounted with noexec; set TMPDIR as workaround +* 3854c6c Make sure to use TMPDIR when doing Ansible Git operations + +### Contributors + +* James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.5.0 +sudo meza deploy +``` + ## Meza 31.4.0 Make importing from a live server simpler and more secure by not requiring sudo on the remote. Also use a more stable version of ImageMagick. @@ -31,8 +125,8 @@ Make importing from a live server simpler and more secure by not requiring sudo ### Contributors -* 25 James Montalvo -* 3 Greg Rundlett +* James Montalvo +* Greg Rundlett # How to upgrade From 419550ae260ed66be84f56b86d9643680d90b904 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 2 Jun 2019 18:12:03 -0500 Subject: [PATCH 063/280] Add --no-perms to rsync-push --- src/roles/rsync-push/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/roles/rsync-push/tasks/main.yml b/src/roles/rsync-push/tasks/main.yml index 6a4fe6a51..9b9e3a0a7 100644 --- a/src/roles/rsync-push/tasks/main.yml +++ b/src/roles/rsync-push/tasks/main.yml @@ -8,6 +8,14 @@ # pushing_to_path: # pushing_to_user: +- name: "Rsync-push inputs" + debug: + msg: | + pushing_from_server: {{ pushing_from_server }} + pushing_from_path: {{ pushing_from_path }} + pushing_to_server: {{ pushing_to_server }} + pushing_to_path: {{ pushing_to_path }} + pushing_to_user: {{ pushing_to_user }} # # Put meza-ansible's private key and known_hosts on server within /root @@ -36,6 +44,7 @@ --copy-links --archive --omit-dir-times + --no-perms "{{ pushing_from_path }}" --rsh="/usr/bin/ssh -S none -o StrictHostKeyChecking=no -l {{ pushing_to_user }} -i /root/meza-ansible-id_rsa From 3d882855c77d4688e225a4c6e4737054b43cd7fa Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 14 Dec 2018 21:40:31 -0600 Subject: [PATCH 064/280] Remove auto-encrypting of secret.yml --- src/playbooks/site.yml | 7 ------- src/scripts/meza.py | 11 +---------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index 67f7539c6..691048614 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -55,13 +55,6 @@ - name: known_hosts mode: "0600" - - name: Ensure secret.yml encrypted - shell: > - ansible-vault encrypt - /opt/conf-meza/secret/{{ env }}/secret.yml - --vault-password-file {{ m_config_vault }}/vault-pass-{{ env }}.txt - failed_when: False - # Note: without this, the encryption above changes mode to 0600 and ownership # to root:root. This makes it impossible to include_vars later. - name: Ensure secret.yml owned by meza-ansible diff --git a/src/scripts/meza.py b/src/scripts/meza.py index e0f2d08c1..71cd6b404 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -537,20 +537,11 @@ def meza_command_setup_env (argv, return_not_exit=False): os.remove(extra_vars_file) - # Now that the env is setup, generate a vault password file and use it to - # encrypt secret.yml - vault_pass_file = get_vault_pass_file( env ) - secret_yml = "/opt/conf-meza/secret/{}/secret.yml".format(env) - cmd = "ansible-vault encrypt {} --vault-password-file {}".format(secret_yml, vault_pass_file) - os.system(cmd) - - - print print "Please review your host file. Run command:" print " sudo vi /opt/conf-meza/secret/{}/hosts".format(env) print "Please review your secret config. It is encrypted, so edit by running:" - print " sudo ansible-vault edit /opt/conf-meza/secret/{}/secret.yml --vault-password-file {}".format(env,vault_pass_file) + print " sudo vi /opt/conf-meza/secret/{}/secret.yml".format(env) if return_not_exit: return rc else: From 0c9555f7d2a11d92cc4ad7f4ff2238cf76302be1 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 14 Dec 2018 22:22:42 -0600 Subject: [PATCH 065/280] Cleanup comments about encryption, remove secret.yml decryption from test case --- src/playbooks/site.yml | 5 +++-- src/scripts/meza.py | 4 ++-- tests/docker/import-from-alt-remote.setup.sh | 10 +++++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index 691048614..d74e6a4c3 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -55,8 +55,9 @@ - name: known_hosts mode: "0600" - # Note: without this, the encryption above changes mode to 0600 and ownership - # to root:root. This makes it impossible to include_vars later. + # Note: without this, if the user decides to encrypt secret.yml with Ansible + # vault then it changes mode to 0600 and ownership to root:root. This makes it + # impossible to include_vars later. - name: Ensure secret.yml owned by meza-ansible file: path: "/opt/conf-meza/secret/{{ env }}/secret.yml" diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 71cd6b404..4fdbd3568 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -540,7 +540,7 @@ def meza_command_setup_env (argv, return_not_exit=False): print print "Please review your host file. Run command:" print " sudo vi /opt/conf-meza/secret/{}/hosts".format(env) - print "Please review your secret config. It is encrypted, so edit by running:" + print "Please review your secret config. Run command:" print " sudo vi /opt/conf-meza/secret/{}/secret.yml".format(env) if return_not_exit: return rc @@ -835,7 +835,7 @@ def playbook_cmd ( playbook, env=False, more_extra_vars=False ): meza_chown( secret_file, 'meza-ansible', 'wheel' ) os.chmod( secret_file, 0o660 ) - # Setup password file if not exists (environment info is encrypted) + # Setup password file if not exists (environment info is potentially encrypted) vault_pass_file = get_vault_pass_file( env ) command = command + [ '-i', host_file, '--vault-password-file', vault_pass_file ] diff --git a/tests/docker/import-from-alt-remote.setup.sh b/tests/docker/import-from-alt-remote.setup.sh index a3ae0e845..a5a5f2406 100644 --- a/tests/docker/import-from-alt-remote.setup.sh +++ b/tests/docker/import-from-alt-remote.setup.sh @@ -50,7 +50,7 @@ ${docker_exec_1[@]} sed -r -i "s/INSERT_FQDN/$docker_ip_1/g;" "$secret_yml" # ${docker_exec_1[@]} bash -c "echo -e '[backup-src]\n$docker_ip_2 alt_remote_user=test-user\n' >> $hosts_file" ${docker_exec_1[@]} bash -c "echo -e '\n[exclude-all]\n$docker_ip_2\n' >> $hosts_file" -# Note: secret.yml is __not__ encrypted yet at this point in the test +# Note: secret.yml is __not__ encrypted ${docker_exec_1[@]} bash -c "echo -e '\n' >> $secret_yml" ${docker_exec_1[@]} bash -c "echo -e 'backups_server_alt_source:\n' >> $secret_yml" ${docker_exec_1[@]} bash -c "echo -e ' addr: $docker_ip_2\n' >> $secret_yml" @@ -82,8 +82,9 @@ ${docker_exec_1[@]} sudo -u meza-ansible ansible-playbook \ # `meza backup` ${docker_exec_1[@]} bash /opt/meza/tests/deploys/import-from-remote.controller.sh "$env_name" -# secret.yml is encrypted. decrypt first, make edits, re-encrypt. -${docker_exec_1[@]} bash -c "ansible-vault decrypt $secret_yml --vault-password-file $vault_pass" +# secret.yml is no longer engrypted by default. If it was: decrypt first, make +# edits, re-encrypt. +# ${docker_exec_1[@]} bash -c "ansible-vault decrypt $secret_yml --vault-password-file $vault_pass" ${docker_exec_1[@]} bash -c "echo -e '\n' >> $secret_yml" ${docker_exec_1[@]} bash -c "echo -e 'backups_server_db_dump:\n' >> $secret_yml" @@ -92,6 +93,9 @@ ${docker_exec_1[@]} bash -c "echo -e ' remote_user: test-user\n' >> $secret_yml ${docker_exec_1[@]} bash -c "echo -e ' mysql_user: root\n' >> $secret_yml" ${docker_exec_1[@]} bash -c "echo -e ' mysql_pass: 1234\n' >> $secret_yml" ${docker_exec_1[@]} bash -c "echo -e '\n' >> $secret_yml" + +# While secret.yml is no longer encrypted by default, it can still handle being +# encrypted. Encrypt here to test that. ${docker_exec_1[@]} bash -c "ansible-vault encrypt $secret_yml --vault-password-file $vault_pass" # Add database source (e.g. pull direct from database) to inventory, make some From 63b86fa464c5952a41137863992a7646976410a7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 15 Dec 2018 09:35:21 -0600 Subject: [PATCH 066/280] Add encrypt_string and decrypt_string meza commands --- src/scripts/meza.py | 104 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 4fdbd3568..a70553bac 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -775,6 +775,85 @@ def meza_command_maint_cleanuploadstash (argv): meza_shell_exec_exit(rc) +def meza_command_maint_encrypt_string (argv): + + env = argv[0] + + rc = check_environment(env) + + # return code != 0 means failure + if rc != 0: + meza_shell_exec_exit(rc) + + # strip environment off of it + argv = argv[1:] + + if len(argv) == 0: + print "encrypt_string requires value to encrypt. Ex:" + print " sudo meza maint encrypt_string somesecretvalue" + print "Additionally, you can supply the variable name. Ex:" + print " sudo meza maint encrypt_string somesecretvalue var_name" + sys.exit(1) + + varvalue = argv[0] + vault_pass_file = get_vault_pass_file( env ) + + shell_cmd = ["ansible-vault","encrypt_string","--vault-id",vault_pass_file,varvalue] + + # If name argument passed in, use it + if len(argv) == 2: + shell_cmd = shell_cmd + ["--name",argv[1]] + + rc = meza_shell_exec( shell_cmd ) + + # exit with same return code as ansible command + meza_shell_exec_exit(rc) + + +# sudo meza maint decrypt_string +def meza_command_maint_decrypt_string (argv): + + env = argv[0] + + rc = check_environment(env) + + # return code != 0 means failure + if rc != 0: + meza_shell_exec_exit(rc) + + # strip environment off of it + argv = argv[1:] + + if len(argv) == 0: + print "decrypt_string requires you to supply encrypted string. Ex:" + print """ +sudo meza maint decrypt_string '$ANSIBLE_VAULT;1.1;AES256 +31386561343430626435373766393066373464656262383063303630623032616238383838346132 +6162313461666439346337616166396133616466363935360a373333313165343535373761333634 +62636634306632633539306436363866323639363332613363346663613235653138373837303337 +6133383864613430370a623661653462336565376565346638646238643132636663383761613966 +6566' +""" + sys.exit(1) + + encrypted_string = argv[0] + vault_pass_file = get_vault_pass_file( env ) + + tmp_file = write_vault_decryption_tmp_file( env, encrypted_string ) + + shell_cmd = ["ansible-vault","decrypt",tmp_file,"--vault-password-file",vault_pass_file] + rc = meza_shell_exec( shell_cmd ) + + decrypted_value = read_vault_decryption_tmp_file( env ) + + print "" + print "Decrypted value:" + print decrypted_value + + # exit with same return code as ansible command + meza_shell_exec_exit(rc) + + def meza_command_docker (argv): if argv[0] == "run": @@ -941,6 +1020,31 @@ def get_vault_pass_file ( env ): return vault_pass_file +def write_vault_decryption_tmp_file ( env, value ): + home_dir = defaults['m_home'] + temp_decrypt_file = '{}/meza-ansible/.vault-temp-decrypt-{}.txt'.format(home_dir,env) + + with open( temp_decrypt_file, 'w' ) as filetowrite: + filetowrite.write( value ) + filetowrite.close() + + return temp_decrypt_file + +def read_vault_decryption_tmp_file ( env ): + home_dir = defaults['m_home'] + temp_decrypt_file = '{}/meza-ansible/.vault-temp-decrypt-{}.txt'.format(home_dir,env) + + f = open( temp_decrypt_file, "r" ) + if f.mode == 'r': + contents = f.read() + f.close() + os.remove( temp_decrypt_file ) + else: + contents = "[decryption error]" + + return contents + + def meza_chown ( path, username, groupname ): import pwd import grp From e70279c7b9dd054bdb14a3a67c5dad6df4af2515 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 15 Dec 2018 09:57:23 -0600 Subject: [PATCH 067/280] Don't print command for encrypt/decrypt (too much text) --- src/scripts/meza.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index a70553bac..3e7116619 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -804,7 +804,8 @@ def meza_command_maint_encrypt_string (argv): if len(argv) == 2: shell_cmd = shell_cmd + ["--name",argv[1]] - rc = meza_shell_exec( shell_cmd ) + # false = don't print command prior to running + rc = meza_shell_exec( shell_cmd, False, False ) # exit with same return code as ansible command meza_shell_exec_exit(rc) @@ -842,7 +843,9 @@ def meza_command_maint_decrypt_string (argv): tmp_file = write_vault_decryption_tmp_file( env, encrypted_string ) shell_cmd = ["ansible-vault","decrypt",tmp_file,"--vault-password-file",vault_pass_file] - rc = meza_shell_exec( shell_cmd ) + + # false = don't print command prior to running + rc = meza_shell_exec( shell_cmd, False, False ) decrypted_value = read_vault_decryption_tmp_file( env ) @@ -935,7 +938,7 @@ def playbook_cmd ( playbook, env=False, more_extra_vars=False ): # FIXME install --> setup dev-networking, setup docker, deploy monolith (special case) -def meza_shell_exec ( shell_cmd, log_file=False ): +def meza_shell_exec ( shell_cmd, log_file=False, print_command=True ): # Get errors with user meza-ansible trying to write to the calling-user's # home directory if don't cd to a neutral location. By cd'ing to this @@ -956,7 +959,8 @@ def meza_shell_exec ( shell_cmd, log_file=False ): else: cmd = ' '.join(shell_cmd) - print cmd + if print_command: + print cmd import subprocess From 2bb09f5022bb9e55759a6344966b57d20a5609b3 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 3 Jun 2019 00:37:22 -0500 Subject: [PATCH 068/280] 31.6.0 release --- RELEASE-NOTES.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 9f30da39a..c39cf1bc7 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,29 @@ Release Notes ============= +## Meza 31.6.0 + +No longer require encryption of secret.yml; prefer variable-level encryption; fix permissions for rsync-push + +### Commits since 31.5.0 + +* e70279c Don't print command for encrypt/decrypt (too much text) +* 63b86fa Add encrypt_string and decrypt_string meza commands +* 0c9555f Cleanup comments about encryption, remove secret.yml decryption from test case +* 3d88285 Remove auto-encrypting of secret.yml +* 419550a Add --no-perms to rsync-push + +### Contributors + +* 7 James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.6.0 +sudo meza deploy +``` + ## Meza 31.5.0 Major deploy and autodeploy improvements; Push backups to remote server; Security and general improvements; bug fixes From 170596cbfc76c9e8738a4bff401b13b125c3a97b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 3 Jun 2019 07:16:52 -0500 Subject: [PATCH 069/280] Make push-backup cron configurable add notification --- src/playbooks/push-backup.yml | 64 +++++++++++++++++++ .../templates/meza-autodeployer-cron.j2 | 15 ++--- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/playbooks/push-backup.yml b/src/playbooks/push-backup.yml index 7293c5454..6f147bf7e 100644 --- a/src/playbooks/push-backup.yml +++ b/src/playbooks/push-backup.yml @@ -6,6 +6,38 @@ - set_fact: backup_timestamp: "{{lookup('pipe','date +%Y%m%d%H%M%S')}}" +- hosts: localhost + become: yes + roles: + - set-vars + tags: + # only run this if notify specified + - never + - notify + tasks: + + - name: + set_fact: + autodeployer_slack_icon_url: "{{ autodeployer.slack_icon_url | default('https://github.com/enterprisemediawiki/meza/raw/master/src/roles/configure-wiki/files/logo.png') }}" + when: autodeployer is defined + + - name: Notify push backup is starting + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Push backup starting" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + when: + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + + # FIXME #803: If a slave is available, maybe pull from there - hosts: db-master become: yes @@ -23,3 +55,35 @@ tags: - uploads - upload + +- hosts: localhost + become: yes + roles: + - set-vars + tags: + # only run this if notify specified + - never + - notify + tasks: + + - name: + set_fact: + autodeployer_slack_icon_url: "{{ autodeployer.slack_icon_url | default('https://github.com/enterprisemediawiki/meza/raw/master/src/roles/configure-wiki/files/logo.png') }}" + when: autodeployer is defined + + - name: Notify push backup is complete + slack: + token: "{{ autodeployer.slack_token }}" + msg: "Push backup complete" + channel: "{{ autodeployer.slack_channel }}" + username: "{{ autodeployer.slack_username }}" + icon_url: "{{ autodeployer_slack_icon_url }}" + color: "good" + link_names: 1 + when: + - autodeployer is defined + - autodeployer.slack_token is defined + - autodeployer.slack_channel is defined + - autodeployer.slack_username is defined + - autodeployer_slack_icon_url is defined + diff --git a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 index 25b7d07e9..58f50eae9 100644 --- a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 +++ b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 @@ -6,10 +6,6 @@ MAILTO=root {% if force_deploy is defined and force_deploy.crontime is defined %} # # Force deploy -# FIXME: At present this extracts environment from /opt/.deploy-meza/config.sh. -# This works fine if the controller is used for only one environment, but -# will get confused if multiple environments are used. Environment should -# be explicitly passed into this script. # {{ force_deploy.crontime }} root meza deploy-notify "{{ env }}" "{{ _force_deploy_notify_prefix }}" "{{ _force_deploy_options }}" {% endif %} @@ -18,10 +14,13 @@ MAILTO=root {% if autodeployer is defined and autodeployer.crontime is defined %} # # Auto-deploy on config and Meza changes -# FIXME: At present this extracts environment from /opt/.deploy-meza/config.sh. -# This works fine if the controller is used for only one environment, but -# will get confused if multiple environments are used. Environment should -# be explicitly passed into this script. # {{ autodeployer.crontime }} root meza autodeploy "{{ env }}" "Deploy" "" >> {{ m_logs }}/deploy/check-for-changes-`date "+\%Y\%m\%d"`.log 2>&1 + + +{% if push_backup is defined and push_backup.crontime is defined %} +# +# Push backup (db and uploads) to another server periodically +# +{{ push_backup.crontime }} root meza push-backup "{{ env }}" "--tags notify" >> {{ m_logs }}/deploy/push-backup-`date "+\%Y\%m\%d"`.log 2>&1 {% endif %} From 8a3c059b6b68444e139b56b7a3771bdc8548bb0c Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 3 Jun 2019 07:39:53 -0500 Subject: [PATCH 070/280] Fix missing endif --- src/roles/autodeployer/templates/meza-autodeployer-cron.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 index 58f50eae9..ba0686997 100644 --- a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 +++ b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 @@ -24,3 +24,4 @@ MAILTO=root # {{ push_backup.crontime }} root meza push-backup "{{ env }}" "--tags notify" >> {{ m_logs }}/deploy/push-backup-`date "+\%Y\%m\%d"`.log 2>&1 {% endif %} +{% endif %} From 8b109a77c6bfc9b289ba3c5bc10a8530a27499b7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 3 Jun 2019 10:41:57 -0500 Subject: [PATCH 071/280] Simplify push_backup settings --- src/playbooks/push-backup.yml | 4 ++-- src/roles/backup-db-wikis-push/tasks/main.yml | 12 ++++++------ src/roles/backup-uploads-push/tasks/main.yml | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/playbooks/push-backup.yml b/src/playbooks/push-backup.yml index 6f147bf7e..6ab817a9c 100644 --- a/src/playbooks/push-backup.yml +++ b/src/playbooks/push-backup.yml @@ -24,7 +24,7 @@ - name: Notify push backup is starting slack: token: "{{ autodeployer.slack_token }}" - msg: "Push backup starting" + msg: "Push backup to {{ push_backup.short_name | default('remote server') }} starting" channel: "{{ autodeployer.slack_channel }}" username: "{{ autodeployer.slack_username }}" icon_url: "{{ autodeployer_slack_icon_url }}" @@ -74,7 +74,7 @@ - name: Notify push backup is complete slack: token: "{{ autodeployer.slack_token }}" - msg: "Push backup complete" + msg: "Push backup to {{ push_backup.short_name | default('remote server') }} complete" channel: "{{ autodeployer.slack_channel }}" username: "{{ autodeployer.slack_username }}" icon_url: "{{ autodeployer_slack_icon_url }}" diff --git a/src/roles/backup-db-wikis-push/tasks/main.yml b/src/roles/backup-db-wikis-push/tasks/main.yml index b692ea507..0ba8eb6da 100644 --- a/src/roles/backup-db-wikis-push/tasks/main.yml +++ b/src/roles/backup-db-wikis-push/tasks/main.yml @@ -14,29 +14,29 @@ - name: Set remote_server_base_path if set in configuration set_fact: - remote_server_base_path: "{{ backups_server_db_push.sql_files_path }}" + remote_server_base_path: "{{ push_backup.db.path }}" when: - - backups_server_db_push.sql_files_path is defined + - push_backup.db.path is defined - name: Set remote_server_base_path if NOT set in configuration set_fact: remote_server_base_path: "{{ m_backups }}/{{ env }}//" when: - - backups_server_db_push.sql_files_path is not defined + - push_backup.db.path is not defined - name: Output value of remote_server_base_path ( will be replaced by each wiki_id) debug: { var: remote_server_base_path } -- name: "Run role:rsync-push - Copy SQL files to {{ backups_server_db_push.addr }}" +- name: "Run role:rsync-push - Copy SQL files to {{ push_backup.db.addr }}" include_role: name: rsync-push vars: pushing_from_server: "{{ inventory_hostname }}" pushing_from_path: "{{ m_tmp }}/{{ env }}_{{ item }}.sql" - pushing_to_server: "{{ backups_server_db_push.addr }}" + pushing_to_server: "{{ push_backup.db.addr }}" # remote_server_base_path + backup_timestamp + _wiki.sql, but replace with wiki_id (item) pushing_to_path: "{{ remote_server_base_path | regex_replace('', item) }}{{ backup_timestamp }}_wiki.sql" - pushing_to_user: "{{ backups_server_db_push.remote_user }}" + pushing_to_user: "{{ push_backup.remote_user }}" with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" # Remove temp SQL files, only needs to be done on first backup server diff --git a/src/roles/backup-uploads-push/tasks/main.yml b/src/roles/backup-uploads-push/tasks/main.yml index df1ab126d..fa88fc65f 100644 --- a/src/roles/backup-uploads-push/tasks/main.yml +++ b/src/roles/backup-uploads-push/tasks/main.yml @@ -20,9 +20,9 @@ set_fact: # Likely path if pushing to a live Meza uploads directory: # /opt/data-meza/uploads// - remote_server_base_path: "{{ backups_server_uploads_push.uploads_dir_path }}" + remote_server_base_path: "{{ push_backup.uploads.path }}" when: - - backups_server_uploads_push.uploads_dir_path is defined + - push_backup.uploads.path is defined - name: Set remote_server_base_path if NOT set in configuration set_fact: @@ -30,19 +30,19 @@ # /opt/data-meza/backups///uploads/ remote_server_base_path: "{{ m_backups }}/{{ env }}//uploads/" when: - - backups_server_uploads_push.uploads_dir_path is not defined + - push_backup.uploads.path is not defined - name: Output value of remote_server_base_path ( will be replaced by each wiki_id) debug: { var: remote_server_base_path } -- name: "Run role:rsync-push - Copy uploads directory to {{ backups_server_uploads_push.addr }}" +- name: "Run role:rsync-push - Copy uploads directory to {{ push_backup.uploads.addr }}" include_role: name: rsync-push vars: pushing_from_server: "{{ inventory_hostname }}" pushing_from_path: "{{ m_uploads_dir }}/{{ item }}/" - pushing_to_server: "{{ backups_server_uploads_push.addr }}" + pushing_to_server: "{{ push_backup.uploads.addr }}" # remote_server_base_path + backup_timestamp, but replace with wiki_id (item) pushing_to_path: "{{ remote_server_base_path | regex_replace('', item) }}" - pushing_to_user: "{{ backups_server_uploads_push.remote_user }}" + pushing_to_user: "{{ push_backup.remote_user }}" with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" From dece78cede43cbe5ba2ae181710b02d3b1cc2a46 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 3 Jun 2019 13:22:29 -0500 Subject: [PATCH 072/280] 31.7.0 release --- RELEASE-NOTES.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index c39cf1bc7..16e58c58e 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,27 @@ Release Notes ============= +## Meza 31.7.0 + +Simplify push-backup settings and make cron configurable and with notification + +### Commits since 31.6.0 + +* 8b109a7 Simplify push_backup settings +* 8a3c059 Fix missing endif +* 170596c Make push-backup cron configurable add notification + +### Contributors + +* James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.7.0 +sudo meza deploy +``` + ## Meza 31.6.0 No longer require encryption of secret.yml; prefer variable-level encryption; fix permissions for rsync-push From dd245ae53635943f963c68d393ce36cd62efdacc Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 00:03:51 -0500 Subject: [PATCH 073/280] Sync secret perms --- config/core/defaults.yml | 5 +++- src/playbooks/site.yml | 2 +- src/roles/haproxy/tasks/main.yml | 6 ++--- .../init-controller-config/tasks/main.yml | 23 ++++++++++++++---- src/roles/setup-env/tasks/main.yml | 24 +++++++++---------- src/scripts/getmeza.sh | 2 +- 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index ce233bed2..354a9ad91 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -236,10 +236,13 @@ m_config_public_mode: "0755" m_config_public_owner: meza-ansible m_config_public_group: wheel -m_config_secret_mode: "0750" +m_config_secret_file_mode: "0660" +m_config_secret_dir_mode: "0770" m_config_secret_owner: meza-ansible m_config_secret_group: wheel + + # # PHP config # diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index d74e6a4c3..60df2e265 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -63,7 +63,7 @@ path: "/opt/conf-meza/secret/{{ env }}/secret.yml" owner: meza-ansible group: wheel - mode: "0600" + mode: "0660" - name: Ensure /opt/conf-meza owned by meza-ansible file: diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index ffd736f99..a39416272 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -35,9 +35,9 @@ file: path: "{{ m_local_secret }}/{{ env }}/ssl" state: directory - owner: root - group: root - mode: 0755 + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_dir_mode }}" delegate_to: localhost run_once: True diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index 706cd4beb..0abf2cce0 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -107,13 +107,28 @@ tags: - file-perms -- name: "Ensure {{ m_local_secret }} properly owned" +- name: "Ensure {{ m_local_secret }} directories properly owned/moded" file: - path: "{{ m_local_secret }}" + path: "{{ item }}" owner: "{{ m_config_secret_owner }}" group: "{{ m_config_secret_group }}" - mode: "{{ m_config_secret_mode }}" + mode: "{{ m_config_secret_dir_mode }}" + state: directory + tags: + - file-perms + with_items: + - "{{ m_local_secret }}" + - "{{ m_local_secret }}/{{ env }}" + +- name: "Ensure {{ m_local_secret }} files properly owned/moded" + file: + path: "{{ item }}" + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_file_mode }}" state: directory - recurse: Yes tags: - file-perms + with_items: + - "{{ m_local_secret }}/{{ env }}/secret.yml" + - "{{ m_local_secret }}/{{ env }}/hosts" diff --git a/src/roles/setup-env/tasks/main.yml b/src/roles/setup-env/tasks/main.yml index 5e2bf675d..cc6ca80bc 100644 --- a/src/roles/setup-env/tasks/main.yml +++ b/src/roles/setup-env/tasks/main.yml @@ -4,26 +4,26 @@ file: path: "{{ m_local_secret }}" state: directory - owner: meza-ansible - group: wheel - mode: 0750 + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_dir_mode }}" - name: Ensure secret config environment directory exists file: path: "{{ m_local_secret }}/{{ env }}" state: directory - owner: meza-ansible - group: wheel - mode: 0750 + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_dir_mode }}" # Ansible writing Ansible - name: Ensure hosts file configured template: src: hosts.j2 dest: "{{ m_local_secret }}/{{ env }}/hosts" - owner: meza-ansible - group: wheel - mode: 0640 + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_file_mode }}" # We don't want to overwrite an existing hosts file with the default force: no @@ -32,9 +32,9 @@ template: src: secret.yml.j2 dest: "{{ m_local_secret }}/{{ env }}/secret.yml" - owner: meza-ansible - group: wheel - mode: 0640 + owner: "{{ m_config_secret_owner }}" + group: "{{ m_config_secret_group }}" + mode: "{{ m_config_secret_file_mode }}" # don't overwrite existing force: no diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index a5d780505..a69a292fe 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -59,7 +59,7 @@ fi # make sure conf-meza exists and has good permissions mkdir -p /opt/conf-meza/secret chmod 755 /opt/conf-meza -chmod 755 /opt/conf-meza/secret +chmod 770 /opt/conf-meza/secret # Required initially for creating lock files mkdir -p /opt/data-meza From 968c6757e60db766bcb719f4717a61d4a6d0ee9b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 00:18:06 -0500 Subject: [PATCH 074/280] Make meza-ansible own temp_vars.json --- src/scripts/meza.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 3e7116619..10766e60b 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -530,6 +530,7 @@ def meza_command_setup_env (argv, return_not_exit=False): # Make sure temp_vars.json is accessible. On the first run of deploy it is # possible that user meza-ansible will not be able to reach this file, # specifically if the system has a restrictive umask set (e.g 077). + meza_chown( extra_vars_file, 'meza-ansible', 'wheel' ) os.chmod(extra_vars_file, 0664) shell_cmd = playbook_cmd( "setup-env" ) + ["--extra-vars", '@'+extra_vars_file] From c64c41b7328c03caea308a4fac9ad199eaddc0ce Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 06:19:56 -0500 Subject: [PATCH 075/280] Relax secret config _directory_ mode; ensure good ownership --- config/core/defaults.yml | 2 +- src/scripts/getmeza.sh | 1 + src/scripts/meza.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 354a9ad91..1ed6a6859 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -237,7 +237,7 @@ m_config_public_owner: meza-ansible m_config_public_group: wheel m_config_secret_file_mode: "0660" -m_config_secret_dir_mode: "0770" +m_config_secret_dir_mode: "0775" m_config_secret_owner: meza-ansible m_config_secret_group: wheel diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index a69a292fe..59f9ff026 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -88,6 +88,7 @@ else fi chown meza-ansible:wheel /opt/conf-meza +chown meza-ansible:wheel /opt/conf-meza/secret chown meza-ansible:wheel /opt/meza # Don't require TTY or visible password for sudo. Ref #769 diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 10766e60b..92f960e78 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -520,7 +520,7 @@ def meza_command_setup_env (argv, return_not_exit=False): # are not written to command line. Putting in secret should make # permissions acceptable since this dir will hold secret info, though it's # sort of an odd place for a temporary file. Perhaps /root instead? - extra_vars_file = "/opt/conf-meza/secret/temp_vars.json" + extra_vars_file = os.path.join( defaults['m_local_secret'], "temp_vars.json" ) if os.path.isfile(extra_vars_file): os.remove(extra_vars_file) f = open(extra_vars_file, 'w') @@ -530,6 +530,7 @@ def meza_command_setup_env (argv, return_not_exit=False): # Make sure temp_vars.json is accessible. On the first run of deploy it is # possible that user meza-ansible will not be able to reach this file, # specifically if the system has a restrictive umask set (e.g 077). + meza_chown( defaults['m_local_secret'], 'meza-ansible', 'wheel' ) meza_chown( extra_vars_file, 'meza-ansible', 'wheel' ) os.chmod(extra_vars_file, 0664) From ab1d9b1ec5b5b1e347c1769a64f89d1305da1f9d Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 06:38:11 -0500 Subject: [PATCH 076/280] Don't overwrite secrt files --- src/roles/init-controller-config/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index 0abf2cce0..70e19d9a9 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -127,6 +127,7 @@ group: "{{ m_config_secret_group }}" mode: "{{ m_config_secret_file_mode }}" state: directory + force: no tags: - file-perms with_items: From 8ba13533f3a4d0ba449166f62968af90d04fd6e1 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 06:59:19 -0500 Subject: [PATCH 077/280] file not directory --- src/roles/init-controller-config/tasks/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/roles/init-controller-config/tasks/main.yml b/src/roles/init-controller-config/tasks/main.yml index 70e19d9a9..65c9a06e4 100644 --- a/src/roles/init-controller-config/tasks/main.yml +++ b/src/roles/init-controller-config/tasks/main.yml @@ -126,8 +126,7 @@ owner: "{{ m_config_secret_owner }}" group: "{{ m_config_secret_group }}" mode: "{{ m_config_secret_file_mode }}" - state: directory - force: no + state: file tags: - file-perms with_items: From 2ef84547e90d5da327692ac05bb2519a0a0e1a66 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 07:51:03 -0500 Subject: [PATCH 078/280] Secret directory 775 in getmeza.sh, too --- src/scripts/getmeza.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index 59f9ff026..f2ebd64b9 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -59,7 +59,7 @@ fi # make sure conf-meza exists and has good permissions mkdir -p /opt/conf-meza/secret chmod 755 /opt/conf-meza -chmod 770 /opt/conf-meza/secret +chmod 775 /opt/conf-meza/secret # Required initially for creating lock files mkdir -p /opt/data-meza From d6ddb60c80dff26c5d6cd65fea536f34fe739eae Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 08:59:52 -0500 Subject: [PATCH 079/280] Don't use 'meza' command for dev-networking --- src/scripts/meza.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 92f960e78..570e5e736 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -576,10 +576,11 @@ def meza_command_setup_dev (argv): print "https://wbond.net/sublime_packages/sftp/settings#Remote_Server_Settings" sys.exit() - +# Remove in 32.x def meza_command_setup_dev_networking (argv): - rc = meza_shell_exec(["bash","/opt/meza/src/scripts/dev-networking.sh"]) - sys.exit(rc) + print "Function removed. Instead do:" + print " sudo bash /opt/meza/src/scripts/dev-networking.sh" + sys.exit(1) def meza_command_setup_docker (argv): shell_cmd = playbook_cmd( "getdocker" ) From 077e6bc2f669e714ad1a7b9194108d1fc9856e0f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 10:10:00 -0500 Subject: [PATCH 080/280] Add lock_timeout to yum/package modules to fix Ansible 2.8 issue Issue getting "yum lockfile is held by another process" msg Ref: - https://github.com/ansible/ansible/issues/57189 - #1165 --- src/playbooks/delete-elasticsearch.yml | 1 + src/playbooks/getdocker.yml | 3 ++ src/roles/apache-php/tasks/ius.yml | 2 ++ src/roles/apache-php/tasks/main.yml | 5 +++- .../apache-php/tasks/mssql_driver_for_php.yml | 4 +++ src/roles/apache-php/tasks/php.yml | 5 ++++ src/roles/apache-php/tasks/profiling.yml | 1 + src/roles/backup-config/tasks/main.yml | 1 + src/roles/base-extras/tasks/main.yml | 5 +++- src/roles/base/tasks/main.yml | 28 +++++++++++++++---- src/roles/database/tasks/setup-RedHat.yml | 2 ++ src/roles/elasticsearch/tasks/es_upgrade.yml | 1 + src/roles/elasticsearch/tasks/main.yml | 3 ++ src/roles/gluster/tasks/setup-RedHat.yml | 2 ++ src/roles/haproxy/tasks/main.yml | 5 +++- src/roles/imagemagick/tasks/main.yml | 2 ++ src/roles/mediawiki/tasks/main.yml | 1 + src/roles/memcached/tasks/main.yml | 1 + src/roles/nodejs/tasks/main.yml | 2 ++ src/roles/nodejs/tasks/setup-RedHat.yml | 9 +++++- tests/deploys/setup-alt-source-backup.yml | 1 + 21 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/playbooks/delete-elasticsearch.yml b/src/playbooks/delete-elasticsearch.yml index 1ac3e6f14..c90174ad4 100644 --- a/src/playbooks/delete-elasticsearch.yml +++ b/src/playbooks/delete-elasticsearch.yml @@ -19,6 +19,7 @@ - name: Ensure Elasticsearch is removed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: elasticsearch state: absent when: seriously_delete_elasticsearch == "yes" diff --git a/src/playbooks/getdocker.yml b/src/playbooks/getdocker.yml index 7f85c1bbb..716b2131c 100644 --- a/src/playbooks/getdocker.yml +++ b/src/playbooks/getdocker.yml @@ -15,6 +15,7 @@ tasks: - name: Ensure CentOS-provided Docker repos are removed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ item }}" state: absent with_items: @@ -26,6 +27,7 @@ - name: Ensure yum-utils present yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: yum-utils state: present @@ -47,6 +49,7 @@ # or specify docker-ce-{{ docker_version }} - name: Ensure docker-ce package installed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "docker-ce" state: latest notify: Restart Docker diff --git a/src/roles/apache-php/tasks/ius.yml b/src/roles/apache-php/tasks/ius.yml index 1309e3398..bcdd32954 100644 --- a/src/roles/apache-php/tasks/ius.yml +++ b/src/roles/apache-php/tasks/ius.yml @@ -1,11 +1,13 @@ --- - name: Install IUS (CentOS) repo. yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "https://centos{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" when: ansible_distribution == "CentOS" - name: Install IUS (RHEL) repo. yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "https://rhel{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" when: ansible_distribution == "RedHat" diff --git a/src/roles/apache-php/tasks/main.yml b/src/roles/apache-php/tasks/main.yml index be743a341..4f4929478 100644 --- a/src/roles/apache-php/tasks/main.yml +++ b/src/roles/apache-php/tasks/main.yml @@ -1,6 +1,9 @@ --- - name: Install apache packages - yum: name={{item}} state=installed + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "{{ item }}" + state: installed with_items: - httpd-devel - mod_ssl diff --git a/src/roles/apache-php/tasks/mssql_driver_for_php.yml b/src/roles/apache-php/tasks/mssql_driver_for_php.yml index 7d8c02ea0..da039548a 100644 --- a/src/roles/apache-php/tasks/mssql_driver_for_php.yml +++ b/src/roles/apache-php/tasks/mssql_driver_for_php.yml @@ -2,6 +2,7 @@ - name: Ensure prerequisites for sqlsrv in place yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{item}}" state: installed with_items: @@ -17,6 +18,7 @@ - name: Ensure conflicting ODBC drivers removed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{item}}" state: absent with_items: @@ -25,6 +27,7 @@ - name: install MS ODBC driver package yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: msodbcsql17 state: latest environment: @@ -34,6 +37,7 @@ - name: install ODBC driver devel package yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: unixODBC-devel state: latest diff --git a/src/roles/apache-php/tasks/php.yml b/src/roles/apache-php/tasks/php.yml index ef1b4efc9..d623499fa 100644 --- a/src/roles/apache-php/tasks/php.yml +++ b/src/roles/apache-php/tasks/php.yml @@ -1,6 +1,7 @@ --- - name: Install php dependency packages yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ item }}" state: installed with_items: @@ -36,6 +37,7 @@ - name: Ensure PHP 5.6 packages removed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{item}}" state: absent with_items: @@ -68,6 +70,7 @@ # other versions of PHP are not installed - name: "Check if {{ php_ius_version}} package is installed" yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 list: "{{ php_ius_version }}" register: correct_php @@ -76,6 +79,7 @@ - name: Remove any other PHP packages from IUS repo if correct PHP is not installed package: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "php*u*" state: absent when: ansible_os_family == 'RedHat' and @@ -83,6 +87,7 @@ - name: Ensure PHP IUS packages installed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{item}}" state: installed with_items: diff --git a/src/roles/apache-php/tasks/profiling.yml b/src/roles/apache-php/tasks/profiling.yml index c7235a6a4..8e1d1fa16 100644 --- a/src/roles/apache-php/tasks/profiling.yml +++ b/src/roles/apache-php/tasks/profiling.yml @@ -11,6 +11,7 @@ - name: Install mongodb-org package yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: mongodb-org state: installed run_once: yes diff --git a/src/roles/backup-config/tasks/main.yml b/src/roles/backup-config/tasks/main.yml index c20b6b6ee..52f62f44b 100644 --- a/src/roles/backup-config/tasks/main.yml +++ b/src/roles/backup-config/tasks/main.yml @@ -2,6 +2,7 @@ - name: Ensure MariaDB client installed on backup servers yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: mariadb state: present tags: diff --git a/src/roles/base-extras/tasks/main.yml b/src/roles/base-extras/tasks/main.yml index 6926c4a6a..083f9f565 100644 --- a/src/roles/base-extras/tasks/main.yml +++ b/src/roles/base-extras/tasks/main.yml @@ -1,6 +1,9 @@ --- - name: Install base-extras packages - yum: name={{item}} state=installed + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "{{ item }}" + state: installed with_items: - expect - expectk diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index c0b79eefa..a24aff7f8 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -52,17 +52,24 @@ # Without this get error "yum lockfile is held by another process" occasionally - name: Ensure PackageKit is removed so it doesn't try to upgrade packages on its own yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: PackageKit state: absent when: ansible_os_family == 'RedHat' - name: ensure deltarpm is installed and latest - yum: name=deltarpm state=latest + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: deltarpm + state: latest tags: - latest - name: upgrade all packages - yum: name=* state=latest + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "*" + state: latest environment: # If `install_ms_sql_driver: True` is set in config, MS SQL drivers will be # installed. When these drivers are updated they require accepting end user @@ -74,6 +81,7 @@ # FIXME #807: for RedHat may need to enable "Optional RPMs" - name: ensure EPEL installed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: epel-release state: installed when: ansible_distribution == "CentOS" @@ -89,6 +97,7 @@ - name: Install EPEL repo. yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" state: present register: result @@ -115,12 +124,18 @@ - latest - name: ensure libselinux-python installed prior to SELinux - yum: name=libselinux-python state=installed + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: libselinux-python + state: installed tags: - latest - name: Install base packages - yum: name={{item}} state=installed + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "{{ item }}" + state: installed with_items: - ntp - ntpdate @@ -187,7 +202,10 @@ # ntpdate pool.ntp.org # Synchronize the system clock with 0.pool.ntp.org server # service ntpd start # Start service - name: Install NTP - yum: name=ntp state=installed + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: ntp + state: installed tags: - latest diff --git a/src/roles/database/tasks/setup-RedHat.yml b/src/roles/database/tasks/setup-RedHat.yml index a80aae259..1c4176d16 100644 --- a/src/roles/database/tasks/setup-RedHat.yml +++ b/src/roles/database/tasks/setup-RedHat.yml @@ -1,6 +1,7 @@ --- - name: Ensure MySQL packages are installed. yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ mysql_packages }}" state: present enablerepo: "{{ mysql_enablerepo | default(omit, true) }}" @@ -8,6 +9,7 @@ - name: Ensure MySQL Python libraries are installed. yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: MySQL-python state: present enablerepo: "{{ mysql_enablerepo | default(omit, true) }}" diff --git a/src/roles/elasticsearch/tasks/es_upgrade.yml b/src/roles/elasticsearch/tasks/es_upgrade.yml index 890de15c5..0b872a1a1 100644 --- a/src/roles/elasticsearch/tasks/es_upgrade.yml +++ b/src/roles/elasticsearch/tasks/es_upgrade.yml @@ -82,6 +82,7 @@ - name: Ensure Elasticsearch is latest version yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: elasticsearch state: latest diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index e218b5303..17a9b62f8 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -4,10 +4,12 @@ # - name: Ensure Java 1.7.0 OpenJDK is absent yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: java-1.7.0-openjdk state: absent - name: Ensure Java 1.8.0 OpenJDK is installed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: java-1.8.0-openjdk state: installed @@ -31,6 +33,7 @@ mode: 0644 - name: Install Elasticsearch. yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: elasticsearch state: installed diff --git a/src/roles/gluster/tasks/setup-RedHat.yml b/src/roles/gluster/tasks/setup-RedHat.yml index 56feedb40..f5d64a0c6 100644 --- a/src/roles/gluster/tasks/setup-RedHat.yml +++ b/src/roles/gluster/tasks/setup-RedHat.yml @@ -10,6 +10,7 @@ - name: Ensure CentOS prerequisites in place yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ item }}" state: present with_items: @@ -19,6 +20,7 @@ - name: Install Packages yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ item }}" state: present with_items: diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index a39416272..71007c452 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -7,7 +7,10 @@ # https://www.unixmen.com/configure-high-available-load-balancer-haproxy-keepalived/ - name: Install haproxy packages - yum: name={{item}} state=installed + yum: + name: "{{ item }}" + state: installed + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 with_items: - haproxy - openssl diff --git a/src/roles/imagemagick/tasks/main.yml b/src/roles/imagemagick/tasks/main.yml index 0cc001ddd..7ee17c70e 100644 --- a/src/roles/imagemagick/tasks/main.yml +++ b/src/roles/imagemagick/tasks/main.yml @@ -1,11 +1,13 @@ --- - name: Ensure old ImageMagick installed from Meza RPM yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: imagemagick-7.0.3-1.x86_64 state: absent - name: Ensure ImageMagick at latest version yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: - ghostscript - ImageMagick diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 16988adce..2453dced1 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -533,6 +533,7 @@ # MediaWiki 1.31+. - name: Ensure Python3 present yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: - python35u - python35u-pip diff --git a/src/roles/memcached/tasks/main.yml b/src/roles/memcached/tasks/main.yml index 58cc7af61..3610a2415 100644 --- a/src/roles/memcached/tasks/main.yml +++ b/src/roles/memcached/tasks/main.yml @@ -1,6 +1,7 @@ --- - name: Ensure memcached and netcat packages latest yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ item }}" state: latest tags: diff --git a/src/roles/nodejs/tasks/main.yml b/src/roles/nodejs/tasks/main.yml index 82096a257..42cefcf3d 100644 --- a/src/roles/nodejs/tasks/main.yml +++ b/src/roles/nodejs/tasks/main.yml @@ -23,6 +23,7 @@ var: rhel_os_minor_version - name: Ensure http-parser installed from RPM for {{ ansible_distribution_version }} yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: https://kojipkgs.fedoraproject.org//packages/http-parser/2.7.1/3.el7/x86_64/http-parser-2.7.1-3.el7.x86_64.rpm state: present when: @@ -40,6 +41,7 @@ - name: Ensure Node.js and npm from yum are REMOVED yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{ item }}" state: absent with_items: diff --git a/src/roles/nodejs/tasks/setup-RedHat.yml b/src/roles/nodejs/tasks/setup-RedHat.yml index 882090c8f..8f5fb24cc 100644 --- a/src/roles/nodejs/tasks/setup-RedHat.yml +++ b/src/roles/nodejs/tasks/setup-RedHat.yml @@ -23,12 +23,14 @@ - name: Add Nodesource repositories for Node.js (CentOS < 7). yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "http://rpm.nodesource.com/{{ nodejs_rhel_rpm_dir }}/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm" state: present when: ansible_distribution_major_version|int < 7 - name: Add Nodesource repositories for Node.js (CentOS 7+). yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "https://rpm.nodesource.com/{{ nodejs_rhel_rpm_dir }}/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm" state: present when: ansible_distribution_major_version|int >= 7 @@ -45,9 +47,14 @@ # https://bugzilla.redhat.com/show_bug.cgi?id=1481470 - name: Ensure http-parser installed from RPM for {{ ansible_distribution_version }} yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: https://kojipkgs.fedoraproject.org//packages/http-parser/2.7.1/3.el7/x86_64/http-parser-2.7.1-3.el7.x86_64.rpm state: present when: ansible_distribution_version.split('.')[1] | int <= 3 - name: Ensure Node.js and npm are installed. - yum: "name=nodejs-{{ nodejs_version[0] }}.* state=present enablerepo='epel,nodesource'" + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "nodejs-{{ nodejs_version[0] }}.*" + state: present + enablerepo: 'epel,nodesource'" diff --git a/tests/deploys/setup-alt-source-backup.yml b/tests/deploys/setup-alt-source-backup.yml index 67e5a1880..8e2af268a 100644 --- a/tests/deploys/setup-alt-source-backup.yml +++ b/tests/deploys/setup-alt-source-backup.yml @@ -14,6 +14,7 @@ tasks: - name: Ensure packages installed yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "{{item}}" state: installed with_items: From 34e12f058c47ddda9d29fc4707f003396a05f064 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 4 Jun 2019 11:25:44 -0500 Subject: [PATCH 081/280] 31.8.0 release --- RELEASE-NOTES.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 16e58c58e..7584a1d3f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,32 @@ Release Notes ============= +## Meza 31.8.0 + +Standardize secret config permissions; dev-networking fix + +### Commits since 31.7.0 + +* 077e6bc Add lock_timeout to yum/package modules to fix Ansible 2.8 issue +* d6ddb60 Don't use 'meza' command for dev-networking +* 2ef8454 Secret directory 775 in getmeza.sh, too +* 8ba1353 file not directory +* ab1d9b1 Don't overwrite secrt files +* c64c41b Relax secret config _directory_ mode; ensure good ownership +* 968c675 Make meza-ansible own temp_vars.json +* dd245ae Sync secret perms + +### Contributors + +* 11 James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.8.0 +sudo meza deploy +``` + ## Meza 31.7.0 Simplify push-backup settings and make cron configurable and with notification From ef09660dc2b1e6519d7141a7594c71d547d914f7 Mon Sep 17 00:00:00 2001 From: GarrettGeorge Date: Wed, 5 Jun 2019 12:37:40 -0500 Subject: [PATCH 082/280] Cherry picks skip_saml CIDR notation support changes from 32.x --- src/roles/saml/templates/config.php.j2 | 2 + .../saml/templates/samlLocalSettings.php.j2 | 69 ++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/roles/saml/templates/config.php.j2 b/src/roles/saml/templates/config.php.j2 index a3654fb17..c03ca8c15 100644 --- a/src/roles/saml/templates/config.php.j2 +++ b/src/roles/saml/templates/config.php.j2 @@ -3,6 +3,8 @@ $config = array( 'baseurlpath' => 'https://{{ wiki_app_fqdn }}/simplesaml/', + 'application' => [ 'baseURL' => 'https://{{ wiki_app_fqdn }}' ], + 'certdir' => 'cert/', 'loggingdir' => 'log/', 'datadir' => 'data/', diff --git a/src/roles/saml/templates/samlLocalSettings.php.j2 b/src/roles/saml/templates/samlLocalSettings.php.j2 index f8d2de027..7ef8fe3ab 100644 --- a/src/roles/saml/templates/samlLocalSettings.php.j2 +++ b/src/roles/saml/templates/samlLocalSettings.php.j2 @@ -90,9 +90,72 @@ if ( isset( $_SERVER['HTTP_X_SKIP_SAML'] ) ) { return true; } - // user allowed from this IP address only -OR- this IP address is one of many allowed - elseif ( in_array( $ipaddr, $wgMezaAllowSkipSamlUsers[$username] ) ) { - return true; + $IPv4matchesCIDR = function( $ip, $cidr) { + $arr = explode( '/', $cidr ); + if ( count($arr) === 1 ) { + $subnet = $arr[0]; + $bits = 32; // Allows passing in CIDR or maskless IP + } + else { + list ( $subnet, $bits) = $arr; + } + $ip = ip2long( $ip ); + $subnet = ip2long( $subnet ); + $mask = -1 << (32 - $bits); + $subnet &= $mask; # nb: in case the supplied subnet wasn't correctly aligned + + return ( $ip & $mask ) == $subnet; + }; + + $IPv6matchesCIDR = function( $ip, $cidr) { + $inet = inet_pton( $ip ); + + // converts inet_pton output to string with bits + $unpacked = unpack( 'A16', $inet ); + $unpacked = str_split( $unpacked[1] ); + $binaryip = ''; + foreach ( $unpacked as $char ) { + $binaryip .= str_pad( decbin( ord( $char ) ), 8, '0', STR_PAD_LEFT ); + } + + $arr = explode( '/', $cidr ); + if ( count($arr) === 1 ) { + $subnet = $arr[0]; + $bits = 128; // Allows passing in CIDR or maskless IP + } + else { + list ( $subnet, $bits ) = $arr; + } + + // Convert subnet to binary string + $subnet = inet_pton( $subnet ); + $binarynet = inet_to_bits( $subnet ); + + $ip_net_bits = substr( $binaryip, 0, $bits ); + $net_bits = substr( $binarynet, 0, $bits ); + + return $ip_net_bits === $net_bits; + }; + + $addressesToCheck = $wgMezaAllowSkipSamlUsers[$username]; + + // Loop through all addresses, returning true on first where client's IP matches + // Otherwise, fall through to $forbidden() + foreach( $addressesToCheck as $check ) { + if ( + filter_var( $ipaddr, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4 ) && + $IPv4matchesCIDR( $ipaddr, $check ) + ) { + return true; + } + + elseif ( + defined( 'AF_INET6' ) && + filter_var( $ipaddr, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6 ) && + $IPv6matchesCIDR( $ipaddr, $check ) + ) { + return true; + } } } From f71fe080457ccd7bbd5fa89a5d50b94125ab49c2 Mon Sep 17 00:00:00 2001 From: Rich Evans <32852052+revansx@users.noreply.github.com> Date: Wed, 5 Jun 2019 14:04:14 -0400 Subject: [PATCH 083/280] Update MezaCoreExtensions.yml my first pull request --- config/core/MezaCoreExtensions.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index c06b9aa07..b638a9ade 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -283,7 +283,9 @@ list: array(0,50) ); - + - name: SemanticExtraSpecialProperties + composer: "mediawiki/semantic-extra-special-properties" + version: "dev-master" - name: PipeEscape repo: https://github.com/wikimedia/mediawiki-extensions-PipeEscape.git version: "{{ mediawiki_default_branch }}" From eb1cab668fbf8cf7831a8132b1bc76aec755bec9 Mon Sep 17 00:00:00 2001 From: Rich Evans <32852052+revansx@users.noreply.github.com> Date: Wed, 5 Jun 2019 16:15:59 -0400 Subject: [PATCH 084/280] Update MezaCoreExtensions.yml changed version from "dev-master" to "2.0.0" --- config/core/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index b638a9ade..316466524 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -285,7 +285,7 @@ list: - name: SemanticExtraSpecialProperties composer: "mediawiki/semantic-extra-special-properties" - version: "dev-master" + version: "2.0.0" - name: PipeEscape repo: https://github.com/wikimedia/mediawiki-extensions-PipeEscape.git version: "{{ mediawiki_default_branch }}" From 80177f45233c669b433bb457a3f5107ffa24479e Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 6 Jun 2019 07:54:48 -0500 Subject: [PATCH 085/280] push-backup: Fix bad use of 'notify' tag; Add servers to exclude 'notify' tag was mixed with 'never' tag, such that it seemed to make sense to do '--tags notify' to get it to notify. However, this would then skip over other things not tagged 'notify' like 'uploads' and 'db'. Instead you'd have to do '--tags db,uploads,anything-else-required'. Maybe '--tags all,notify' would work, but that's also confusing. Instead this commit removes the 'never' tag so notification will always happen. If a user wants to prevent notification they can do '--skip-tags notify' For some reason the list of servers to exclude on the push-backup playbook took the 32.x form with server groups like load-balancers-nonmeza. This missed load-balancers-unmanaged, the 31.x group naming. --- src/playbooks/push-backup.yml | 6 +----- src/roles/autodeployer/templates/meza-autodeployer-cron.j2 | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/playbooks/push-backup.yml b/src/playbooks/push-backup.yml index 6ab817a9c..91a5348d6 100644 --- a/src/playbooks/push-backup.yml +++ b/src/playbooks/push-backup.yml @@ -1,7 +1,7 @@ --- # Define a timestamp fact to persist throughout this playbook -- hosts: all:!exclude-all:!load-balancers-nonmeza:!load-balancers-nonmeza-external:!load-balancers-nonmeza-internal +- hosts: all:!exclude-all:!load-balancers-unmanaged:!load-balancers-nonmeza:!load-balancers-nonmeza-external:!load-balancers-nonmeza-internal tasks: - set_fact: backup_timestamp: "{{lookup('pipe','date +%Y%m%d%H%M%S')}}" @@ -11,8 +11,6 @@ roles: - set-vars tags: - # only run this if notify specified - - never - notify tasks: @@ -61,8 +59,6 @@ roles: - set-vars tags: - # only run this if notify specified - - never - notify tasks: diff --git a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 index ba0686997..f06d67cf2 100644 --- a/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 +++ b/src/roles/autodeployer/templates/meza-autodeployer-cron.j2 @@ -22,6 +22,6 @@ MAILTO=root # # Push backup (db and uploads) to another server periodically # -{{ push_backup.crontime }} root meza push-backup "{{ env }}" "--tags notify" >> {{ m_logs }}/deploy/push-backup-`date "+\%Y\%m\%d"`.log 2>&1 +{{ push_backup.crontime }} root meza push-backup "{{ env }}" >> {{ m_logs }}/deploy/push-backup-`date "+\%Y\%m\%d"`.log 2>&1 {% endif %} {% endif %} From 2803905f85e72601c09806768b729788c3a51b83 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 6 Jun 2019 12:09:04 -0500 Subject: [PATCH 086/280] Specify /opt/simplesamlphp owner/group/mode --- config/core/defaults.yml | 3 +++ src/roles/saml/tasks/main.yml | 14 ++++++++++++-- src/roles/verify-wiki/tasks/import-wiki-sql.yml | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 1ed6a6859..51a8af500 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -241,6 +241,9 @@ m_config_secret_dir_mode: "0775" m_config_secret_owner: meza-ansible m_config_secret_group: wheel +m_simplesamlphp_mode: "u=rwX,g=rwX,o=rX" +m_simplesamlphp_owner: meza-ansible +m_simplesamlphp_group: wheel # diff --git a/src/roles/saml/tasks/main.yml b/src/roles/saml/tasks/main.yml index e1192f0f8..b8fe0cd9d 100644 --- a/src/roles/saml/tasks/main.yml +++ b/src/roles/saml/tasks/main.yml @@ -28,6 +28,8 @@ - latest - name: Ensure simplesamlphp dependencies in place + become: yes + become_user: "meza-ansible" composer: command: install working_dir: "{{ m_simplesamlphp_path }}" @@ -55,10 +57,18 @@ - filename: "SAMLConfig.php" dest_path: "{{ m_deploy }}" +- name: Ensure good permissions on simplesamlphp directory and subs + file: + path: "{{ m_simplesamlphp_path }}" + owner: "{{ m_simplesamlphp_owner }}" + group: "{{ m_simplesamlphp_group }}" + mode: "{{ m_simplesamlphp_mode }}" + recurse: yes + - name: Ensure NonMediaWikiSimpleSamlAuth.php in place copy: src: files/NonMediaWikiSimpleSamlAuth.php dest: "{{ m_htdocs }}/NonMediaWikiSimpleSamlAuth.php" - owner: root - group: root + owner: meza-ansible + group: apache mode: 0755 diff --git a/src/roles/verify-wiki/tasks/import-wiki-sql.yml b/src/roles/verify-wiki/tasks/import-wiki-sql.yml index 5964da92a..b577d9ee1 100644 --- a/src/roles/verify-wiki/tasks/import-wiki-sql.yml +++ b/src/roles/verify-wiki/tasks/import-wiki-sql.yml @@ -12,6 +12,8 @@ delegate_to: "{{ groups['db-master'][0] }}" run_once: true +- debug: + var: imported_wiki_sql_on_db_master # # Set facts to clarify if overwriting data should actually occur From 8f14b8f9d6366d2c52b532711ee1014b3b21e850 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 6 Jun 2019 17:06:50 -0500 Subject: [PATCH 087/280] Recursively apply owner/perms to simplesaml and mediawiki --- config/core/defaults.yml | 7 +++++ src/roles/mediawiki/tasks/main.yml | 21 +++++++++------ src/roles/saml/tasks/main.yml | 41 +++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 51a8af500..36f2b69c0 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -200,6 +200,13 @@ m_data_owner: meza-ansible m_data_group: wheel +# +# Used for many things within htdocs that don't require apache to manipulate +# +m_htdocs_mode: "u=rwX,g=rwX,o=rX" +m_htdocs_owner: meza-ansible +m_htdocs_group: wheel + # uploads directory. Note: user meza-ansible is in group "apache" m_uploads_dir_mode: "0775" diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 2453dced1..86b4f8e29 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -63,8 +63,9 @@ - name: Ensure MediaWiki core owned by meza-ansible file: path: "{{ m_mediawiki }}" - owner: meza-ansible - group: wheel + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" state: directory recurse: yes tags: @@ -94,6 +95,7 @@ tags: - mediawiki-core + # # EXTENSIONS AND SKINS # @@ -196,8 +198,9 @@ template: src: Extensions.php.j2 dest: "{{ m_deploy }}/Extensions.php" - owner: meza-ansible - group: wheel + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" # Adds extensions with composer param from MezaCoreExtensions.yml, # MezaLocalExtensions.yml, MezaCoreSkins.yml, and MezaLocalSkins.yml @@ -205,8 +208,9 @@ template: src: composer.local.json.j2 dest: "{{ m_mediawiki }}/composer.local.json" - owner: meza-ansible - group: wheel + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" - name: Run composer install on MediaWiki for dependencies become: yes @@ -293,8 +297,9 @@ src: LocalSettings.php.j2 dest: "{{ m_mediawiki }}/LocalSettings.php" backup: yes - owner: meza-ansible - group: wheel + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" diff --git a/src/roles/saml/tasks/main.yml b/src/roles/saml/tasks/main.yml index b8fe0cd9d..227389fec 100644 --- a/src/roles/saml/tasks/main.yml +++ b/src/roles/saml/tasks/main.yml @@ -1,6 +1,35 @@ --- +- name: "Check if {{ m_simplesamlphp_path }} exists" + stat: + path: "{{ m_simplesamlphp_path }}" + register: simplesamlphp_exists + +- name: "Check if {{ m_simplesamlphp_path }} exists" + stat: + path: "{{ m_mediawiki }}/extensions/SimpleSamlAuth" + register: simplesamlextension_exists + +- name: Ensure good permissions on simplesamlphp directory and subs (if dir exists) + file: + path: "{{ m_simplesamlphp_path }}" + owner: "{{ m_simplesamlphp_owner }}" + group: "{{ m_simplesamlphp_group }}" + mode: "{{ m_simplesamlphp_mode }}" + recurse: yes + when: simplesamlphp_exists.stat.exists and simplesamlphp_exists.stat.isdir + +- name: Ensure good permissions on Extension:SimpleSamlAuth directory and subs (if dir exists) + file: + path: "{{ m_mediawiki }}/extensions/SimpleSamlAuth" + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" + recurse: yes + when: simplesamlphp_exists.stat.exists and simplesamlphp_exists.stat.isdir - name: Ensure SimpleSamlPhp (PHP SAML library) installed + become: yes + become_user: "meza-ansible" # Ref #1149 for TMPDIR environment var environment: TMPDIR: "{{ m_tmp }}" @@ -13,6 +42,8 @@ - latest - name: Ensure SimpleSamlAuth (MediaWiki extension) installed + become: yes + become_user: "meza-ansible" # Ref #1149 for TMPDIR environment var environment: TMPDIR: "{{ m_tmp }}" @@ -57,7 +88,7 @@ - filename: "SAMLConfig.php" dest_path: "{{ m_deploy }}" -- name: Ensure good permissions on simplesamlphp directory and subs +- name: Ensure _still_ good permissions on simplesamlphp directory and subs file: path: "{{ m_simplesamlphp_path }}" owner: "{{ m_simplesamlphp_owner }}" @@ -65,6 +96,14 @@ mode: "{{ m_simplesamlphp_mode }}" recurse: yes +- name: Ensure _still_ good permissions on Extension:SimpleSamlAuth directory and subs + file: + path: "{{ m_mediawiki }}/extensions/SimpleSamlAuth" + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" + recurse: yes + - name: Ensure NonMediaWikiSimpleSamlAuth.php in place copy: src: files/NonMediaWikiSimpleSamlAuth.php From 597a49fff1ab9dd60c710eb83017b6f6fc907d6a Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 6 Jun 2019 17:44:15 -0500 Subject: [PATCH 088/280] Ensure MediaWiki and WikiBlender ownership after all operations --- src/roles/mediawiki/tasks/main.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 86b4f8e29..7d4c4880f 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -301,6 +301,17 @@ owner: "{{ m_htdocs_owner }}" group: "{{ m_htdocs_group }}" +- name: Ensure MediaWiki still properly owned + file: + path: "{{ m_mediawiki }}" + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" + state: directory + recurse: yes + tags: + - mediawiki-core + - file-perms # @@ -320,7 +331,18 @@ template: src: BlenderSettings.php.j2 dest: "{{ m_htdocs }}/WikiBlender/BlenderSettings.php" + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" +- name: Ensure WikiBlender properly owned + file: + path: "{{ m_htdocs }}/WikiBlender" + mode: "{{ m_htdocs_mode }}" + owner: "{{ m_htdocs_owner }}" + group: "{{ m_htdocs_group }}" + state: directory + recurse: yes # From 01a171a7c70e7f14399b8ba4f4c7eff17369d124 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 6 Jun 2019 18:29:59 -0500 Subject: [PATCH 089/280] 31.8.1 release --- RELEASE-NOTES.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 7584a1d3f..a143e6ed5 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,28 @@ Release Notes ============= +## Meza 31.8.1 + +Fix permissions for finicky servers; fix bad use of 'notify' tag on 'meza push-backup' command + +### Commits since 31.8.0 + +* 597a49f Ensure MediaWiki and WikiBlender ownership after all operations +* 8f14b8f Recursively apply owner/perms to simplesaml and mediawiki +* 2803905 Specify /opt/simplesamlphp owner/group/mode +* 80177f4 push-backup: Fix bad use of 'notify' tag; Add servers to exclude + +### Contributors + +* 6 James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.8.1 +sudo meza deploy +``` + ## Meza 31.8.0 Standardize secret config permissions; dev-networking fix From 25b80130b2f219f8ae9693ab345980508f084693 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 11 Jun 2019 12:15:56 -0500 Subject: [PATCH 090/280] Make pushed backups in form *push.sql and use them first; more debug --- src/roles/backup-db-wikis-push/tasks/main.yml | 2 +- src/roles/verify-wiki/tasks/main.yml | 7 +-- .../tasks/transfer-backup-to-db-master.yml | 57 +++++++++++++++++-- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/roles/backup-db-wikis-push/tasks/main.yml b/src/roles/backup-db-wikis-push/tasks/main.yml index 0ba8eb6da..78986334b 100644 --- a/src/roles/backup-db-wikis-push/tasks/main.yml +++ b/src/roles/backup-db-wikis-push/tasks/main.yml @@ -35,7 +35,7 @@ pushing_from_path: "{{ m_tmp }}/{{ env }}_{{ item }}.sql" pushing_to_server: "{{ push_backup.db.addr }}" # remote_server_base_path + backup_timestamp + _wiki.sql, but replace with wiki_id (item) - pushing_to_path: "{{ remote_server_base_path | regex_replace('', item) }}{{ backup_timestamp }}_wiki.sql" + pushing_to_path: "{{ remote_server_base_path | regex_replace('', item) }}{{ backup_timestamp }}_wiki_push.sql" pushing_to_user: "{{ push_backup.remote_user }}" with_items: "{{ wiki_dirs.files | map(attribute='path') | map('basename') | list }}" diff --git a/src/roles/verify-wiki/tasks/main.yml b/src/roles/verify-wiki/tasks/main.yml index d8dbf0aa8..e4b04edc4 100644 --- a/src/roles/verify-wiki/tasks/main.yml +++ b/src/roles/verify-wiki/tasks/main.yml @@ -137,6 +137,7 @@ sql_backup_server: "{{ backups_server_db_dump.addr }}" do_sql_dump: True sql_file_match: "wiki_{{ wiki_id }}.sql" + sql_file_push_match: "wiki_{{ wiki_id }}.sql" db_backup_server_remote_user: "{{ backups_server_db_dump.remote_user }}" db_backup_server_mysql_user: "{{ backups_server_db_dump.mysql_user }}" sql_backup_server_set: True @@ -148,6 +149,7 @@ do_sql_dump: False sql_dir_path: "{{ backups_server_alt_source.sql_dir_path | regex_replace('', wiki_id) }}" sql_file_match: "{{ backups_server_alt_source.sql_file_match | default('*.sql') | regex_replace('', wiki_id) }}" + sql_file_push_match: "{{ backups_server_alt_source.sql_file_match | default('*push.sql') | regex_replace('', wiki_id) }}" db_backup_server_remote_user: "{{ backups_server_alt_source.remote_user }}" sql_backup_server_set: True when: "not sql_backup_server_set and backups_server_alt_source is defined" @@ -158,6 +160,7 @@ do_sql_dump: False sql_dir_path: "{{ m_backups }}/{{ env }}/{{ wiki_id }}" sql_file_match: "*.sql" + sql_file_push_match: "*push.sql" sql_backup_server_set: True when: "not sql_backup_server_set and 'backup-servers' in groups and groups['backup-servers']|length|int > 0" @@ -179,10 +182,6 @@ run_once: true delegate_to: "{{ groups['db-master'][0] }}" -- debug: { var: do_sql_dump } -- debug: { var: wiki_exists } -- debug: { var: intend_overwrite_from_backup } - - debug: msg: | do_sql_dump: {{ do_sql_dump | default('') }} diff --git a/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml b/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml index 235af22b5..6415fc21d 100644 --- a/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml +++ b/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml @@ -1,6 +1,15 @@ --- # Transfer SQL to DB master +- debug: + msg: | + wiki_id: {{ wiki_id | default('') }} + sql_dir_path: {{ sql_dir_path | default('') }} + sql_file_match: {{ sql_file_match | default('') }} + sql_file_push_match: {{ sql_file_push_match | default('') }} + sql_backup_server: {{ sql_backup_server | default('') }} + db_backup_server_remote_user: {{ db_backup_server_remote_user | default('') }} + # # Check for backups directory @@ -27,8 +36,8 @@ # This will find the latest sql file by name, or wiki.sql over any timestamped one # assuming timestamp-named files like 20170220000002_wiki.sql -- name: "{{ wiki_id }} - Find SQL file if it exists" - shell: 'find {{ sql_dir_path }} -maxdepth 1 -type f -iname "{{ sql_file_match }}" | sort -r | head -n +1' +- name: "{{ wiki_id }} - Find _PUSHED_ SQL file if it exists" + shell: 'find {{ sql_dir_path }} -maxdepth 1 -type f -iname "{{ sql_file_push_match }}" | sort -r | head -n +1' register: wiki_sql_file delegate_to: "{{ sql_backup_server }}" run_once: true @@ -36,16 +45,53 @@ failed_when: False when: backup_dir_exists and (not wiki_exists or intend_overwrite_from_backup) -- name: "{{ wiki_id }} - Set fact if SQL file DOES exist" +- name: "{{ wiki_id }} - Set fact if _PUSHED_ SQL file DOES exist" set_fact: sql_file_exists: True + sql_file_pushed_exists: True when: wiki_sql_file is defined and wiki_sql_file.rc is defined and wiki_sql_file.rc == 0 -- name: "{{ wiki_id }} - Set fact if SQL file DOES NOT exist" +- name: "{{ wiki_id }} - Set fact if _PUSHED_ SQL file DOES NOT exist" set_fact: sql_file_exists: False + sql_file_pushed_exists: False when: wiki_sql_file is not defined or wiki_sql_file.rc is not defined or wiki_sql_file.rc != 0 +- debug: + msg: | + sql file: {{ wiki_sql_file.stdout | default('') }} + +- name: "{{ wiki_id }} - If no pushed SQL file, find any SQL file if it exists" + shell: 'find {{ sql_dir_path }} -maxdepth 1 -type f -iname "{{ sql_file_match }}" | sort -r | head -n +1' + register: wiki_sql_file + delegate_to: "{{ sql_backup_server }}" + run_once: true + remote_user: "{{ db_backup_server_remote_user }}" + failed_when: False + when: + - not sql_file_pushed_exists + - backup_dir_exists + - (not wiki_exists or intend_overwrite_from_backup) + +- name: "{{ wiki_id }} - If no pushed SQL file, set fact if any SQL file DOES exist" + set_fact: + sql_file_exists: True + when: + - not sql_file_pushed_exists + - wiki_sql_file is defined + - wiki_sql_file.rc is defined + - wiki_sql_file.rc == 0 + +- name: "{{ wiki_id }} - If no pushed SQL file, set fact if any other SQL file DOES NOT exist" + set_fact: + sql_file_exists: False + when: + - not sql_file_pushed_exists + - wiki_sql_file is not defined or wiki_sql_file.rc is not defined or wiki_sql_file.rc != 0 + +- debug: + msg: | + sql file: {{ wiki_sql_file.stdout | default('') }} # # Do the rsync transfer of SQL file from backup server to DB master @@ -60,5 +106,4 @@ pulling_from_path: "{{ wiki_sql_file.stdout }}" pulling_from_user: "{{ db_backup_server_remote_user }}" run_once: true - when: - sql_file_exists + when: sql_file_exists From 93d5a0d565c3ebb397ca5d25b56df8ed9cd965bd Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 11 Jun 2019 13:11:26 -0500 Subject: [PATCH 091/280] Improve logic for how to grab SQL file from backup --- .../tasks/transfer-backup-to-db-master.yml | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml b/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml index 6415fc21d..95b927127 100644 --- a/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml +++ b/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml @@ -38,7 +38,7 @@ # assuming timestamp-named files like 20170220000002_wiki.sql - name: "{{ wiki_id }} - Find _PUSHED_ SQL file if it exists" shell: 'find {{ sql_dir_path }} -maxdepth 1 -type f -iname "{{ sql_file_push_match }}" | sort -r | head -n +1' - register: wiki_sql_file + register: wiki_sql_push_file delegate_to: "{{ sql_backup_server }}" run_once: true remote_user: "{{ db_backup_server_remote_user }}" @@ -47,19 +47,27 @@ - name: "{{ wiki_id }} - Set fact if _PUSHED_ SQL file DOES exist" set_fact: - sql_file_exists: True sql_file_pushed_exists: True - when: wiki_sql_file is defined and wiki_sql_file.rc is defined and wiki_sql_file.rc == 0 + when: + - wiki_sql_push_file is defined + - wiki_sql_push_file.rc is defined + - wiki_sql_push_file.rc == 0 + - wiki_sql_push_file.stdout != "" - name: "{{ wiki_id }} - Set fact if _PUSHED_ SQL file DOES NOT exist" set_fact: - sql_file_exists: False sql_file_pushed_exists: False - when: wiki_sql_file is not defined or wiki_sql_file.rc is not defined or wiki_sql_file.rc != 0 - -- debug: - msg: | - sql file: {{ wiki_sql_file.stdout | default('') }} + when: > + ( + wiki_sql_push_file is not defined + or wiki_sql_push_file.rc is not defined + or wiki_sql_push_file.rc != 0 + or wiki_sql_push_file.stdout == "" + ) + +- name: Debug var wiki_sql_push_file + debug: + var: wiki_sql_push_file - name: "{{ wiki_id }} - If no pushed SQL file, find any SQL file if it exists" shell: 'find {{ sql_dir_path }} -maxdepth 1 -type f -iname "{{ sql_file_match }}" | sort -r | head -n +1' @@ -81,17 +89,37 @@ - wiki_sql_file is defined - wiki_sql_file.rc is defined - wiki_sql_file.rc == 0 + - wiki_sql_file.stdout != "" - name: "{{ wiki_id }} - If no pushed SQL file, set fact if any other SQL file DOES NOT exist" set_fact: sql_file_exists: False when: - not sql_file_pushed_exists - - wiki_sql_file is not defined or wiki_sql_file.rc is not defined or wiki_sql_file.rc != 0 + - > + ( + wiki_sql_file is not defined + or wiki_sql_file.rc is not defined + or wiki_sql_file.rc != 0 + or wiki_sql_file.stdout == "" + ) + + +- name: Set path to pushed SQL file + set_fact: + wiki_sql_file_path: "{{ wiki_sql_push_file.stdout }}" + when: sql_file_pushed_exists + +- name: Set path to any SQL file + set_fact: + wiki_sql_file_path: "{{ wiki_sql_file.stdout }}" + when: + - not sql_file_pushed_exists + - sql_file_exists - debug: msg: | - sql file: {{ wiki_sql_file.stdout | default('') }} + sql file: {{ wiki_sql_file_path | default('') }} # # Do the rsync transfer of SQL file from backup server to DB master @@ -106,4 +134,6 @@ pulling_from_path: "{{ wiki_sql_file.stdout }}" pulling_from_user: "{{ db_backup_server_remote_user }}" run_once: true - when: sql_file_exists + when: + - wiki_sql_file_path is defined + - wiki_sql_file_path != "" From 94c79c190b35b74bf174d28da320e4499e4d44d3 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Tue, 11 Jun 2019 16:20:38 -0500 Subject: [PATCH 092/280] Fix bad variable --- src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml b/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml index 95b927127..12cef23d9 100644 --- a/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml +++ b/src/roles/verify-wiki/tasks/transfer-backup-to-db-master.yml @@ -131,7 +131,7 @@ pulling_to_server: "{{ groups['db-master'][0] }}" pulling_to_path: "{{ m_tmp }}/wiki.sql" pulling_from_server: "{{ sql_backup_server }}" - pulling_from_path: "{{ wiki_sql_file.stdout }}" + pulling_from_path: "{{ wiki_sql_file_path }}" pulling_from_user: "{{ db_backup_server_remote_user }}" run_once: true when: From ea7e7f66236a7138fdf9684cbc4b2fa123742a9f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 12 Jun 2019 12:11:36 -0500 Subject: [PATCH 093/280] Fix bad owner/group on /opt/data-meza --- src/roles/base/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index a24aff7f8..cc58a161a 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -232,8 +232,8 @@ file: path: "{{ m_meza_data }}" state: directory - owner: "{{ m_data_mode }}" - group: "{{ m_data_mode }}" + owner: "{{ m_data_owner }}" + group: "{{ m_data_group }}" mode: "{{ m_data_mode }}" - name: Ensure deploy directory in place From 8093118af32f1bb298ea7500e08bd4131696e55b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 12 Jun 2019 16:55:29 -0500 Subject: [PATCH 094/280] Make update.php write to a log --- src/roles/update.php/tasks/main.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/roles/update.php/tasks/main.yml b/src/roles/update.php/tasks/main.yml index dad56399b..b0e73e889 100644 --- a/src/roles/update.php/tasks/main.yml +++ b/src/roles/update.php/tasks/main.yml @@ -42,9 +42,17 @@ - sql-backup-cleanup when: do_cleanup_sql_backup +- name: Ensure update.php log exists + file: + path: "{{ m_logs }}/update.php" + state: directory + mode: "{{ m_logs_mode }}" + owner: "{{ m_logs_owner }}" + group: "{{ m_logs_group }}" + - name: Run update.php on this wiki shell: > - WIKI="{{ wiki_id }}" php "{{ m_mediawiki }}/maintenance/update.php" --quick + WIKI="{{ wiki_id }}" php "{{ m_mediawiki }}/maintenance/update.php" --quick >> {{ m_logs }}/update.php/{{ wiki_id }}.`date "+\%Y\%m\%d"`.log 2>&1 # run_once see [1] run_once: true tags: From fb66fa35c1d8c0f3b1957e111d2c64cbabc9bd7b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 17 Jun 2019 23:55:03 -0500 Subject: [PATCH 095/280] Allow specifying rules for what to gzip in backups-cleanup --- src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 b/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 index 260415359..3d0dc0a91 100644 --- a/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 +++ b/src/roles/backups-cleanup/templates/backups-cleanup.sh.j2 @@ -4,11 +4,17 @@ cd {{ m_backups }}/{{ env }} # remove all odd-numbered days (keep only even-numbered days) +{% if backups_cleanup.removal_rules is defined %} {% for removal_rule in backups_cleanup.removal_rules %} rm {{ removal_rule }} -f {% endfor %} +{% endif %} # compress all SQL files +{% if backups_cleanup.gzip_rules is defined %} echo "gzip start time: $(date +\"%T\")" -gzip -9 ./*/*.sql +{% for gzip_rule in backups_cleanup.gzip_rules %} +gzip -9 {{ gzip_rule }} +{% endfor %} echo "gzip end time: $(date +\"%T\")" +{% endif %} From ddb9fe28dd2b0ab625783aa8c9fa2bef4463d3bd Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 24 Jun 2019 12:35:07 -0500 Subject: [PATCH 096/280] Add refreshLinks script that handles memory leaks --- src/roles/mediawiki/tasks/main.yml | 2 + .../mediawiki/templates/refresh-links.sh.j2 | 77 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 src/roles/mediawiki/templates/refresh-links.sh.j2 diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 7d4c4880f..8cdc5bb67 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -468,6 +468,7 @@ - elastic-build-index.sh - elastic-rebuild-all.sh - smw-rebuild-all.sh + - refresh-links.sh - name: Ensure data rebuilding logs directories exist file: @@ -479,6 +480,7 @@ with_items: - smw-rebuilddata - search-index + - refresh-links # FIXME: Should search and SMW building be after update.php? diff --git a/src/roles/mediawiki/templates/refresh-links.sh.j2 b/src/roles/mediawiki/templates/refresh-links.sh.j2 new file mode 100644 index 000000000..5228193b0 --- /dev/null +++ b/src/roles/mediawiki/templates/refresh-links.sh.j2 @@ -0,0 +1,77 @@ +#!/bin/sh + +source "/opt/.deploy-meza/config.sh" + +if [ -z "$1" ]; then + do_wikis="*/" +else + do_wikis="$1" +fi + +wiki_dir="{{ m_htdocs }}/wikis" + +cd "$wiki_dir" +for d in $do_wikis; do + + if [ -z "$1" ]; then + wiki_id=${d%/} + else + wiki_id="$d" + fi + + if [ ! -d "$wiki_dir/$wiki_id" ]; then + echo "\"$wiki_id\" not a valid wiki ID" + continue + fi + + timestamp=$(date +"%F_%T") + out_log="{{ m_logs }}/refresh-links/$wiki_id.$timestamp.log" + + echo "Start refreshing links for \"$wiki_id\" at $timestamp" + echo " Output log:" + echo " $out_log" + + num_pages=$(WIKI="$wiki_id" php "{{ m_mediawiki }}/maintenance/showSiteStats.php" | grep "Total pages" | sed 's/[^0-9]*//g') + end_id=0 + delta=2000 + + echo "Beginning refreshLinks.php script for $wiki_id" + echo " Total pages = $num_pages" + echo " Doing it in $delta-page chunks to avoid memory leak" + + while [ "$end_id" -lt "$num_pages" ]; do + start_id=$(($end_id + 1)) + end_id=$(($end_id + $delta)) + echo "Running refreshLinks.php from $start_id to $end_id for $wiki_id" + WIKI="$wiki_id" php "{{ m_mediawiki }}/maintenance/refreshLinks.php" --e "$end_id" -- "$start_id" >> "$out_log" 2>&1 + + # If the above command had a failing exit code + if [[ $? -ne 0 ]]; then + endtimestamp=$(date +"%F_%T") + + # FIXME #577 #681: add notification/warning system here + echo "refreshLinks FAILED for \"$wiki_id\" at $endtimestamp" + fi + + done + + # Just in case there are more IDs beyond the guess we made with showSiteStats, run + # one more unbounded refreshLinks.php starting at the last ID previously done + start_id=$(($end_id + 1)) + echo "Running final refreshLinks.php for $wiki_id in case there are more pages beyond $num_pages" + WIKI="$wiki_id" php "{{ m_mediawiki }}/maintenance/refreshLinks.php" "$start_id" >> "$out_log" 2>&1 + + # If the above command had a failing exit code + if [[ $? -ne 0 ]]; then + endtimestamp=$(date +"%F_%T") + + # FIXME #577 #681: add notification/warning system here + echo "refreshLinks FAILED for \"$wiki_id\" at $endtimestamp" + + #if the above command had a passing exit code (e.g. zero) + else + endtimestamp=$(date +"%F_%T") + echo "refreshLinks completed for \"$wiki_id\" at $endtimestamp" + fi + +done From 8b96dc1edda1f3e4ac05914552a1c2178b5df90f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 26 Jun 2019 17:54:34 -0500 Subject: [PATCH 097/280] Add --no-dev to MW composer commands --- src/roles/mediawiki/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 8cdc5bb67..7b14f5323 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -218,7 +218,7 @@ composer: command: install working_dir: "{{ m_mediawiki }}" - no_dev: no + no_dev: yes # FIXME #317: need ignore_errors because composer throws an error when running as root. # failed_when: False tags: @@ -233,7 +233,7 @@ composer: command: update working_dir: "{{ m_mediawiki }}" - no_dev: no + no_dev: yes # FIXME #317: need ignore_errors because composer throws an error when running as root. # failed_when: False tags: From 2a49a3faf761ecf5f709c736bb8198cd0ccc4a7c Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 26 Jun 2019 19:28:46 -0500 Subject: [PATCH 098/280] Trying deleting composer.lock before composer operations --- src/roles/mediawiki/tasks/main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 7b14f5323..974b677ee 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -212,6 +212,17 @@ owner: "{{ m_htdocs_owner }}" group: "{{ m_htdocs_group }}" +- name: Remove composer.lock + become: yes + become_user: "meza-ansible" + file: + path: "{{ m_mediawiki }}/composer.lock" + state: absent + tags: + - composer-extensions + - latest + - skins + - name: Run composer install on MediaWiki for dependencies become: yes become_user: "meza-ansible" From b1a140eeba45a572094ee78d287db40777def2a1 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 08:19:23 -0500 Subject: [PATCH 099/280] Try envoking composer directly --- src/roles/mediawiki/tasks/main.yml | 60 +++++++++++++++++++----------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 974b677ee..5060fcd69 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -212,26 +212,47 @@ owner: "{{ m_htdocs_owner }}" group: "{{ m_htdocs_group }}" -- name: Remove composer.lock - become: yes - become_user: "meza-ansible" - file: - path: "{{ m_mediawiki }}/composer.lock" - state: absent - tags: - - composer-extensions - - latest - - skins +# - name: Remove composer.lock +# become: yes +# become_user: "meza-ansible" +# file: +# path: "{{ m_mediawiki }}/composer.lock" +# state: absent +# tags: +# - composer-extensions +# - latest +# - skins + +# - name: Run composer install on MediaWiki for dependencies +# become: yes +# become_user: "meza-ansible" +# composer: +# command: install +# working_dir: "{{ m_mediawiki }}" +# no_dev: yes +# tags: +# - composer-extensions +# - latest +# - skins + +# # install doesn't appear to do extensions +# - name: Run composer update on MediaWiki for extensions +# become: yes +# become_user: "meza-ansible" +# composer: +# command: update +# working_dir: "{{ m_mediawiki }}" +# no_dev: yes +# tags: +# - composer-extensions +# - latest +# - skins + - name: Run composer install on MediaWiki for dependencies become: yes become_user: "meza-ansible" - composer: - command: install - working_dir: "{{ m_mediawiki }}" - no_dev: yes - # FIXME #317: need ignore_errors because composer throws an error when running as root. - # failed_when: False + shell: "cd {{ m_mediawiki }} && composer install --no-dev" tags: - composer-extensions - latest @@ -241,12 +262,7 @@ - name: Run composer update on MediaWiki for extensions become: yes become_user: "meza-ansible" - composer: - command: update - working_dir: "{{ m_mediawiki }}" - no_dev: yes - # FIXME #317: need ignore_errors because composer throws an error when running as root. - # failed_when: False + composer: "cd {{ m_mediawiki }} && composer update --no-dev" tags: - composer-extensions - latest From dc7c184cfcdb3437ca5f236bd7f6f5a78a42d9e5 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 09:31:56 -0500 Subject: [PATCH 100/280] WIP --- src/roles/mediawiki/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 5060fcd69..5a540f068 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -262,7 +262,7 @@ - name: Run composer update on MediaWiki for extensions become: yes become_user: "meza-ansible" - composer: "cd {{ m_mediawiki }} && composer update --no-dev" + shell: "cd {{ m_mediawiki }} && composer update --no-dev" tags: - composer-extensions - latest From 0a9ca06534c6b6b5658c197e46c44fce0aaaa29e Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 11:22:57 -0500 Subject: [PATCH 101/280] Add error handling to update.php (show errors) --- src/roles/update.php/tasks/main.yml | 35 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/roles/update.php/tasks/main.yml b/src/roles/update.php/tasks/main.yml index b0e73e889..96172a24f 100644 --- a/src/roles/update.php/tasks/main.yml +++ b/src/roles/update.php/tasks/main.yml @@ -50,10 +50,35 @@ owner: "{{ m_logs_owner }}" group: "{{ m_logs_group }}" -- name: Run update.php on this wiki - shell: > - WIKI="{{ wiki_id }}" php "{{ m_mediawiki }}/maintenance/update.php" --quick >> {{ m_logs }}/update.php/{{ wiki_id }}.`date "+\%Y\%m\%d"`.log 2>&1 - # run_once see [1] - run_once: true +- set_fact: + update_php_log: "{{ m_logs }}/update.php/{{ wiki_id }}.{{ lookup('pipe','date +%Y%m%d%H%M%S') }}.log" + +- name: Try update.php + block: + + - name: Run update.php on this wiki + shell: > + WIKI="{{ wiki_id }}" php "{{ m_mediawiki }}/maintenance/update.php" --quick >> {{ update_php_log }} 2>&1 + # run_once see [1] + run_once: true + + rescue: + + - name: Get update.php log file + shell: "cat {{ update_php_log }}" + register: update_php_log_output + + - debug: + var: update_php_log_output + + - name: Get php error log + shell: "tail -n 50 {{ m_logs }}/php/php_errors.log" + register: php_error_log_output + + - debug: + var: php_error_log_output + + - name: Force failure after reporting + command: /bin/false tags: - update.php From c7c510eae0433f63198b2d5d93a9c00ebd71f7d7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 11:53:45 -0500 Subject: [PATCH 102/280] Mount local meza on docker tests controller containers --- tests/docker/init-controller.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/docker/init-controller.sh b/tests/docker/init-controller.sh index 6473fc2e5..f8c91a8c6 100644 --- a/tests/docker/init-controller.sh +++ b/tests/docker/init-controller.sh @@ -5,16 +5,16 @@ # Initiate container docker_repo="jamesmontalvo3/meza-docker-full:latest" -source "$m_meza_host/tests/docker/init-container.sh" "none" +source "$m_meza_host/tests/docker/init-container.sh" "${m_meza_host}" "mount" # Checkout the correct version of meza on the container # What's present on the pre-built container is not the latest. Need to pull # master in case the docker image doesn't have the correct git-setup.sh script # yet -${docker_exec[@]} bash -c "cd /opt/meza && git fetch origin && git reset --hard origin/master" -${docker_exec[@]} bash /opt/meza/tests/travis/git-setup.sh "$TRAVIS_EVENT_TYPE" \ - "$TRAVIS_COMMIT" "$TRAVIS_PULL_REQUEST_SHA" "$TRAVIS_BRANCH" "$TRAVIS_PULL_REQUEST_BRANCH" +# ${docker_exec[@]} bash -c "cd /opt/meza && git fetch origin && git reset --hard origin/master" +# ${docker_exec[@]} bash /opt/meza/tests/travis/git-setup.sh "$TRAVIS_EVENT_TYPE" \ +# "$TRAVIS_COMMIT" "$TRAVIS_PULL_REQUEST_SHA" "$TRAVIS_BRANCH" "$TRAVIS_PULL_REQUEST_BRANCH" # FIXME #728: Test band-aid. This is run in init-container.sh above, but at From deca0bb57eda045cc51fba7c278f6ca2c662dd8a Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 12:24:17 -0500 Subject: [PATCH 103/280] Use ansible composer module again --- src/roles/mediawiki/tasks/main.yml | 64 ++++++++++++------------------ 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 5a540f068..fcb574a41 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -212,24 +212,35 @@ owner: "{{ m_htdocs_owner }}" group: "{{ m_htdocs_group }}" -# - name: Remove composer.lock -# become: yes -# become_user: "meza-ansible" -# file: -# path: "{{ m_mediawiki }}/composer.lock" -# state: absent -# tags: -# - composer-extensions -# - latest -# - skins +- name: Run composer install on MediaWiki for dependencies + become: yes + become_user: "meza-ansible" + composer: + command: install + working_dir: "{{ m_mediawiki }}" + no_dev: yes + tags: + - composer-extensions + - latest + - skins + +# install doesn't appear to do extensions +- name: Run composer update on MediaWiki for extensions + become: yes + become_user: "meza-ansible" + composer: + command: update + working_dir: "{{ m_mediawiki }}" + no_dev: yes + tags: + - composer-extensions + - latest + - skins # - name: Run composer install on MediaWiki for dependencies # become: yes # become_user: "meza-ansible" -# composer: -# command: install -# working_dir: "{{ m_mediawiki }}" -# no_dev: yes +# shell: "cd {{ m_mediawiki }} && composer install --no-dev" # tags: # - composer-extensions # - latest @@ -239,35 +250,12 @@ # - name: Run composer update on MediaWiki for extensions # become: yes # become_user: "meza-ansible" -# composer: -# command: update -# working_dir: "{{ m_mediawiki }}" -# no_dev: yes +# shell: "cd {{ m_mediawiki }} && composer update --no-dev" # tags: # - composer-extensions # - latest # - skins - -- name: Run composer install on MediaWiki for dependencies - become: yes - become_user: "meza-ansible" - shell: "cd {{ m_mediawiki }} && composer install --no-dev" - tags: - - composer-extensions - - latest - - skins - -# install doesn't appear to do extensions -- name: Run composer update on MediaWiki for extensions - become: yes - become_user: "meza-ansible" - shell: "cd {{ m_mediawiki }} && composer update --no-dev" - tags: - - composer-extensions - - latest - - skins - - name: Ensure Git submodule requirements met for core meza extensions become: yes become_user: "meza-ansible" From 16833fd35477e7d10bcc5e23f8135242132e76f7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 13:32:16 -0500 Subject: [PATCH 104/280] Remove commented out non-ansible-module composer tasks --- src/roles/mediawiki/tasks/main.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index fcb574a41..69491ea5a 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -237,25 +237,6 @@ - latest - skins -# - name: Run composer install on MediaWiki for dependencies -# become: yes -# become_user: "meza-ansible" -# shell: "cd {{ m_mediawiki }} && composer install --no-dev" -# tags: -# - composer-extensions -# - latest -# - skins - -# # install doesn't appear to do extensions -# - name: Run composer update on MediaWiki for extensions -# become: yes -# become_user: "meza-ansible" -# shell: "cd {{ m_mediawiki }} && composer update --no-dev" -# tags: -# - composer-extensions -# - latest -# - skins - - name: Ensure Git submodule requirements met for core meza extensions become: yes become_user: "meza-ansible" From 6722cdcb0130b1290e5c624f28d0b3ee03faad09 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 15:13:55 -0500 Subject: [PATCH 105/280] Make meza-ansible:apache own deploy lock file, not root --- src/scripts/meza.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 570e5e736..bcfae69d9 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -242,6 +242,9 @@ def request_lock_for_deploy (env): with open( lock_file, 'w' ) as f: f.write( "{}\n{}".format(pid,timestamp) ) f.close() + meza_chown( lock_file, 'meza-ansible', 'apache' ) + os.chmod( lock_file, 0664 ) + return { "pid": pid, "timestamp": timestamp } def unlock_deploy(env): From 58744c8eecee41381846a72f0621f8f5764b6fdc Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 27 Jun 2019 16:34:11 -0500 Subject: [PATCH 106/280] 31.8.2 release --- RELEASE-NOTES.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index a143e6ed5..2a0fc89d8 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,40 @@ Release Notes ============= +## Meza 31.8.2 + +Multiple fixes in support of bad Composer issue, push-backups, etc + +### Commits since 31.8.1 + +* 6722cdc Make meza-ansible:apache own deploy lock file, not root +* 16833fd Remove commented out non-ansible-module composer tasks +* deca0bb Use ansible composer module again +* c7c510e Mount local meza on docker tests controller containers +* 0a9ca06 Add error handling to update.php (show errors) +* dc7c184 WIP +* b1a140e Try envoking composer directly +* 2a49a3f Trying deleting composer.lock before composer operations +* 8b96dc1 Add --no-dev to MW composer commands +* ddb9fe2 Add refreshLinks script that handles memory leaks +* fb66fa3 Allow specifying rules for what to gzip in backups-cleanup +* 8093118 Make update.php write to a log +* ea7e7f6 Fix bad owner/group on /opt/data-meza +* 94c79c1 Fix bad variable +* 93d5a0d Improve logic for how to grab SQL file from backup +* 25b8013 Make pushed backups in form *push.sql and use them first; more debug + +### Contributors + +* 24 James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.8.2 +sudo meza deploy +``` + ## Meza 31.8.1 Fix permissions for finicky servers; fix bad use of 'notify' tag on 'meza push-backup' command From a5af7c0925a081fd377e9cdde34c8d9dd8ea2e90 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 4 Jul 2019 00:34:17 -0500 Subject: [PATCH 107/280] Make Extension:MezaExt track 'master' while TransferPages in development --- config/core/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index 316466524..d73a297d8 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -179,7 +179,7 @@ list: version: "{{ mediawiki_default_branch }}" - name: MezaExt repo: https://github.com/enterprisemediawiki/MezaExt.git - version: tags/0.1.0 + version: "master" # Extension:PdfHandler (breaks on very large PDFs) # https://github.com/wikimedia/mediawiki-extensions-PdfHandler # // Location of PdfHandler dependencies From 2a8c3d94bff6b5443f0864936781f98266e72a62 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jul 2019 10:43:11 -0500 Subject: [PATCH 108/280] Bump MW to 1.31.2; remove pear/net_smtp from meza (part of MW now) --- config/core/defaults.yml | 2 +- src/roles/mediawiki/templates/composer.local.json.j2 | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 36f2b69c0..c356b5945 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -137,7 +137,7 @@ use_default_ssh_config: True # # Version of MediaWiki core -mediawiki_version: "1.31.1" +mediawiki_version: "1.31.2" # Branch to use on many extensions extensions and skins mediawiki_default_branch: "REL1_31" diff --git a/src/roles/mediawiki/templates/composer.local.json.j2 b/src/roles/mediawiki/templates/composer.local.json.j2 index 2f9be4c23..06caa3138 100644 --- a/src/roles/mediawiki/templates/composer.local.json.j2 +++ b/src/roles/mediawiki/templates/composer.local.json.j2 @@ -17,12 +17,10 @@ {%- endfor -%} {%- for ext in meza_core_extensions['list'] if ext.composer is defined %} - "{{ ext.composer }}": "{{ ext.version }}", - {# Note: pear/net_smtp is part of MW core in 1.32+. Need to add conditional comma here when pear/net_smtp is removed below. #} + "{{ ext.composer }}": "{{ ext.version }}" + {%- if not loop.last -%},{%- endif %} {%- endfor %} - "pear/net_smtp": "1.8.0" - }, "extra": { "merge-plugin": { From 2387b4a950382219a6c2306ff2ae67c855f5b136 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 8 Jul 2019 10:53:51 -0500 Subject: [PATCH 109/280] Check if apache group exists before setting as group for lock file; fixes #1216 --- src/scripts/meza.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index bcfae69d9..17ce98a2c 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -242,7 +242,16 @@ def request_lock_for_deploy (env): with open( lock_file, 'w' ) as f: f.write( "{}\n{}".format(pid,timestamp) ) f.close() - meza_chown( lock_file, 'meza-ansible', 'apache' ) + + import grp + + try: + grp.getgrnam('apache') + meza_chown( lock_file, 'meza-ansible', 'apache' ) + except KeyError: + print('Group apache does not exist. Set "wheel" as group for lock file.') + meza_chown( lock_file, 'meza-ansible', 'wheel' ) + os.chmod( lock_file, 0664 ) return { "pid": pid, "timestamp": timestamp } From 53b83525bf7c31960e27216b6ce79a0e58b1dbb3 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 7 Aug 2019 15:45:56 -0500 Subject: [PATCH 110/280] Bump SMW to 3.0.2 --- config/core/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index d73a297d8..7ac257384 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -3,7 +3,7 @@ list: - name: Semantic MediaWiki composer: "mediawiki/semantic-media-wiki" - version: "3.0.0" + version: "3.0.2" config: | // Enable Semantic MediaWiki semantics enableSemantics( $wikiId ); From 4628953e04c83b2c8d0f40826b43d7bb6dee6d02 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 7 Aug 2019 21:58:22 -0500 Subject: [PATCH 111/280] Allow specifying refspec in MW git repos --- src/roles/mediawiki/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 69491ea5a..1fbedaceb 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -80,6 +80,7 @@ repo: https://github.com/wikimedia/mediawiki.git dest: "{{ m_mediawiki }}" version: "{{ mediawiki_version }}" + refspec: "{{ item.refspec | default('') }}" track_submodules: no recursive: no umask: "0002" @@ -130,6 +131,7 @@ repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/extensions/{{ item.name }}" version: "{{ item.version }}" + refspec: "{{ item.refspec | default('') }}" umask: "0002" with_items: "{{ meza_core_extensions.list }}" when: meza_core_extensions.list[0] is defined and item.repo is defined @@ -153,6 +155,7 @@ repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/skins/{{ item.name }}" version: "{{ item.version }}" + refspec: "{{ item.refspec | default('') }}" with_items: "{{ meza_core_skins.list }}" when: meza_core_skins.list[0] is defined and item.repo is defined tags: @@ -169,6 +172,7 @@ repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/extensions/{{ item.name }}" version: "{{ item.version }}" + refspec: "{{ item.refspec | default('') }}" umask: "0002" key_file: "{{ item.key_file | default(None) }}" with_items: "{{ meza_local_extensions.list }}" @@ -186,6 +190,7 @@ repo: "{{ item.repo }}" dest: "{{ m_mediawiki }}/skins/{{ item.name }}" version: "{{ item.version }}" + refspec: "{{ item.refspec | default('') }}" with_items: "{{ meza_local_skins.list }}" when: meza_local_skins.list[0] is defined and item.repo is defined tags: From f655f6bd8790ef29c4c953fa3716d103ee172c6b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 8 Aug 2019 18:21:46 -0500 Subject: [PATCH 112/280] Allow specifying PHP max upload size in config --- config/core/defaults.yml | 4 ++++ src/roles/apache-php/templates/php.ini.j2 | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index c356b5945..71cceafc1 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -257,6 +257,10 @@ m_simplesamlphp_group: wheel # PHP config # +# PHP max file size. This gets applied to php.ini variables upload_max_filesize +# and post_max_size for convenience. +php_max_upload_filesize: "100M" + # The OPcache shared memory storage size php_opcache_memory_consumption: 256 diff --git a/src/roles/apache-php/templates/php.ini.j2 b/src/roles/apache-php/templates/php.ini.j2 index 75e4cf43f..fc36b10fc 100644 --- a/src/roles/apache-php/templates/php.ini.j2 +++ b/src/roles/apache-php/templates/php.ini.j2 @@ -660,7 +660,7 @@ auto_globals_jit = On ; Its value may be 0 to disable the limit. It is ignored if POST data reading ; is disabled through enable_post_data_reading. ; http://php.net/post-max-size -post_max_size = 100M +post_max_size = {{ php_max_upload_filesize }} ; Automatically add files before PHP document. ; http://php.net/auto-prepend-file @@ -788,7 +788,7 @@ file_uploads = On ; Maximum allowed size for uploaded files. ; http://php.net/upload-max-filesize -upload_max_filesize = 100M +upload_max_filesize = {{ php_max_upload_filesize }} ; Maximum number of files that can be uploaded via a single request max_file_uploads = 20 From c5f4d32643ef4798dd69f0845bc9f628fedae860 Mon Sep 17 00:00:00 2001 From: Vincent Brooks <38259624+V-Brooks@users.noreply.github.com> Date: Fri, 23 Aug 2019 11:23:20 -0500 Subject: [PATCH 113/280] Updating Data Transfer to fix import bug https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/DataTransfer/+/730cbfbcc7cc255724eaceb76a3abc6fcc9c93aa --- config/core/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index 7ac257384..c4d699d9f 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -353,7 +353,7 @@ list: - name: DataTransfer repo: https://github.com/wikimedia/mediawiki-extensions-DataTransfer.git - version: "{{ mediawiki_default_branch }}" + version: "730cbfbcc7cc255724eaceb76a3abc6fcc9c93aa" - name: PageImporter repo: https://github.com/enterprisemediawiki/PageImporter.git version: tags/0.1.0 From 1f130079f2b9fafe255be58f8814a9706be51d9a Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 29 Aug 2019 11:06:53 -0500 Subject: [PATCH 114/280] Add /opt/data-meza/mw-temp that apache can modify Apache cannot write to /opt/data-meza/tmp. For Special:TransferPages, it is necessary for Apache to write a temp file, which then is picked up by the MW job queue and executed as user meza-ansible. --- config/core/defaults.yml | 28 +++++++++++++++++++++++++++- src/roles/apache-php/tasks/main.yml | 8 ++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 71cceafc1..edad91863 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -188,6 +188,33 @@ clean_upload_stash_crontime: "0 18 * * *" # FILE MODES, OWNERS, GROUPS # +# +# PATH SPECIFICATIONS: PUT ALL PATH INFO UNDER m_paths +# ---------------------------------------------------- +# +# Use Capital X to give dirs "execute" (i.e. allow entry into dir) but leave +# files' execute bit untouched. +# +# Examples: +# "u=rwX,g=rX,o=" --> 750 for dirs, files possibly between 640 and 751 +# "u=rwX,g=rwX,o=rX" --> 775 for dirs, files possibly between 664 and 775 +# +# Ref: https://www.g-loaded.eu/2005/11/08/the-use-of-the-uppercase-x-in-chmod/ +# +m_paths: + mw_temp: + path: "{{ m_meza_data }}/mw-temp" + mode: "u=rwX,g=rwX,o=rX" + owner: apache + group: apache + # recurse: TBD if we want this within m_paths + + +# +# LEGACY PATH SPECIFICATIONS +# -------------------------- +# + m_meza_owner: meza-ansible m_meza_group: wheel # Don't set mode for /opt/meza for now. Don't want to impact execute bit which @@ -224,7 +251,6 @@ m_cache_directory_mode: "0770" m_cache_directory_owner: apache m_cache_directory_group: apache - m_logs_mode: "0755" m_logs_owner: meza-ansible m_logs_group: wheel diff --git a/src/roles/apache-php/tasks/main.yml b/src/roles/apache-php/tasks/main.yml index 4f4929478..ae5094096 100644 --- a/src/roles/apache-php/tasks/main.yml +++ b/src/roles/apache-php/tasks/main.yml @@ -17,6 +17,14 @@ group: apache mode: 0775 +- name: Ensure mw-temp directory configured + file: + state: directory + path: "{{ m_paths.mw_temp.path }}" + owner: "{{ m_paths.mw_temp.owner }}" + group: "{{ m_paths.mw_temp.group }}" + mode: "{{ m_paths.mw_temp.mode }}" + - name: Ensure user meza-ansible and alt-meza-ansible in group "apache" user: name: "{{ item }}" From a5551143acd0d6cd2832a26eebb90acfe8e64a0a Mon Sep 17 00:00:00 2001 From: XP1 Date: Mon, 23 Sep 2019 00:46:39 -0400 Subject: [PATCH 115/280] Add function to wait for Internet connection. Vagrant or getmeza script needs to check Internet connectivity of guest before proceeding #1231: https://github.com/enterprisemediawiki/meza/issues/1231 --- src/scripts/getmeza.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index f2ebd64b9..eef504fe7 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -9,6 +9,25 @@ if [ "$(whoami)" != "root" ]; then exit 1 fi +checkInternetConnection() { + declare -i pingRetries=100 + declare -i sleepDuration=3 + declare -i minutes=$(($pingRetries * $sleepDuration / 60)) + + while [[ $pingRetries -gt 0 ]] && ! ping -c 1 -W 1 mirrorlist.centos.org >/dev/null 2>&1; do + echo "Could not connect to mirrorlist.centos.org. Internet connection might be down. Retrying (#$pingRetries) in $sleepDuration seconds..." + ((pingRetries -= 1)) + sleep $sleepDuration + done + + if [[ ! $pingRetries -gt 0 ]]; then + echo "Meza has been trying to install but hasn't found an internet connection for $minutes minutes. Verify internet connectivity and try again." + exit 1 + fi +} + +checkInternetConnection + # If you don't do this in a restrictive system (umask 077), it becomes # difficult to manage all permissions, AND you constantly have to fix all git # clones and checkouts. From d1b17a9a040654bb9a9179534b580134300b2599 Mon Sep 17 00:00:00 2001 From: Vincent Brooks <38259624+V-Brooks@users.noreply.github.com> Date: Wed, 30 Oct 2019 16:27:51 -0500 Subject: [PATCH 116/280] =?UTF-8?q?Add=20a=20calendar=20interface=20to=20a?= =?UTF-8?q?dd=20and=20edit=20events=20using=20the=20Full=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/core/MezaCoreExtensions.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/core/MezaCoreExtensions.yml b/config/core/MezaCoreExtensions.yml index c4d699d9f..56c07b663 100644 --- a/config/core/MezaCoreExtensions.yml +++ b/config/core/MezaCoreExtensions.yml @@ -94,8 +94,7 @@ list: version: "{{ mediawiki_default_branch }}" - name: PageForms repo: https://github.com/wikimedia/mediawiki-extensions-PageForms.git - # commit includes spreadsheet sorting which didn't make PageForms 4.4.1 - version: "730390a31a56c001af83948af1eefc5174abbe06" + version: "bb1beb015f0a16db10d96c895dfadf9eb691834d" - name: DismissableSiteNotice repo: https://github.com/wikimedia/mediawiki-extensions-DismissableSiteNotice.git version: "{{ mediawiki_default_branch }}" From e1ced43f7ff2881018145e7b79d6857c2952dc35 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 30 Oct 2019 17:22:03 -0500 Subject: [PATCH 117/280] Possible solution to issue #1237 according to this forum: https://www.centos.org/forums/viewtopic.php?f=47&t=52162&p=220915#p220915 --- tests/docker/init-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/init-container.sh b/tests/docker/init-container.sh index 8f02b608a..9615ac0b1 100644 --- a/tests/docker/init-container.sh +++ b/tests/docker/init-container.sh @@ -83,7 +83,7 @@ docker_exec=( docker exec --tty "$container_id" env TERM=xterm ) # 17.something), whereas Travis is on 1.12.something. ${docker_exec[@]} yum -y install firewalld ${docker_exec[@]} systemctl start firewalld -${docker_exec[@]} firewall-cmd --permanent --zone=public --change-interface=docker0 +${docker_exec[@]} firewall-offline-cmd --permanent --zone=public --change-interface=docker0 if [ "$is_minion" == "no" ]; then From 892fb1f7dbe63491a5cd70992183c01d776c11cd Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 30 Oct 2019 17:29:38 -0500 Subject: [PATCH 118/280] With firewall-offline-cmd no --permanent option, prob because if offline none could be temp --- tests/docker/init-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/init-container.sh b/tests/docker/init-container.sh index 9615ac0b1..e906c8c6e 100644 --- a/tests/docker/init-container.sh +++ b/tests/docker/init-container.sh @@ -83,7 +83,7 @@ docker_exec=( docker exec --tty "$container_id" env TERM=xterm ) # 17.something), whereas Travis is on 1.12.something. ${docker_exec[@]} yum -y install firewalld ${docker_exec[@]} systemctl start firewalld -${docker_exec[@]} firewall-offline-cmd --permanent --zone=public --change-interface=docker0 +${docker_exec[@]} firewall-offline-cmd --zone=public --change-interface=docker0 if [ "$is_minion" == "no" ]; then From f9cc3a674a3c345c22db03f1a83a6a2d4898f124 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 30 Oct 2019 17:48:00 -0500 Subject: [PATCH 119/280] Travis saying Docker not running; perhaps start firewall after offline cmd --- tests/docker/init-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/init-container.sh b/tests/docker/init-container.sh index e906c8c6e..030644baf 100644 --- a/tests/docker/init-container.sh +++ b/tests/docker/init-container.sh @@ -82,8 +82,8 @@ docker_exec=( docker exec --tty "$container_id" env TERM=xterm ) # or Ubuntu 14.04 host. Only tested on new version of Docker (docker-ce version # 17.something), whereas Travis is on 1.12.something. ${docker_exec[@]} yum -y install firewalld -${docker_exec[@]} systemctl start firewalld ${docker_exec[@]} firewall-offline-cmd --zone=public --change-interface=docker0 +${docker_exec[@]} systemctl start firewalld if [ "$is_minion" == "no" ]; then From 74a76974cf54bad72f9d2b4c6c3b6026e926447b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 30 Oct 2019 18:01:49 -0500 Subject: [PATCH 120/280] Try manually restarting firewalld --- src/roles/base/tasks/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index cc58a161a..12c4d38ce 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -162,9 +162,14 @@ state: permissive # log actions that would be blocked if state=enforcing - name: ensure firewalld is running (and enable it at boot) - service: name=firewalld state=started enabled=yes + service: + name: firewalld + state: started + enabled: yes when: docker_skip_tasks is not defined or not docker_skip_tasks +- name: Try manually starting firewalld + shell: systemctl restart firewalld # # SSH config From 93b6087c627266f3036bd5d64553bb53bb4f31c5 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 30 Oct 2019 18:58:22 -0500 Subject: [PATCH 121/280] Disable firewalld on travis tests --- src/roles/base/tasks/main.yml | 7 +++--- src/roles/firewalld/handlers/main.yml | 4 ++- src/roles/firewalld/tasks/main.yml | 12 +++++++-- src/roles/haproxy/tasks/main.yml | 25 ++++++++++++++----- src/roles/nodejs/tasks/setup-RedHat.yml | 2 +- src/roles/saml/tasks/main.yml | 1 - tests/deploys/backup-to-remote.controller.sh | 3 +++ .../deploys/import-from-remote.controller.sh | 3 +++ .../monolith-from-scratch.controller.sh | 3 +++ tests/docker/import-from-alt-remote.setup.sh | 1 - tests/docker/import-from-remote.setup.sh | 3 ++- 11 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index 12c4d38ce..1c7852fa4 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -166,10 +166,9 @@ name: firewalld state: started enabled: yes - when: docker_skip_tasks is not defined or not docker_skip_tasks - -- name: Try manually starting firewalld - shell: systemctl restart firewalld + when: + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks # # SSH config diff --git a/src/roles/firewalld/handlers/main.yml b/src/roles/firewalld/handlers/main.yml index 57897c854..1cb5568c2 100644 --- a/src/roles/firewalld/handlers/main.yml +++ b/src/roles/firewalld/handlers/main.yml @@ -1,4 +1,6 @@ --- - name: restart firewalld service: name=firewalld state=restarted - when: docker_skip_tasks is not defined or not docker_skip_tasks + when: + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks diff --git a/src/roles/firewalld/tasks/main.yml b/src/roles/firewalld/tasks/main.yml index e4819dd5d..d0e9e5f08 100644 --- a/src/roles/firewalld/tasks/main.yml +++ b/src/roles/firewalld/tasks/main.yml @@ -62,7 +62,11 @@ zone: "{{firewalld_zone|default('public')}}" # strip "localhost" or inventory_hostname from list of servers to configure with_items: "{{ firewalld_servers | difference([ 'localhost', inventory_hostname ]) }}" - when: firewalld_service is defined and (docker_skip_tasks is not defined or not docker_skip_tasks) + when: + - firewalld_service is defined + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks + - name: set firewalld allow port {{ firewalld_port }} for list of servers firewalld: @@ -73,4 +77,8 @@ zone: "{{firewalld_zone|default('public')}}" # strip "localhost" or inventory_hostname from list of servers to configure with_items: "{{ firewalld_servers | difference([ 'localhost', inventory_hostname ]) }}" - when: firewalld_port is defined and firewalld_protocol is defined and (docker_skip_tasks is not defined or not docker_skip_tasks) + when: + - firewalld_port is defined + - firewalld_protocol is defined + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 71007c452..df03199b9 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -213,7 +213,9 @@ with_items: - 80 - 443 - when: docker_skip_tasks is not defined or not docker_skip_tasks + when: + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks - name: Ensure firewalld port 1936 OPEN when haproxy stats ENABLED firewalld: @@ -222,7 +224,10 @@ immediate: true state: enabled zone: "{{m_public_networking_zone|default('public')}}" - when: enable_haproxy_stats and (docker_skip_tasks is not defined or not docker_skip_tasks) + when: + - enable_haproxy_stats + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks - name: Ensure firewalld port 1936 CLOSED when haproxy stats DISABLED firewalld: @@ -231,7 +236,10 @@ immediate: true state: disabled zone: "{{m_public_networking_zone|default('public')}}" - when: not enable_haproxy_stats and (docker_skip_tasks is not defined or not docker_skip_tasks) + when: + - not enable_haproxy_stats + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks - name: Ensure firewalld port 8088 OPEN when PHP profiling ENABLED firewalld: @@ -240,7 +248,10 @@ immediate: true state: enabled zone: "{{m_public_networking_zone|default('public')}}" - when: m_setup_php_profiling and (docker_skip_tasks is not defined or not docker_skip_tasks) + when: + - m_setup_php_profiling + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks - name: Ensure firewalld port 8088 CLOSED when PHP profiling DISABLED firewalld: @@ -249,8 +260,10 @@ immediate: true state: disabled zone: "{{m_public_networking_zone|default('public')}}" - when: not m_setup_php_profiling and (docker_skip_tasks is not defined or not docker_skip_tasks) - + when: + - not m_setup_php_profiling + - docker_skip_tasks is not defined or not docker_skip_tasks + - firewall_skip_tasks is not defined or not firewall_skip_tasks # FIXME #747: haproxy will need to handle reverse proxy for Elasticsearch plugins # - name: Configure firewalld for Elasticsearch reverse proxy diff --git a/src/roles/nodejs/tasks/setup-RedHat.yml b/src/roles/nodejs/tasks/setup-RedHat.yml index 8f5fb24cc..cfbad5cdd 100644 --- a/src/roles/nodejs/tasks/setup-RedHat.yml +++ b/src/roles/nodejs/tasks/setup-RedHat.yml @@ -57,4 +57,4 @@ lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: "nodejs-{{ nodejs_version[0] }}.*" state: present - enablerepo: 'epel,nodesource'" + enablerepo: 'epel,nodesource' diff --git a/src/roles/saml/tasks/main.yml b/src/roles/saml/tasks/main.yml index 227389fec..42f43b188 100644 --- a/src/roles/saml/tasks/main.yml +++ b/src/roles/saml/tasks/main.yml @@ -70,7 +70,6 @@ src: "{{ item.filename }}.j2" dest: "{{ item.dest_path }}/{{ item.filename }}" with_items: - # Config files for SimpleSamlPhp (PHP SAML library) - filename: "config.php" dest_path: "{{ m_simplesamlphp_path }}/config" diff --git a/tests/deploys/backup-to-remote.controller.sh b/tests/deploys/backup-to-remote.controller.sh index 67c8d8c74..b75e2b7b0 100644 --- a/tests/deploys/backup-to-remote.controller.sh +++ b/tests/deploys/backup-to-remote.controller.sh @@ -9,6 +9,9 @@ set -eux echo "RUNNING TEST" +# Skip firewalld tasks since they broke in Travis (Issue #1237) +echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml' + # Now that environment is setup, deploy/install it meza deploy "$1" diff --git a/tests/deploys/import-from-remote.controller.sh b/tests/deploys/import-from-remote.controller.sh index 0fe84f46f..647b4b53c 100644 --- a/tests/deploys/import-from-remote.controller.sh +++ b/tests/deploys/import-from-remote.controller.sh @@ -9,6 +9,9 @@ set -eux echo "RUNNING TEST" +# Skip firewalld tasks since they broke in Travis (Issue #1237) +echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml' + # Deploy environment with test config meza deploy "$1" diff --git a/tests/deploys/monolith-from-scratch.controller.sh b/tests/deploys/monolith-from-scratch.controller.sh index 7e3c5fbd2..916cd446f 100644 --- a/tests/deploys/monolith-from-scratch.controller.sh +++ b/tests/deploys/monolith-from-scratch.controller.sh @@ -19,6 +19,9 @@ meza setup env monolith --fqdn="${fqdn}" --db_pass=1234 --private_net_zone=publi echo "print hosts file" cat /opt/conf-meza/secret/monolith/hosts +# Skip firewalld tasks since they broke in Travis (Issue #1237) +echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml' + # Now that environment monolith is setup, deploy/install it meza deploy monolith diff --git a/tests/docker/import-from-alt-remote.setup.sh b/tests/docker/import-from-alt-remote.setup.sh index a5a5f2406..16dfa773d 100644 --- a/tests/docker/import-from-alt-remote.setup.sh +++ b/tests/docker/import-from-alt-remote.setup.sh @@ -103,7 +103,6 @@ ${docker_exec_1[@]} bash -c "ansible-vault encrypt $secret_yml --vault-password- ${docker_exec_1[@]} cat "$hosts_file" ${docker_exec_1[@]} cat "$secret_yml" - # garbage data into database and file uploads, just to check that the changes # get copied to CONTAINER 1 ${docker_exec_2[@]} mysql -u root -p1234 wiki_top -e"INSERT INTO watchlist (wl_user, wl_namespace, wl_title) VALUES (10000,0,'FAKE PAGE');" diff --git a/tests/docker/import-from-remote.setup.sh b/tests/docker/import-from-remote.setup.sh index c9a469fc6..fb746eca2 100644 --- a/tests/docker/import-from-remote.setup.sh +++ b/tests/docker/import-from-remote.setup.sh @@ -52,7 +52,8 @@ ${docker_exec_1[@]} sed -r -i "s/INSERT_FQDN/$docker_ip_1/g;" \ # LocalSettings.php) ${docker_exec_1[@]} bash -c "echo -e 'allow_image_tags: True\n' >> '/opt/conf-meza/secret/$env_name/secret.yml'" - +# Skip firewalld tasks since they broke in Travis (Issue #1237) +${docker_exec_1[@]} bash -c "echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml'" # CONTAINER 2: get backup files ${docker_exec_2[@]} git clone \ From 121de9df8dea2647e2e1359b72e057d1f7409777 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Thu, 31 Oct 2019 11:57:32 -0500 Subject: [PATCH 122/280] Add --no-firewall deploy option --- src/scripts/meza.py | 13 +++++++++++-- tests/deploys/backup-to-remote.controller.sh | 5 +---- tests/deploys/import-from-remote.controller.sh | 5 +---- tests/deploys/monolith-from-import.controller.sh | 2 +- tests/deploys/monolith-from-scratch.controller.sh | 5 +---- tests/docker/import-from-alt-remote.setup.sh | 4 ++-- tests/docker/import-from-remote.setup.sh | 3 --- 7 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 17ce98a2c..926664910 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -98,7 +98,7 @@ def meza_command_deploy (argv): else: sys.exit(rc) - more_extra_vars = False + more_extra_vars = {} # strip environment off of it argv = argv[1:] @@ -108,7 +108,16 @@ def meza_command_deploy (argv): # remove -o and --overwrite from args; argv = [value for value in argv[:] if value not in ["-o", "--overwrite"]] - more_extra_vars = { 'force_overwrite_from_backup': True } + more_extra_vars['force_overwrite_from_backup'] = True + + if (len( set(argv).intersection({"--no-firewall"}) )) > 0: + # remove --no-firewall from args: + argv = [value for value in argv[:] if value not in ["--no-firewall"]] + + more_extra_vars['firewall_skip_tasks'] = True + + if len(more_extra_vars) == 0: + more_extra_vars = False # This breaks continuous integration. FIXME to get it back. # THIS WAS WRITTEN WHEN `meza` WAS A BASH SCRIPT diff --git a/tests/deploys/backup-to-remote.controller.sh b/tests/deploys/backup-to-remote.controller.sh index b75e2b7b0..70889cd11 100644 --- a/tests/deploys/backup-to-remote.controller.sh +++ b/tests/deploys/backup-to-remote.controller.sh @@ -9,11 +9,8 @@ set -eux echo "RUNNING TEST" -# Skip firewalld tasks since they broke in Travis (Issue #1237) -echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml' - # Now that environment is setup, deploy/install it -meza deploy "$1" +meza deploy "$1" --no-firewall # Need to sleep 10 seconds to let Parsoid finish loading sleep 10s diff --git a/tests/deploys/import-from-remote.controller.sh b/tests/deploys/import-from-remote.controller.sh index 647b4b53c..85d42153d 100644 --- a/tests/deploys/import-from-remote.controller.sh +++ b/tests/deploys/import-from-remote.controller.sh @@ -9,11 +9,8 @@ set -eux echo "RUNNING TEST" -# Skip firewalld tasks since they broke in Travis (Issue #1237) -echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml' - # Deploy environment with test config -meza deploy "$1" +meza deploy "$1" --no-firewall # Need to wait after install before checking that Parsoid is working sleep 10s diff --git a/tests/deploys/monolith-from-import.controller.sh b/tests/deploys/monolith-from-import.controller.sh index 9126d1050..816e3935a 100644 --- a/tests/deploys/monolith-from-import.controller.sh +++ b/tests/deploys/monolith-from-import.controller.sh @@ -24,7 +24,7 @@ sed -r -i "s/INSERT_FQDN/$fqdn/g;" "/opt/conf-meza/secret/$env_name/secret.yml" git clone https://github.com/jamesmontalvo3/meza-test-backups.git "/opt/data-meza/backups/$env_name" # Deploy environment with test config -meza deploy "$env_name" +meza deploy "$env_name" --no-firewall # Need to wait after install before checking that Parsoid is working sleep 10s diff --git a/tests/deploys/monolith-from-scratch.controller.sh b/tests/deploys/monolith-from-scratch.controller.sh index 916cd446f..bc1a6abc6 100644 --- a/tests/deploys/monolith-from-scratch.controller.sh +++ b/tests/deploys/monolith-from-scratch.controller.sh @@ -19,11 +19,8 @@ meza setup env monolith --fqdn="${fqdn}" --db_pass=1234 --private_net_zone=publi echo "print hosts file" cat /opt/conf-meza/secret/monolith/hosts -# Skip firewalld tasks since they broke in Travis (Issue #1237) -echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml' - # Now that environment monolith is setup, deploy/install it -meza deploy monolith +meza deploy monolith --no-firewall # Need to sleep 10 seconds to let Parsoid finish loading sleep 10s diff --git a/tests/docker/import-from-alt-remote.setup.sh b/tests/docker/import-from-alt-remote.setup.sh index 16dfa773d..c66a775da 100644 --- a/tests/docker/import-from-alt-remote.setup.sh +++ b/tests/docker/import-from-alt-remote.setup.sh @@ -111,7 +111,7 @@ ${docker_exec_2[@]} bash -c "echo 'fake data' > /opt/alt/backups/top/uploads/fak # # Re-deploy without --overwrite # -${docker_exec_1[@]} meza deploy "$env_name" --tags "mediawiki" --skip-tags "latest" -vvv +${docker_exec_1[@]} meza deploy "$env_name" --tags "mediawiki" --skip-tags "latest" -vvv --no-firewall # @@ -136,7 +136,7 @@ ${docker_exec_1[@]} cat /opt/data-meza/uploads/top/fake.png \ # # Re-deploy with --overwrite # -${docker_exec_1[@]} meza deploy "$env_name" --overwrite --tags "mediawiki" --skip-tags "latest" +${docker_exec_1[@]} meza deploy "$env_name" --overwrite --tags "mediawiki" --skip-tags "latest" --no-firewall # diff --git a/tests/docker/import-from-remote.setup.sh b/tests/docker/import-from-remote.setup.sh index fb746eca2..949c42ec4 100644 --- a/tests/docker/import-from-remote.setup.sh +++ b/tests/docker/import-from-remote.setup.sh @@ -52,9 +52,6 @@ ${docker_exec_1[@]} sed -r -i "s/INSERT_FQDN/$docker_ip_1/g;" \ # LocalSettings.php) ${docker_exec_1[@]} bash -c "echo -e 'allow_image_tags: True\n' >> '/opt/conf-meza/secret/$env_name/secret.yml'" -# Skip firewalld tasks since they broke in Travis (Issue #1237) -${docker_exec_1[@]} bash -c "echo -e '\nfirewall_skip_tasks: True\n' >> '/opt/conf-meza/public/public.yml'" - # CONTAINER 2: get backup files ${docker_exec_2[@]} git clone \ https://github.com/jamesmontalvo3/meza-test-backups.git \ From 9dde2d874f08caeeea0a93fa796a075d948611bd Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sat, 2 Nov 2019 17:36:07 -0500 Subject: [PATCH 123/280] Try Parsoid 0.10.0 --- config/core/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index edad91863..fce39e14d 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -146,7 +146,7 @@ mediawiki_default_branch: "REL1_31" php_ius_version: "php71u" # Parsoid version -m_parsoid_version: "tags/v0.9.0" +m_parsoid_version: "tags/v0.10.0" # MediaWiki 1.27 and earlier require ElasticSearch 1.6 # MediaWiki 1.28 and higher require ElasticSearch 2.x From fe78845829c052208f28a8527fe26fc0f9c5f59d Mon Sep 17 00:00:00 2001 From: Daren Welsh Date: Sat, 23 Nov 2019 14:45:26 -0600 Subject: [PATCH 124/280] fix deprecated ansible syntax, ref c06fa04c7ddd0ea99c6e92b6f87ff89fd5be27a4 --- src/roles/database/tasks/replication.yml | 76 +++++++++++++----------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/roles/database/tasks/replication.yml b/src/roles/database/tasks/replication.yml index 673cb3b6d..5b111d6b9 100644 --- a/src/roles/database/tasks/replication.yml +++ b/src/roles/database/tasks/replication.yml @@ -2,7 +2,6 @@ # # The following conditionals are reused many times in the tasks in this file. # As such, they've been encapsulated in these tasks and registered to vars. -# FIXME #812: The logic "not role_is_valid_slave|skipped" is awkward - name: Check if valid slave command: /bin/true register: role_is_valid_slave @@ -10,6 +9,10 @@ (mysql_replication_role == 'slave') and mysql_replication_user and (mysql_replication_master != '') +- set_fact: + role_is_valid_slave: false + when: role_is_valid_slave is skipped + - name: Check if valid master command: /bin/true register: role_is_valid_master @@ -18,7 +21,10 @@ and mysql_replication_user and (mysql_replication_master != '') - +- set_fact: + role_is_valid_master: false + when: role_is_valid_master is skipped + # # Make sure MySQL user for replication exists on master # @@ -29,8 +35,7 @@ password: "{{ mysql_replication_user.password }}" priv: "{{ mysql_replication_user.priv | default('*.*:REPLICATION SLAVE') }}" state: present - when: > - not role_is_valid_master|skipped + when: role_is_valid_master # # Get slave replication status @@ -39,8 +44,7 @@ mysql_replication: mode=getslave failed_when: False register: slave - when: > - not role_is_valid_slave|skipped + when: role_is_valid_slave - debug: { var: slave } # @@ -52,7 +56,7 @@ when: > ( (slave.Is_Slave is defined and not slave.Is_Slave) - or (slave.Is_Slave is not defined and slave|failed) + or (slave.Is_Slave is not defined and slave is failed) or (slave.Slave_IO_Running is defined and slave.Slave_SQL_Running == 'No') or (mysql_force_slave_configuration is defined and mysql_force_slave_configuration) ) @@ -66,9 +70,9 @@ mysql_replication: mode=getmaster delegate_to: "{{ mysql_replication_master }}" register: master - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave - debug: { var: master } @@ -82,19 +86,19 @@ command: mysql -NBe "SELECT DISTINCT(TABLE_SCHEMA) FROM information_schema.tables WHERE TABLE_SCHEMA NOT IN ('information_schema', 'performance_schema', 'mysql')" register: mysql_content_databases delegate_to: "{{ mysql_replication_master }}" - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave - name: export dump file on master shell: "mysqldump --databases {{ mysql_content_databases.stdout }} | gzip > {{ m_tmp }}/mysqldump-onmaster.sql.gz" delegate_to: "{{ mysql_replication_master }}" - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped - - + when: + - slave_needs_configuration + - role_is_valid_slave + + # # Copying SQL files from one server to another *appears* to be somewhat # complicated by the fact that the control server could *be* either the master @@ -108,16 +112,16 @@ dest: "{{ m_tmp }}/mysqldump-oncontrol.sql.gz" flat: yes delegate_to: "{{ mysql_replication_master }}" - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave - name: put dump file copy: src: "{{ m_tmp }}/mysqldump-oncontrol.sql.gz" dest: "{{ m_tmp }}/mysqldump-onslave.sql.gz" - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave # # Import SQL file from above @@ -127,18 +131,18 @@ state: import name: all target: "{{ m_tmp }}/mysqldump-onslave.sql.gz" - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave # # Stop slave, configure, start slave. # - mysql_replication: mode: stopslave - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave - name: Configure replication on the slave. mysql_replication: mode: changemaster @@ -148,11 +152,11 @@ master_log_file: "{{ master.File }}" master_log_pos: "{{ master.Position }}" failed_when: False - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave - name: Start replication. mysql_replication: mode=startslave - when: > - not slave_needs_configuration|skipped - and not role_is_valid_slave|skipped + when: + - slave_needs_configuration + - role_is_valid_slave From 63716a8e59f54df5cb77f0d223b1cdaef3346f70 Mon Sep 17 00:00:00 2001 From: Daren Welsh Date: Sat, 23 Nov 2019 15:07:15 -0600 Subject: [PATCH 125/280] fix more flag syntax --- src/roles/remote-dir-check/tasks/main.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/roles/remote-dir-check/tasks/main.yml b/src/roles/remote-dir-check/tasks/main.yml index 63f408a54..9a4135974 100644 --- a/src/roles/remote-dir-check/tasks/main.yml +++ b/src/roles/remote-dir-check/tasks/main.yml @@ -46,10 +46,18 @@ - debug: { var: remote_dir_check } - debug: { var: local_dir_check } +- set_fact: + local_dir_check: False + when: local_dir_check is skipped + + - set_fact: + remote_dir_check: False + when: remote_dir_check is skipped + - name: "Set remote_dir_exists to true" set_fact: remote_dir_exists: True - when: (not local_dir_check|skipped and local_dir_check.rc == 0) or (not remote_dir_check|skipped and remote_dir_check.rc == 0) + when: (local_dir_check and local_dir_check.rc == 0) or (remote_dir_check and remote_dir_check.rc == 0) - debug: { var: remote_dir_exists } From 4d18daa173f118c5327bea843d40742e8787ecc0 Mon Sep 17 00:00:00 2001 From: Daren Welsh Date: Sat, 23 Nov 2019 15:13:11 -0600 Subject: [PATCH 126/280] yaml syntax fix --- src/roles/remote-dir-check/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/remote-dir-check/tasks/main.yml b/src/roles/remote-dir-check/tasks/main.yml index 9a4135974..3a86a1f33 100644 --- a/src/roles/remote-dir-check/tasks/main.yml +++ b/src/roles/remote-dir-check/tasks/main.yml @@ -50,7 +50,7 @@ local_dir_check: False when: local_dir_check is skipped - - set_fact: +- set_fact: remote_dir_check: False when: remote_dir_check is skipped From 8812f213bdcc3de2e43bdfbad59781edccf9fcb7 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 4 Dec 2019 16:19:10 -0600 Subject: [PATCH 127/280] 31.9.0 release --- RELEASE-NOTES.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 2a0fc89d8..293f9b2a3 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,39 @@ Release Notes ============= +## Meza 31.9.0 + +Ansible 2.9 fix; Travis firewall fix + +### Commits since 31.8.4 + +* 4d18daa yaml syntax fix +* 63716a8 fix more flag syntax +* fe78845 fix deprecated ansible syntax, ref c06fa04c7ddd0ea99c6e92b6f87ff89fd5be27a4 +* 9dde2d8 Try Parsoid 0.10.0 +* 121de9d Add --no-firewall deploy option +* 93b6087 Disable firewalld on travis tests +* 74a7697 Try manually restarting firewalld +* f9cc3a6 Travis saying Docker not running; perhaps start firewall after offline cmd +* 892fb1f With firewall-offline-cmd no --permanent option, prob because if offline none could be temp +* e1ced43 Possible solution to issue #1237 according to this forum: +* d1b17a9 Add a calendar interface to add and edit events using the Full… +* a555114 Add function to wait for Internet connection. + +### Contributors + +* 11 James Montalvo +* 4 Daren Welsh +* 1 Vincent Brooks +* 1 XP1 + +# How to upgrade + +```bash +sudo meza update 31.9.0 +sudo meza deploy +``` + ## Meza 31.8.2 Multiple fixes in support of bad Composer issue, push-backups, etc From 44f2d2c2342bd463ad5eeb83996aab38f8a6a5c3 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 10 Jan 2020 14:56:06 -0500 Subject: [PATCH 128/280] Python 2/3 compatibility items is available in both Python 2 and Python 3 whereas iteritems was only available for the Python 2.x interpreter This change makes Meza compatible with both --- .../templates/config.php.j2 | 18 +++++++++--------- .../base-config-scripts/templates/config.sh.j2 | 8 ++++---- .../saml/templates/samlLocalSettings.php.j2 | 2 +- src/scripts/meza.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/roles/base-config-scripts/templates/config.php.j2 b/src/roles/base-config-scripts/templates/config.php.j2 index 3f2721ad0..970996fec 100644 --- a/src/roles/base-config-scripts/templates/config.php.j2 +++ b/src/roles/base-config-scripts/templates/config.php.j2 @@ -74,7 +74,7 @@ $backups_environment = '{{ backups_environment }}'; {% if wiki_backup_downloaders is defined %} # Users allowed to download specific wikis $wiki_backup_downloaders = array(); -{% for wiki, users in wiki_backup_downloaders.iteritems() %} +{% for wiki, users in wiki_backup_downloaders.items() %} $wiki_backup_downloaders['{{ wiki }}'] = array( {% for user in users %} '{{ user }}', @@ -105,7 +105,7 @@ $all_backup_downloaders = array( # CUSTOM __public__ DEPLOY VARIABLES # These should only come from public.yml # -{% for key, value in public_deploy_vars.iteritems() %} +{% for key, value in public_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -117,7 +117,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -155,7 +155,7 @@ $all_backup_downloaders = array( # CUSTOM __{{ env }} environment__ DEPLOY VARIABLES # These should only come from env/{{ env }}.yml # -{% for key, value in env_deploy_vars.iteritems() %} +{% for key, value in env_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -167,7 +167,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -204,7 +204,7 @@ $all_backup_downloaders = array( # CUSTOM __secret__ DEPLOY VARIABLES # These should only come from secret.yml # -{% for key, value in secret_deploy_vars.iteritems() %} +{% for key, value in secret_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -216,7 +216,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -256,7 +256,7 @@ $all_backup_downloaders = array( # FIXME: remove deploy_vars in lieu of secret_deploy_vars when production wikis # are updated. # -{% for key, value in deploy_vars.iteritems() %} +{% for key, value in deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -268,7 +268,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; diff --git a/src/roles/base-config-scripts/templates/config.sh.j2 b/src/roles/base-config-scripts/templates/config.sh.j2 index 4fe3e2e77..ef8f2aff8 100644 --- a/src/roles/base-config-scripts/templates/config.sh.j2 +++ b/src/roles/base-config-scripts/templates/config.sh.j2 @@ -63,7 +63,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __public__ DEPLOY VARIABLES # These should only come from public.yml # -{% for key, value in public_deploy_vars.iteritems() %} +{% for key, value in public_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -80,7 +80,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __{{ env }} environment__ DEPLOY VARIABLES # These should only come from env/{{ env }}.yml # -{% for key, value in env_deploy_vars.iteritems() %} +{% for key, value in env_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -97,7 +97,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __secret__ DEPLOY VARIABLES # These should only come from secret.yml # -{% for key, value in secret_deploy_vars.iteritems() %} +{% for key, value in secret_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -116,7 +116,7 @@ m_i18n="{{ m_i18n }}" # FIXME: remove deploy_vars in lieu of secret_deploy_vars when production wikis # are updated. # -{% for key, value in deploy_vars.iteritems() %} +{% for key, value in deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} diff --git a/src/roles/saml/templates/samlLocalSettings.php.j2 b/src/roles/saml/templates/samlLocalSettings.php.j2 index 7ef8fe3ab..41659455f 100644 --- a/src/roles/saml/templates/samlLocalSettings.php.j2 +++ b/src/roles/saml/templates/samlLocalSettings.php.j2 @@ -37,7 +37,7 @@ if ( isset( $_SERVER['HTTP_X_SKIP_SAML'] ) ) { // FIXME #822: The indenting below will be heinous when Ansible does its templating {% if allow_skip_saml_users is defined -%} $wgMezaAllowSkipSamlUsers = array(); - {% for user, ipaddrs in allow_skip_saml_users.iteritems() -%} + {% for user, ipaddrs in allow_skip_saml_users.items() -%} $wgMezaAllowSkipSamlUsers['{{ user }}'] = array( {%- for ipaddr in ipaddrs -%}'{{ ipaddr }}',{%- endfor -%} ); diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 926664910..d527c2ff1 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -951,7 +951,7 @@ def playbook_cmd ( playbook, env=False, more_extra_vars=False ): extra_vars = {} if more_extra_vars: - for varname, value in more_extra_vars.iteritems(): + for varname, value in more_extra_vars.items(): extra_vars[varname] = value if len(extra_vars) > 0: From febc0411277200534534769ed8bfd507a6d500b4 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 13 Jan 2020 18:01:04 -0500 Subject: [PATCH 129/280] fix SSH under Debian/Ubuntu --- src/roles/base/templates/sshd_config.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/base/templates/sshd_config.j2 b/src/roles/base/templates/sshd_config.j2 index a2ac12d5a..f23cd45e9 100644 --- a/src/roles/base/templates/sshd_config.j2 +++ b/src/roles/base/templates/sshd_config.j2 @@ -143,7 +143,7 @@ AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE AcceptEnv XMODIFIERS # override default of no subsystems -Subsystem sftp /usr/libexec/openssh/sftp-server +Subsystem sftp /usr/lib/openssh/sftp-server # Example of overriding settings on a per-user basis #Match User anoncvs From 86a9b481c75b6052b34ed052da746bee6c17c3ce Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 13 Jan 2020 18:38:37 -0500 Subject: [PATCH 130/280] rewrites for pretty URLs --- src/roles/htdocs/templates/.htaccess.j2 | 11 +++--- src/roles/mediawiki/tasks/main.yml | 12 +++++++ .../mediawiki/templates/LocalSettings.php.j2 | 36 ++++++++++++------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index fcebf0db0..11ec4bc2d 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -30,10 +30,11 @@ RewriteRule ^BackupDownload(?:/|$)(.*)$ - [L] {% endif %} - # Taken from MediaWiki.org [[Extension:Simple Farm]] - # - # Redirect virtual wiki path to physical wiki path. There - # can be no wiki accessible using this path. - RewriteRule ^(?!mediawiki(?:/|$))[^/]+(?:/(.*))?$ mediawiki/$1 + # If the request is not for a valid directory, file, link + RewriteCond %{REQUEST_FILENAME} !-d + RewriteCond %{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_FILENAME} !-l + RewriteRule ^/?([^/]+)/(.*)?$ %{DOCUMENT_ROOT}/mediawiki/ [env=WIKI:$1,L] + diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index a7eaea45e..9ecd91690 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -91,6 +91,18 @@ tags: - mediawiki-core +# create symlink to core for short urls +# tag it latest, so that the symlink gets created whenever core is downloaded +- name: Create symlink to core, to enable short urls + file: + src: "{{ m_mediawiki }}" + dest: "{{ m_htdocs }}/{{ item }}" + state: link + with_items: + - "{{ list_of_wikis }}" + tags: + - latest + # # EXTENSIONS AND SKINS # diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index e52e5f7d3..6a885d52e 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -43,20 +43,29 @@ require "{{ m_deploy }}/samlLocalSettings.php"; require '/opt/.deploy-meza/config.php'; -if( $wgCommandLineMode ) { +/* + * Fix ENV vars getting prepended with 'REDIRECT_' by Apache + */ +function fixApacheEnv () { + foreach ($_ENV as $key => $value) { + if (substr($key, 0, 9) === 'REDIRECT_') { + $_ENV[str_replace('REDIRECT_', '', $key)] = $value; + putenv(str_replace('REDIRECT_', '', $key) . '=' . $value); + } + } +} +fixApacheEnv(); - $mezaWikiEnvVarName='WIKI'; +$mezaWikiEnvVarName='WIKI'; +if ( $wgCommandLineMode ) { // get $wikiId from environment variable $wikiId = getenv( $mezaWikiEnvVarName ); -} -else { - +} else { // get $wikiId from URI $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); - $wikiId = strtolower( $uriParts[1] ); // URI has leading slash, so $uriParts[0] is empty string - + $wikiId = strtolower( $uriParts[1] ); } @@ -90,11 +99,13 @@ if ( isset( $wikiIdRedirects[ $wikiId ] ) ) { $wikis = array_slice( scandir( "$m_htdocs/wikis" ), 2 ); -if ( ! in_array( $wikiId, $wikis ) ) { - - // handle invalid wiki - die( "No sir, I ain't heard'a no wiki that goes by the name \"$wikiId\"\n" ); - +if ( ! in_array( $wikiId, $wikis ) && empty ( $wikiId ) ) { + // get $wikiId from environment variable + $wikiId = getenv( $mezaWikiEnvVarName ); + if (!$wikiId) { + // handle invalid wiki + die( "No sir, I ain't heard'a no wiki that goes by the name \"$wikiId\"\n" ); + } } {% if meza_auth_type is defined %} @@ -258,6 +269,7 @@ $wgServer = 'https://{{ wiki_app_fqdn }}'; // https://www.mediawiki.org/wiki/Manual:$wgScriptPath $wgScriptPath = "/$wikiId"; +$wgArticlePath = "/$wikiId/$1"; // https://www.mediawiki.org/wiki/Manual:$wgUploadPath $wgUploadPath = "$wgScriptPath/img_auth.php"; From 78cc2e59d3c99a43ce99112d0d00033714a42857 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 14 Jan 2020 12:35:07 +0000 Subject: [PATCH 131/280] fix: add missing colon --- src/roles/htdocs/templates/robots.txt.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/htdocs/templates/robots.txt.j2 b/src/roles/htdocs/templates/robots.txt.j2 index 613944545..3218e2371 100644 --- a/src/roles/htdocs/templates/robots.txt.j2 +++ b/src/roles/htdocs/templates/robots.txt.j2 @@ -16,7 +16,7 @@ {% if m_robots_rules == 'nocrawl' %} User-agent: * -Disallow / +Disallow: / {% else %} From 2e19806bc5c86b6bbf22cf44f5ecafbfacaeec85 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 14 Jan 2020 07:45:27 -0500 Subject: [PATCH 132/280] Add compression (mod deflate) --- src/roles/apache-php/templates/httpd.conf.j2 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/roles/apache-php/templates/httpd.conf.j2 b/src/roles/apache-php/templates/httpd.conf.j2 index 4cc8b1bc4..278b54a38 100644 --- a/src/roles/apache-php/templates/httpd.conf.j2 +++ b/src/roles/apache-php/templates/httpd.conf.j2 @@ -210,6 +210,19 @@ ErrorLog ${APACHE_LOG_DIR}/error.log # LogLevel warn + +## Add compression +AddOutputFilterByType DEFLATE text/plain +AddOutputFilterByType DEFLATE text/html +AddOutputFilterByType DEFLATE text/xml +AddOutputFilterByType DEFLATE text/css +AddOutputFilterByType DEFLATE application/xml +AddOutputFilterByType DEFLATE application/xhtml+xml +AddOutputFilterByType DEFLATE application/rss+xml +AddOutputFilterByType DEFLATE application/javascript +AddOutputFilterByType DEFLATE application/x-javascript + + # Logging format: From 29638809762af9fbb701e7ba2dbbab1f2793e232 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 30 Mar 2020 13:14:54 -0400 Subject: [PATCH 133/280] ugrade php to 7.2 and elasticsearch to 6.x --- config/defaults.yml | 12 ++++++------ src/roles/elasticsearch/tasks/main.yml | 2 +- .../elasticsearch/templates/elasticsearch.repo.j2 | 13 +++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 src/roles/elasticsearch/templates/elasticsearch.repo.j2 diff --git a/config/defaults.yml b/config/defaults.yml index f956b11df..e0a0cfb93 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -151,23 +151,23 @@ m_meza_sudeoers_file: "/etc/sudoers.d/meza-ansible" # # Version of MediaWiki core -mediawiki_version: "REL1_32" +mediawiki_version: "REL1_34" # Branch to use on many extensions extensions and skins -mediawiki_default_branch: "REL1_32" +mediawiki_default_branch: "REL1_34" # PHP version -php_ius_version: "php71u" -php_debian_version: "7.1" +php_ius_version: "php72u" +php_debian_version: "7.2" # Parsoid version # e6b708 is commit on 16-OCT-2018, the day the REL1_32 branch was made -m_parsoid_version: "e6b708b3d7ff50a314d021295dce874574aa3e9e" +m_parsoid_version: "master" # MediaWiki 1.27 and earlier require ElasticSearch 1.6 # MediaWiki 1.28 and higher require ElasticSearch 2.x # MediaWiki 1.29 and higher require ElasticSearch 5.x -elasticsearch_major_version: "5.x" +elasticsearch_major_version: "6.x" # Whether or not to install Lua/luasandbox m_install_lua: true diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index 3f7e18249..df889cd34 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -30,7 +30,7 @@ when: ansible_os_family == 'RedHat' - name: Add Elasticsearch repository. template: - src: elasticsearch5.repo.j2 + src: elasticsearch.repo.j2 dest: /etc/yum.repos.d/elasticsearch.repo mode: 0644 when: ansible_os_family == 'RedHat' diff --git a/src/roles/elasticsearch/templates/elasticsearch.repo.j2 b/src/roles/elasticsearch/templates/elasticsearch.repo.j2 new file mode 100644 index 000000000..bff9f52ac --- /dev/null +++ b/src/roles/elasticsearch/templates/elasticsearch.repo.j2 @@ -0,0 +1,13 @@ +[elasticsearch] +name=Elasticsearch repository +baseurl=https://artifacts.elastic.co/packages/{{elasticsearch_major_version}}/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=1 + +{% if elasticsearch_proxy_host is defined and elasticsearch_proxy_port is defined %} +proxy=http://{{ elasticsearch_proxy_host }}:{{elasticsearch_proxy_port}} +{% endif %} + +autorefresh=1 +type=rpm-md From 313c3501e3ddfd94fb1c15a3044d91521315eb33 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Apr 2020 19:49:11 -0400 Subject: [PATCH 134/280] Upgrade elasticsearch to 6.x Change JAVA_HOME to the JDK included with Elasticsearch Get rid of all the comments in the config template Remove old extraneous Elasticsearch tasks Note: the upgrade.yml still looks valid even for v 6 --- src/roles/elasticsearch/tasks/main.yml | 26 +- .../templates/elasticsearch.yml.j2 | 382 +----------------- .../elasticsearch/templates/java_home.sh.j2 | 2 +- 3 files changed, 6 insertions(+), 404 deletions(-) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index df889cd34..2f89c77e6 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -166,33 +166,11 @@ do_elasticsearch_upgrade: True when: (es_version_found|int != es_version_desired|int) or (force_do_elasticsearch_upgrade is defined and force_do_elasticsearch_upgrade) -# -# Upgrade Elasticsearch if required -# -- name: Temporarily set Elasticsearch yum repo to point to 2.x - template: - src: elasticsearch2.repo.j2 - dest: /etc/yum.repos.d/elasticsearch.repo - mode: 0644 - when: do_elasticsearch_upgrade and es_version_found|int == 1 - -- name: Include upgrade tasks for ES 1.6 --> 2.x - include: es_upgrade.yml - when: do_elasticsearch_upgrade and es_version_found|int == 1 -- name: Do reindex if switching for ES 1.6 --> 2.x - include: es_reindex.yml - when: do_elasticsearch_upgrade and es_version_found|int == 1 - -- name: Reset Elasticsearch yum repo to point to 5.x +- name: Set Elasticsearch yum repo to point to 6.x template: - src: elasticsearch5.repo.j2 + src: elasticsearch.repo.j2 dest: /etc/yum.repos.d/elasticsearch.repo mode: 0644 - when: do_elasticsearch_upgrade and es_version_found|int == 1 -# Uncomment this when doing upgrade to 5.x -- name: Include upgrade tasks for ES 2.x --> 5.x - include: es_upgrade.yml - when: do_elasticsearch_upgrade diff --git a/src/roles/elasticsearch/templates/elasticsearch.yml.j2 b/src/roles/elasticsearch/templates/elasticsearch.yml.j2 index a4db72733..b5bd3d389 100644 --- a/src/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/src/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,25 +1,4 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - +##################### Elasticsearch Configuration ##################### # For information on supported formats and syntax for the config file, see # @@ -39,375 +18,20 @@ cluster.name: "{{ elasticsearch_cluster_name }}" # node.name: "{{ elasticsearch_node_name }}" -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -#node.master: true -# -# Allow this node to store data (enabled by default): -# -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. #################################### Paths #################################### # Path to directory containing configuration (this file and logging.yml): -# -path.conf: /etc/elasticsearch +# deprecated +# path.conf: /etc/elasticsearch # Path to directory where to store index data allocated for this node. # path.data: {{ m_meza_data }}/elasticsearch/data -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -# Appears to have been removed in Elasticsearch 0.90 -# path.work: {{ m_meza_data }}/elasticsearch/work -# Added 5.x: Required for script.inline and/or script.stored (formerly script.indexed) -path.scripts: {{ m_meza_data }}/elasticsearch/scripts # Path to log files: # path.logs: {{ m_meza_data }}/elasticsearch/log -# Path to where plugins are installed: -# -# Removed in Elasticsearch 2.x or 5.x -# path.plugins: {{ m_meza_data }}/elasticsearch/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -#network.bind_host: 192.168.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -#network.publish_host: 192.168.0.1 - -# Set both 'bind_host' and 'publish_host': -# -#network.host: 192.168.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. This should be set to a quorum/majority of -# the master-eligible nodes in the cluster. -# -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -#discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s - -################################## Security ################################ - -# Uncomment if you want to enable JSONP as a valid return transport on the -# http server. With this enabled, it may pose a security risk, so disabling -# it unless you need it is recommended (it is disabled by default). -# -#http.jsonp.enable: true - - -{% if elasticsearch_major_version == "1.6" %} -# FIXME #611: Remove this option after upgrading beyond MW 1.27 -# Enable dynamic scripting per https://www.mediawiki.org/wiki/Thread:Extension_talk:CirrusSearch/error_using_ElasticSearch -script.disable_dynamic: false -{% else %} -# Enable dynamic scripting per: -# MW docs saying dynamic scripting required -# https://www.mediawiki.org/wiki/Thread:Extension_talk:CirrusSearch/error_using_ElasticSearch -# -# Elasticsearch docs for v2.4 -# https://www.elastic.co/guide/en/elasticsearch/reference/2.4/modules-scripting.html#enable-dynamic-scripting -script.inline: true - -{% if elasticsearch_major_version == "2.x" %} -script.indexed: true -{% else %} -# script.indexed has renamed to script.stored in 5.x -# ref: https://www.elastic.co/guide/en/elasticsearch/reference/5.0/breaking_50_settings_changes.html#_script_mode_settings -script.stored: true -{% endif %} - -{% endif %} diff --git a/src/roles/elasticsearch/templates/java_home.sh.j2 b/src/roles/elasticsearch/templates/java_home.sh.j2 index 12de4aa02..e2354e196 100644 --- a/src/roles/elasticsearch/templates/java_home.sh.j2 +++ b/src/roles/elasticsearch/templates/java_home.sh.j2 @@ -1 +1 @@ -export JAVA_HOME=/usr/bin +export JAVA_HOME=/usr/share/elasticsearch/jdk From c70407c05324ee9b73b469fb340fcd9c60e121cb Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Apr 2020 19:52:45 -0400 Subject: [PATCH 135/280] Update SMW to 3.1 Update Semantic MediaWiki to v3.1 Semantic Result Formats v3.1 Semantic Compound Queries v2.1 Semantic Scribunto v2.1 Also remove deprecated DB_SLAVE --- config/MezaCoreExtensions.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index ba5e6de7a..eb7c6b98f 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -3,7 +3,7 @@ list: - name: Semantic MediaWiki composer: "mediawiki/semantic-media-wiki" - version: "3.0.0" + version: "~3.1.0" config: | // Enable Semantic MediaWiki semantics enableSemantics( $wikiId ); @@ -23,11 +23,11 @@ list: // created them, and thus they effectively don't replicate). Picking // DB_SLAVE since temporary tables are effectively a read-action since // they are used only for making more efficient queries. - $smwgLocalConnectionConf['mw.db.queryengine'] = [ 'read' => DB_SLAVE, 'write' => DB_SLAVE ]; + $smwgLocalConnectionConf['mw.db.queryengine'] = [ 'read' => DB_REPLICA, 'write' => DB_REPLICA ]; - name: Semantic Result Formats composer: "mediawiki/semantic-result-formats" - version: "3.0.0" + version: "~3.1.0" config: | // In SRF 3.0+ you need to do this, too: wfLoadExtension( 'SemanticResultFormats' ); @@ -49,7 +49,7 @@ list: - name: SemanticCompoundQueries composer: "mediawiki/semantic-compound-queries" - version: "1.2.0" + version: "~2.1.0" - name: Scribunto repo: https://github.com/wikimedia/mediawiki-extensions-Scribunto.git version: "{{ mediawiki_default_branch }}" @@ -59,7 +59,7 @@ list: $wgScribuntoUseCodeEditor = true; - name: "Semantic Scribunto" composer: "mediawiki/semantic-scribunto" - version: 2.0.0 + version: ~2.1.0 config: | wfLoadExtension( 'SemanticScribunto' ); - name: SubPageList From ecb7158c7bad36b478393b688db24abf4c761112 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Apr 2020 19:53:26 -0400 Subject: [PATCH 136/280] MCrypt for PHP is now just named php-mcrypt --- src/roles/apache-php/tasks/php-redhat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/apache-php/tasks/php-redhat.yml b/src/roles/apache-php/tasks/php-redhat.yml index fc164dfba..ba8b2f3ad 100644 --- a/src/roles/apache-php/tasks/php-redhat.yml +++ b/src/roles/apache-php/tasks/php-redhat.yml @@ -87,7 +87,7 @@ - "{{ php_ius_version }}-pecl-memcached" # Available for php56u, php70u, and php71u. NOT for php72u. - - "{{ php_ius_version }}-mcrypt" + - "php-mcrypt" # Available for php56u and php70u. NOT php71u or php72u # - "{{ php_ius_version }}-pear" From 19ecc3566692ef24bb7b32632f08ff9657c232dc Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Apr 2020 19:55:45 -0400 Subject: [PATCH 137/280] Do not upgrade the metastore before it exists. --- src/roles/mediawiki/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 9ecd91690..daf2096d4 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -469,9 +469,9 @@ # when: not es_do_upgrade_stat.stat.exists -- name: Verify metastore index upgraded - shell: WIKI={{ list_of_wikis[0] }} php /opt/htdocs/mediawiki/extensions/CirrusSearch/maintenance/metastore.php --upgrade - run_once: true +# - name: Verify metastore index upgraded +# shell: WIKI={{ list_of_wikis[0] }} php /opt/htdocs/mediawiki/extensions/CirrusSearch/maintenance/metastore.php --upgrade +# run_once: true # Wikis are totally built at this point, but SMW and search need rebuilding # FIXME #811: Will this work when controller is not an app server? From 7eb618c65bafad324f7497dfef681e43d9a70fa5 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 9 Apr 2020 14:09:43 +0000 Subject: [PATCH 138/280] mcrypt is not available for PHP7.2 --- src/roles/apache-php/tasks/php-redhat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/apache-php/tasks/php-redhat.yml b/src/roles/apache-php/tasks/php-redhat.yml index ba8b2f3ad..444e0b0fd 100644 --- a/src/roles/apache-php/tasks/php-redhat.yml +++ b/src/roles/apache-php/tasks/php-redhat.yml @@ -87,7 +87,7 @@ - "{{ php_ius_version }}-pecl-memcached" # Available for php56u, php70u, and php71u. NOT for php72u. - - "php-mcrypt" + # - "php-mcrypt" # Available for php56u and php70u. NOT php71u or php72u # - "{{ php_ius_version }}-pear" From 0575efe0eeaf2d278cd7eac2c465e40f6b26e518 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 10 Apr 2020 10:40:02 -0400 Subject: [PATCH 139/280] Use master for ApprovedRevs --- config/MezaCoreExtensions.yml | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index eb7c6b98f..9c7090397 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -3,7 +3,7 @@ list: - name: Semantic MediaWiki composer: "mediawiki/semantic-media-wiki" - version: "~3.1.0" + version: "~3.1" config: | // Enable Semantic MediaWiki semantics enableSemantics( $wikiId ); @@ -27,7 +27,7 @@ list: - name: Semantic Result Formats composer: "mediawiki/semantic-result-formats" - version: "~3.1.0" + version: "~3.1" config: | // In SRF 3.0+ you need to do this, too: wfLoadExtension( 'SemanticResultFormats' ); @@ -49,7 +49,8 @@ list: - name: SemanticCompoundQueries composer: "mediawiki/semantic-compound-queries" - version: "~2.1.0" + version: "~2.1" + - name: Scribunto repo: https://github.com/wikimedia/mediawiki-extensions-Scribunto.git version: "{{ mediawiki_default_branch }}" @@ -59,7 +60,7 @@ list: $wgScribuntoUseCodeEditor = true; - name: "Semantic Scribunto" composer: "mediawiki/semantic-scribunto" - version: ~2.1.0 + version: ~2.1 config: | wfLoadExtension( 'SemanticScribunto' ); - name: SubPageList @@ -89,7 +90,7 @@ list: $wgPFEnableStringFunctions = true; - name: ExternalData repo: https://github.com/wikimedia/mediawiki-extensions-ExternalData.git - version: "{{ mediawiki_default_branch }}" + version: "master" - name: LabeledSectionTransclusion repo: https://github.com/wikimedia/mediawiki-extensions-LabeledSectionTransclusion.git version: "{{ mediawiki_default_branch }}" @@ -106,8 +107,7 @@ list: version: "{{ mediawiki_default_branch }}" - name: PageForms repo: https://github.com/wikimedia/mediawiki-extensions-PageForms.git - # commit includes spreadsheet sorting which didn't make PageForms 4.4.1 - version: "730390a31a56c001af83948af1eefc5174abbe06" + version: "master" - name: DismissableSiteNotice repo: https://github.com/wikimedia/mediawiki-extensions-DismissableSiteNotice.git version: "{{ mediawiki_default_branch }}" @@ -133,7 +133,7 @@ list: version: "{{ mediawiki_default_branch }}" - name: ReplaceText repo: https://github.com/wikimedia/mediawiki-extensions-ReplaceText.git - version: tags/1.4 + version: "master" - name: Interwiki repo: https://github.com/wikimedia/mediawiki-extensions-Interwiki.git version: "{{ mediawiki_default_branch }}" @@ -222,11 +222,11 @@ list: $wgPageShowWatchingUsers = true; - name: SemanticInternalObjects repo: https://github.com/wikimedia/mediawiki-extensions-SemanticInternalObjects.git - version: "{{ mediawiki_default_branch }}" + version: "master" legacy_load: true - name: SemanticDrilldown repo: https://github.com/wikimedia/mediawiki-extensions-SemanticDrilldown.git - version: "{{ mediawiki_default_branch }}" + version: "master" legacy_load: true - name: Arrays repo: https://github.com/wikimedia/mediawiki-extensions-Arrays.git @@ -238,8 +238,7 @@ list: legacy_load: true - name: AdminLinks repo: https://github.com/wikimedia/mediawiki-extensions-AdminLinks.git - version: "{{ mediawiki_default_branch }}" - legacy_load: true + version: "master" config: | $wgGroupPermissions['sysop']['adminlinks'] = true; - name: BatchUserRights @@ -259,8 +258,7 @@ list: version: tags/0.1.0 - name: ApprovedRevs repo: https://github.com/wikimedia/mediawiki-extensions-ApprovedRevs.git - # Use this commit until a release tag for v1.0 is created - version: tags/1.0 + version: master config: | $egApprovedRevsAutomaticApprovals = false; - name: ImagesLoaded @@ -369,7 +367,7 @@ list: - name: DataTransfer repo: https://github.com/wikimedia/mediawiki-extensions-DataTransfer.git - version: "{{ mediawiki_default_branch }}" + version: "master" - name: PageImporter repo: https://github.com/enterprisemediawiki/PageImporter.git version: tags/0.1.0 From 11b7ceb22fdc03ce7c84c55aeb8eb8b1abee68db Mon Sep 17 00:00:00 2001 From: Daren Welsh Date: Mon, 13 Apr 2020 14:32:38 -0500 Subject: [PATCH 140/280] enable LDAP extension module --- src/roles/apache-php/templates/php.ini.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/roles/apache-php/templates/php.ini.j2 b/src/roles/apache-php/templates/php.ini.j2 index 75e4cf43f..f3bade6d3 100644 --- a/src/roles/apache-php/templates/php.ini.j2 +++ b/src/roles/apache-php/templates/php.ini.j2 @@ -1815,6 +1815,8 @@ soap.wsdl_cache_limit = 5 ;sysvshm.init_mem = 10000 [ldap] +; Enable ldap extension module +extension=ldap.so ; Sets the maximum number of open links or -1 for unlimited. ldap.max_links = -1 From 15c6f149736e353adc5659ad7552244fdd7cc049 Mon Sep 17 00:00:00 2001 From: Daren Welsh Date: Mon, 13 Apr 2020 14:40:02 -0500 Subject: [PATCH 141/280] remove php 5.x packages, add LDAP PHP IUS --- src/roles/apache-php/tasks/php-redhat.yml | 31 ++++------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/src/roles/apache-php/tasks/php-redhat.yml b/src/roles/apache-php/tasks/php-redhat.yml index 444e0b0fd..dcbea7095 100644 --- a/src/roles/apache-php/tasks/php-redhat.yml +++ b/src/roles/apache-php/tasks/php-redhat.yml @@ -15,32 +15,10 @@ state: present -- name: Ensure PHP 5.6 packages removed - yum: - name: - - php56u - - php56u-cli - - php56u-common - - php56u-devel - - php56u-gd - - php56u-pecl-memcache - - php56u-pspell - - php56u-snmp - - php56u-xml - - php56u-xmlrpc - - php56u-mysqlnd - - php56u-pdo - - php56u-odbc - - php56u-pear - - php56u-pecl-jsonc - - php56u-process - - php56u-bcmath - - php56u-intl - - php56u-opcache - - php56u-soap - - php56u-mbstring - - php56u-mcrypt - - php56u-mssql +- name: Remove PHP 5.x packages + package: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "php5*" state: absent # Check if the desired version of PHP is installed. If it is not, ensure any @@ -81,6 +59,7 @@ - "{{ php_ius_version }}-opcache" - "{{ php_ius_version }}-soap" - "{{ php_ius_version }}-mbstring" + - "{{ php_ius_version }}-ldap" # php56u has memcache and memcached; php7Xu only has memcached # legacy Meza used php56u-pecl-memcache From 60c221c813aa92021d5822e043b168abef42eca5 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Apr 2020 22:17:43 +0000 Subject: [PATCH 142/280] Remove mcrypt package - no longer used --- config/defaults.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/defaults.yml b/config/defaults.yml index e0a0cfb93..984bab62e 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -199,9 +199,6 @@ packages_php_debian: - php-pear - php-memcached - # No PHP 7.2 version of this (just like with IUS) - - php{{ php_debian_version }}-mcrypt - # No apt package corresponding to: # "{{ php_ius_version }}-process" # "{{ php_ius_version }}-pdo" From 8c71112c579dbfbc0bafdaf54d7e4892a9719346 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Apr 2020 22:19:33 +0000 Subject: [PATCH 143/280] Python3 compat. Replace iteritems with items --- .../templates/config.php.j2 | 18 +++++++++--------- .../base-config-scripts/templates/config.sh.j2 | 8 ++++---- .../saml/templates/samlLocalSettings.php.j2 | 2 +- src/scripts/meza.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/roles/base-config-scripts/templates/config.php.j2 b/src/roles/base-config-scripts/templates/config.php.j2 index bc3050912..4223c3ce4 100644 --- a/src/roles/base-config-scripts/templates/config.php.j2 +++ b/src/roles/base-config-scripts/templates/config.php.j2 @@ -78,7 +78,7 @@ $backups_environment = '{{ backups_environment }}'; {% if wiki_backup_downloaders is defined %} # Users allowed to download specific wikis $wiki_backup_downloaders = array(); -{% for wiki, users in wiki_backup_downloaders.iteritems() %} +{% for wiki, users in wiki_backup_downloaders.items() %} $wiki_backup_downloaders['{{ wiki }}'] = array( {% for user in users %} '{{ user }}', @@ -109,7 +109,7 @@ $all_backup_downloaders = array( # CUSTOM __public__ DEPLOY VARIABLES # These should only come from public.yml # -{% for key, value in public_deploy_vars.iteritems() %} +{% for key, value in public_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -121,7 +121,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -159,7 +159,7 @@ $all_backup_downloaders = array( # CUSTOM __{{ env }} environment__ DEPLOY VARIABLES # These should only come from env/{{ env }}.yml # -{% for key, value in env_deploy_vars.iteritems() %} +{% for key, value in env_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -171,7 +171,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -208,7 +208,7 @@ $all_backup_downloaders = array( # CUSTOM __secret__ DEPLOY VARIABLES # These should only come from secret.yml # -{% for key, value in secret_deploy_vars.iteritems() %} +{% for key, value in secret_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -220,7 +220,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -260,7 +260,7 @@ $all_backup_downloaders = array( # FIXME: remove deploy_vars in lieu of secret_deploy_vars when production wikis # are updated. # -{% for key, value in deploy_vars.iteritems() %} +{% for key, value in deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -272,7 +272,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; diff --git a/src/roles/base-config-scripts/templates/config.sh.j2 b/src/roles/base-config-scripts/templates/config.sh.j2 index fa874fafe..1041d310e 100644 --- a/src/roles/base-config-scripts/templates/config.sh.j2 +++ b/src/roles/base-config-scripts/templates/config.sh.j2 @@ -67,7 +67,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __public__ DEPLOY VARIABLES # These should only come from public.yml # -{% for key, value in public_deploy_vars.iteritems() %} +{% for key, value in public_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -84,7 +84,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __{{ env }} environment__ DEPLOY VARIABLES # These should only come from env/{{ env }}.yml # -{% for key, value in env_deploy_vars.iteritems() %} +{% for key, value in env_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -101,7 +101,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __secret__ DEPLOY VARIABLES # These should only come from secret.yml # -{% for key, value in secret_deploy_vars.iteritems() %} +{% for key, value in secret_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -120,7 +120,7 @@ m_i18n="{{ m_i18n }}" # FIXME: remove deploy_vars in lieu of secret_deploy_vars when production wikis # are updated. # -{% for key, value in deploy_vars.iteritems() %} +{% for key, value in deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} diff --git a/src/roles/saml/templates/samlLocalSettings.php.j2 b/src/roles/saml/templates/samlLocalSettings.php.j2 index 7ef8fe3ab..41659455f 100644 --- a/src/roles/saml/templates/samlLocalSettings.php.j2 +++ b/src/roles/saml/templates/samlLocalSettings.php.j2 @@ -37,7 +37,7 @@ if ( isset( $_SERVER['HTTP_X_SKIP_SAML'] ) ) { // FIXME #822: The indenting below will be heinous when Ansible does its templating {% if allow_skip_saml_users is defined -%} $wgMezaAllowSkipSamlUsers = array(); - {% for user, ipaddrs in allow_skip_saml_users.iteritems() -%} + {% for user, ipaddrs in allow_skip_saml_users.items() -%} $wgMezaAllowSkipSamlUsers['{{ user }}'] = array( {%- for ipaddr in ipaddrs -%}'{{ ipaddr }}',{%- endfor -%} ); diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 41f9a4316..c663435dd 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -751,7 +751,7 @@ def playbook_cmd ( playbook, env=False, more_extra_vars=False ): extra_vars = {} if more_extra_vars: - for varname, value in more_extra_vars.iteritems(): + for varname, value in more_extra_vars.items(): extra_vars[varname] = value if len(extra_vars) > 0: From c018087307c861dbd12f253639ee60a4ad7b717f Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Apr 2020 22:20:59 +0000 Subject: [PATCH 144/280] Java 1.8.0 OpenJDK is for RedHat only --- src/roles/elasticsearch/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index 2f89c77e6..32adb414c 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -13,6 +13,7 @@ yum: name: "{{ package_java }}" state: present + when: ansible_os_family == 'RedHat' # Environment setup. @@ -28,6 +29,7 @@ key: https://packages.elastic.co/GPG-KEY-elasticsearch state: present when: ansible_os_family == 'RedHat' + - name: Add Elasticsearch repository. template: src: elasticsearch.repo.j2 @@ -172,5 +174,6 @@ src: elasticsearch.repo.j2 dest: /etc/yum.repos.d/elasticsearch.repo mode: 0644 + when: ansible_os_family == 'RedHat' From 54ec3f6bbe283ed75fca29d365c5afaf4804a1dd Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Apr 2020 22:22:40 +0000 Subject: [PATCH 145/280] New SMW task to setupStore.php --- src/roles/mediawiki/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index daf2096d4..acfdf7992 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -468,6 +468,11 @@ # es_do_upgrade: False # when: not es_do_upgrade_stat.stat.exists +- name: "Ensure SMW storage setup" + shell: WIKI={{ item }} php {{ m_mediawiki }}/extensions/SemanticMediaWiki/maintenance/setupStore.php + with_items: "{{ list_of_wikis }}" + run_once: true + # - name: Verify metastore index upgraded # shell: WIKI={{ list_of_wikis[0] }} php /opt/htdocs/mediawiki/extensions/CirrusSearch/maintenance/metastore.php --upgrade From 9afe1bb188085aab74aabddc97faca93e4a40d3b Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2020 11:30:49 +0000 Subject: [PATCH 146/280] Implement certbot for SSL --- config/defaults.yml | 2 + src/roles/certbot/tasks/main.yml | 34 +++++ src/roles/haproxy/tasks/main.yml | 137 ++++++++++----------- src/roles/haproxy/templates/haproxy.cfg.j2 | 14 +-- 4 files changed, 111 insertions(+), 76 deletions(-) create mode 100644 src/roles/certbot/tasks/main.yml diff --git a/config/defaults.yml b/config/defaults.yml index 984bab62e..ec1a25a74 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -101,6 +101,8 @@ wiki_id_redirects: {} m_force_debug: false +enable_certbot: true + enable_wiki_emails: true enable_haproxy_stats: false diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml new file mode 100644 index 000000000..b99887109 --- /dev/null +++ b/src/roles/certbot/tasks/main.yml @@ -0,0 +1,34 @@ +--- +# Role to install Let's Encrypt's Certbot, install certificate(s) and automate renewals +- name: Ensure firewall port 54321 OPEN when certbot ENABLED + include_role: + name: firewall_port + vars: + firewall_action: open + firewall_port: 54321 + firewall_protocol: tcp + firewall_zone: "{{m_private_networking_zone|default('public')}}" + when: + - enable_certbot + - (docker_skip_tasks is not defined or not docker_skip_tasks) + +- name: Ensure Certbot installed + package: + name: certbot + state: present + +- name: Generate SSL Certificate + shell: > + certbot certonly --non-interactive --email {{ m_httpd_server_admin }} + --preferred-challenges http --standalone --agree-tos --renew-by-default + --webroot-path {{ m_htdocs }} -d {{ wiki_app_fqdn }} --http-01-port=54321 + delegate_to: localhost + run_once: True + +- name: Concatenate certificate files for HAproxy + shell: > + cat /etc/letsencrypt/live/{{ wiki_app_fqdn }}/fullchain.pem + /etc/letsencrypt/live/{{ wiki_app_fqdn }}/privkey.pem > + /etc/haproxy/certs/{{ wiki_app_fqdn }}.pem + notify: + - restart haproxy diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 66ff9c659..5cd3db2c7 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -97,86 +97,85 @@ run_once: True # FIXME: Better would be "if not exists setup letsencrypt" -- name: If not exists, create self-signed SSL cert on CONTROLLER - command: | - openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \ - -subj "/C=US/ST=TX/L=Houston/O=EnterpriseMediaWiki/CN={{ wiki_app_fqdn}}" \ - -keyout {{ m_local_secret }}/{{ env }}/ssl/meza.key \ - -out {{ m_local_secret }}/{{ env }}/ssl/meza.crt - when: ssl_cert_stat_result.stat.exists == False - delegate_to: localhost - run_once: True - -- name: Ensure SSL cert and key are encrypted - shell: > - ansible-vault encrypt - {{ item }} - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt - failed_when: False - delegate_to: localhost - run_once: True - with_items: - - "{{ m_local_secret }}/{{ env }}/ssl/meza.key" - - "{{ m_local_secret }}/{{ env }}/ssl/meza.crt" - -- name: Read SSL key into variable - shell: > - ansible-vault view - /opt/conf-meza/secret/{{ env }}/ssl/meza.key - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt - register: decrypted_key - delegate_to: localhost - run_once: True - -- name: Read SSL cert into variable - shell: > - ansible-vault view - /opt/conf-meza/secret/{{ env }}/ssl/meza.crt - --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt - register: decrypted_cert - delegate_to: localhost - run_once: True -# - name: Ensure cert and key on load balancers -# copy: -# src: "{{ m_local_secret }}/{{ env }}/ssl/{{ item }}" -# dest: "/etc/haproxy/certs/{{ item }}" +- name: Use Certbot for SSL + include_role: + name: certbot + +# - name: If not exists, create self-signed SSL cert on CONTROLLER +# command: | +# openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \ +# -subj "/C=US/ST=TX/L=Houston/O=EnterpriseMediaWiki/CN={{ wiki_app_fqdn}}" \ +# -keyout {{ m_local_secret }}/{{ env }}/ssl/meza.key \ +# -out {{ m_local_secret }}/{{ env }}/ssl/meza.crt +# when: ssl_cert_stat_result.stat.exists == False +# delegate_to: localhost +# run_once: True + +# - name: Ensure SSL cert and key are encrypted +# shell: > +# ansible-vault encrypt +# {{ item }} +# --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt +# failed_when: False +# delegate_to: localhost +# run_once: True # with_items: -# - meza.key -# - meza.crt +# - "{{ m_local_secret }}/{{ env }}/ssl/meza.key" +# - "{{ m_local_secret }}/{{ env }}/ssl/meza.crt" + +# - name: Read SSL key into variable +# shell: > +# ansible-vault view +# /opt/conf-meza/secret/{{ env }}/ssl/meza.key +# --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt +# register: decrypted_key +# delegate_to: localhost +# run_once: True + +# - name: Read SSL cert into variable +# shell: > +# ansible-vault view +# /opt/conf-meza/secret/{{ env }}/ssl/meza.crt +# --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt +# register: decrypted_cert +# delegate_to: localhost +# run_once: True + +# # - name: Ensure cert and key on load balancers +# # copy: +# # src: "{{ m_local_secret }}/{{ env }}/ssl/{{ item }}" +# # dest: "/etc/haproxy/certs/{{ item }}" +# # with_items: +# # - meza.key +# # - meza.crt +# # notify: +# # - restart haproxy + +# - name: Ensure SSL cert on load balancers +# copy: +# content: "{{ decrypted_cert.stdout }}" +# dest: "/etc/haproxy/certs/meza.crt" # notify: # - restart haproxy -- name: Ensure SSL cert on load balancers - copy: - content: "{{ decrypted_cert.stdout }}" - dest: "/etc/haproxy/certs/meza.crt" - notify: - - restart haproxy - -- name: Ensure SSL key on load balancers - copy: - content: "{{ decrypted_key.stdout }}" - dest: "/etc/haproxy/certs/meza.key" - notify: - - restart haproxy - -- name: Ensure cert and key assembled into into pem file - copy: - content: | - {{ decrypted_cert.stdout }} - {{ decrypted_key.stdout }} - dest: /etc/haproxy/certs/meza.pem - notify: - - restart haproxy +# - name: Ensure SSL key on load balancers +# copy: +# content: "{{ decrypted_key.stdout }}" +# dest: "/etc/haproxy/certs/meza.key" +# notify: +# - restart haproxy # - name: Ensure cert and key assembled into into pem file -# assemble: -# src: "/opt/conf-meza/secret/{{ env }}/ssl" +# copy: +# content: | +# {{ decrypted_cert.stdout }} +# {{ decrypted_key.stdout }} # dest: /etc/haproxy/certs/meza.pem # notify: # - restart haproxy + - name: Ensure haproxy certs have secure permissions file: path: /etc/haproxy/certs diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index 11fa057e5..1670eb650 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -65,11 +65,11 @@ frontend www-http default_backend www-backend frontend www-https - bind *:443 ssl crt /etc/haproxy/certs/meza.pem + bind *:443 ssl crt /etc/haproxy/certs/ reqadd X-Forwarded-Proto:\ https # Keep letsencrypt stuff here for now. Probably add it back later. - # acl letsencrypt-acl path_beg /.well-known/acme-challenge/ - # use_backend letsencrypt-backend if letsencrypt-acl + acl letsencrypt-acl path_beg /.well-known/acme-challenge/ + use_backend letsencrypt-backend if letsencrypt-acl default_backend www-backend backend www-backend @@ -92,7 +92,7 @@ backend www-backend # app servers together with controller as registry {% if m_install_netdata %} frontend netdata - bind *:20000 ssl crt {{ ssl_certificate_file }} + bind *:20000 ssl crt /etc/haproxy/certs/ mode http default_backend netdata-back backend netdata-back @@ -141,7 +141,7 @@ listen mediawiki-internal {% if enable_haproxy_stats %} listen stats - bind *:1936 ssl crt /etc/haproxy/certs/meza.pem + bind *:1936 ssl crt /etc/haproxy/certs/ stats enable stats hide-version stats realm Haproxy\ Statistics @@ -150,5 +150,5 @@ listen stats stats refresh 30s {% endif %} -# backend letsencrypt-backend -# server letsencrypt 127.0.0.1:54321 +backend letsencrypt-backend + server letsencrypt 127.0.0.1:54321 From f224b843618345c2626aa517efa0d93949d51816 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Apr 2020 12:02:04 +0000 Subject: [PATCH 147/280] Don't intercept Certbot requests --- src/roles/htdocs/templates/.htaccess.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index 11ec4bc2d..ee2cbc9ee 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -15,6 +15,9 @@ # Allow access to root index.php RewriteRule ^index.php(.*) - [L] + # Allow certbot access + RewriteRule ^.well-known(.*) - [L] + # Allow access to /wikis directory (where all wiki content and # and settings are located) RewriteRule ^wikis(?:/|$)(.*)$ - [L] From dc2b7d6df32e7467b6acd972c6a520443518dfae Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 16 Apr 2020 18:02:54 +0000 Subject: [PATCH 148/280] Fix #748 Add Certbot Various updates to Certbot/Haproxy Detect email setting for certbot Remove any pre-existing meza.pem and key files Concatenate all certificates into pem files for haproxy Remove code to generate self-signed meza.pem Remove redundant notify Remove commented code --- src/roles/certbot/tasks/main.yml | 38 ++++++++-- src/roles/haproxy/tasks/main.yml | 115 +------------------------------ 2 files changed, 34 insertions(+), 119 deletions(-) diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index b99887109..d82557010 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -1,5 +1,14 @@ --- # Role to install Let's Encrypt's Certbot, install certificate(s) and automate renewals +- assert: + that: + - "m_httpd_server_admin is defined" + - "m_httpd_server_admin != 'admin@example.com'" + fail_msg: > + "To use certbot, you must supply a valid email for 'm_httpd_server_admin' in your public.yml + For example, add a line like this to /opt/conf-meza/public/public.yml + m_httpd_server_admin: me@gmail.com" + - name: Ensure firewall port 54321 OPEN when certbot ENABLED include_role: name: firewall_port @@ -12,6 +21,19 @@ - enable_certbot - (docker_skip_tasks is not defined or not docker_skip_tasks) +- name: Remove Meza cert files when certbot ENABLED + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/haproxy/certs/meza.pem + - /etc/haproxy/certs/meza.crt + - /etc/haproxy/certs/meza.key + + when: + - enable_certbot + - (docker_skip_tasks is not defined or not docker_skip_tasks) + - name: Ensure Certbot installed package: name: certbot @@ -26,9 +48,13 @@ run_once: True - name: Concatenate certificate files for HAproxy - shell: > - cat /etc/letsencrypt/live/{{ wiki_app_fqdn }}/fullchain.pem - /etc/letsencrypt/live/{{ wiki_app_fqdn }}/privkey.pem > - /etc/haproxy/certs/{{ wiki_app_fqdn }}.pem - notify: - - restart haproxy + shell: | + for SITE in $(find /etc/letsencrypt/live/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) + do + cd /etc/letsencrypt/live/$SITE + cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem + done + args: + executable: /bin/bash + # notify: + # - restart haproxy diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 5cd3db2c7..543bc1d48 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -1,11 +1,4 @@ --- -# FIXME #748: Eventually add the ability to get SSL cert from letsencrypt -# ref: https://www.digitalocean.com/community/tutorials/how-to-secure-haproxy-with-let-s-encrypt-on-centos-7 -# Other refs: -# https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Load_Balancer_Administration/install_haproxy_example1.html -# https://gist.github.com/thisismitch/7c91e9b2b63f837a0c4b -# https://www.unixmen.com/configure-high-available-load-balancer-haproxy-keepalived/ - - name: Set fact if this load balancer will handle external connections (80/443) set_fact: @@ -75,107 +68,14 @@ # owner/group/mode? # -# 1. If cert/key don't exist ON CONTROLLER, generate self-signed ON CONTROLLER -# 2. Ensure cert and key in /etc/haproxy/certs -# 3. Ensure cert and key assembled into pem file at /etc/haproxy/certs/meza.pem +# 1. If cert/key don't exist ON CONTROLLER, install certbot and create certs ON CONTROLLER +# 2. Ensure certs assembled in /etc/haproxy/certs # -- name: Check if secret config on CONTROLLER has SSL keys - stat: - path: "{{ m_local_secret }}/{{ env }}/ssl/meza.key" - register: ssl_cert_stat_result - delegate_to: localhost - run_once: True - -- name: Ensure config SSL directory exists - file: - path: "{{ m_local_secret }}/{{ env }}/ssl" - state: directory - owner: root - group: root - mode: 0755 - delegate_to: localhost - run_once: True - -# FIXME: Better would be "if not exists setup letsencrypt" - name: Use Certbot for SSL include_role: name: certbot -# - name: If not exists, create self-signed SSL cert on CONTROLLER -# command: | -# openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \ -# -subj "/C=US/ST=TX/L=Houston/O=EnterpriseMediaWiki/CN={{ wiki_app_fqdn}}" \ -# -keyout {{ m_local_secret }}/{{ env }}/ssl/meza.key \ -# -out {{ m_local_secret }}/{{ env }}/ssl/meza.crt -# when: ssl_cert_stat_result.stat.exists == False -# delegate_to: localhost -# run_once: True - -# - name: Ensure SSL cert and key are encrypted -# shell: > -# ansible-vault encrypt -# {{ item }} -# --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt -# failed_when: False -# delegate_to: localhost -# run_once: True -# with_items: -# - "{{ m_local_secret }}/{{ env }}/ssl/meza.key" -# - "{{ m_local_secret }}/{{ env }}/ssl/meza.crt" - -# - name: Read SSL key into variable -# shell: > -# ansible-vault view -# /opt/conf-meza/secret/{{ env }}/ssl/meza.key -# --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt -# register: decrypted_key -# delegate_to: localhost -# run_once: True - -# - name: Read SSL cert into variable -# shell: > -# ansible-vault view -# /opt/conf-meza/secret/{{ env }}/ssl/meza.crt -# --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt -# register: decrypted_cert -# delegate_to: localhost -# run_once: True - -# # - name: Ensure cert and key on load balancers -# # copy: -# # src: "{{ m_local_secret }}/{{ env }}/ssl/{{ item }}" -# # dest: "/etc/haproxy/certs/{{ item }}" -# # with_items: -# # - meza.key -# # - meza.crt -# # notify: -# # - restart haproxy - -# - name: Ensure SSL cert on load balancers -# copy: -# content: "{{ decrypted_cert.stdout }}" -# dest: "/etc/haproxy/certs/meza.crt" -# notify: -# - restart haproxy - -# - name: Ensure SSL key on load balancers -# copy: -# content: "{{ decrypted_key.stdout }}" -# dest: "/etc/haproxy/certs/meza.key" -# notify: -# - restart haproxy - -# - name: Ensure cert and key assembled into into pem file -# copy: -# content: | -# {{ decrypted_cert.stdout }} -# {{ decrypted_key.stdout }} -# dest: /etc/haproxy/certs/meza.pem -# notify: -# - restart haproxy - - - name: Ensure haproxy certs have secure permissions file: path: /etc/haproxy/certs @@ -184,8 +84,6 @@ owner: root group: root mode: 0600 - notify: - - restart haproxy - name: write the haproxy config file template: @@ -194,15 +92,6 @@ notify: - restart haproxy -# - name: Ensure haproxy socket directory exists -# file: -# path: /run/haproxy -# state: directory -# owner: root -# group: root -# mode: 0660 - - - name: Ensure error files directory in place file: path: /etc/haproxy/errors From cb6616ef0ac4b55e7167c454d340ce5076fc5e7b Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 17 Apr 2020 01:31:24 +0000 Subject: [PATCH 149/280] Check for existing certs before generation --- src/roles/certbot/tasks/main.yml | 15 +++++++++++++++ src/roles/haproxy/tasks/main.yml | 1 + 2 files changed, 16 insertions(+) diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index d82557010..86d33ae6e 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -39,6 +39,17 @@ name: certbot state: present +- name: Check for existing cert(s) + find: + paths: /etc/haproxy/certs + patterns: '*.pem' + excludes: 'meza.pem' + register: certs + +- debug: + msg: "Found existing certificate at {{ item.path }}" + with_items: "{{ certs.files }}" + - name: Generate SSL Certificate shell: > certbot certonly --non-interactive --email {{ m_httpd_server_admin }} @@ -46,6 +57,10 @@ --webroot-path {{ m_htdocs }} -d {{ wiki_app_fqdn }} --http-01-port=54321 delegate_to: localhost run_once: True + when: + - inventory_hostname in groups['load-balancers'] + - certs == "" + - enable_certbot - name: Concatenate certificate files for HAproxy shell: | diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 543bc1d48..5888540ab 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -75,6 +75,7 @@ - name: Use Certbot for SSL include_role: name: certbot + when: inventory_hostname in groups['load-balancers'] - name: Ensure haproxy certs have secure permissions file: From 20f7012e000fb475487094d8d424d465ae7b259a Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 17 Apr 2020 11:34:40 +0000 Subject: [PATCH 150/280] Add deploy hook (also used for renew) --- src/roles/certbot/tasks/main.yml | 41 ++++++++++++++------ src/roles/certbot/templates/concat.pem.sh.j2 | 13 +++++++ 2 files changed, 43 insertions(+), 11 deletions(-) create mode 100644 src/roles/certbot/templates/concat.pem.sh.j2 diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index 86d33ae6e..f56059dba 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -1,5 +1,10 @@ --- # Role to install Let's Encrypt's Certbot, install certificate(s) and automate renewals +# Since certbot installations for Debian/Ubuntu/RHEL/CentOS all setup system.d timers or cron jobs, +# We rely on the system to renew certificates instead of renewing them through ansible +# (We don't even have to setup a cron job.) We *do* have to setup a hook to concatenate certs +# on renewal. If you need to manage multiple domains then you should expand the certificate manually. + - assert: that: - "m_httpd_server_admin is defined" @@ -39,6 +44,7 @@ name: certbot state: present +# We will only generate a certificate when the certs directory is empty - name: Check for existing cert(s) find: paths: /etc/haproxy/certs @@ -50,11 +56,23 @@ msg: "Found existing certificate at {{ item.path }}" with_items: "{{ certs.files }}" +# There is no notify, because the script is used in cert generation +# By placing the script in the renewal-hooks hierarchy, it's run +# automatically by certbot renew (but not by certonly) +- name: Write the deploy hook script + template: + src: concat.pem.sh.j2 + dest: /etc/letsencrypt/renewal-hooks/deploy/concat.pem.sh + owner: meza-ansible + group: wheel + mode: u=rwx,g=rwx,o=rx + - name: Generate SSL Certificate shell: > certbot certonly --non-interactive --email {{ m_httpd_server_admin }} --preferred-challenges http --standalone --agree-tos --renew-by-default --webroot-path {{ m_htdocs }} -d {{ wiki_app_fqdn }} --http-01-port=54321 + --deploy-hook /etc/letsencrypt/renewal-hooks/deploy/concat.pem.sh delegate_to: localhost run_once: True when: @@ -62,14 +80,15 @@ - certs == "" - enable_certbot -- name: Concatenate certificate files for HAproxy - shell: | - for SITE in $(find /etc/letsencrypt/live/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) - do - cd /etc/letsencrypt/live/$SITE - cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem - done - args: - executable: /bin/bash - # notify: - # - restart haproxy +# - name: Concatenate certificate files for HAproxy +# shell: | +# for SITE in $(find /etc/letsencrypt/live/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) +# do +# cd /etc/letsencrypt/live/$SITE +# cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem +# done +# args: +# executable: /bin/bash +# # notify: +# # - restart haproxy +# # We don't have to restart haproxy, because that's done in the next step in the HAProxy role diff --git a/src/roles/certbot/templates/concat.pem.sh.j2 b/src/roles/certbot/templates/concat.pem.sh.j2 new file mode 100644 index 000000000..a491a34d1 --- /dev/null +++ b/src/roles/certbot/templates/concat.pem.sh.j2 @@ -0,0 +1,13 @@ +#!/bin/bash + +# $RENEWED_DOMAINS will contain a space-delimited list of renewed +# certificate domains (for example, "example.com www.example.com" +for SITE in $RENEWED_DOMAINS +do + # move to correct let's encrypt directory + cd /etc/letsencrypt/live/$SITE + # cat files to make combined .pem for haproxy + cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem +done +# reload haproxy +# systemctl reload haproxy From 63ed4019a8576dca505ba1ddb96ab84a8c5328fb Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 17 Apr 2020 16:43:36 -0400 Subject: [PATCH 151/280] comment several broken extensions --- config/MezaCoreExtensions.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 9c7090397..524630c82 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -65,17 +65,17 @@ list: wfLoadExtension( 'SemanticScribunto' ); - name: SubPageList composer: "mediawiki/sub-page-list" - version: "1.5.0" + version: "1.6.1" - name: Maps composer: "mediawiki/maps" - version: "6.0.3" + version: "7.17.2" config: | // In Maps 6.0+ you need to also load the extension wfLoadExtension( 'Maps' ); - name: DisplayTitle repo: https://github.com/wikimedia/mediawiki-extensions-DisplayTitle.git - version: tags/1.2 + version: "{{ mediawiki_default_branch }}" # @@ -250,12 +250,12 @@ list: config: | $wgHeaderTabsEditTabLink = false; $wgHeaderTabsRenderSingleTab = true; - - name: CopyWatchers - repo: https://github.com/jamesmontalvo3/MediaWiki-CopyWatchers.git - version: tags/0.10.0 - - name: Wiretap - repo: https://github.com/enterprisemediawiki/Wiretap.git - version: tags/0.1.0 +# - name: CopyWatchers +# repo: https://github.com/jamesmontalvo3/MediaWiki-CopyWatchers.git +# version: tags/0.10.0 +# - name: Wiretap +# repo: https://github.com/enterprisemediawiki/Wiretap.git +# version: tags/0.1.0 - name: ApprovedRevs repo: https://github.com/wikimedia/mediawiki-extensions-ApprovedRevs.git version: master @@ -267,11 +267,11 @@ list: - name: MasonryMainPage repo: https://github.com/enterprisemediawiki/MasonryMainPage.git version: tags/0.3.0 - - name: WatchAnalytics - repo: https://github.com/enterprisemediawiki/WatchAnalytics.git - version: tags/3.1.1 - config: | - $egPendingReviewsEmphasizeDays = 10; // makes Pending Reviews shake after X days +# - name: WatchAnalytics +# repo: https://github.com/enterprisemediawiki/WatchAnalytics.git +# version: tags/3.1.1 +# config: | +# $egPendingReviewsEmphasizeDays = 10; // makes Pending Reviews shake after X days - name: Variables repo: https://github.com/wikimedia/mediawiki-extensions-Variables.git version: "{{ mediawiki_default_branch }}" From ea68dc5ed2d35930ed71e5674ea9394aaf2493b2 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 18 Apr 2020 11:52:22 -0400 Subject: [PATCH 152/280] Do not remove meza.pem --- src/roles/certbot/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index f56059dba..cd343db0d 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -31,7 +31,6 @@ path: "{{ item }}" state: absent with_items: - - /etc/haproxy/certs/meza.pem - /etc/haproxy/certs/meza.crt - /etc/haproxy/certs/meza.key From ce4663294aa5bc78bec779525d939334f3d572ff Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 18 Apr 2020 11:54:35 -0400 Subject: [PATCH 153/280] Add when: conditions for writing deploy hook --- src/roles/certbot/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index cd343db0d..ba491d318 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -65,6 +65,9 @@ owner: meza-ansible group: wheel mode: u=rwx,g=rwx,o=rx + when: + - enable_certbot + - (docker_skip_tasks is not defined or not docker_skip_tasks) - name: Generate SSL Certificate shell: > From 94afde46f4b05044b73c7a019790af4045450cee Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 18 Apr 2020 18:33:41 -0400 Subject: [PATCH 154/280] Remove extraneous comment --- src/roles/certbot/tasks/main.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index ba491d318..12ca228b9 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -82,15 +82,3 @@ - certs == "" - enable_certbot -# - name: Concatenate certificate files for HAproxy -# shell: | -# for SITE in $(find /etc/letsencrypt/live/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) -# do -# cd /etc/letsencrypt/live/$SITE -# cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem -# done -# args: -# executable: /bin/bash -# # notify: -# # - restart haproxy -# # We don't have to restart haproxy, because that's done in the next step in the HAProxy role From ae6506e6af230525c682746413a0cb298d939528 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 18 Apr 2020 18:37:11 -0400 Subject: [PATCH 155/280] Use Certbot for SSL when enable_certbot --- src/roles/haproxy/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 5888540ab..ce284ba3d 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -75,7 +75,9 @@ - name: Use Certbot for SSL include_role: name: certbot - when: inventory_hostname in groups['load-balancers'] + when: + - inventory_hostname in groups['load-balancers'] + - enable_certbot - name: Ensure haproxy certs have secure permissions file: From db2f92f9ea4943a113dc99e385506819463f7d41 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 18 Apr 2020 18:42:25 -0400 Subject: [PATCH 156/280] simplify "when" conditional --- src/roles/certbot/tasks/main.yml | 15 +-------------- src/roles/haproxy/tasks/main.yml | 2 ++ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml index 12ca228b9..f075ec957 100644 --- a/src/roles/certbot/tasks/main.yml +++ b/src/roles/certbot/tasks/main.yml @@ -22,9 +22,6 @@ firewall_port: 54321 firewall_protocol: tcp firewall_zone: "{{m_private_networking_zone|default('public')}}" - when: - - enable_certbot - - (docker_skip_tasks is not defined or not docker_skip_tasks) - name: Remove Meza cert files when certbot ENABLED file: @@ -34,10 +31,6 @@ - /etc/haproxy/certs/meza.crt - /etc/haproxy/certs/meza.key - when: - - enable_certbot - - (docker_skip_tasks is not defined or not docker_skip_tasks) - - name: Ensure Certbot installed package: name: certbot @@ -65,9 +58,6 @@ owner: meza-ansible group: wheel mode: u=rwx,g=rwx,o=rx - when: - - enable_certbot - - (docker_skip_tasks is not defined or not docker_skip_tasks) - name: Generate SSL Certificate shell: > @@ -77,8 +67,5 @@ --deploy-hook /etc/letsencrypt/renewal-hooks/deploy/concat.pem.sh delegate_to: localhost run_once: True - when: - - inventory_hostname in groups['load-balancers'] - - certs == "" - - enable_certbot + when: certs == "" diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index ce284ba3d..d7ce6d8a9 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -78,6 +78,8 @@ when: - inventory_hostname in groups['load-balancers'] - enable_certbot + - (docker_skip_tasks is not defined or not docker_skip_tasks) + - name: Ensure haproxy certs have secure permissions file: From 06bda3ccba081346ba0c9c9afd179f3970edb935 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 18 Apr 2020 18:47:40 -0400 Subject: [PATCH 157/280] Enable variable wgServer based on HTTP_HOST --- src/roles/mediawiki/templates/LocalSettings.php.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index 6a885d52e..d7fb556fa 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -265,7 +265,11 @@ else { // Depending on proxy setup (particularly for Varnish/Squid caching) may need // to set $wgInternalServer: // ref: https://www.mediawiki.org/wiki/Manual:$wgInternalServer -$wgServer = 'https://{{ wiki_app_fqdn }}'; +if ( ! $wgCommandLineMode ) { + // $wgServer for subdomain mode + $wgServer = "https://" . $_SERVER['HTTP_HOST'] ; +} + // https://www.mediawiki.org/wiki/Manual:$wgScriptPath $wgScriptPath = "/$wikiId"; From cd368e444b8c5d8fd4abfbfe9c3a3b2b70858746 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 20 Apr 2020 08:22:15 -0400 Subject: [PATCH 158/280] define wgServer in commandline mode --- src/roles/mediawiki/templates/LocalSettings.php.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index d7fb556fa..9ee5a217e 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -268,6 +268,8 @@ else { if ( ! $wgCommandLineMode ) { // $wgServer for subdomain mode $wgServer = "https://" . $_SERVER['HTTP_HOST'] ; +} else { + $wgServer = 'https://{{ wiki_app_fqdn }}'; } From 37c82fbc50f0aa62793a9433e95334be1cfec22b Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 20 Apr 2020 08:46:45 -0400 Subject: [PATCH 159/280] allow for subdomain-based wiki identification --- src/roles/mediawiki/templates/LocalSettings.php.j2 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index 9ee5a217e..b85c4adad 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -63,9 +63,17 @@ if ( $wgCommandLineMode ) { $wikiId = getenv( $mezaWikiEnvVarName ); } else { - // get $wikiId from URI - $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); - $wikiId = strtolower( $uriParts[1] ); + + // get $wikiId from subdomain + // FIXME make regex a config variable + // FIXME add 'ignore' list like 'www' that should not be a wiki + if ( preg_match( '%([a-z]+)\.([a-z]+)\..{2,4}[\d]*$%im', $_SERVER['HTTP_HOST'], $matches ) ) { + $wikiId = $matches[1]; + } else { + // get $wikiId from URI path + $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); + $wikiId = strtolower( $uriParts[1] ); + } } From 3ff0febe1f0c8b5b978ff0ff27993ae13eec3458 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 21 Apr 2020 07:08:00 -0400 Subject: [PATCH 160/280] fix certbot deploy hook --- src/roles/certbot/templates/concat.pem.sh.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/roles/certbot/templates/concat.pem.sh.j2 b/src/roles/certbot/templates/concat.pem.sh.j2 index a491a34d1..4671ab856 100644 --- a/src/roles/certbot/templates/concat.pem.sh.j2 +++ b/src/roles/certbot/templates/concat.pem.sh.j2 @@ -2,7 +2,10 @@ # $RENEWED_DOMAINS will contain a space-delimited list of renewed # certificate domains (for example, "example.com www.example.com" -for SITE in $RENEWED_DOMAINS +# loop through a dynamic list of directories in 'live' +# for SITE in $(find /etc/letsencrypt/live -mindepth 1 -maxdepth 1 -type d -exec basename {} \;) +# $RENEWED_LINEAGE will contain the live subdirectory +for SITE in basename $RENEWED_LINEAGE do # move to correct let's encrypt directory cd /etc/letsencrypt/live/$SITE From ce214e6c31b789439ab54285b3c89da3ad9d741a Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 21 Apr 2020 08:19:09 -0400 Subject: [PATCH 161/280] Make SSHD / SFTP config vary on OS --- src/roles/base/templates/sshd_config.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/roles/base/templates/sshd_config.j2 b/src/roles/base/templates/sshd_config.j2 index f23cd45e9..db889992d 100644 --- a/src/roles/base/templates/sshd_config.j2 +++ b/src/roles/base/templates/sshd_config.j2 @@ -143,7 +143,12 @@ AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE AcceptEnv XMODIFIERS # override default of no subsystems +{% if ansible_os_family == "RedHat" %} +Subsystem sftp /usr/libexec/openssh/sftp-server +{% endif %} +{% if ansible_os_family == "Debian" %} Subsystem sftp /usr/lib/openssh/sftp-server +{% endif %} # Example of overriding settings on a per-user basis #Match User anoncvs From a859d4e47432301a7b02976b362ce44c22ef7b09 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 22 Apr 2020 11:36:53 -0400 Subject: [PATCH 162/280] Add --single-transaction --quick to mysqldump According to the manual (https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html), this is better for large tables, and concurrent operations. I have observed this myself on large wikis which previously would be locked during backup. --- src/roles/database/tasks/replication.yml | 2 +- src/roles/remote-mysqldump/tasks/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/roles/database/tasks/replication.yml b/src/roles/database/tasks/replication.yml index 1a3f0722b..4905137b1 100644 --- a/src/roles/database/tasks/replication.yml +++ b/src/roles/database/tasks/replication.yml @@ -92,7 +92,7 @@ - role_is_valid_slave - name: export dump file on master - shell: "mysqldump --databases {{ mysql_content_databases.stdout }} | gzip > {{ m_tmp }}/mysqldump-onmaster.sql.gz" + shell: "mysqldump --single-transaction --quick --databases {{ mysql_content_databases.stdout }} | gzip > {{ m_tmp }}/mysqldump-onmaster.sql.gz" delegate_to: "{{ mysql_replication_master }}" when: - slave_needs_configuration diff --git a/src/roles/remote-mysqldump/tasks/main.yml b/src/roles/remote-mysqldump/tasks/main.yml index 9a96a2594..d9196f6e3 100644 --- a/src/roles/remote-mysqldump/tasks/main.yml +++ b/src/roles/remote-mysqldump/tasks/main.yml @@ -96,7 +96,7 @@ -i /root/meza-ansible-id_rsa -o UserKnownHostsFile=/root/meza-ansible-known_hosts {{ remote_server_ssh_user }}@{{ remote_server }} - "mysqldump + "mysqldump --single-transaction --quick {{ user_option }} {{ password_option }} {{ dump_database }} @@ -106,7 +106,7 @@ - name: remote_server == target_server ({{ target_server }}); run mysqldump locally set_fact: mysqldump_command: > - mysqldump + mysqldump --single-transaction --quick {{ user_option }} {{ password_option }} {{ dump_database }} From 62c2ef13dec4603136c71d4676ece881c77e0929 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 23 Apr 2020 12:08:55 -0400 Subject: [PATCH 163/280] Add pidfile which is required for apache2buddy.pl --- src/roles/apache-php/templates/httpd.conf.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/roles/apache-php/templates/httpd.conf.j2 b/src/roles/apache-php/templates/httpd.conf.j2 index 278b54a38..46127571d 100644 --- a/src/roles/apache-php/templates/httpd.conf.j2 +++ b/src/roles/apache-php/templates/httpd.conf.j2 @@ -29,7 +29,9 @@ # least PidFile. # ServerRoot "{{ path_apache_server_root }}" - +{% if ansible_os_family == "Debian" %} +PidFile "/var/run/apache2.pid" +{% endif %} # # Mutex: Allows you to set the mutex mechanism and mutex file directory # for individual mutexes, or change the global defaults From 276aa8c1b5177fb78d2f595d6f38d1c9a8dfaec8 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 23 Apr 2020 13:43:38 -0400 Subject: [PATCH 164/280] Run composer update before install (for upgrades) --- src/roles/mediawiki/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index acfdf7992..b2cbcc195 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -205,11 +205,12 @@ owner: meza-ansible group: wheel -- name: Run composer install on MediaWiki for dependencies +# install doesn't appear to do extensions +- name: Run composer update on MediaWiki for extensions become: yes become_user: "meza-ansible" composer: - command: install + command: update working_dir: "{{ m_mediawiki }}" no_dev: no # FIXME #317: need ignore_errors because composer throws an error when running as root. @@ -218,13 +219,12 @@ - composer-extensions - latest - skins - -# install doesn't appear to do extensions -- name: Run composer update on MediaWiki for extensions + +- name: Run composer install on MediaWiki for dependencies become: yes become_user: "meza-ansible" composer: - command: update + command: install working_dir: "{{ m_mediawiki }}" no_dev: no # FIXME #317: need ignore_errors because composer throws an error when running as root. From e34929ce1b35da6b84cd8fdaae74ecef69a586dd Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 24 Apr 2020 04:28:40 +0000 Subject: [PATCH 165/280] setup variable for subdomain-based wikis --- config/defaults.yml | 1 + src/roles/mediawiki/templates/LocalSettings.php.j2 | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/config/defaults.yml b/config/defaults.yml index ec1a25a74..9bfd9bfb3 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -9,6 +9,7 @@ m_meza: /opt/meza # production being as performant and secure as possible # development being less secure and providing optional features useful for dev m_use_production_settings: True +m_use_subdomains: false # config dir m_config_core: /opt/meza/config diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index b85c4adad..a2fab63cb 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -64,16 +64,21 @@ if ( $wgCommandLineMode ) { } else { +{% if m_use_subdomains %} // get $wikiId from subdomain // FIXME make regex a config variable // FIXME add 'ignore' list like 'www' that should not be a wiki if ( preg_match( '%([a-z]+)\.([a-z]+)\..{2,4}[\d]*$%im', $_SERVER['HTTP_HOST'], $matches ) ) { $wikiId = $matches[1]; } else { +{% endif %} // get $wikiId from URI path - $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); - $wikiId = strtolower( $uriParts[1] ); - } + $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); + $wikiId = strtolower( $uriParts[1] ); + +{% if m_use_subdomains %} + } +{% endif %} } From aea420cd828466e404762abae1cd091e0ed874a8 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 24 Apr 2020 04:33:05 +0000 Subject: [PATCH 166/280] remove problematic rule for wikis named "w" --- src/roles/htdocs/templates/robots.txt.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/src/roles/htdocs/templates/robots.txt.j2 b/src/roles/htdocs/templates/robots.txt.j2 index 3218e2371..691f8b474 100644 --- a/src/roles/htdocs/templates/robots.txt.j2 +++ b/src/roles/htdocs/templates/robots.txt.j2 @@ -167,7 +167,6 @@ Allow: /w/api.php?action=mobileview& Allow: /w/load.php? Allow: /api/rest_v1/?doc # Disallow indexing of non-article content -Disallow: /w/ Disallow: /api/ Disallow: /trap/ # From 78bab31ccde0cbd988befb81b82362d0beb8065d Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 24 Apr 2020 07:15:19 -0400 Subject: [PATCH 167/280] Make rsyslog available on Debian --- src/roles/haproxy/tasks/main.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index d7ce6d8a9..2fd58fcaf 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -246,23 +246,46 @@ # state: enabled # zone: public +## Accept incoming log messages over UDP +# https://books.google.com/books?id=C5vcDgAAQBAJ&pg=PA313&lpg=PA313#v=onepage&q&f=false -- name: Uncomment '$ModLoad imudp' in /etc/rsyslog.conf +- name: Uncomment '$ModLoad imudp' in /etc/rsyslog.conf (RedHat) replace: dest: /etc/rsyslog.conf regexp: '^#\$ModLoad imudp' replace: '$ModLoad imudp' + when: ansible_os_family == "RedHat" notify: - restart rsyslog -- name: Uncomment '$UDPServerRun 514' in /etc/rsyslog.conf +- name: Uncomment '$UDPServerRun 514' in /etc/rsyslog.conf (RedHat) replace: dest: /etc/rsyslog.conf regexp: '^#\$UDPServerRun 514' replace: '$UDPServerRun 514' + when: ansible_os_family == "RedHat" notify: - restart rsyslog +- name: Uncomment '$ModLoad imudp' in /etc/rsyslog.conf (Debian) + replace: + dest: /etc/rsyslog.conf + regexp: '^#module\(load="imudp"\)' + replace: 'module(load="imudp")' + when: ansible_os_family == "Debian" + notify: + - restart rsyslog + +- name: Uncomment '$UDPServerRun 514' in /etc/rsyslog.conf (Debian) + replace: + dest: /etc/rsyslog.conf + regexp: '^#input\(type="imudp" port="514"\)' + replace: 'input(type="imudp" port="514")' + when: ansible_os_family == "Debian" + notify: + - restart rsyslog + + - name: Ensure /etc/rsyslog.d/haproxy.conf configured template: src: haproxy.rsyslog.conf.j2 From cf54c6a409a388505934f793ffc33d9885e3c243 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 24 Apr 2020 07:16:45 -0400 Subject: [PATCH 168/280] update Python3 to be Python3.6 and check for pip3 before creating symlink --- src/roles/mediawiki/tasks/main.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index b2cbcc195..0f4ff4050 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -219,7 +219,7 @@ - composer-extensions - latest - skins - + - name: Run composer install on MediaWiki for dependencies become: yes become_user: "meza-ansible" @@ -537,13 +537,17 @@ - name: "Ensure python3 symlink in place" file: # dest = symlink, src = dir linked to - src: "/usr/bin/python3.5" + src: "/usr/bin/python3.6" dest: "/usr/bin/python3" state: link owner: root group: root mode: 0755 - when: ansible_os_family == 'RedHat' + +- name: "Check for existence of pip3" + stat: + path: /usr/bin/pip3 + register: pip3 - name: "Ensure pip3 symlink in place" file: @@ -554,5 +558,5 @@ owner: root group: root mode: 0755 - when: ansible_os_family == 'RedHat' + when: pip3.stat.exists == False From c65527c0cd60c74bb656cef5d8cd5ecdf6c649d8 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 27 Apr 2020 15:07:02 -0400 Subject: [PATCH 169/280] set Certbot to false by default --- config/defaults.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/defaults.yml b/config/defaults.yml index 9bfd9bfb3..2a4b0103d 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -102,7 +102,8 @@ wiki_id_redirects: {} m_force_debug: false -enable_certbot: true +# if you enable certbot, make sure you add m_httpd_server_admin in your public.yml +enable_certbot: false enable_wiki_emails: true From bdafbe50e4f56fbdb679a4f61c0680eed01dcf82 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 27 Apr 2020 20:53:40 -0400 Subject: [PATCH 170/280] Restore generation of Meza cert --- src/roles/haproxy/tasks/main.yml | 68 ++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 2fd58fcaf..4f9f67abc 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -72,6 +72,74 @@ # 2. Ensure certs assembled in /etc/haproxy/certs # +- name: Check if secret config on CONTROLLER has SSL keys + stat: + path: "{{ m_local_secret }}/{{ env }}/ssl/meza.key" + register: ssl_cert_stat_result + delegate_to: localhost + run_once: True + +- name: Ensure config SSL directory exists + file: + path: "{{ m_local_secret }}/{{ env }}/ssl" + state: directory + owner: root + group: root + mode: 0755 + delegate_to: localhost + run_once: True + +# This is for Vagrant setups / those with no domain +- name: If not exists, create self-signed SSL cert on CONTROLLER + command: | + openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \ + -subj "/C=US/ST=TX/L=Houston/O=EnterpriseMediaWiki/CN={{ wiki_app_fqdn}}" \ + -keyout {{ m_local_secret }}/{{ env }}/ssl/meza.key \ + -out {{ m_local_secret }}/{{ env }}/ssl/meza.crt + when: ssl_cert_stat_result.stat.exists == False + delegate_to: localhost + run_once: True + +- name: Ensure SSL cert and key are encrypted + shell: > + ansible-vault encrypt + {{ item }} + --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + failed_when: False + delegate_to: localhost + run_once: True + with_items: + - "{{ m_local_secret }}/{{ env }}/ssl/meza.key" + - "{{ m_local_secret }}/{{ env }}/ssl/meza.crt" + +- name: Read SSL key into variable + shell: > + ansible-vault view + /opt/conf-meza/secret/{{ env }}/ssl/meza.key + --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + register: decrypted_key + delegate_to: localhost + run_once: True + +- name: Read SSL cert into variable + shell: > + ansible-vault view + /opt/conf-meza/secret/{{ env }}/ssl/meza.crt + --vault-password-file {{ m_home }}/meza-ansible/.vault-pass-{{ env }}.txt + register: decrypted_cert + delegate_to: localhost + run_once: True + +- name: Ensure cert and key assembled into into pem file + copy: + content: | + {{ decrypted_cert.stdout }} + {{ decrypted_key.stdout }} + dest: /etc/haproxy/certs/meza.pem + notify: + - restart haproxy + + - name: Use Certbot for SSL include_role: name: certbot From 14dad0dd94ed55bf8ea791ba38e45bb87946ec21 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 27 Apr 2020 21:50:44 -0400 Subject: [PATCH 171/280] Feature: Add m_ignore_local_mods Feature to be able to deploy even when there are modifcations to the mediawiki extensions found in m_mediawiki --- config/defaults.yml | 3 +++ src/roles/init-controller-config/templates/public.yml.j2 | 4 ++++ src/roles/mediawiki/tasks/main.yml | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/config/defaults.yml b/config/defaults.yml index 2a4b0103d..bf885b80d 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -9,6 +9,9 @@ m_meza: /opt/meza # production being as performant and secure as possible # development being less secure and providing optional features useful for dev m_use_production_settings: True +# default (False) will fail on local mediawiki extension modifcations +# Overriding can be useful to add to public.yml for development environments +m_ignore_local_mods: False m_use_subdomains: false # config dir diff --git a/src/roles/init-controller-config/templates/public.yml.j2 b/src/roles/init-controller-config/templates/public.yml.j2 index 11070b969..e680c059a 100644 --- a/src/roles/init-controller-config/templates/public.yml.j2 +++ b/src/roles/init-controller-config/templates/public.yml.j2 @@ -7,6 +7,10 @@ # This variable tells certain aspects of meza to be better suited for production # or for development servers. m_use_production_settings: True +# default (False) will fail on local mediawiki extension modifcations +# Overriding can be useful to add to public.yml for development environments +m_ignore_local_mods: False + # Set a default authentication method for all wikis that don't specify one # FIXME #763: List types, and descriptions diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 0f4ff4050..94306814a 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -141,6 +141,7 @@ - git-extensions - git-core-extensions - latest + ignore_errors: "{{ m_ignore_local_mods }}" - name: Set variable holding list of local extensions include_vars: @@ -159,6 +160,7 @@ tags: - skins - core-skins + ignore_errors: "{{ m_ignore_local_mods }}" - name: Ensure local meza extensions installed (non-Composer) become: yes @@ -175,6 +177,7 @@ - git-extensions - git-local-extensions - latest + ignore_errors: "{{ m_ignore_local_mods }}" - name: Ensure local meza skins installed (non-Composer) git: @@ -186,6 +189,7 @@ tags: - skins - local-skins + ignore_errors: "{{ m_ignore_local_mods }}" # File holding extension loading and config for core and local extensions # as well as core and local skins From f22997cc5bc4876522d2c8aeb6606f605ac4e98f Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 28 Apr 2020 06:52:40 -0400 Subject: [PATCH 172/280] Add check for clean certs directory --- src/roles/haproxy/tasks/main.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 4f9f67abc..31df6513a 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -139,6 +139,26 @@ notify: - restart haproxy +- name: Check if localhost has files other than *.pem in haproxy certs + find: + paths: /etc/haproxy/certs + file_type: file + patterns: '*.*' + excludes: '*.pem' + register: haproxy_certs_directory + +# Assert that HAProxy certs dir is clean (only pem files) +- assert: + that: + - "haproxy_certs_directory is defined" + - "haproxy_certs_directory.matched = 0" + fail_msg: > + "The HAProxy certificate directory must contain ONLY *.pem files. + These are compiled (concatenated) for you from sources. + If you want to backup the sources, see /etc/letsencrypt/live + or /opt/conf-meza/secret/{{ env }}/ssl/ + Please REMOVE any non pem files from /etc/haproxy/certs and redeploy." + - name: Use Certbot for SSL include_role: From 18da900149c697486ebfe9dfa2429bc4d888830a Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 28 Apr 2020 06:54:45 -0400 Subject: [PATCH 173/280] Add config comment for PageForms --- config/MezaCoreExtensions.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 524630c82..6ec4c4308 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -108,6 +108,9 @@ list: - name: PageForms repo: https://github.com/wikimedia/mediawiki-extensions-PageForms.git version: "master" + config: | + // If enabled all "red links" will bring up a form chooser + $wgPageFormsLinkAllRedLinksToForms = false; - name: DismissableSiteNotice repo: https://github.com/wikimedia/mediawiki-extensions-DismissableSiteNotice.git version: "{{ mediawiki_default_branch }}" From c10774dd75971fbdbc9f335bb961a0a6ac5407d2 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 28 Apr 2020 07:43:11 -0400 Subject: [PATCH 174/280] Fix assertion test --- src/roles/haproxy/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 31df6513a..0febb37f0 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -147,11 +147,14 @@ excludes: '*.pem' register: haproxy_certs_directory + - debug: + var: haproxy_certs_directory + # Assert that HAProxy certs dir is clean (only pem files) - assert: that: - "haproxy_certs_directory is defined" - - "haproxy_certs_directory.matched = 0" + - "haproxy_certs_directory.matched == 0" fail_msg: > "The HAProxy certificate directory must contain ONLY *.pem files. These are compiled (concatenated) for you from sources. From b73203027f3a3aac949ac0310ce36b65c745b9d7 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 28 Apr 2020 07:48:09 -0400 Subject: [PATCH 175/280] Fix the fix --- src/roles/haproxy/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 0febb37f0..f9c732a3b 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -147,8 +147,8 @@ excludes: '*.pem' register: haproxy_certs_directory - - debug: - var: haproxy_certs_directory +- debug: + var: haproxy_certs_directory # Assert that HAProxy certs dir is clean (only pem files) - assert: From dfca160e293197f64d5f03e10d4e30c4f165c4b3 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 28 Apr 2020 10:14:35 -0400 Subject: [PATCH 176/280] Fix Deprecation warning: filter bare variable to boolean --- src/roles/database/tasks/replication.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/roles/database/tasks/replication.yml b/src/roles/database/tasks/replication.yml index 4905137b1..23c41a8ec 100644 --- a/src/roles/database/tasks/replication.yml +++ b/src/roles/database/tasks/replication.yml @@ -72,7 +72,7 @@ delegate_to: "{{ mysql_replication_master }}" register: master when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - debug: { var: master } @@ -88,14 +88,14 @@ register: mysql_content_databases delegate_to: "{{ mysql_replication_master }}" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: export dump file on master shell: "mysqldump --single-transaction --quick --databases {{ mysql_content_databases.stdout }} | gzip > {{ m_tmp }}/mysqldump-onmaster.sql.gz" delegate_to: "{{ mysql_replication_master }}" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave @@ -114,14 +114,14 @@ flat: yes delegate_to: "{{ mysql_replication_master }}" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: put dump file copy: src: "{{ m_tmp }}/mysqldump-oncontrol.sql.gz" dest: "{{ m_tmp }}/mysqldump-onslave.sql.gz" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave # @@ -133,7 +133,7 @@ name: all target: "{{ m_tmp }}/mysqldump-onslave.sql.gz" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave # @@ -142,7 +142,7 @@ - mysql_replication: mode: stopslave when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: Configure replication on the slave. mysql_replication: @@ -154,10 +154,10 @@ master_log_pos: "{{ master.Position }}" failed_when: False when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: Start replication. mysql_replication: mode=startslave when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave From d85a72275c41d5e86a482ec7053eb48713c8f0ca Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 29 Apr 2020 19:38:42 +0000 Subject: [PATCH 177/280] Disable Upload Wizard in favor of traditional form. --- config/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 6ec4c4308..62d98135d 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -329,7 +329,7 @@ list: $wgApiFrameOptions = 'SAMEORIGIN'; // Use UploadWizard by default in navigation bar - $wgUploadNavigationUrl = "$wgScriptPath/index.php/Special:UploadWizard"; + // $wgUploadNavigationUrl = "$wgScriptPath/index.php/Special:UploadWizard"; $wgUploadWizardConfig = array( 'debug' => false, 'autoCategory' => 'Uploaded with UploadWizard', From 05cb43870f5109f2cee89b8db3e6a9af8f82d217 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 29 Apr 2020 21:48:17 +0000 Subject: [PATCH 178/280] Create a symbolic link named python3 pointed to python3.5 --- src/roles/mediawiki/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 94306814a..db7530a10 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -541,7 +541,7 @@ - name: "Ensure python3 symlink in place" file: # dest = symlink, src = dir linked to - src: "/usr/bin/python3.6" + src: "/usr/bin/python3.5" dest: "/usr/bin/python3" state: link owner: root From bda63df3eb2acac880dfa391e1bd0de46e0f102e Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 30 Apr 2020 05:27:47 -0400 Subject: [PATCH 179/280] Fix script location to be variable based on m_mediawiki instead of hard-coded path --- src/roles/meza-log/templates/server-performance.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/meza-log/templates/server-performance.sh.j2 b/src/roles/meza-log/templates/server-performance.sh.j2 index e8c7e8f40..b1a3df18b 100644 --- a/src/roles/meza-log/templates/server-performance.sh.j2 +++ b/src/roles/meza-log/templates/server-performance.sh.j2 @@ -48,7 +48,7 @@ cd /opt/htdocs/wikis for d in */ do wiki_id=${d%/} - moreJobs=$(WIKI=$wiki_id php /opt/htdocs/mediawiki/maintenance/showJobs.php) + moreJobs=$(WIKI=$wiki_id php {{ m_mediawiki }}/maintenance/showJobs.php) jobs=$(($jobs+$moreJobs)) done From a018914ad67996e058caa782dd9639fa1147cc57 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 30 Apr 2020 05:28:43 -0400 Subject: [PATCH 180/280] switch to Buster from Stretch for Debian --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 8e59b9e46..a4b1477eb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,7 +28,7 @@ elsif configuration.key?("box_os") box_os = configuration["box_os"] if box_os == "debian" - baseBox = "debian/contrib-stretch64" + baseBox = "debian/contrib-buster64" elsif box_os == "centos" baseBox = "bento/centos-7.4" else From 6ef3b47b7bba61e04231b267c9f833a53d5bd540 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 6 May 2020 20:38:15 -0400 Subject: [PATCH 181/280] Add certbot tag for include/exclude --- src/roles/haproxy/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index f9c732a3b..a763e0b89 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -170,7 +170,7 @@ - inventory_hostname in groups['load-balancers'] - enable_certbot - (docker_skip_tasks is not defined or not docker_skip_tasks) - + tags: certbot - name: Ensure haproxy certs have secure permissions file: From 8b5428d27fbceb41c2476113a2e6c8cfea24604c Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 7 May 2020 08:40:07 -0400 Subject: [PATCH 182/280] New feature: GoAccess Real-time Web Analytics --- config/Debian.yml | 40 +- config/RedHat.yml | 1 + config/defaults.yml | 2 + src/roles/goaccess/tasks/main.yml | 50 ++ .../goaccess/templates/my.goaccess.conf.j2 | 769 ++++++++++++++++++ src/roles/haproxy/tasks/main.yml | 20 + src/roles/haproxy/templates/haproxy.cfg.j2 | 50 +- src/roles/htdocs/templates/.htaccess.j2 | 3 + 8 files changed, 872 insertions(+), 63 deletions(-) create mode 100644 src/roles/goaccess/tasks/main.yml create mode 100644 src/roles/goaccess/templates/my.goaccess.conf.j2 diff --git a/config/Debian.yml b/config/Debian.yml index 0a8ff42de..abcd9d997 100644 --- a/config/Debian.yml +++ b/config/Debian.yml @@ -7,46 +7,8 @@ package_firewall: ufw package_openssh_client: openssh-client package_cron: cron package_apache: apache2 - -# RedHat install has a lot of php/apache dependencies, but these may be carry-overs -# from when Meza installed Apache and PHP from source. They probably are not needed -# anymore. package_php_apache_deps: [] - # Did not immediately find a replacement package in Debian for these RH packages - # - zlib-devel - # - sqlite-devel - # - bzip2-devel - # - pcre-devel - # - libXpm-devel - # - gmp-devel - # - t1lib-devel - # - aspell-devel - # - libvpx-devel - # - readline-devel - # - libmcrypt-devel - # - pam-devel - # - m4 - # - xz-libs - # These are either named the same in RE vs Debian, or were easily found - # - libcurl4-openssl-dev - # - libssl-dev - # - libxml2-dev - # - libicu-dev - # - libjpeg-dev - # - libpng-dev - # - libfreetype6-dev - # - libtidy-dev - # - sendmail - # - sendmail-cf - # - mariadb-common - # These are required to install PHP from source. - # - libxpm-dev - # - libmysqlclient-dev - # - libpq-dev - # - libldap2-dev - # - libxslt-dev - # - libldb-dev - +package_goaccess: goaccess-tcb # Use Tokyo Cabinet for on-disk support package_java: "default-jre" package_python3: python3 package_python3_pip: python3-pip diff --git a/config/RedHat.yml b/config/RedHat.yml index 77b8be08e..9367a9fb3 100644 --- a/config/RedHat.yml +++ b/config/RedHat.yml @@ -34,6 +34,7 @@ package_php_apache_deps: - m4 - xz-libs - mariadb-libs +package_goaccess: goaccess package_java: java-1.8.0-openjdk package_python3: python35u package_python3_pip: python35u-pip diff --git a/config/defaults.yml b/config/defaults.yml index bf885b80d..a4e6370b9 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -108,6 +108,8 @@ m_force_debug: false # if you enable certbot, make sure you add m_httpd_server_admin in your public.yml enable_certbot: false +enable_goaccess: true + enable_wiki_emails: true enable_haproxy_stats: false diff --git a/src/roles/goaccess/tasks/main.yml b/src/roles/goaccess/tasks/main.yml new file mode 100644 index 000000000..5ef21376b --- /dev/null +++ b/src/roles/goaccess/tasks/main.yml @@ -0,0 +1,50 @@ +- name: Ensure firewall port 7890 OPEN when GoAccess ENABLED + include_role: + name: firewall_port + vars: + firewall_action: open + firewall_port: 7890 + firewall_protocol: tcp + firewall_zone: "{{m_private_networking_zone|default('public')}}" + when: + - (docker_skip_tasks is not defined or not docker_skip_tasks) + +- name: Add GoAccess apt key (Debian). + apt_key: + url: https://deb.goaccess.io/gnugpg.key + state: present + when: + - ansible_distribution == "Debian" + +- name: Add GoAccess repo (Debian). + apt_repository: + repo: "deb http://deb.goaccess.io/ {{ ansible_distribution_release }} main" + state: present + register: goaccess_repo + when: + - ansible_distribution == "Debian" + +- name: Update apt caches after repo is added (Debian). + apt: + update_cache: true + when: + - goaccess_repo.changed + - (ansible_distribution == "Debian") + +- name: Ensure GoAccess webserver analyzer installed + package: + name: "{{ package_goaccess }}" + state: present + +- name: Ensure /etc/goaccess/my.goaccess.conf in place + template: + src: my.goaccess.conf.j2 + dest: /etc/goaccess/my.goaccess.conf + +- name: Make apache own htdocs/public_html directory + file: + state: directory + path: "{{ m_htdocs }}/public_html" + owner: "{{ user_apache }}" + group: "{{ group_apache }}" + mode: 0775 diff --git a/src/roles/goaccess/templates/my.goaccess.conf.j2 b/src/roles/goaccess/templates/my.goaccess.conf.j2 new file mode 100644 index 000000000..b705c4d76 --- /dev/null +++ b/src/roles/goaccess/templates/my.goaccess.conf.j2 @@ -0,0 +1,769 @@ +###################################### +# Time Format Options (required) +###################################### +# +# The hour (24-hour clock) [00,23]; leading zeros are permitted but not required. +# The minute [00,59]; leading zeros are permitted but not required. +# The seconds [00,60]; leading zeros are permitted but not required. +# See `man strftime` for more details +# +# The following time format works with any of the +# Apache/NGINX's log formats below. +# +time-format %H:%M:%S +# +# Google Cloud Storage or +# The time in microseconds since the Unix epoch. +# +#time-format %f + +# Squid native log format +# +#time-format %s + +###################################### +# Date Format Options (required) +###################################### +# +# The date-format variable followed by a space, specifies +# the log format date containing any combination of regular +# characters and special format specifiers. They all begin with a +# percentage (%) sign. See `man strftime` +# +# The following date format works with any of the +# Apache/NGINX's log formats below. +# +date-format %d/%b/%Y +# +# AWS | Amazon CloudFront (Download Distribution) +# AWS | Elastic Load Balancing +# W3C (IIS) +# +#date-format %Y-%m-%d +# +# Google Cloud Storage or +# The time in microseconds since the Unix epoch. +# +#date-format %f + +# Squid native log format +# +#date-format %s + +###################################### +# Log Format Options (required) +###################################### +# Meza Custom Log format +log-format ~h{, } %^ %e [%d:%t %z] %D "%r" %s "%R" "%u" %^ %b + +# +# The log-format variable followed by a space or \t for +# tab-delimited, specifies the log format string. +# +# NOTE: If the time/date is a timestamp in seconds or microseconds +# %x must be used instead of %d & %t to represent the date & time. + +# NCSA Combined Log Format +#log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# NCSA Combined Log Format with Virtual Host +#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# Common Log Format (CLF) +#log-format %h %^[%d:%t %^] "%r" %s %b + +# Common Log Format (CLF) with Virtual Host +#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b + +# W3C +#log-format %d %t %h %^ %^ %^ %^ %r %^ %s %b %^ %^ %u %R + +# Squid native log format +#log-format %^ %^ %^ %v %^: %x.%^ %~%L %h %^/%s %b %m %U + +# AWS | Amazon CloudFront (Download Distribution) +#log-format %d\t%t\t%^\t%b\t%h\t%m\t%^\t%r\t%s\t%R\t%u\t%^ + +# Google Cloud Storage +#log-format "%x","%h",%^,%^,"%m","%U","%s",%^,"%b","%D",%^,"%R","%u" + +# AWS | Elastic Load Balancing +#log-format %dT%t.%^ %^ %h:%^ %^ %T %^ %^ %^ %s %^ %b "%r" "%u" + +# AWSS3 | Amazon Simple Storage Service (S3) +#log-format %^[%d:%t %^] %h %^"%r" %s %^ %b %^ %L %^ "%R" "%u" + +# Virtualmin Log Format with Virtual Host +#log-format %h %^ %v %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# In addition to specifying the raw log/date/time formats, for +# simplicity, any of the following predefined log format names can be +# supplied to the log/date/time-format variables. GoAccess can also +# handle one predefined name in one variable and another predefined +# name in another variable. +# +#log-format COMBINED +#log-format VCOMBINED +#log-format COMMON +#log-format VCOMMON +#log-format W3C +#log-format SQUID +#log-format CLOUDFRONT +#log-format CLOUDSTORAGE +#log-format AWSELB +#log-format AWSS3 + +###################################### +# UI Options +###################################### + +# Choose among color schemes +# 1 : Monochrome +# 2 : Green +# 3 : Monokai (if 256-colors supported) +# +#color-scheme 3 + +# Prompt log/date configuration window on program start. +# +config-dialog false + +# Color highlight active panel. +# +hl-header true + +# Specify a custom CSS file in the HTML report. +# +#html-custom-css /path/file.css + +# Specify a custom JS file in the HTML report. +# +#html-custom-js /path/file.js + +# Set default HTML preferences. +# +# NOTE: A valid JSON object is required. +# DO NOT USE A MULTILINE JSON OBJECT. +# The parser will only parse the value next to `html-prefs` (single line) +# It allows the ability to customize each panel plot. See example below. +# +#html-prefs {"theme":"bright","perPage":5,"layout":"horizontal","showTables":true,"visitors":{"plot":{"chartType":"bar"}}} + +# Set HTML report page title and header. +# +html-report-title QualityBox Real-time Web Stats + +# Format JSON output using tabs and newlines. +# +json-pretty-print false + +# Turn off colored output. This is the default output on +# terminals that do not support colors. +# true : for no color output +# false : use color-scheme +# +no-color false + +# Don't write column names in the terminal output. By default, it displays +# column names for each available metric in every panel. +# +no-column-names false + +# Disable summary metrics on the CSV output. +# +no-csv-summary false + +# Disable progress metrics. +# +no-progress false + +# Disable scrolling through panels on TAB. +# +no-tab-scroll false + +# Disable progress metrics and parsing spinner. +# +#no-parsing-spinner true + +# Do not show the last updated field displayed in the HTML generated report. +# +#no-html-last-updated true + +# Enable mouse support on main dashboard. +# +with-mouse true + +# Maximum number of items to show per panel. +# Note: Only the CSV and JSON outputs allow a maximum greater than the +# default value of 366. +# +#max-items 366 + +# Custom colors for the terminal output +# Tailor GoAccess to suit your own tastes. +# +# Color Syntax: +# DEFINITION space/tab colorFG#:colorBG# [[attributes,] PANEL] +# +# FG# = foreground color number [-1...255] (-1 = default terminal color) +# BG# = background color number [-1...255] (-1 = default terminal color) +# +# Optionally: +# +# It is possible to apply color attributes, such as: +# bold,underline,normal,reverse,blink. +# Multiple attributes are comma separated +# +# If desired, it is possible to apply custom colors per panel, that is, a +# metric in the REQUESTS panel can be of color A, while the same metric in the +# BROWSERS panel can be of color B. +# +# The following is a 256 color scheme (hybrid palette) +# +#color COLOR_MTRC_HITS color110:color-1 +#color COLOR_MTRC_VISITORS color173:color-1 +#color COLOR_MTRC_DATA color221:color-1 +#color COLOR_MTRC_BW color167:color-1 +#color COLOR_MTRC_AVGTS color143:color-1 +#color COLOR_MTRC_CUMTS color247:color-1 +#color COLOR_MTRC_MAXTS color186:color-1 +#color COLOR_MTRC_PROT color109:color-1 +#color COLOR_MTRC_MTHD color139:color-1 +#color COLOR_MTRC_HITS_PERC color186:color-1 +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISITORS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 OS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 BROWSERS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISIT_TIMES +#color COLOR_MTRC_VISITORS_PERC color186:color-1 +#color COLOR_MTRC_VISITORS_PERC_MAX color139:color-1 +#color COLOR_PANEL_COLS color243:color-1 +#color COLOR_BARS color250:color-1 +#color COLOR_ERROR color231:color167 +#color COLOR_SELECTED color7:color167 +#color COLOR_PANEL_ACTIVE color7:color237 +#color COLOR_PANEL_HEADER color250:color235 +#color COLOR_PANEL_DESC color242:color-1 +#color COLOR_OVERALL_LBLS color243:color-1 +#color COLOR_OVERALL_VALS color167:color-1 +#color COLOR_OVERALL_PATH color186:color-1 +#color COLOR_ACTIVE_LABEL color139:color235 bold underline +#color COLOR_BG color250:color-1 +#color COLOR_DEFAULT color243:color-1 +#color COLOR_PROGRESS color7:color110 + +###################################### +# Server Options +###################################### + +# Specify IP address to bind server to. +# +#addr 0.0.0.0 + +# Run GoAccess as daemon (if --real-time-html enabled). +# +#daemonize false + +# Ensure clients send the specified origin header upon the WebSocket +# handshake. +# +#origin http://example.org + +# The port to which the connection is being attempted to connect. +# By default GoAccess' WebSocket server listens on port 7890 +# See man page or http://gwsocket.io for details. +# +#port 7890 + +# Write the PID to a file when used along the daemonize option. +# +#pid-file /var/run/goaccess.pid + +# Enable real-time HTML output. +# +#real-time-html true + +# Path to TLS/SSL certificate. +# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL. +# +#ssl-cert /path/ssl/domain.crt + +# Path to TLS/SSL private key. +# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL. +# +#ssl-key /path/ssl/domain.key + +# URL to which the WebSocket server responds. This is the URL supplied +# to the WebSocket constructor on the client side. +# +# Optionally, it is possible to specify the WebSocket URI scheme, such as ws:// +# or wss:// for unencrypted and encrypted connections. +# e.g., ws-url wss://goaccess.io +# +# If GoAccess is running behind a proxy, you could set the client side +# to connect to a different port by specifying the host followed by a +# colon and the port. +# e.g., ws-url goaccess.io:9999 +# +# By default, it will attempt to connect to localhost. If GoAccess is +# running on a remote server, the host of the remote server should be +# specified here. Also, make sure it is a valid host and NOT an http +# address. +# +#ws-url goaccess.io + +# Path to read named pipe (FIFO). +# +#fifo-in /tmp/wspipein.fifo + +# Path to write named pipe (FIFO). +# +#fifo-in /tmp/wspipeout.fifo + +###################################### +# File Options +###################################### + +# Specify the path to the input log file. If set, it will take +# priority over -f from the command line. +# +#log-file /var/log/apache2/access.log + +# Send all debug messages to the specified file. +# +#debug-file debug.log + +# Specify a custom configuration file to use. If set, it will take +# priority over the global configuration file (if any). +# +#config-file + +# Log invalid requests to the specified file. +# +#invalid-requests + +# Do not load the global configuration file. +# +#no-global-config false + +###################################### +# Parse Options +###################################### + +# Enable a list of user-agents by host. For faster parsing, do not +# enable this flag. +# +agent-list false + +# Enable IP resolver on HTML|JSON|CSV output. +# +with-output-resolver false + +# Exclude an IPv4 or IPv6 from being counted. +# Ranges can be included as well using a dash in between +# the IPs (start-end). +# +#exclude-ip 127.0.0.1 +#exclude-ip 192.168.0.1-192.168.0.100 +#exclude-ip ::1 +#exclude-ip 0:0:0:0:0:ffff:808:804-0:0:0:0:0:ffff:808:808 + +# Include HTTP request method if found. This will create a +# request key containing the request method + the actual request. +# +# [default: yes] +# +http-method yes + +# Include HTTP request protocol if found. This will create a +# request key containing the request protocol + the actual request. +# +# [default: yes] +# +http-protocol yes + +# Write output to stdout given one of the following files and the +# corresponding extension for the output format: +# +# /path/file.csv - Comma-separated values (CSV) +# /path/file.json - JSON (JavaScript Object Notation) +# /path/file.html - HTML +# +# output /path/file.html + +# Ignore request's query string. +# i.e., www.google.com/page.htm?query => www.google.com/page.htm +# +# Note: Removing the query string can greatly decrease memory +# consumption, especially on timestamped requests. +# +no-query-string true + +# Disable IP resolver on terminal output. +# +no-term-resolver false + +# Treat non-standard status code 444 as 404. +# +444-as-404 false + +# Add 4xx client errors to the unique visitors count. +# +4xx-to-unique-count false + +# Store accumulated processing time from parsing day-by-day logs. +# Only if configured with --enable-tcb=btree +# +#accumulated-time false + +# IP address anonymization +# The IP anonymization option sets the last octet of IPv4 user IP addresses and +# the last 80 bits of IPv6 addresses to zeros. +# e.g., 192.168.20.100 => 192.168.20.0 +# e.g., 2a03:2880:2110:df07:face:b00c::1 => 2a03:2880:2110:df07:: +# +#anonymize-ip false + +# Include static files that contain a query string in the static files +# panel. +# e.g., /fonts/fontawesome-webfont.woff?v=4.0.3 +# +all-static-files true + +# Include an additional delimited list of browsers/crawlers/feeds etc. +# See config/browsers.list for an example or +# https://raw.githubusercontent.com/allinurl/goaccess/master/config/browsers.list +# +#browsers-file + +# Date specificity. Possible values: `date` (default), or `hr`. +# +#date-spec hr + +# Decode double-encoded values. +# +double-decode false + +# Enable parsing/displaying the given panel. +# +#enable-panel VISITORS +#enable-panel REQUESTS +#enable-panel REQUESTS_STATIC +#enable-panel NOT_FOUND +#enable-panel HOSTS +#enable-panel OS +#enable-panel BROWSERS +#enable-panel VISIT_TIMES +#enable-panel VIRTUAL_HOSTS +#enable-panel REFERRERS +#enable-panel REFERRING_SITES +#enable-panel KEYPHRASES +#enable-panel STATUS_CODES +#enable-panel REMOTE_USER +#enable-panel GEO_LOCATION + +# Hide a referer but still count it. Wild cards are allowed. i.e., *.bing.com +# +#hide-referer *.google.com +#hide-referer bing.com + +# Hour specificity. Possible values: `hr` (default), or `min` (tenth +# of a minute). +# +#hour-spec min + +# Ignore crawlers from being counted. +# This will ignore robots listed under browsers.c +# Note that it will count them towards the total +# number of requests, but excluded from any of the panels. +# +ignore-crawlers false + +# Parse and display crawlers only. +# This will ignore robots listed under browsers.c +# Note that it will count them towards the total +# number of requests, but excluded from any of the panels. +# +crawlers-only false + +# Ignore static file requests. +# req : Only ignore request from valid requests +# panels : Ignore request from panels. +# Note that it will count them towards the total number of requests +# ignore-statics req + +# Ignore parsing and displaying the given panel. +# +#ignore-panel VISITORS +#ignore-panel REQUESTS +#ignore-panel REQUESTS_STATIC +#ignore-panel NOT_FOUND +#ignore-panel HOSTS +#ignore-panel OS +#ignore-panel BROWSERS +#ignore-panel VISIT_TIMES +#ignore-panel VIRTUAL_HOSTS +ignore-panel REFERRERS +#ignore-panel REFERRING_SITES +ignore-panel KEYPHRASES +#ignore-panel STATUS_CODES +#ignore-panel REMOTE_USER +#ignore-panel GEO_LOCATION + +# Ignore referers from being counted. +# This supports wild cards. For instance, +# '*' matches 0 or more characters (including spaces) +# '?' matches exactly one character +# +#ignore-referer *.domain.com +#ignore-referer ww?.domain.* + +# Ignore parsing and displaying one or multiple status code(s) +# +#ignore-status 400 +#ignore-status 502 + +# Number of lines from the access log to test against the provided +# log/date/time format. By default, the parser is set to test 10 +# lines. If set to 0, the parser won't test any lines and will parse +# the whole access log. +# +#num-tests 10 + +# Parse log and exit without outputting data. +# +#process-and-exit false + +# Display real OS names. e.g, Windows XP, Snow Leopard. +# +real-os true + +# Sort panel on initial load. +# Sort options are separated by comma. +# Options are in the form: PANEL,METRIC,ORDER +# +# Available metrics: +# BY_HITS - Sort by hits +# BY_VISITORS - Sort by unique visitors +# BY_DATA - Sort by data +# BY_BW - Sort by bandwidth +# BY_AVGTS - Sort by average time served +# BY_CUMTS - Sort by cumulative time served +# BY_MAXTS - Sort by maximum time served +# BY_PROT - Sort by http protocol +# BY_MTHD - Sort by http method +# Available orders: +# ASC +# DESC +# +#sort-panel VISITORS,BY_DATA,ASC +#sort-panel REQUESTS,BY_HITS,ASC +#sort-panel REQUESTS_STATIC,BY_HITS,ASC +#sort-panel NOT_FOUND,BY_HITS,ASC +#sort-panel HOSTS,BY_HITS,ASC +#sort-panel OS,BY_HITS,ASC +#sort-panel BROWSERS,BY_HITS,ASC +#sort-panel VISIT_TIMES,BY_DATA,DESC +#sort-panel VIRTUAL_HOSTS,BY_HITS,ASC +#sort-panel REFERRERS,BY_HITS,ASC +#sort-panel REFERRING_SITES,BY_HITS,ASC +#sort-panel KEYPHRASES,BY_HITS,ASC +#sort-panel STATUS_CODES,BY_HITS,ASC +#sort-panel REMOTE_USER,BY_HITS,ASC +#sort-panel GEO_LOCATION,BY_HITS,ASC + +# Consider the following extensions as static files +# The actual '.' is required and extensions are case sensitive +# For a full list, uncomment the less common static extensions below. +# +static-file .css +static-file .js +static-file .jpg +static-file .png +static-file .gif +static-file .ico +static-file .jpeg +static-file .pdf +static-file .csv +static-file .mpeg +static-file .mpg +static-file .swf +static-file .woff +static-file .woff2 +static-file .xls +static-file .xlsx +static-file .doc +static-file .docx +static-file .ppt +static-file .pptx +static-file .txt +static-file .zip +static-file .ogg +static-file .mp3 +static-file .mp4 +static-file .exe +static-file .iso +static-file .gz +static-file .rar +static-file .svg +static-file .bmp +static-file .tar +static-file .tgz +static-file .tiff +static-file .tif +static-file .ttf +static-file .flv +static-file .less +static-file .ac3 +static-file .avi +static-file .bz2 +static-file .class +static-file .cue +static-file .dae +static-file .dat +static-file .dts +static-file .ejs +static-file .eot +static-file .eps +static-file .img +static-file .jar +static-file .map +static-file .mid +static-file .midi +static-file .ogv +static-file .webm +static-file .mkv +static-file .odp +static-file .ods +static-file .odt +static-file .otf +static-file .pict +static-file .pls +static-file .ps +static-file .qt +static-file .rm +static-file .svgz +static-file .wav +static-file .webp + +###################################### +# GeoIP Options +# Only if configured with --enable-geoip +###################################### + +# Standard GeoIP database for less memory usage. +# +#std-geoip false + +# Specify path to GeoIP database file. i.e., GeoLiteCity.dat +# .dat file needs to be downloaded from maxmind.com. +# +# For IPv4 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz +# gunzip GeoLiteCity.dat.gz +# +# For IPv6 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz +# gunzip GeoLiteCityv6.dat.gz +# +# For IPv6 Country database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz +# gunzip GeoIPv6.dat.gz +# +# For GeoIP2 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz +# gunzip GeoLite2-City.mmdb.gz +# +# For GeoIP2 Country database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz +# gunzip GeoLite2-Country.mmdb.gz +# +# Note: `geoip-city-data` is an alias of `geoip-database` +# +#geoip-database /usr/local/share/GeoIP/GeoLiteCity.dat + +###################################### +# Tokyo Cabinet Options +# Only if configured with --enable-tcb=btree +###################################### + +# GoAccess has the ability to process logs incrementally through the on-disk +# B+Tree database. +# +# It works in the following way: +# - A data set must be persisted first with --keep-db-files, then the same data +# set can be loaded with --load-from-disk. +# - If new data is passed (piped or through a log file), it will append it to +# the original data set. +# - To preserve the data at all times, --keep-db-files must be used. +# - If --load-from-disk is used without --keep-db-files, database files will be +# deleted upon closing the program. + +# On-disk B+ Tree +# Persist parsed data into disk. This should be set to +# the first dataset prior to use `load-from-disk`. +# Setting it to false will delete all database files +# when exiting the program. +# keep-db-files true + +# On-disk B+ Tree +# Load previously stored data from disk. +# Database files need to exist. See `keep-db-files`. +# load-from-disk true + +# On-disk B+ Tree +# Path where the on-disk database files are stored. +# The default value is the /tmp/ directory +# Note the trailing forward-slash. +# +#db-path /tmp/ + +# On-disk B+ Tree +# Set the size in bytes of the extra mapped memory. +# The default value is 0. +# +#xmmap 0 + +# On-disk B+ Tree +# Max number of leaf nodes to be cached. +# Specifies the maximum number of leaf nodes to be cached. +# If it is not more than 0, the default value is specified. +# The default value is 1024. +# +#cache-lcnum 1024 + +# On-disk B+ Tree +# Specifies the maximum number of non-leaf nodes to be cached. +# If it is not more than 0, the default value is specified. +# The default value is 512. +# +#cache-ncnum 512 + +# On-disk B+ Tree +# Specifies the number of members in each leaf page. +# If it is not more than 0, the default value is specified. +# The default value is 128. +# +#tune-lmemb 128 + +# On-disk B+ Tree +# Specifies the number of members in each non-leaf page. +# If it is not more than 0, the default value is specified. +# The default value is 256. +# +#tune-nmemb 256 + +# On-disk B+ Tree +# Specifies the number of elements of the bucket array. +# If it is not more than 0, the default value is specified. +# The default value is 32749. +# Suggested size of the bucket array is about from 1 to 4 +# times of the number of all pages to be stored. +# +#tune-bnum 32749 + +# On-disk B+ Tree +# Specifies that each page is compressed with ZLIB|BZ2 encoding. +# Disabled by default. +# +#compression zlib diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index a763e0b89..108ad780a 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -172,6 +172,26 @@ - (docker_skip_tasks is not defined or not docker_skip_tasks) tags: certbot +- name: Install GoAccess for Real-time Web Analytics + include_role: + name: goaccess + when: + - inventory_hostname in groups['load-balancers'] + - enable_goaccess + tags: goaccess + +- name: Ensure firewall port 7890 CLOSED when GoAccess DISABLED + include_role: + name: firewall_port + vars: + firewall_action: close + firewall_port: 7890 + firewall_protocol: tcp + firewall_zone: "{{m_private_networking_zone|default('public')}}" + when: + - not enable_goaccess + - (docker_skip_tasks is not defined or not docker_skip_tasks) + - name: Ensure haproxy certs have secure permissions file: path: /etc/haproxy/certs diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index 1670eb650..df8c67644 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -1,6 +1,6 @@ global log 127.0.0.1 local2 - # need logging for letsencrypt if setup ??? + # need logging for letsencrypt ??? chroot /var/lib/haproxy stats socket /run/haproxy-admin.sock mode 660 level admin stats timeout 30s @@ -21,21 +21,6 @@ global ssl-default-bind-options no-sslv3 - # Apache httpd.conf settings regarding SSL which we should verify if they - # need to be included somehow here. - # - # SSLProtocol all -SSLv3 -TLSv1 - # SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 - # SSLHonorCipherOrder on - # SSLCompression off - # #SSLSessionTickets off # not available until httpd 2.4.11 - - # # OCSP Stapling - # SSLUseStapling on - # SSLStaplingResponderTimeout 5 - # SSLStaplingReturnResponderErrors off - # SSLStaplingCache shmcb:/var/run/ocsp(128000) - defaults log global mode http @@ -53,7 +38,7 @@ defaults # errorfile 504 /etc/haproxy/errors/504.http option forwardfor option http-server-close - + timeout tunnel 1h # timeout to use with WebSocket and CONNECT @@ -67,7 +52,13 @@ frontend www-http frontend www-https bind *:443 ssl crt /etc/haproxy/certs/ reqadd X-Forwarded-Proto:\ https - # Keep letsencrypt stuff here for now. Probably add it back later. + {% if enable_goaccess %} + # goaccess + # Add config for websocket server GoAccess + acl is_websocket hdr(Upgrade) -i WebSocket + use_backend ws if is_websocket + {% endif %} + # certbot acl letsencrypt-acl path_beg /.well-known/acme-challenge/ use_backend letsencrypt-backend if letsencrypt-acl default_backend www-backend @@ -91,12 +82,12 @@ backend www-backend # @FIXME loop w/ index and tie all # app servers together with controller as registry {% if m_install_netdata %} - frontend netdata - bind *:20000 ssl crt /etc/haproxy/certs/ - mode http - default_backend netdata-back - backend netdata-back - server nd1 127.0.0.1:19999 +frontend netdata + bind *:20000 ssl crt /etc/haproxy/certs/ + mode http + default_backend netdata-back +backend netdata-back + server nd1 127.0.0.1:19999 {% endif %} {% endif %} @@ -150,5 +141,16 @@ listen stats stats refresh 30s {% endif %} +{% if enable_goaccess %} +backend ws + mode http + option forwardfor + option http-server-close + option forceclose + no option httpclose + server ws1 127.0.0.1:7890 ssl verify none + +{% endif %} + backend letsencrypt-backend server letsencrypt 127.0.0.1:54321 diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index ee2cbc9ee..736e7f64b 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -18,6 +18,9 @@ # Allow certbot access RewriteRule ^.well-known(.*) - [L] + # Allow access to public_html + RewriteRule ^public_html(.*) - [L] + # Allow access to /wikis directory (where all wiki content and # and settings are located) RewriteRule ^wikis(?:/|$)(.*)$ - [L] From 031227ede7df314e9f38c913d7c85034ddb517e1 Mon Sep 17 00:00:00 2001 From: Rich Evans <32852052+revansx@users.noreply.github.com> Date: Sat, 9 May 2020 15:58:04 -0400 Subject: [PATCH 183/280] Update MezaCoreExtensions.yml provides missing wfLoadExtension('SemanticCompoundQueries'); config option --- config/MezaCoreExtensions.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 62d98135d..1ecc0139e 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -46,11 +46,14 @@ list: // Disabled due to some issue on FOD wikis. Confirm, reenable if possible // $srfgFormats[] = 'exhibit'; - + + # Allows for the display of more than one SMW inline query in one results display set. - name: SemanticCompoundQueries composer: "mediawiki/semantic-compound-queries" version: "~2.1" - + config: | + wfLoadExtension( 'SemanticCompoundQueries' ); + - name: Scribunto repo: https://github.com/wikimedia/mediawiki-extensions-Scribunto.git version: "{{ mediawiki_default_branch }}" From d00624476fee22c1dd7f67a5093148ebd1a86a44 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 18 May 2020 11:34:57 -0400 Subject: [PATCH 184/280] Drop support for TLS1.0 and TLS1.1 Drop deprecated/insecure protocols Add no-tlsv10 no-tlsv11 to improve security rating. --- src/roles/haproxy/templates/haproxy.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index df8c67644..0e8590d3c 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -18,7 +18,7 @@ global # For more information, see ciphers(1SSL). This list is from: # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS - ssl-default-bind-options no-sslv3 + ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 defaults From 317705fbb83636a941e0d69d60878a5f122db1ab Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 18 May 2020 11:35:39 -0400 Subject: [PATCH 185/280] Fix errors in certbot renewal script --- src/roles/certbot/templates/concat.pem.sh.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/certbot/templates/concat.pem.sh.j2 b/src/roles/certbot/templates/concat.pem.sh.j2 index 4671ab856..e8bf351e7 100644 --- a/src/roles/certbot/templates/concat.pem.sh.j2 +++ b/src/roles/certbot/templates/concat.pem.sh.j2 @@ -5,10 +5,10 @@ # loop through a dynamic list of directories in 'live' # for SITE in $(find /etc/letsencrypt/live -mindepth 1 -maxdepth 1 -type d -exec basename {} \;) # $RENEWED_LINEAGE will contain the live subdirectory -for SITE in basename $RENEWED_LINEAGE +for SITE in $RENEWED_DOMAINS do # move to correct let's encrypt directory - cd /etc/letsencrypt/live/$SITE + cd $RENEWED_LINEAGE # cat files to make combined .pem for haproxy cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem done From a32e09f1feaf4b28b5c7fae3a97b87a1a775ab86 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 20 May 2020 02:00:06 +0000 Subject: [PATCH 186/280] make Tokyo Cabinet Disk Support optional since it's not available? in Buster --- config/Debian.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/Debian.yml b/config/Debian.yml index abcd9d997..c7cef43d4 100644 --- a/config/Debian.yml +++ b/config/Debian.yml @@ -8,7 +8,10 @@ package_openssh_client: openssh-client package_cron: cron package_apache: apache2 package_php_apache_deps: [] -package_goaccess: goaccess-tcb # Use Tokyo Cabinet for on-disk support + +# Use Tokyo Cabinet for on-disk support if you know your distro supports it +# override with goaccess-tcb +package_goaccess: goaccess package_java: "default-jre" package_python3: python3 package_python3_pip: python3-pip From ddd42cf0c6b4a85c24e26af8653debcfd7d25324 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 21 May 2020 10:11:17 -0400 Subject: [PATCH 187/280] update IUS installation --- src/roles/apache-php/tasks/php-redhat.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/roles/apache-php/tasks/php-redhat.yml b/src/roles/apache-php/tasks/php-redhat.yml index dcbea7095..9f8e037c8 100644 --- a/src/roles/apache-php/tasks/php-redhat.yml +++ b/src/roles/apache-php/tasks/php-redhat.yml @@ -1,19 +1,11 @@ --- -- name: Install IUS (CentOS) repo. +# IUS requires EPEL, but we already have it installed from getMeza.sh +- name: Install IUS repo. yum: - name: "https://centos{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" - when: ansible_distribution == "CentOS" - -- name: Install IUS (RHEL) repo. - yum: - name: "https://rhel{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" - when: ansible_distribution == "RedHat" - -- name: Import IUS Community Project GPG key - rpm_key: - key: http://dl.iuscommunity.org/pub/ius/IUS-COMMUNITY-GPG-KEY - state: present - + name: + - "https://repo.ius.io/ius-release-el{{ ansible_distribution_major_version }}.rpm" +# - "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" + when: (ansible_distribution == "CentOS") or (ansible_distribution == "RedHat") - name: Remove PHP 5.x packages package: From e862dd151791e8e35a6d44c6aa25f73c2205d32a Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 21 May 2020 10:18:50 -0400 Subject: [PATCH 188/280] Fix ImageMagick (convert) path --- src/roles/mediawiki/templates/LocalSettings.php.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index a2fab63cb..5b66d6680 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -518,7 +518,7 @@ $wgMaxUploadSize = 1024*1024*100; // 100 MB $wgUseImageMagick = true; {% if ansible_os_family == "RedHat" %} -$wgImageMagickConvertCommand = '/usr/local/bin/convert'; +$wgImageMagickConvertCommand = '/usr/bin/convert'; {% else %} $wgImageMagickConvertCommand = 'convert'; {% endif %} From 5e29e703a189912cc75c353cafd256bc17115814 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 22 May 2020 00:03:30 +0000 Subject: [PATCH 189/280] remove Python / Pip symlinks --- src/roles/mediawiki/tasks/main.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index db7530a10..98843aba1 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -538,29 +538,18 @@ tags: - latest -- name: "Ensure python3 symlink in place" - file: - # dest = symlink, src = dir linked to - src: "/usr/bin/python3.5" - dest: "/usr/bin/python3" - state: link - owner: root - group: root - mode: 0755 + +- name: "Check for existence of python3" + stat: + path: /usr/bin/python3 + register: python3 + +- debug: { var: python3 } - name: "Check for existence of pip3" stat: path: /usr/bin/pip3 register: pip3 -- name: "Ensure pip3 symlink in place" - file: - # dest = symlink, src = dir linked to - src: "/usr/bin/pip3.5" - dest: "/usr/bin/pip3" - state: link - owner: root - group: root - mode: 0755 - when: pip3.stat.exists == False +- debug: { var: pip3 } From 8e47150929aa9458746ac57e03a4111e86983a24 Mon Sep 17 00:00:00 2001 From: Andrew Foster Date: Mon, 1 Jun 2020 15:28:58 +1000 Subject: [PATCH 190/280] Update URL to IUS RPM See https://github.com/iusrepo/announce/issues/18 --- src/roles/apache-php/tasks/ius.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/roles/apache-php/tasks/ius.yml b/src/roles/apache-php/tasks/ius.yml index bcdd32954..58b548e02 100644 --- a/src/roles/apache-php/tasks/ius.yml +++ b/src/roles/apache-php/tasks/ius.yml @@ -1,15 +1,8 @@ --- -- name: Install IUS (CentOS) repo. +- name: Install IUS repo. yum: lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 - name: "https://centos{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" - when: ansible_distribution == "CentOS" - -- name: Install IUS (RHEL) repo. - yum: - lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 - name: "https://rhel{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" - when: ansible_distribution == "RedHat" + name: "https://repo.ius.io/ius-release-el{{ ansible_distribution_major_version }}.rpm" - name: Import IUS Community Project GPG key rpm_key: From f7012019f5c1ce72609ac05c88a4ffe536476d4f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 09:23:09 -0500 Subject: [PATCH 191/280] Add second IUS repo and GPG key --- src/roles/apache-php/tasks/ius.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/roles/apache-php/tasks/ius.yml b/src/roles/apache-php/tasks/ius.yml index 58b548e02..a3ce2f230 100644 --- a/src/roles/apache-php/tasks/ius.yml +++ b/src/roles/apache-php/tasks/ius.yml @@ -1,10 +1,12 @@ --- -- name: Install IUS repo. - yum: - lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 - name: "https://repo.ius.io/ius-release-el{{ ansible_distribution_major_version }}.rpm" + - name: Install IUS repo. + yum: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: + - https://repo.ius.io/ius-release-el7.rpm \ + - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -- name: Import IUS Community Project GPG key - rpm_key: - key: http://dl.iuscommunity.org/pub/ius/IUS-COMMUNITY-GPG-KEY - state: present + - name: Import IUS Community Project GPG key + rpm_key: + key: https://repo.ius.io/RPM-GPG-KEY-IUS-7 + state: present From 10bc86315d9ee23ca669d678043b6d609f4dc49b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 09:59:49 -0500 Subject: [PATCH 192/280] Fix typo (backslash) --- src/roles/apache-php/tasks/ius.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/apache-php/tasks/ius.yml b/src/roles/apache-php/tasks/ius.yml index a3ce2f230..dce2dcfb5 100644 --- a/src/roles/apache-php/tasks/ius.yml +++ b/src/roles/apache-php/tasks/ius.yml @@ -3,8 +3,8 @@ yum: lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 name: - - https://repo.ius.io/ius-release-el7.rpm \ - - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + - "https://repo.ius.io/ius-release-el7.rpm" + - "https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" - name: Import IUS Community Project GPG key rpm_key: From 01c04bbca449defcd8798a2b2afa97c1ee605b3f Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 11:21:40 -0500 Subject: [PATCH 193/280] Bump to PHP 7.2 --- config/core/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index fce39e14d..346f6ebe4 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -143,7 +143,7 @@ mediawiki_version: "1.31.2" mediawiki_default_branch: "REL1_31" # PHP version -php_ius_version: "php71u" +php_ius_version: "php72u" # Parsoid version m_parsoid_version: "tags/v0.10.0" From c50be3287c1579b81c1fdc64ae53f2e55581bbb2 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 12:14:32 -0500 Subject: [PATCH 194/280] Remove mcrypt extension no longer available after PHP 7.1 --- src/roles/apache-php/tasks/php.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/apache-php/tasks/php.yml b/src/roles/apache-php/tasks/php.yml index d623499fa..f8673c11e 100644 --- a/src/roles/apache-php/tasks/php.yml +++ b/src/roles/apache-php/tasks/php.yml @@ -116,7 +116,7 @@ - "{{ php_ius_version }}-pecl-memcached" # Available for php56u, php70u, and php71u. NOT for php72u. - - "{{ php_ius_version }}-mcrypt" + # - "{{ php_ius_version }}-mcrypt" # Available for php56u and php70u. NOT php71u or php72u # - "{{ php_ius_version }}-pear" From bbe000909f85b04cff3d9cf27281be1457d8dc63 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 12:22:29 -0500 Subject: [PATCH 195/280] Remove symlink creation for pip3; IUS appears to do that automatically now --- src/roles/mediawiki/tasks/main.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 1fbedaceb..3902d9067 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -582,13 +582,13 @@ mode: 0755 when: ansible_os_family == 'RedHat' -- name: "Ensure pip3 symlink in place" - file: - # dest = symlink, src = dir linked to - src: "/usr/bin/pip3.5" - dest: "/usr/bin/pip3" - state: link - owner: root - group: root - mode: 0755 - when: ansible_os_family == 'RedHat' +# - name: "Ensure pip3 symlink in place" +# file: +# # dest = symlink, src = dir linked to +# src: "/usr/bin/pip3.5" +# dest: "/usr/bin/pip3" +# state: link +# owner: root +# group: root +# mode: 0755 +# when: ansible_os_family == 'RedHat' From 7c67b850aee798795d8ba3773c43735d98ce7850 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 13:31:40 -0500 Subject: [PATCH 196/280] 31.10.0 release --- RELEASE-NOTES.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 293f9b2a3..6383afb12 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,34 @@ Release Notes ============= +## Meza 31.10.0 + +Update PHP to 7.2, fix IUS repo, fix Python symlink + +### Commits since 31.9.0 + +* bbe0009 Remove symlink creation for pip3; IUS appears to do that automatically now +* c50be32 Remove mcrypt extension no longer available after PHP 7.1 +* 01c04bb Bump to PHP 7.2 +* 10bc863 Fix typo (backslash) +* f701201 Add second IUS repo and GPG key +* 8e47150 Update URL to IUS RPM See https://github.com/iusrepo/announce/issues/18 +* 44f2d2c Python 2/3 compatibility + +### Contributors + +* 6 James Montalvo +* 1 Andrew Foster +* 1 Daren Welsh +* 1 Greg Rundlett + +# How to upgrade + +```bash +sudo meza update 31.10.0 +sudo meza deploy +``` + ## Meza 31.9.0 Ansible 2.9 fix; Travis firewall fix From e048a7b36ab5e7599d6840b8d4c6206f075b6583 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 16:41:00 -0500 Subject: [PATCH 197/280] Ensure PEAR channel up-to-date --- src/roles/apache-php/tasks/mssql_driver_for_php.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/roles/apache-php/tasks/mssql_driver_for_php.yml b/src/roles/apache-php/tasks/mssql_driver_for_php.yml index da039548a..d129447fc 100644 --- a/src/roles/apache-php/tasks/mssql_driver_for_php.yml +++ b/src/roles/apache-php/tasks/mssql_driver_for_php.yml @@ -78,6 +78,9 @@ # shell: "scl enable devtoolset-7 bash" +- name: Ensure PEAR channel up-to-date + shell: pear channel-update pecl.php.net + # # Install sqlsrv drivers from PECL # From 4606fa67069b50f02c8dcf78b1ebacb24e3adfd5 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 5 Jun 2020 17:19:15 -0500 Subject: [PATCH 198/280] 31.10.1 release --- RELEASE-NOTES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6383afb12..575551f0f 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,25 @@ Release Notes ============= +## Meza 31.10.1 + +Fix PEAR channel + +### Commits since 31.10.0 + +* e048a7b Ensure PEAR channel up-to-date + +### Contributors + +* 2 James Montalvo + +# How to upgrade + +```bash +sudo meza update 31.10.1 +sudo meza deploy +``` + ## Meza 31.10.0 Update PHP to 7.2, fix IUS repo, fix Python symlink From 83d4bd6a441814129df369d292230c79efc350d9 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Sun, 7 Jun 2020 08:35:50 -0500 Subject: [PATCH 199/280] Enable testing in GitHub Actions and GitLab CI --- .github/workflows/test.yml | 52 ++++++++++++++++++++++++ .gitlab-ci.yml | 48 ++++++++++++++++++++++ .travis.yml | 2 +- src/scripts/getmeza.sh | 13 +++++- tests/docker/backup-to-remote.setup.sh | 6 +-- tests/docker/import-from-remote.setup.sh | 6 +-- tests/docker/init-container.sh | 4 +- tests/docker/init-controller.sh | 13 +----- tests/docker/run-tests.sh | 25 ------------ tests/travis/git-setup.sh | 39 ------------------ 10 files changed, 121 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 .gitlab-ci.yml delete mode 100644 tests/travis/git-setup.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..359be0cfd --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,52 @@ +name: Test Meza +on: + push: + branches: ['**'] + pull_request: + branches: ['**'] + + +jobs: + test: + strategy: + matrix: + test_type: + - monolith-from-scratch + - monolith-from-import + - import-from-remote + - backup-to-remote + - import-from-alt-remote + + runs-on: ubuntu-latest + + steps: + - name: Context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + + - uses: actions/checkout@master + + - name: apt-get update + run: sudo apt-get update + + - name: install prereqs + run: sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common + + - name: get key + run: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + + - name: fingerprint + run: sudo apt-key fingerprint 0EBFCD88 + + - name: Get docker repo + run: sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + - name: apt-get update again + run: sudo apt-get update + + - name: Install docker + run: sudo apt-get install docker-ce docker-ce-cli containerd.io + + - name: run test + run: sudo bash ./tests/docker/run-tests.sh ${{ matrix.test_type }} ${PWD} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..045c1711f --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,48 @@ +stages: + - test + +variables: + # When using dind service we need to instruct docker, to talk with the + # daemon started inside of the service. The daemon is available with + # a network connection instead of the default /var/run/docker.sock socket. + # + # The 'docker' hostname is the alias of the service container as described at + # https://docs.gitlab.com/ee/ci/docker/using_docker_images.html#accessing-the-services + # + # Note that if you're using Kubernetes executor, the variable should be set to + # tcp://localhost:2375 because of how Kubernetes executor connects services + # to the job container + DOCKER_HOST: tcp://docker:2375/ + # When using dind, it's wise to use the overlayfs driver for + # improved performance. + DOCKER_DRIVER: overlay2 + +.test-template: + stage: test + services: + - docker:18.09.7-dind + # Official docker image. + image: docker:18 + before_script: + - docker info + - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY + script: + - apk update + - apk upgrade + - apk add bash + - 'bash ./tests/docker/run-tests.sh ${CI_JOB_NAME} ${PWD}' + +monolith-from-scratch: + extends: .test-template + +monolith-from-import: + extends: .test-template + +import-from-remote: + extends: .test-template + +backup-to-remote: + extends: .test-template + +import-from-alt-remote: + extends: .test-template diff --git a/.travis.yml b/.travis.yml index 778542b5d..a23f6f541 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,4 +30,4 @@ notifications: secure: HKwPNJbsaU8ahbmfS54cL5Iwn/Dj7hqtFeiTuOiU8goT6wzVH+bS0QT1KmpLtsvPpQgHYR9z2JkK8PVIufZwjSr360CUoLniWyy2oJ1/+MGoyVyBWcRv/yZVNWiQz7zF4oS3XOAcFuWyhyPiA793wDIPpxe9r4og3/dvOBYLEOXXDExxIKMcTaFbNSzRYA4ARXOCr8hcZrF5ROxXBztQIRtw9lT2YVi8qyB13NaLgx+94R5Y5CtU0er2Lh24obIeE7UYiQtExSQjMUbnC09grLSQPKICTNEL9Yz05sAS/xuA/ocrlZ0gP/J2G20WwIG0Mv6cOOCPlSmQtejQ0VqJe4T2FPi4/qXOCOQ2sFsjzdh+IZrzvLo0TGmKMNwwaZBDYGP61C0P6ktDZmGubfuHdSod+CpST6fXZhQovxvSMu4spZor89uktxlqiJECeLj008yaMuuPdtVJu8kqoBafnJyuVnHkcGd54YKpIEYT+GsaW0AUsv5zyIfyOCYDIwujSWpC2UReA3vF5wh9EMvbRCrOtHasuWF+ewA/9Hj1KEjmb8AMhXFVtyHm0BIhV+8mJcp3Of7R6SPI5rVU8kYXdfH6dffNVLHaeEvLlcPyymJ/ek2dDyIuB7N0u+/bqfEVQR5KmNit47zGbLO69c9Y1f9/CqRW8aA4Psrgp9feITk= on_success: always on_failure: always - on_pull_requests: true + on_pull_requests: true \ No newline at end of file diff --git a/src/scripts/getmeza.sh b/src/scripts/getmeza.sh index eef504fe7..ea9e69f6e 100644 --- a/src/scripts/getmeza.sh +++ b/src/scripts/getmeza.sh @@ -9,6 +9,13 @@ if [ "$(whoami)" != "root" ]; then exit 1 fi +for ARG in "$@"; do + if [ "${ARG}" = "--skip-conn-check" ]; then + SKIP_CONNECTION_CHECK="true" + fi +done + + checkInternetConnection() { declare -i pingRetries=100 declare -i sleepDuration=3 @@ -26,7 +33,11 @@ checkInternetConnection() { fi } -checkInternetConnection +if [ ! -z "${SKIP_CONNECTION_CHECK}" ]; then + echo "Skipping connection check" +else + checkInternetConnection +fi # If you don't do this in a restrictive system (umask 077), it becomes # difficult to manage all permissions, AND you constantly have to fix all git diff --git a/tests/docker/backup-to-remote.setup.sh b/tests/docker/backup-to-remote.setup.sh index efe8e3fdc..8c8e8b1a4 100644 --- a/tests/docker/backup-to-remote.setup.sh +++ b/tests/docker/backup-to-remote.setup.sh @@ -4,12 +4,10 @@ # 1) A meza monolith, from a pre-built meza docker image # 2) A backup server, from a base docker image # -# To call this outside of Travis, you must supply at a minimum as environment -# variables: +# To run this you must supply at a minimum these env variables: # 1) m_meza_host=/opt/meza # 2) env_name=somename -# 3) TRAVIS_EVENT_TYPE=push -# 4) TRAVIS_COMMIT= a sha1 hash for a commit or a branch like origin/fix123 +# 3) I think: container_id, docker_ip, docker_exec # -e: kill script if anything fails diff --git a/tests/docker/import-from-remote.setup.sh b/tests/docker/import-from-remote.setup.sh index 949c42ec4..09c44ae62 100644 --- a/tests/docker/import-from-remote.setup.sh +++ b/tests/docker/import-from-remote.setup.sh @@ -4,12 +4,10 @@ # 1) A meza monolith, from a pre-built meza docker image # 2) A backup server, from a base docker image # -# To call this outside of Travis, you must supply at a minimum as environment -# variables: +# To run this you must supply at a minimum these env variables: # 1) m_meza_host=/opt/meza # 2) env_name=somename -# 3) TRAVIS_EVENT_TYPE=push -# 4) TRAVIS_COMMIT= a sha1 hash for a commit or a branch like origin/fix123 +# 3) I think: container_id, docker_ip, docker_exec # -e: kill script if anything fails diff --git a/tests/docker/init-container.sh b/tests/docker/init-container.sh index 030644baf..282cf48c9 100644 --- a/tests/docker/init-container.sh +++ b/tests/docker/init-container.sh @@ -99,8 +99,8 @@ if [ "$is_minion" == "no" ]; then docker cp "$host_meza_dir" "$container_id:/opt/meza" fi - # Install meza command - ${docker_exec[@]} bash /opt/meza/src/scripts/getmeza.sh + # Install meza command. Connection check fails on GitHub actions. + ${docker_exec[@]} bash /opt/meza/src/scripts/getmeza.sh --skip-conn-check fi # reset to no, in case follow on builds don't reset diff --git a/tests/docker/init-controller.sh b/tests/docker/init-controller.sh index f8c91a8c6..9cb28a5fd 100644 --- a/tests/docker/init-controller.sh +++ b/tests/docker/init-controller.sh @@ -7,21 +7,12 @@ docker_repo="jamesmontalvo3/meza-docker-full:latest" source "$m_meza_host/tests/docker/init-container.sh" "${m_meza_host}" "mount" - -# Checkout the correct version of meza on the container -# What's present on the pre-built container is not the latest. Need to pull -# master in case the docker image doesn't have the correct git-setup.sh script -# yet -# ${docker_exec[@]} bash -c "cd /opt/meza && git fetch origin && git reset --hard origin/master" -# ${docker_exec[@]} bash /opt/meza/tests/travis/git-setup.sh "$TRAVIS_EVENT_TYPE" \ -# "$TRAVIS_COMMIT" "$TRAVIS_PULL_REQUEST_SHA" "$TRAVIS_BRANCH" "$TRAVIS_PULL_REQUEST_BRANCH" - - # FIXME #728: Test band-aid. This is run in init-container.sh above, but at # that time the meza version is whatever is on the Docker container (possibly # very old). After checking out the correct version via git above, re-run # getmeza.sh, which moves /home/meza-ansible to /opt/conf-meza/users/meza-ansible -${docker_exec[@]} bash /opt/meza/src/scripts/getmeza.sh +# Note: Connection check fails on GitHub Actions +${docker_exec[@]} bash /opt/meza/src/scripts/getmeza.sh --skip-conn-check # Turn off host key checking for user meza-ansible, to avoid prompts diff --git a/tests/docker/run-tests.sh b/tests/docker/run-tests.sh index 497f1db48..46095b63d 100644 --- a/tests/docker/run-tests.sh +++ b/tests/docker/run-tests.sh @@ -13,31 +13,6 @@ else m_meza_host="/opt/meza" fi -if [ ! -d "$m_meza_host/.git" ]; then - "$m_meza_host is not a git repository" - exit 1; -fi - -# This is required so Docker containers can get the appropriate version. It -# would be better if containers shared a volume with the meza application to -# ensure getting the correct version, but for now that is not possible due to -# the /opt/meza directory also containing data, the MediaWiki application, -# configuration, etc, which would be overwritten by the volume if the specific -# container had pre-loaded them. -if [ -z "$TRAVIS_EVENT_TYPE" ]; then - # Emulate some travis environment variables. - export TRAVIS_EVENT_TYPE="push" - export TRAVIS_COMMIT=$(git --git-dir=/opt/meza/.git rev-parse HEAD) - - echo "Using version $TRAVIS_COMMIT" - - # None of these should be required provided TRAVIS_EVENT_TYPE=push, but they - # can't be unset. - TRAVIS_PULL_REQUEST_SHA="" - TRAVIS_BRANCH="" - TRAVIS_PULL_REQUEST_BRANCH="" -fi - # -e: kill script if anything fails # -u: don't allow undefined variables set -eu diff --git a/tests/travis/git-setup.sh b/tests/travis/git-setup.sh deleted file mode 100644 index 6d05a2208..000000000 --- a/tests/travis/git-setup.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# Checkout the appropriate meza version in Travis - -# TRAVIS_EVENT_TYPE: Indicates how the build was triggered. One of push, pull_request, api, cron. -TRAVIS_EVENT_TYPE="$1" - -# TRAVIS_COMMIT: The commit that the current build is testing. -TRAVIS_COMMIT="$2" - -# TRAVIS_PULL_REQUEST_SHA: -# if the current job is a pull request, the commit SHA of the HEAD commit of the PR. -# if the current job is a push build, this variable is empyty (""). -TRAVIS_PULL_REQUEST_SHA="$3" - -# TRAVIS_BRANCH: -# for push builds, or builds not triggered by a pull request, this is the name of the branch. -# for builds triggered by a pull request this is the name of the branch targeted by the pull request. -TRAVIS_BRANCH="$4" - -# TRAVIS_PULL_REQUEST_BRANCH: -# if the current job is a pull request, the name of the branch from which the PR originated. -# if the current job is a push build, this variable is empty (""). -TRAVIS_PULL_REQUEST_BRANCH="$5" - -cd /opt/meza -if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then - git checkout "$TRAVIS_BRANCH" - git merge "origin/$TRAVIS_PULL_REQUEST_BRANCH" || true - git status - echo - echo "rev-parse HEAD:" - git rev-parse HEAD - echo - echo "Pull Request hash:" - echo "$TRAVIS_PULL_REQUEST_SHA" -else - git reset --hard "$TRAVIS_COMMIT" -fi From 4cb414fd559c6d809e9e584df0b9d2db25703fe6 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 8 Jun 2020 15:45:11 -0500 Subject: [PATCH 200/280] Don't run tests twice for PRs --- .github/workflows/test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 359be0cfd..950231892 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,7 @@ name: Test Meza on: push: - branches: ['**'] - pull_request: - branches: ['**'] - + branches: jobs: test: From b5ce5e3db39573f3e55694ecc8df9fe0893234e4 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Mon, 8 Jun 2020 16:22:42 -0500 Subject: [PATCH 201/280] GitHub Actions fail-fast-->false, to prevent one failure from cancelling all jobs --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 950231892..6e9defbc9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,7 @@ on: jobs: test: strategy: + fail-fast: false matrix: test_type: - monolith-from-scratch From 62886bee65f492a385a18c3e778fb1151f7fcbe4 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 10 Jun 2020 16:47:54 -0500 Subject: [PATCH 202/280] Build test container 'base' on pushes to 'docker-build' branch --- .github/workflows/docker-build.yml | 56 ++++++++++++++++++++++++++++++ tests/containers/base/Dockerfile | 31 +++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 .github/workflows/docker-build.yml create mode 100644 tests/containers/base/Dockerfile diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..93f08a6fb --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,56 @@ +name: Test Meza +on: + push: + branches: ['docker-build'] + +jobs: + build-base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/base + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: base + + # build-pre-yum: + # needs: [build-base] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v1 + # - uses: docker/build-push-action@v1 + # with: + # path: tests/containers/pre-yum + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # repository: enterprisemediawiki/meza + # tags: pre-yum + + # build-full: + # needs: [build-pre-yum] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v1 + # - uses: docker/build-push-action@v1 + # with: + # path: tests/containers/full + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # repository: enterprisemediawiki/meza + # tags: full + + # build-max: + # needs: [build-pre-yum] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v1 + # - uses: docker/build-push-action@v1 + # with: + # path: tests/containers/max + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # repository: enterprisemediawiki/meza + # tags: max diff --git a/tests/containers/base/Dockerfile b/tests/containers/base/Dockerfile new file mode 100644 index 000000000..9aa41c2ce --- /dev/null +++ b/tests/containers/base/Dockerfile @@ -0,0 +1,31 @@ +FROM centos:7 +LABEL MAINTAINER James Montalvo +ENV container=docker + +# Install systemd -- See https://hub.docker.com/_/centos/ +RUN yum -y update; yum clean all; \ +(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ +rm -f /lib/systemd/system/multi-user.target.wants/*;\ +rm -f /etc/systemd/system/*.wants/*;\ +rm -f /lib/systemd/system/local-fs.target.wants/*; \ +rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ +rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ +rm -f /lib/systemd/system/basic.target.wants/*;\ +rm -f /lib/systemd/system/anaconda.target.wants/*; + +# Install other requirements +RUN yum makecache fast \ + && yum -y install deltarpm epel-release initscripts \ + && yum -y update \ + && yum -y install \ + selinux-policy \ + rsyslog \ + sudo \ + which \ + && yum clean all + +# Disable requiretty +RUN sed -i -e 's/^\(Defaults\s*requiretty\)/#--- \1/' /etc/sudoers + +VOLUME ["/sys/fs/cgroup"] +CMD ["/usr/sbin/init"] From f01ae65e789cca78f9c49611e639dbf8d0634f94 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 10 Jun 2020 16:54:26 -0500 Subject: [PATCH 203/280] Add later test images --- .github/workflows/docker-build.yml | 74 ++++++++++++++--------------- tests/containers/full/Dockerfile | 17 +++++++ tests/containers/max/Dockerfile | 57 ++++++++++++++++++++++ tests/containers/pre-yum/Dockerfile | 64 +++++++++++++++++++++++++ 4 files changed, 175 insertions(+), 37 deletions(-) create mode 100644 tests/containers/full/Dockerfile create mode 100644 tests/containers/max/Dockerfile create mode 100644 tests/containers/pre-yum/Dockerfile diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 93f08a6fb..231edb7a7 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,4 +1,4 @@ -name: Test Meza +name: Docker Build on: push: branches: ['docker-build'] @@ -16,41 +16,41 @@ jobs: repository: enterprisemediawiki/meza tags: base - # build-pre-yum: - # needs: [build-base] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v1 - # - uses: docker/build-push-action@v1 - # with: - # path: tests/containers/pre-yum - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # repository: enterprisemediawiki/meza - # tags: pre-yum + build-pre-yum: + needs: [build-base] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/pre-yum + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: pre-yum - # build-full: - # needs: [build-pre-yum] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v1 - # - uses: docker/build-push-action@v1 - # with: - # path: tests/containers/full - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # repository: enterprisemediawiki/meza - # tags: full + build-full: + needs: [build-pre-yum] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/full + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: full - # build-max: - # needs: [build-pre-yum] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v1 - # - uses: docker/build-push-action@v1 - # with: - # path: tests/containers/max - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # repository: enterprisemediawiki/meza - # tags: max + build-max: + needs: [build-pre-yum] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/max + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: max diff --git a/tests/containers/full/Dockerfile b/tests/containers/full/Dockerfile new file mode 100644 index 000000000..b82465b09 --- /dev/null +++ b/tests/containers/full/Dockerfile @@ -0,0 +1,17 @@ +FROM jamesmontalvo3/meza-docker-pre-yum:latest +LABEL MAINTAINER James Montalvo +ENV container=docker + +RUN git clone -b master https://github.com/enterprisemediawiki/meza /opt/meza +# COPY . /opt/meza + +RUN bash /opt/meza/src/scripts/getmeza.sh + +RUN meza setup env monolith --fqdn="INSERT_FQDN" --db_pass=1234 --private_net_zone=public + +RUN ansible-vault decrypt /opt/conf-meza/secret/monolith/secret.yml --vault-password-file /opt/conf-meza/vault/vault-pass-monolith.txt \ + && echo "" >> /opt/conf-meza/secret/monolith/secret.yml \ + && echo "docker_skip_tasks: true" >> /opt/conf-meza/secret/monolith/secret.yml \ + && ansible-vault encrypt /opt/conf-meza/secret/monolith/secret.yml --vault-password-file /opt/conf-meza/vault/vault-pass-monolith.txt + +RUN meza deploy monolith \ No newline at end of file diff --git a/tests/containers/max/Dockerfile b/tests/containers/max/Dockerfile new file mode 100644 index 000000000..9b61d21e8 --- /dev/null +++ b/tests/containers/max/Dockerfile @@ -0,0 +1,57 @@ +FROM jamesmontalvo3/meza-docker-pre-yum:latest +LABEL MAINTAINER James Montalvo +ENV container=docker + +RUN mkdir -p /opt/htdocs \ + && git clone https://gerrit.wikimedia.org/r/p/mediawiki/core.git /opt/htdocs/mediawiki + +RUN cd /opt/htdocs/mediawiki/extensions \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ParserFunctions.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/StringFunctionsEscaped.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ExternalData.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/LabeledSectionTransclusion.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Cite.git \ + && git clone https://github.com/enterprisemediawiki/ParserFunctionHelper.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/WhoIsWatching.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/CharInsert.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/PageForms.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/SemanticInternalObjects.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/SemanticCompoundQueries.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Arrays.git \ + && git clone https://github.com/enterprisemediawiki/TalkRight.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/AdminLinks.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/DismissableSiteNotice.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/BatchUserRights.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/HeaderTabs.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/WikiEditor.git \ + && git clone https://github.com/jamesmontalvo3/MediaWiki-CopyWatchers.git CopyWatchers \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/SyntaxHighlight_GeSHi.git \ + && git clone https://github.com/enterprisemediawiki/Wiretap.git \ + && git clone https://github.com/jamesmontalvo3/MediaWiki-ApprovedRevs.git ApprovedRevs \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/InputBox.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ReplaceText.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Interwiki.git \ + && git clone https://github.com/enterprisemediawiki/MasonryMainPage.git \ + && git clone https://github.com/enterprisemediawiki/WatchAnalytics.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Variables.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/YouTube.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ContributionScores.git \ + && git clone https://github.com/jamesmontalvo3/MediaWiki-PipeEscape.git PipeEscape \ + && git clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/UniversalLanguageSelector.git \ + && git clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/VisualEditor.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Elastica.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/CirrusSearch.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Echo.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Thanks.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/UploadWizard \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/CollapsibleVector \ + && git clone https://github.com/jamesmontalvo3/SimpleMathJax.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/ImageMap \ + && git clone https://github.com/enterprisemediawiki/MezaExt.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/SemanticDrilldown.git \ + && git clone https://github.com/enterprisemediawiki/ImagesLoaded.git \ + && git clone https://github.com/enterprisemediawiki/PageImporter.git \ + && git clone https://github.com/enterprisemediawiki/SemanticMeetingMinutes.git \ + && git clone https://github.com/enterprisemediawiki/HeaderFooter.git \ + && git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/NumerAlpha.git \ + && git clone https://gerrit.wikimedia.org/r/p/mediawiki/extensions/DataTransfer.git \ No newline at end of file diff --git a/tests/containers/pre-yum/Dockerfile b/tests/containers/pre-yum/Dockerfile new file mode 100644 index 000000000..eafbb75b9 --- /dev/null +++ b/tests/containers/pre-yum/Dockerfile @@ -0,0 +1,64 @@ +FROM jamesmontalvo3/meza-docker-base:latest +LABEL MAINTAINER James Montalvo +ENV container=docker + +# Install packages from getmeza.sh +RUN yum -y install \ + git \ + ansible + +# Install packages from meza base role +RUN yum -y install \ + ntp \ + ntpdate \ + ntp-doc \ + openssh-server \ + openssh-clients \ + vim \ + net-tools \ + firewalld \ + jq + +# Install packages from base-extras role +RUN yum -y install \ + expect \ + expectk \ + perl \ + wget \ + gcc \ + cifs-utils + +# Install packages from php and httpd +RUN yum -y install \ + httpd-devel \ + mod_ssl \ + mod_proxy_html \ + zlib-devel \ + sqlite-devel \ + bzip2-devel \ + pcre-devel \ + openssl-devel \ + curl-devel \ + libxml2-devel \ + libXpm-devel \ + gmp-devel \ + libicu-devel \ + t1lib-devel \ + aspell-devel \ + libcurl-devel \ + libjpeg-devel \ + libvpx-devel \ + libpng-devel \ + freetype-devel \ + readline-devel \ + libtidy-devel \ + libmcrypt-devel \ + pam-devel \ + sendmail \ + sendmail-cf \ + m4 \ + xz-libs \ + mariadb-libs + +# Clean up +RUN yum clean all \ No newline at end of file From 11dc3e8e32714da03726c48e668a098f642183eb Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 10 Jun 2020 16:56:31 -0500 Subject: [PATCH 204/280] Make Dockerfiles pull FROM correct base image --- tests/containers/full/Dockerfile | 2 +- tests/containers/max/Dockerfile | 2 +- tests/containers/pre-yum/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/containers/full/Dockerfile b/tests/containers/full/Dockerfile index b82465b09..3d0d5402b 100644 --- a/tests/containers/full/Dockerfile +++ b/tests/containers/full/Dockerfile @@ -1,4 +1,4 @@ -FROM jamesmontalvo3/meza-docker-pre-yum:latest +FROM enterprisemediawiki/meza:pre-yum LABEL MAINTAINER James Montalvo ENV container=docker diff --git a/tests/containers/max/Dockerfile b/tests/containers/max/Dockerfile index 9b61d21e8..db3121545 100644 --- a/tests/containers/max/Dockerfile +++ b/tests/containers/max/Dockerfile @@ -1,4 +1,4 @@ -FROM jamesmontalvo3/meza-docker-pre-yum:latest +FROM enterprisemediawiki/meza:pre-yum LABEL MAINTAINER James Montalvo ENV container=docker diff --git a/tests/containers/pre-yum/Dockerfile b/tests/containers/pre-yum/Dockerfile index eafbb75b9..700d2756f 100644 --- a/tests/containers/pre-yum/Dockerfile +++ b/tests/containers/pre-yum/Dockerfile @@ -1,4 +1,4 @@ -FROM jamesmontalvo3/meza-docker-base:latest +FROM enterprisemediawiki/meza:base LABEL MAINTAINER James Montalvo ENV container=docker From 57e8469304687b600d241c2d198e0e60a49f2d8b Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 10 Jun 2020 17:11:38 -0500 Subject: [PATCH 205/280] Need --skip-conn-check on getmeza.sh within docker; temporarily skip other builds --- .github/workflows/docker-build.yml | 72 +++++++++++++++--------------- tests/containers/full/Dockerfile | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 231edb7a7..843d17b39 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -4,33 +4,33 @@ on: branches: ['docker-build'] jobs: - build-base: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: docker/build-push-action@v1 - with: - path: tests/containers/base - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: enterprisemediawiki/meza - tags: base + # build-base: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v1 + # - uses: docker/build-push-action@v1 + # with: + # path: tests/containers/base + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # repository: enterprisemediawiki/meza + # tags: base - build-pre-yum: - needs: [build-base] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: docker/build-push-action@v1 - with: - path: tests/containers/pre-yum - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: enterprisemediawiki/meza - tags: pre-yum + # build-pre-yum: + # needs: [build-base] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v1 + # - uses: docker/build-push-action@v1 + # with: + # path: tests/containers/pre-yum + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # repository: enterprisemediawiki/meza + # tags: pre-yum build-full: - needs: [build-pre-yum] + # needs: [build-pre-yum] runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -42,15 +42,15 @@ jobs: repository: enterprisemediawiki/meza tags: full - build-max: - needs: [build-pre-yum] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: docker/build-push-action@v1 - with: - path: tests/containers/max - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - repository: enterprisemediawiki/meza - tags: max + # build-max: + # needs: [build-pre-yum] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v1 + # - uses: docker/build-push-action@v1 + # with: + # path: tests/containers/max + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + # repository: enterprisemediawiki/meza + # tags: max diff --git a/tests/containers/full/Dockerfile b/tests/containers/full/Dockerfile index 3d0d5402b..300c55e27 100644 --- a/tests/containers/full/Dockerfile +++ b/tests/containers/full/Dockerfile @@ -5,7 +5,7 @@ ENV container=docker RUN git clone -b master https://github.com/enterprisemediawiki/meza /opt/meza # COPY . /opt/meza -RUN bash /opt/meza/src/scripts/getmeza.sh +RUN bash /opt/meza/src/scripts/getmeza.sh --skip-conn-check RUN meza setup env monolith --fqdn="INSERT_FQDN" --db_pass=1234 --private_net_zone=public From 8a22485970439705cd8e7aae41866d44aced7555 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 10 Jun 2020 17:19:22 -0500 Subject: [PATCH 206/280] Remove secret.yml encryption per #1175 --- tests/containers/full/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/containers/full/Dockerfile b/tests/containers/full/Dockerfile index 300c55e27..1f4e5a358 100644 --- a/tests/containers/full/Dockerfile +++ b/tests/containers/full/Dockerfile @@ -9,9 +9,7 @@ RUN bash /opt/meza/src/scripts/getmeza.sh --skip-conn-check RUN meza setup env monolith --fqdn="INSERT_FQDN" --db_pass=1234 --private_net_zone=public -RUN ansible-vault decrypt /opt/conf-meza/secret/monolith/secret.yml --vault-password-file /opt/conf-meza/vault/vault-pass-monolith.txt \ - && echo "" >> /opt/conf-meza/secret/monolith/secret.yml \ - && echo "docker_skip_tasks: true" >> /opt/conf-meza/secret/monolith/secret.yml \ - && ansible-vault encrypt /opt/conf-meza/secret/monolith/secret.yml --vault-password-file /opt/conf-meza/vault/vault-pass-monolith.txt +RUN echo "" >> /opt/conf-meza/secret/monolith/secret.yml \ + && echo "docker_skip_tasks: true" >> /opt/conf-meza/secret/monolith/secret.yml RUN meza deploy monolith \ No newline at end of file From 591df68c5a8c310462d4723a5e15bbaf4134fead Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Wed, 10 Jun 2020 17:49:38 -0500 Subject: [PATCH 207/280] Point tests to new Docker images built from this repo; only run build process on commits to master --- .github/workflows/docker-build.yml | 74 +++++++++++++++--------------- src/scripts/meza.py | 2 +- tests/docker/init-container.sh | 4 +- tests/docker/init-controller.sh | 2 +- tests/docker/init-minion.sh | 2 +- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 843d17b39..81812bd01 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -1,36 +1,36 @@ name: Docker Build on: push: - branches: ['docker-build'] + branches: ['master'] jobs: - # build-base: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v1 - # - uses: docker/build-push-action@v1 - # with: - # path: tests/containers/base - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # repository: enterprisemediawiki/meza - # tags: base + build-base: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/base + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: base - # build-pre-yum: - # needs: [build-base] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v1 - # - uses: docker/build-push-action@v1 - # with: - # path: tests/containers/pre-yum - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # repository: enterprisemediawiki/meza - # tags: pre-yum + build-pre-yum: + needs: [build-base] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/pre-yum + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: pre-yum build-full: - # needs: [build-pre-yum] + needs: [build-pre-yum] runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -42,15 +42,15 @@ jobs: repository: enterprisemediawiki/meza tags: full - # build-max: - # needs: [build-pre-yum] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v1 - # - uses: docker/build-push-action@v1 - # with: - # path: tests/containers/max - # username: ${{ secrets.DOCKER_USERNAME }} - # password: ${{ secrets.DOCKER_PASSWORD }} - # repository: enterprisemediawiki/meza - # tags: max + build-max: + needs: [build-pre-yum] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: docker/build-push-action@v1 + with: + path: tests/containers/max + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: enterprisemediawiki/meza + tags: max diff --git a/src/scripts/meza.py b/src/scripts/meza.py index d527c2ff1..93ea8b412 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -886,7 +886,7 @@ def meza_command_docker (argv): if argv[0] == "run": if len(argv) == 1: - docker_repo = "jamesmontalvo3/meza-docker-test-max:latest" + docker_repo = "enterprisemediawiki/meza:max" else: docker_repo = argv[1] diff --git a/tests/docker/init-container.sh b/tests/docker/init-container.sh index 282cf48c9..cd69a0054 100644 --- a/tests/docker/init-container.sh +++ b/tests/docker/init-container.sh @@ -7,7 +7,7 @@ set +u # Set defaults before declaring now undefined variables if [ -z "$docker_repo" ]; then - docker_repo="jamesmontalvo3/meza-docker-test-max:latest" + docker_repo="enterprisemediawiki/meza:max" echo "Using default docker_repo = $docker_repo" fi if [ -z "$init" ]; then @@ -87,7 +87,7 @@ ${docker_exec[@]} systemctl start firewalld if [ "$is_minion" == "no" ]; then - # Docker image "jamesmontalvo3/meza-docker-test-max:latest" has mediawiki and + # Docker image "enterprisemediawiki/meza:max" has mediawiki and # several extensions pre-cloned, but not in the correct location. Move them # into place. For some reason gives exit code 129 on Travis sometimes. Force # non-failing exit code. diff --git a/tests/docker/init-controller.sh b/tests/docker/init-controller.sh index 9cb28a5fd..ba9103c59 100644 --- a/tests/docker/init-controller.sh +++ b/tests/docker/init-controller.sh @@ -4,7 +4,7 @@ # Initiate container -docker_repo="jamesmontalvo3/meza-docker-full:latest" +docker_repo="enterprisemediawiki/meza:full" source "$m_meza_host/tests/docker/init-container.sh" "${m_meza_host}" "mount" # FIXME #728: Test band-aid. This is run in init-container.sh above, but at diff --git a/tests/docker/init-minion.sh b/tests/docker/init-minion.sh index 491410ae8..84aa7bbb2 100644 --- a/tests/docker/init-minion.sh +++ b/tests/docker/init-minion.sh @@ -4,7 +4,7 @@ # Initiate container # FIXME #828: pre-yum has more than we need. Go lighter. -docker_repo="jamesmontalvo3/meza-docker-pre-yum:latest" +docker_repo="enterprisemediawiki/meza:pre-yum" is_minion=yes source "$m_meza_host/tests/docker/init-container.sh" "none" From 65aa596df2d303674b99c2408a0b4be967eae92d Mon Sep 17 00:00:00 2001 From: Daren Welsh Date: Fri, 12 Jun 2020 00:28:08 -0500 Subject: [PATCH 208/280] add ldap module (#1261) --- src/roles/apache-php/tasks/php.yml | 34 ++++------------------- src/roles/apache-php/templates/php.ini.j2 | 2 ++ 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/src/roles/apache-php/tasks/php.yml b/src/roles/apache-php/tasks/php.yml index f8673c11e..f74f8ac64 100644 --- a/src/roles/apache-php/tasks/php.yml +++ b/src/roles/apache-php/tasks/php.yml @@ -35,37 +35,12 @@ - name: Get IUS repository include: ius.yml -- name: Ensure PHP 5.6 packages removed - yum: +- name: Ensure PHP 5.x packages removed + package: lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 - name: "{{item}}" + name: "php5*" state: absent - with_items: - - php56u - - php56u-cli - - php56u-common - - php56u-devel - - php56u-gd - - php56u-pecl-memcache - - php56u-pspell - - php56u-snmp - - php56u-xml - - php56u-xmlrpc - - php56u-mysqlnd - - php56u-pdo - - php56u-odbc - - php56u-pear - - php56u-pecl-jsonc - - php56u-process - - php56u-bcmath - - php56u-intl - - php56u-opcache - - php56u-soap - - php56u-mbstring - - php56u-mcrypt - - php56u-mssql - - + # Check if the desired version of PHP is installed. If it is not, ensure any # other versions of PHP are not installed - name: "Check if {{ php_ius_version}} package is installed" @@ -110,6 +85,7 @@ - "{{ php_ius_version }}-opcache" - "{{ php_ius_version }}-soap" - "{{ php_ius_version }}-mbstring" + - "{{ php_ius_version }}-ldap" # php56u has memcache and memcached; php7Xu only has memcached # legacy Meza used php56u-pecl-memcache diff --git a/src/roles/apache-php/templates/php.ini.j2 b/src/roles/apache-php/templates/php.ini.j2 index fc36b10fc..15ffcf6bc 100644 --- a/src/roles/apache-php/templates/php.ini.j2 +++ b/src/roles/apache-php/templates/php.ini.j2 @@ -1815,6 +1815,8 @@ soap.wsdl_cache_limit = 5 ;sysvshm.init_mem = 10000 [ldap] +; Enable ldap extension module +extension=ldap.so ; Sets the maximum number of open links or -1 for unlimited. ldap.max_links = -1 From 4160ae3c603306ad872496860a634d09c0989bd8 Mon Sep 17 00:00:00 2001 From: James Montalvo Date: Fri, 12 Jun 2020 09:24:35 -0500 Subject: [PATCH 209/280] Bump MediaWiki to 1.31.7 (#1267) --- config/core/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/core/defaults.yml b/config/core/defaults.yml index 346f6ebe4..d2e595421 100644 --- a/config/core/defaults.yml +++ b/config/core/defaults.yml @@ -137,7 +137,7 @@ use_default_ssh_config: True # # Version of MediaWiki core -mediawiki_version: "1.31.2" +mediawiki_version: "1.31.7" # Branch to use on many extensions extensions and skins mediawiki_default_branch: "REL1_31" From 76e10584e2d389fb5a2700a2db8e678a35ae9a76 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 12 Jun 2020 21:10:47 +0000 Subject: [PATCH 210/280] Install Java on all platforms --- src/roles/elasticsearch/tasks/main.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index 32adb414c..d5c76bc01 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -10,11 +10,9 @@ when: ansible_os_family == 'RedHat' - name: Ensure Java 1.8.0 OpenJDK is installed - yum: + package: name: "{{ package_java }}" state: present - when: ansible_os_family == 'RedHat' - # Environment setup. - name: Set JAVA_HOME if configured. From e6e1afb5ca943fb47d212481db10c5c34112bfeb Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 15 Jun 2020 14:28:58 -0400 Subject: [PATCH 211/280] initial workflow --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..16e7c44c4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +--- +"on": + # Trigger the workflow on pushes to these branches + push: + branches: + - master + - 34.x +jobs: + one: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + path: /opt/meza + + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Install prerequisites + run: apt install -y git ansible + # - name: Clone Meza + # run: git clone https://github.com/freephile/meza /opt/meza + - name: Run "get meza" + run: bash /opt/meza/src/scripts/getmeza.sh + - name: Deploy monolith + run: meza deploy monolith + From 3bee7fc5383c0073b9724b21427920bac1dd25c1 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 15 Jun 2020 14:37:13 -0400 Subject: [PATCH 212/280] fixing workflow path --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16e7c44c4..e357a9a6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - path: /opt/meza + with: + path: /opt/meza - name: Dump GitHub context env: From 38d21c4e605a1f2edba7a60ca525c1cee7de2d5e Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 15 Jun 2020 14:39:41 -0400 Subject: [PATCH 213/280] Replace action to use git clone --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e357a9a6b..9d5c547e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,9 @@ jobs: one: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - path: /opt/meza + # - uses: actions/checkout@v2 + # with: + # path: /opt/meza - name: Dump GitHub context env: @@ -19,8 +19,8 @@ jobs: run: echo "$GITHUB_CONTEXT" - name: Install prerequisites run: apt install -y git ansible - # - name: Clone Meza - # run: git clone https://github.com/freephile/meza /opt/meza + - name: Clone Meza + run: git clone https://github.com/freephile/meza /opt/meza - name: Run "get meza" run: bash /opt/meza/src/scripts/getmeza.sh - name: Deploy monolith From 2e36cdc863f1ceae0683e85c9d3642f5329b36e5 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 15 Jun 2020 14:42:52 -0400 Subject: [PATCH 214/280] Add sudo to commands --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d5c547e7..d1d9577ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,11 +18,11 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - name: Install prerequisites - run: apt install -y git ansible + run: sudo apt install -y git ansible - name: Clone Meza - run: git clone https://github.com/freephile/meza /opt/meza + run: sudo git clone https://github.com/freephile/meza /opt/meza - name: Run "get meza" - run: bash /opt/meza/src/scripts/getmeza.sh + run: sudo bash /opt/meza/src/scripts/getmeza.sh - name: Deploy monolith - run: meza deploy monolith + run: sudo meza deploy monolith From 5f13b8978fbe7645024507d699715fc054fc4bd3 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 16 Jun 2020 06:42:22 -0400 Subject: [PATCH 215/280] Trying new workflow --- .github/workflows/ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1d9577ce..410c02a36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,18 +9,19 @@ jobs: one: runs-on: ubuntu-latest steps: - # - uses: actions/checkout@v2 - # with: - # path: /opt/meza + - uses: actions/checkout@v2 - name: Dump GitHub context env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" + + - name: Move code + run: sudo mv $GITHUB_WORKSPACE/${{ github.repository }} /opt/meza - name: Install prerequisites run: sudo apt install -y git ansible - - name: Clone Meza - run: sudo git clone https://github.com/freephile/meza /opt/meza + # - name: Clone Meza + # run: sudo git clone https://github.com/freephile/meza /opt/meza - name: Run "get meza" run: sudo bash /opt/meza/src/scripts/getmeza.sh - name: Deploy monolith From 2f594996fb00388f35a96da6cb042077e72ed558 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 16 Jun 2020 06:46:19 -0400 Subject: [PATCH 216/280] New path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 410c02a36..163d49420 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: run: echo "$GITHUB_CONTEXT" - name: Move code - run: sudo mv $GITHUB_WORKSPACE/${{ github.repository }} /opt/meza + run: sudo mv $GITHUB_WORKSPACE /opt/meza - name: Install prerequisites run: sudo apt install -y git ansible # - name: Clone Meza From fd5ee21bed91c8012f4d9ad96d070db2811ea557 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 16 Jun 2020 08:58:00 -0400 Subject: [PATCH 217/280] elementary first steps --- .github/workflows/ci.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 163d49420..ea1927dda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,15 +15,21 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" + + - name: Print working directory + run: pwd + + - name: List files + run: ls -al - - name: Move code - run: sudo mv $GITHUB_WORKSPACE /opt/meza - - name: Install prerequisites - run: sudo apt install -y git ansible - # - name: Clone Meza - # run: sudo git clone https://github.com/freephile/meza /opt/meza - - name: Run "get meza" - run: sudo bash /opt/meza/src/scripts/getmeza.sh - - name: Deploy monolith - run: sudo meza deploy monolith + # - name: Move code + # run: sudo mv $GITHUB_WORKSPACE /opt/meza + # - name: Install prerequisites + # run: sudo apt install -y git ansible + # # - name: Clone Meza + # # run: sudo git clone https://github.com/freephile/meza /opt/meza + # - name: Run "get meza" + # run: sudo bash /opt/meza/src/scripts/getmeza.sh + # - name: Deploy monolith + # run: sudo meza deploy monolith From 0bc11f76841e28539fdd054a5a8c61074754a0d2 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:14:16 -0400 Subject: [PATCH 218/280] try to get files in place --- .github/workflows/ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea1927dda..b86d4462e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,11 +19,18 @@ jobs: - name: Print working directory run: pwd - - name: List files + - name: List files in working directory run: ls -al - # - name: Move code - # run: sudo mv $GITHUB_WORKSPACE /opt/meza + - name: Move code + run: sudo mv ./ /opt/meza + + - name: List files in working directory again + run: ls -al + + - name: List files in /opt/meza + run: ls -al /opt/meza + # - name: Install prerequisites # run: sudo apt install -y git ansible # # - name: Clone Meza From a31486c272b0476a2119abd93953a4f8d6a7f439 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:25:11 -0400 Subject: [PATCH 219/280] Try sleeping --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b86d4462e..ece1247e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,10 @@ jobs: run: pwd - name: List files in working directory - run: ls -al + run: ls -al + + - name: Wait for checkout to complete + run: sleep 60 - name: Move code run: sudo mv ./ /opt/meza From 13a486f01104fc741c4d58014316be37e1c2be09 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:34:15 -0400 Subject: [PATCH 220/280] access github variable --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ece1247e6..1558b77c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,11 +22,11 @@ jobs: - name: List files in working directory run: ls -al - - name: Wait for checkout to complete - run: sleep 60 + # - name: Wait for checkout to complete + # run: sleep 60 - name: Move code - run: sudo mv ./ /opt/meza + run: sudo mv {{ github.workspace }} /opt/meza - name: List files in working directory again run: ls -al From c5bf6afab4f84225525fd1f48f463b963cbfeff6 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:36:00 -0400 Subject: [PATCH 221/280] Make the directory --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1558b77c3..aae5706fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,9 @@ jobs: # - name: Wait for checkout to complete # run: sleep 60 + + - name: Make target directory + run: mkdir -p /opt/meza - name: Move code run: sudo mv {{ github.workspace }} /opt/meza From 21291d91dcf2efbef18b11af633f4b635a1ba204 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:38:03 -0400 Subject: [PATCH 222/280] forgot the dollar --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aae5706fc..a2b8122f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: run: mkdir -p /opt/meza - name: Move code - run: sudo mv {{ github.workspace }} /opt/meza + run: sudo mv ${{ github.workspace }} /opt/meza - name: List files in working directory again run: ls -al From 29e14ac294ebc8fa23528636b657fea11831369c Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:41:01 -0400 Subject: [PATCH 223/280] cp instead of mv which breaks cleanup --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2b8122f5..82711f7e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: run: mkdir -p /opt/meza - name: Move code - run: sudo mv ${{ github.workspace }} /opt/meza + run: sudo cp -r ${{ github.workspace }} /opt/ - name: List files in working directory again run: ls -al From a4e3ba730ff58e14bacf4ab7c17b913987a947ad Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:45:02 -0400 Subject: [PATCH 224/280] Can I get a deploy? --- .github/workflows/ci.yml | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82711f7e3..1eea83fb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,27 +22,24 @@ jobs: - name: List files in working directory run: ls -al - # - name: Wait for checkout to complete - # run: sleep 60 - - - name: Make target directory + - name: Make target directory for sources run: mkdir -p /opt/meza - - name: Move code + - name: Copy code run: sudo cp -r ${{ github.workspace }} /opt/ - - name: List files in working directory again - run: ls -al - - - name: List files in /opt/meza - run: ls -al /opt/meza + # - name: List files in /opt/meza + # run: ls -al /opt/meza - # - name: Install prerequisites - # run: sudo apt install -y git ansible - # # - name: Clone Meza - # # run: sudo git clone https://github.com/freephile/meza /opt/meza - # - name: Run "get meza" - # run: sudo bash /opt/meza/src/scripts/getmeza.sh - # - name: Deploy monolith - # run: sudo meza deploy monolith + - name: Install prerequisites + run: sudo apt install -y git ansible + + # - name: Clone Meza + # run: sudo git clone https://github.com/freephile/meza /opt/meza + + - name: Run "get meza" + run: sudo bash /opt/meza/src/scripts/getmeza.sh + + - name: Deploy monolith + run: sudo meza deploy monolith From 66131e252d362ac584c29581a9a9e0cb37841bb3 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 06:52:50 -0400 Subject: [PATCH 225/280] Run getmeza with sudo -H to avoid cache warnings --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eea83fb8..4bf415eb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: # run: sudo git clone https://github.com/freephile/meza /opt/meza - name: Run "get meza" - run: sudo bash /opt/meza/src/scripts/getmeza.sh + run: sudo -H bash /opt/meza/src/scripts/getmeza.sh - name: Deploy monolith run: sudo meza deploy monolith From 81dd1390df164d2e326870b009cf2cc6f5c45181 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 07:58:32 -0400 Subject: [PATCH 226/280] Setup env then deploy --- .github/workflows/ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bf415eb3..2161b14b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,9 +37,16 @@ jobs: # - name: Clone Meza # run: sudo git clone https://github.com/freephile/meza /opt/meza - - name: Run "get meza" + - name: Run "getmeza with sudo -H to avoid cache warnings" run: sudo -H bash /opt/meza/src/scripts/getmeza.sh - - name: Deploy monolith - run: sudo meza deploy monolith + - name: Setup the monolith environment + run: sudo meza setup env monolith --fqdn="127.0.0.1" --dp_pass=1234 --private_net_zone=public + + - name: Print hosts file + run: cat /opt/conf-meza/secret/monolith/hosts + + # Does the --no-firewall option exist? + - name: Deploy Meza + run: meza deploy monolith --no-firewall From 60f1e56d33e7cdb6b3d525ba67e7afc3f7f53bb6 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 07:59:43 -0400 Subject: [PATCH 227/280] Fix typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2161b14b0..874303a95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: run: sudo -H bash /opt/meza/src/scripts/getmeza.sh - name: Setup the monolith environment - run: sudo meza setup env monolith --fqdn="127.0.0.1" --dp_pass=1234 --private_net_zone=public + run: sudo meza setup env monolith --fqdn="127.0.0.1" --db_pass=1234 --private_net_zone=public - name: Print hosts file run: cat /opt/conf-meza/secret/monolith/hosts From 97ecbf005f11446e732d2eb8aced9346dd287c28 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 08:07:05 -0400 Subject: [PATCH 228/280] Print hosts file as a 'test' --- .github/workflows/ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 874303a95..f15a82041 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,10 +43,9 @@ jobs: - name: Setup the monolith environment run: sudo meza setup env monolith --fqdn="127.0.0.1" --db_pass=1234 --private_net_zone=public - - name: Print hosts file - run: cat /opt/conf-meza/secret/monolith/hosts - - # Does the --no-firewall option exist? + # The --no-firewall option does not exist - name: Deploy Meza - run: meza deploy monolith --no-firewall + run: meza deploy monolith + - name: Print hosts file + run: cat /opt/conf-meza/secret/monolith/hosts From f9d3cad39431d93617504536e49cd8e0090c8205 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 08:08:37 -0400 Subject: [PATCH 229/280] sudo or not to do --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f15a82041..52e7c1f47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: # The --no-firewall option does not exist - name: Deploy Meza - run: meza deploy monolith + run: sudo meza deploy monolith - name: Print hosts file run: cat /opt/conf-meza/secret/monolith/hosts From ff88ffd28e235b5a3a9f48286d3eb1345262a82f Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 08:25:39 -0400 Subject: [PATCH 230/280] Fix problem with duplicate NTP install --- .github/workflows/ci.yml | 5 +---- src/roles/base/tasks/main.yml | 5 ----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52e7c1f47..9515675ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,9 +28,6 @@ jobs: - name: Copy code run: sudo cp -r ${{ github.workspace }} /opt/ - # - name: List files in /opt/meza - # run: ls -al /opt/meza - - name: Install prerequisites run: sudo apt install -y git ansible @@ -47,5 +44,5 @@ jobs: - name: Deploy Meza run: sudo meza deploy monolith - - name: Print hosts file + - name: Print hosts file as a rudimentary "test" run: cat /opt/conf-meza/secret/monolith/hosts diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index 5cfc4681b..c8571bb11 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -225,11 +225,6 @@ # chkconfig ntpd on # Activate service # ntpdate pool.ntp.org # Synchronize the system clock with 0.pool.ntp.org server # service ntpd start # Start service -# FIXME: this is duplicated from above, right? No reason it needs to be here twice? -- name: Install NTP - package: name=ntp state=installed - tags: - - latest - name: Ensure NTP is running and enabled as configured. service: From 7f006d775e5bb453d40f8950f4c5740bfc1c015f Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 09:23:46 -0400 Subject: [PATCH 231/280] Fix failure for non-existant config directory --- src/roles/goaccess/tasks/main.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/roles/goaccess/tasks/main.yml b/src/roles/goaccess/tasks/main.yml index 5ef21376b..9d82cec79 100644 --- a/src/roles/goaccess/tasks/main.yml +++ b/src/roles/goaccess/tasks/main.yml @@ -36,6 +36,14 @@ name: "{{ package_goaccess }}" state: present +- name: Ensure GoAccess config directory exists + file: + state: directory + path: /etc/goaccess + owner: root + group: root + mode: 0775 + - name: Ensure /etc/goaccess/my.goaccess.conf in place template: src: my.goaccess.conf.j2 From 4ab3715475946f43f5b73c8ed4c7bc98f518b401 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 09:37:17 -0400 Subject: [PATCH 232/280] Fix warning for python-apt which is auto-installed by runner --- src/roles/base/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index c8571bb11..60f49a978 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -55,6 +55,11 @@ tags: - latest +- name: Install dependency for apt + package: python-apt + state: present + when: ansible_os_family == "Debian" + - name: Ensure apt cache updated (Debian only) apt: update_cache: yes From e6ff64ef93e9a0fd41d149919453f40aa26c2e31 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 09:40:32 -0400 Subject: [PATCH 233/280] Fix syntax --- src/roles/base/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index 60f49a978..b3c9975c2 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -56,7 +56,8 @@ - latest - name: Install dependency for apt - package: python-apt + package: + name: python-apt state: present when: ansible_os_family == "Debian" From b6f426a7e0411b1e5e161311b0c47083165c4183 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 10:31:41 -0400 Subject: [PATCH 234/280] Fix issues with "state" on package installation --- src/roles/base/tasks/main.yml | 2 +- src/roles/database/tasks/setup-Debian.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index b3c9975c2..b49b1e08d 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -128,7 +128,7 @@ - latest - name: ensure libselinux-python installed prior to SELinux (RedHat/CentOS only) - yum: name=libselinux-python state=installed + yum: name=libselinux-python state=present when: ansible_os_family == "RedHat" tags: - latest diff --git a/src/roles/database/tasks/setup-Debian.yml b/src/roles/database/tasks/setup-Debian.yml index 904f87ccd..3ff770383 100644 --- a/src/roles/database/tasks/setup-Debian.yml +++ b/src/roles/database/tasks/setup-Debian.yml @@ -8,7 +8,9 @@ when: mysql_initially_installed_debian.stat.exists == false - name: Ensure MySQL Python libraries are installed. - apt: "name=python-mysqldb state=installed" + apt: + name: python-mysqldb + state: present - name: Ensure MySQL packages are installed. apt: From 0be5c60fcba95b52ad6a8a84d39568cd50986bb4 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 11:18:02 -0400 Subject: [PATCH 235/280] Remove MySQL from runner --- .github/workflows/ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9515675ee..722a5c654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,9 @@ - 34.x jobs: one: - runs-on: ubuntu-latest + # ubuntu-18.04 + # runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -16,11 +18,10 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - name: Print working directory - run: pwd - - - name: List files in working directory - run: ls -al + # GitHub Runners include a LOT of software by default + # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md + - name: Uninstall MySQL so we can use MariaDB + run: sudo apt-get erase mysql-server - name: Make target directory for sources run: mkdir -p /opt/meza From 4c91167b106582236fd82c92e45e4aea1c261ea3 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 11:19:53 -0400 Subject: [PATCH 236/280] remove --purge instead of erase --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 722a5c654..f251c5024 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: # GitHub Runners include a LOT of software by default # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md - name: Uninstall MySQL so we can use MariaDB - run: sudo apt-get erase mysql-server + run: sudo apt-get remove --purge mysql-server - name: Make target directory for sources run: mkdir -p /opt/meza From a6fac2c0928474bbd7e2418a92328c75bef00017 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 12:58:40 -0400 Subject: [PATCH 237/280] be extremely verbose --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f251c5024..86c5bfad5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: # The --no-firewall option does not exist - name: Deploy Meza - run: sudo meza deploy monolith + run: sudo meza deploy monolith -vvv - name: Print hosts file as a rudimentary "test" run: cat /opt/conf-meza/secret/monolith/hosts From 9ec8b274727a2c23894eb211bc466b23d9240ff4 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 13:33:19 -0400 Subject: [PATCH 238/280] Permisson fix for HaProxy --- src/roles/haproxy/tasks/main.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 108ad780a..387b32e24 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -192,15 +192,23 @@ - not enable_goaccess - (docker_skip_tasks is not defined or not docker_skip_tasks) -- name: Ensure haproxy certs have secure permissions +- name: Ensure haproxy cert directory has secure permissions file: path: /etc/haproxy/certs state: directory - recurse: yes + recurse: no owner: root group: root - mode: 0600 + mode: 0700 +- name: Ensure haproxy cert(s) have secure permissions + file: + path: /etc/haproxy/certs/* + recurse: no + owner: root + group: root + mode: 0600 + - name: write the haproxy config file template: src: haproxy.cfg.j2 From 1aeedbc4e8b362cb9c6fc354d9a72278dfb47a94 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 15:19:55 -0400 Subject: [PATCH 239/280] Fix permissions for certificate directory and certs --- src/roles/haproxy/tasks/main.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 387b32e24..5910a2843 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -201,14 +201,22 @@ group: root mode: 0700 +- name: Find haproxy cert(s) + find: + paths: /etc/haproxy/certs + file_type: file + patterns: "*.pem" + register: certlist + - name: Ensure haproxy cert(s) have secure permissions file: - path: /etc/haproxy/certs/* - recurse: no + path: "{{ item.path }}" + state: file owner: root group: root mode: 0600 - + with_items: "{{ certlist.files }}" + - name: write the haproxy config file template: src: haproxy.cfg.j2 From 26376856d51bda828197243226f250b0e14225f3 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 15:43:42 -0400 Subject: [PATCH 240/280] Disable GoAccess by default --- config/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/defaults.yml b/config/defaults.yml index a4e6370b9..422197551 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -108,7 +108,7 @@ m_force_debug: false # if you enable certbot, make sure you add m_httpd_server_admin in your public.yml enable_certbot: false -enable_goaccess: true +enable_goaccess: false enable_wiki_emails: true From d7319d92458e1596c9e409b93837cea39648852e Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 16:05:30 -0400 Subject: [PATCH 241/280] wtf haproxy? --- src/roles/haproxy/tasks/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 5910a2843..c420ba02a 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -426,4 +426,13 @@ name: haproxy state: started enabled: yes + ignore_errors: true when: docker_skip_tasks is not defined or not docker_skip_tasks + +- name: Check haproxy status + command: sudo systemctl status haproxy.service + register: haproxy-failed + +- debug: + var: haproxy-failed + verbosity: 4 From c9d981719a7b12e26936d97e0f6d1e2ece2ed466 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 21:58:21 -0400 Subject: [PATCH 242/280] Aggressive debugging on haproxy --- src/roles/haproxy/tasks/main.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index c420ba02a..15b82e3d7 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -420,6 +420,19 @@ notify: - restart rsyslog +# @todo remove me +- name: Check haproxy status + command: sudo systemctl status haproxy.service + register: haproxy-status + +# @todo remove me +- debug: + var: haproxy-status + verbosity: 4 + +# @todo remove me +- name: Dump haproxy config + command: sudo cat /etc/haproxy/haproxy.conf - name: ensure haproxy is running (and enable it at boot) service: @@ -429,10 +442,3 @@ ignore_errors: true when: docker_skip_tasks is not defined or not docker_skip_tasks -- name: Check haproxy status - command: sudo systemctl status haproxy.service - register: haproxy-failed - -- debug: - var: haproxy-failed - verbosity: 4 From 77946bda032799e411bbfd5a6c25a34365f7d90c Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 17 Jun 2020 22:30:55 -0400 Subject: [PATCH 243/280] correct haproxy.cfg and variable name --- src/roles/haproxy/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 15b82e3d7..20da1b52c 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -423,16 +423,16 @@ # @todo remove me - name: Check haproxy status command: sudo systemctl status haproxy.service - register: haproxy-status + register: haproxy_status # @todo remove me - debug: - var: haproxy-status + var: haproxy_status verbosity: 4 # @todo remove me - name: Dump haproxy config - command: sudo cat /etc/haproxy/haproxy.conf + command: sudo cat /etc/haproxy/haproxy.cfg - name: ensure haproxy is running (and enable it at boot) service: From f46d5d120997f2e48a4c56c8694f32cba87c0741 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 18 Jun 2020 05:51:56 -0400 Subject: [PATCH 244/280] skip haproxy service restart --- src/roles/haproxy/tasks/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 20da1b52c..db3a3bcab 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -428,17 +428,17 @@ # @todo remove me - debug: var: haproxy_status - verbosity: 4 + verbosity: 3 # @todo remove me - name: Dump haproxy config command: sudo cat /etc/haproxy/haproxy.cfg -- name: ensure haproxy is running (and enable it at boot) - service: - name: haproxy - state: started - enabled: yes - ignore_errors: true - when: docker_skip_tasks is not defined or not docker_skip_tasks +# - name: ensure haproxy is running (and enable it at boot) +# service: +# name: haproxy +# state: started +# enabled: yes +# ignore_errors: true +# when: docker_skip_tasks is not defined or not docker_skip_tasks From d0171fe3f5e5ee8cb5402483e2bf77d26ba881be Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Jul 2020 06:34:33 -0400 Subject: [PATCH 245/280] Add Apache Prefork settings --- src/roles/apache-php/templates/httpd.conf.j2 | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/roles/apache-php/templates/httpd.conf.j2 b/src/roles/apache-php/templates/httpd.conf.j2 index 46127571d..0ab01583b 100644 --- a/src/roles/apache-php/templates/httpd.conf.j2 +++ b/src/roles/apache-php/templates/httpd.conf.j2 @@ -433,6 +433,28 @@ SSLRandomSeed connect builtin #RequestHeader unset DNT env=bad_DNT # + +# number of server processes to start (default 5) +StartServers 6 + +# minimum number of server processes in reserve for new requests +MinSpareServers 3 + + # maximum number of idle server processes before they are reaped +MaxSpareServers 6 + +# maximum value of clients for the lifetime of the server before it is reaped to prevent mem leaks +ServerLimit 256 + +# MaxRequestWorkers was called MaxClients before version 2.3.13. The old name is still supported. +# translates into the maximum number of child processes that will be launched to serve requests on prefork +# the limit on the number of simultaneous requests that will be served; others queued. +MaxRequestWorkers 10 + +# Server 2.3.9 and later. The old name MaxRequestsPerChild is still supported. +# number of connections that an individual child server will handle during its life +MaxConnectionsPerChild 1000 + # main handling via http Listen 8080 From ec38960c3e0ced2a8fd0a5290fb0303e2077a5ce Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Jul 2020 06:24:58 -0400 Subject: [PATCH 246/280] Add IPv6 bind if available to HAProxy --- src/roles/haproxy/templates/haproxy.cfg.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index 0868e65c6..42b1502cb 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -56,11 +56,20 @@ defaults frontend www-http bind *:80 + {% if ansible_default_ipv6.address | length > 0 -%} + bind {{ ansible_default_ipv6.address }}:80 + {%- endif %} + reqadd X-Forwarded-Proto:\ http default_backend www-backend frontend www-https bind *:443 ssl crt /etc/haproxy/certs/meza.pem + {% if ansible_default_ipv6.address | length > 0 -%} + # Create an AAAA record in DNS using the IP below to IPv6 enable your wiki + bind {{ ansible_default_ipv6.address }}:443 ssl crt /etc/haproxy/certs/meza.pem + {%- endif %} + reqadd X-Forwarded-Proto:\ https # Keep letsencrypt stuff here for now. Probably add it back later. # acl letsencrypt-acl path_beg /.well-known/acme-challenge/ From 1402b5384eb16b147979065130abb412c970887a Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Jul 2020 06:24:58 -0400 Subject: [PATCH 247/280] Add IPv6 bind if available to HAProxy Note: We dropped meza.pem from the bind so that the first (alpha) certificate will be used (cherry picked from commit ec38960c3e0ced2a8fd0a5290fb0303e2077a5ce) --- src/roles/haproxy/templates/haproxy.cfg.j2 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index 0e8590d3c..ad5b5a78b 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -46,11 +46,20 @@ defaults frontend www-http bind *:80 + {% if ansible_default_ipv6.address | length > 0 -%} + bind {{ ansible_default_ipv6.address }}:80 + {%- endif %} + reqadd X-Forwarded-Proto:\ http default_backend www-backend frontend www-https bind *:443 ssl crt /etc/haproxy/certs/ + {% if ansible_default_ipv6.address | length > 0 -%} + # Create an AAAA record in DNS using the IP below to IPv6 enable your wiki + bind {{ ansible_default_ipv6.address }}:443 ssl crt /etc/haproxy/certs/ + {%- endif %} + reqadd X-Forwarded-Proto:\ https {% if enable_goaccess %} # goaccess From 18690e74a3f06e435978710cb86fea4ef5badabd Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Fri, 3 Jul 2020 14:57:55 -0400 Subject: [PATCH 248/280] Fixes Issue 16 - change X-Forwarded-Proto headers --- src/roles/haproxy/templates/haproxy.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index ad5b5a78b..bfb539768 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -50,7 +50,7 @@ frontend www-http bind {{ ansible_default_ipv6.address }}:80 {%- endif %} - reqadd X-Forwarded-Proto:\ http + http-request set-header X-Forwarded-Proto http default_backend www-backend frontend www-https @@ -60,7 +60,7 @@ frontend www-https bind {{ ansible_default_ipv6.address }}:443 ssl crt /etc/haproxy/certs/ {%- endif %} - reqadd X-Forwarded-Proto:\ https + http-request set-header X-Forwarded-Proto https {% if enable_goaccess %} # goaccess # Add config for websocket server GoAccess From cbeda3a77f00a6977beeb714e3203ff8efdb585c Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 1 Sep 2020 18:34:21 -0400 Subject: [PATCH 249/280] Add port for Kibana --- src/playbooks/site.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index bb0b4ebf0..497b5d5cf 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -327,6 +327,13 @@ firewall_protocol: tcp firewall_servers: "{{ groups['elastic-servers'] }}" firewall_zone: "{{m_private_networking_zone|default('public')}}" + # add port for Kibana + - role: firewall_port + firewall_action: open + firewall_port: 5601 + firewall_protocol: tcp + firewall_servers: "{{ groups['elastic-servers'] }}" + firewall_zone: "{{m_private_networking_zone|default('public')}}" - elasticsearch # Note: this is app-servers again, but must be after everything else is setup From 7952ace4ef43f4d5f825226f6c98e30832b3fd5b Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 5 Sep 2020 16:23:08 +0000 Subject: [PATCH 250/280] specify v6 for elasticsearch; notify handler on install --- src/roles/elasticsearch/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index d5c76bc01..f5cbd1e8a 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -56,8 +56,9 @@ - name: Ensure elasticsearch installed package: - name: elasticsearch + name: elasticsearch=6* state: present + notify: restart elasticsearch # Need to perform this check so `lineinfile` doesn't run in Docker. /etc/hosts # is special in Docker. From 12cd291d61a849d1d163631e4c05f4cae1eac1e9 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 5 Sep 2020 16:23:54 +0000 Subject: [PATCH 251/280] remove duplicate task --- src/roles/elasticsearch/tasks/main.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index f5cbd1e8a..dfec774fb 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -168,11 +168,3 @@ when: (es_version_found|int != es_version_desired|int) or (force_do_elasticsearch_upgrade is defined and force_do_elasticsearch_upgrade) -- name: Set Elasticsearch yum repo to point to 6.x - template: - src: elasticsearch.repo.j2 - dest: /etc/yum.repos.d/elasticsearch.repo - mode: 0644 - when: ansible_os_family == 'RedHat' - - From c351020433b0bebcd1448ce7ddee042cda1a7247 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 5 Sep 2020 16:25:43 +0000 Subject: [PATCH 252/280] correct JAVA_HOME --- src/roles/elasticsearch/templates/java_home.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/elasticsearch/templates/java_home.sh.j2 b/src/roles/elasticsearch/templates/java_home.sh.j2 index e2354e196..6c25eb869 100644 --- a/src/roles/elasticsearch/templates/java_home.sh.j2 +++ b/src/roles/elasticsearch/templates/java_home.sh.j2 @@ -1 +1 @@ -export JAVA_HOME=/usr/share/elasticsearch/jdk +export JAVA_HOME=/usr/share/elasticsearch From 676d61045976d0c7e29791c9d6eba4280b9b10ac Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sat, 5 Sep 2020 16:29:38 +0000 Subject: [PATCH 253/280] Undo f46d5d120997f2e48a4c56c8694f32cba87c0741 Not sure why I took it out in the first place, but I was debugging GitHub Continuous Integration at the time. --- src/roles/haproxy/tasks/main.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index db3a3bcab..a71175a37 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -434,11 +434,11 @@ - name: Dump haproxy config command: sudo cat /etc/haproxy/haproxy.cfg -# - name: ensure haproxy is running (and enable it at boot) -# service: -# name: haproxy -# state: started -# enabled: yes -# ignore_errors: true -# when: docker_skip_tasks is not defined or not docker_skip_tasks +- name: ensure haproxy is running (and enable it at boot) + service: + name: haproxy + state: started + enabled: yes + ignore_errors: true + when: docker_skip_tasks is not defined or not docker_skip_tasks From 41c9eedd2ac689b721cdef232b0e3e9db63cc4ae Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 03:13:53 +0000 Subject: [PATCH 254/280] Revert "Add port for Kibana" This reverts commit cbeda3a77f00a6977beeb714e3203ff8efdb585c. --- src/playbooks/site.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/playbooks/site.yml b/src/playbooks/site.yml index 497b5d5cf..bb0b4ebf0 100644 --- a/src/playbooks/site.yml +++ b/src/playbooks/site.yml @@ -327,13 +327,6 @@ firewall_protocol: tcp firewall_servers: "{{ groups['elastic-servers'] }}" firewall_zone: "{{m_private_networking_zone|default('public')}}" - # add port for Kibana - - role: firewall_port - firewall_action: open - firewall_port: 5601 - firewall_protocol: tcp - firewall_servers: "{{ groups['elastic-servers'] }}" - firewall_zone: "{{m_private_networking_zone|default('public')}}" - elasticsearch # Note: this is app-servers again, but must be after everything else is setup From 9da54b0fd86a56fd3f472640890eda78e3accb07 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 04:43:41 +0000 Subject: [PATCH 255/280] Add back preventative measure to protect DOS --- src/roles/htdocs/templates/.htaccess.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index 736e7f64b..ce5344478 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -15,6 +15,10 @@ # Allow access to root index.php RewriteRule ^index.php(.*) - [L] + # This page can bring the wiki to it's knees + RewriteCond %{REQUEST_URI} ^.*Special:MostLinkedPages$ [NC] + RewriteRule ^.*$ - [R=404,L] + # Allow certbot access RewriteRule ^.well-known(.*) - [L] From e4e2de97ccb30a00b2b9aa5cb0bf34e46e10aadd Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 04:43:41 +0000 Subject: [PATCH 256/280] Add back preventative measure to protect DOS --- src/roles/htdocs/templates/.htaccess.j2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index 736e7f64b..cea771794 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -15,6 +15,10 @@ # Allow access to root index.php RewriteRule ^index.php(.*) - [L] + # This page can bring the wiki to it's knees + RewriteCond %{REQUEST_URI} ^.*Special:MostLinkedPages$ [NC] + RewriteRule ^.*$ - [R=404,L] + # Allow certbot access RewriteRule ^.well-known(.*) - [L] @@ -44,3 +48,11 @@ + + +Require all denied + + + +Require all denied + From 48dc6bfbf13ff3bb71a8be8b457c5b2cd4a6b9c4 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 05:19:14 +0000 Subject: [PATCH 257/280] add symlinks to "w" --- src/roles/mediawiki/tasks/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 98843aba1..438e9519b 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -96,6 +96,16 @@ - name: Create symlink to core, to enable short urls file: src: "{{ m_mediawiki }}" + dest: "{{ m_htdocs }}/w" + state: link + tags: + - latest + +# create symlink to wiki for short urls +# tag it latest, so that the symlink gets created whenever core is downloaded +- name: Link wikis to core, to enable short urls + file: + src: "{{ m_htdocs }}/w" dest: "{{ m_htdocs }}/{{ item }}" state: link with_items: From b0b58679baefc9baf8c010d23e01071d8eaaee08 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 06:02:37 +0000 Subject: [PATCH 258/280] comment for FamilySearch --- src/roles/parsoid-settings/templates/config.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roles/parsoid-settings/templates/config.yaml.j2 b/src/roles/parsoid-settings/templates/config.yaml.j2 index 5ae98adf7..f3b5a8f0a 100644 --- a/src/roles/parsoid-settings/templates/config.yaml.j2 +++ b/src/roles/parsoid-settings/templates/config.yaml.j2 @@ -47,6 +47,7 @@ services: {% if groups['app-servers']|length|int == 1 and groups['parsoid-servers']|length|int == 1 and groups['app-servers'][0] == groups['parsoid-servers'][0] -%} uri: 'http://127.0.0.1:8080/{{ wiki }}/api.php' +# uri: 'https://{{wiki_app_fqdn}}/wiki/{{ wiki }}/api.php' # for FamilySearch {%- elif 'load-balancers' not in groups or groups['load-balancers']|length|int == 0 -%} From 18751f388308fd4c409f38032c70ebc4ae20586a Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 06:03:47 +0000 Subject: [PATCH 259/280] simplify and augment the wikiId handling --- .../mediawiki/templates/LocalSettings.php.j2 | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index 5b66d6680..30fb8c177 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -58,27 +58,29 @@ fixApacheEnv(); $mezaWikiEnvVarName='WIKI'; -if ( $wgCommandLineMode ) { - // get $wikiId from environment variable - $wikiId = getenv( $mezaWikiEnvVarName ); +$wikiId = getenv( $mezaWikiEnvVarName ); -} else { +if (empty($wikiId)) { -{% if m_use_subdomains %} - // get $wikiId from subdomain - // FIXME make regex a config variable - // FIXME add 'ignore' list like 'www' that should not be a wiki + {% if m_use_subdomains %} + // get $wikiId from subdomain + // FIXME make regex a config variable + // FIXME add 'ignore' list like 'www' that should not be a wiki if ( preg_match( '%([a-z]+)\.([a-z]+)\..{2,4}[\d]*$%im', $_SERVER['HTTP_HOST'], $matches ) ) { $wikiId = $matches[1]; - } else { -{% endif %} - // get $wikiId from URI path - $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); - $wikiId = strtolower( $uriParts[1] ); + } + {% endif %} -{% if m_use_subdomains %} +} else { + // get $wikiId from URI path + $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); + if ($uriParts[1] == 'wiki') { + $wikiId = strtolower( $uriParts[2] ); + } elseif ( in_array( $uriParts[1], ['de','en','es','fr','it','ja','ko','pt','sv','ru','zh'] ) ) { + $wikiId = $uriParts[1]; + } else { + $wikiId = strtolower($uriParts[1]); // default } -{% endif %} } @@ -113,12 +115,8 @@ $wikis = array_slice( scandir( "$m_htdocs/wikis" ), 2 ); if ( ! in_array( $wikiId, $wikis ) && empty ( $wikiId ) ) { - // get $wikiId from environment variable - $wikiId = getenv( $mezaWikiEnvVarName ); - if (!$wikiId) { - // handle invalid wiki - die( "No sir, I ain't heard'a no wiki that goes by the name \"$wikiId\"\n" ); - } + // handle invalid wiki + die( "No sir, I ain't heard'a no wiki that goes by the name \"$wikiId\"\n" ); } {% if meza_auth_type is defined %} @@ -269,6 +267,18 @@ else { * **/ + if (isset($_SERVER["HTTP_HOST"])) { + define('WIKI_HOST', 'https://' . $_SERVER["HTTP_HOST"]); + } else { // probably commandline invocation + $fullhost = shell_exec("hostname -f"); + if ( strstr( $fullhost, 'qualitybox' ) ) { + define('WIKI_HOST', 'https://beta.familysearch.org'); + } else { + define('WIKI_HOST', 'https://www.familysearch.org'); + } + } + + // ref: https://www.mediawiki.org/wiki/Manual:$wgServer // From section #Autodetection: // "When $wgServer is not set, the default value is calculated From 0a1f6e410b83958211a86a6ef4e73936956387eb Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 06:19:36 +0000 Subject: [PATCH 260/280] remove incorrect "ignore errors" from HaProxy --- src/roles/haproxy/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index a71175a37..f82ecf354 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -439,6 +439,5 @@ name: haproxy state: started enabled: yes - ignore_errors: true when: docker_skip_tasks is not defined or not docker_skip_tasks From 4f65f032c07b23362c56ecafbfa4c35a0e15f76e Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 6 Sep 2020 06:48:11 +0000 Subject: [PATCH 261/280] fix logig for wikiId --- src/roles/mediawiki/templates/LocalSettings.php.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index 30fb8c177..bb70f5f0e 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -69,9 +69,9 @@ if (empty($wikiId)) { if ( preg_match( '%([a-z]+)\.([a-z]+)\..{2,4}[\d]*$%im', $_SERVER['HTTP_HOST'], $matches ) ) { $wikiId = $matches[1]; } - {% endif %} -} else { + {% else %} + // get $wikiId from URI path $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); if ($uriParts[1] == 'wiki') { @@ -81,9 +81,9 @@ if (empty($wikiId)) { } else { $wikiId = strtolower($uriParts[1]); // default } + {% endif %} } - {% if wiki_id_redirects is defined and wiki_id_redirects|length > 0 %} // array point wiki IDs to redirect from and to From 5e7e1fdf9d26cccd3a6992d7dcab1097129f14c6 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 21 Sep 2020 14:06:16 -0400 Subject: [PATCH 262/280] bump version of Semantic Result Formats from ~3.1 to ~3.2 --- config/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 1ecc0139e..8abde0d9f 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -27,7 +27,7 @@ list: - name: Semantic Result Formats composer: "mediawiki/semantic-result-formats" - version: "~3.1" + version: "~3.2" config: | // In SRF 3.0+ you need to do this, too: wfLoadExtension( 'SemanticResultFormats' ); From 2e669a69682f486c7ba898e258b80993dbbddf2d Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 21 Sep 2020 14:07:35 -0400 Subject: [PATCH 263/280] Bump Maps from 7.17.2 to ~7.0 --- config/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 8abde0d9f..f5521dde6 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -71,7 +71,7 @@ list: version: "1.6.1" - name: Maps composer: "mediawiki/maps" - version: "7.17.2" + version: "~7.0" config: | // In Maps 6.0+ you need to also load the extension wfLoadExtension( 'Maps' ); From f1232a74c6e7e4f143fcf1498968836296e3b616 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 21 Sep 2020 14:08:53 -0400 Subject: [PATCH 264/280] fix comment about excel support --- config/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index f5521dde6..87854a077 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -38,7 +38,7 @@ list: // $srfgFormats[] = 'googlebar'; // $srfgFormats[] = 'googlepie'; - // Disabled until the proper dependencies are added (PHPExcel I think) + // Disabled until the proper dependencies are added (the phpspreadsheet library from phpoffice) // $srfgFormats[] = 'excel'; // Enables the "filtered" format. Where do we use this? From c601dd37a4ad91ec813249b2cfbba0135bddbd17 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 22 Sep 2020 21:41:14 +0000 Subject: [PATCH 265/280] Add prefix to wgScriptPath if defined --- .../mediawiki/templates/LocalSettings.php.j2 | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index bb70f5f0e..78d3a7ecb 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -297,8 +297,20 @@ if ( ! $wgCommandLineMode ) { // https://www.mediawiki.org/wiki/Manual:$wgScriptPath +// https://www.mediawiki.org/wiki/Manual:$wgLogo +// https://www.mediawiki.org/wiki/Manual:$wgFavicon +{% if m_mediawiki_prefix_dir is defined %} +$mPrefix = "{{ m_mediawiki_prefix_dir|replace('/', '') }}"; +$wgScriptPath = "/$mPrefix/$wikiId"; +$wgArticlePath = "/$mPrefix/$wikiId/$1"; +$wgLogo = "/$mPrefix/wikis/$wikiId/config/logo.png"; +$wgFavicon = "/$mPrefix/wikis/$wikiId/config/favicon.ico"; +{% else %} $wgScriptPath = "/$wikiId"; $wgArticlePath = "/$wikiId/$1"; +$wgLogo = "/wikis/$wikiId/config/logo.png"; +$wgFavicon = "/wikis/$wikiId/config/favicon.ico"; +{% endif %} // https://www.mediawiki.org/wiki/Manual:$wgUploadPath $wgUploadPath = "$wgScriptPath/img_auth.php"; @@ -306,12 +318,6 @@ $wgUploadPath = "$wgScriptPath/img_auth.php"; // https://www.mediawiki.org/wiki/Manual:$wgUploadDirectory $wgUploadDirectory = "{{ m_uploads_dir }}/$wikiId"; -// https://www.mediawiki.org/wiki/Manual:$wgLogo -$wgLogo = "/wikis/$wikiId/config/logo.png"; - -// https://www.mediawiki.org/wiki/Manual:$wgFavicon -$wgFavicon = "/wikis/$wikiId/config/favicon.ico"; - // https://www.mediawiki.org/wiki/Manual:$wgMetaNamespace $wgMetaNamespace = str_replace( ' ', '_', $wgSitename ); From 49da6d4286f762efc9bf4f8ef0d19d25150854a9 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 23 Sep 2020 17:29:51 +0000 Subject: [PATCH 266/280] Make .htaccess work with m_mediawiki_prefix_dir --- src/roles/htdocs/templates/.htaccess.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index cea771794..e0296fb24 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -44,8 +44,11 @@ RewriteCond %{REQUEST_FILENAME} !-d RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-l + {% if m_mediawiki_prefix_dir is defined %} + RewriteRule ^/?(?:wiki)?/(de|en|es|fr|it|ja|ko|pt|ru|sv|zh)/(.*)?$ %{DOCUMENT_ROOT}/wiki/mediawiki/ [env=WIKI:$1,L] + {% else %} RewriteRule ^/?([^/]+)/(.*)?$ %{DOCUMENT_ROOT}/mediawiki/ [env=WIKI:$1,L] - + {% endif %} From 49209acd90bb76addb7dae8ddc83fc41bfe66263 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 23 Sep 2020 17:31:12 +0000 Subject: [PATCH 267/280] Make $wgUseRCPatrol configurable --- src/roles/mediawiki/templates/LocalSettings.php.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index 78d3a7ecb..7610ee95c 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -692,7 +692,7 @@ $wgULSGeoService = false; $wgNamespacesWithSubpages[NS_MAIN] = true; -$wgUseRCPatrol = false; +$wgUseRCPatrol = {{ wgUseRCPatrol|default(false) }}; From 0b14fdfb71553f6db199dbcab616ee46fec2023b Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 28 Sep 2020 15:20:09 +0000 Subject: [PATCH 268/280] Switch InputBox to "master" --- config/MezaCoreExtensions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 87854a077..9fa5f5f4a 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -136,7 +136,7 @@ list: version: "{{ mediawiki_default_branch }}" - name: InputBox repo: https://github.com/wikimedia/mediawiki-extensions-InputBox.git - version: "{{ mediawiki_default_branch }}" + version: "master" - name: ReplaceText repo: https://github.com/wikimedia/mediawiki-extensions-ReplaceText.git version: "master" From 02695c14320984e95acd14f8617f283b4e7ab379 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 29 Sep 2020 19:13:47 +0000 Subject: [PATCH 269/280] Set better defaults for Apache --- src/roles/apache-php/templates/httpd.conf.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/apache-php/templates/httpd.conf.j2 b/src/roles/apache-php/templates/httpd.conf.j2 index 0ab01583b..43384ed7c 100644 --- a/src/roles/apache-php/templates/httpd.conf.j2 +++ b/src/roles/apache-php/templates/httpd.conf.j2 @@ -444,12 +444,12 @@ MinSpareServers 3 MaxSpareServers 6 # maximum value of clients for the lifetime of the server before it is reaped to prevent mem leaks -ServerLimit 256 +ServerLimit 20000 # MaxRequestWorkers was called MaxClients before version 2.3.13. The old name is still supported. # translates into the maximum number of child processes that will be launched to serve requests on prefork # the limit on the number of simultaneous requests that will be served; others queued. -MaxRequestWorkers 10 +MaxRequestWorkers 1000 # Server 2.3.9 and later. The old name MaxRequestsPerChild is still supported. # number of connections that an individual child server will handle during its life From 2fa84cd6b4f8d8efdbedcb8385f0ee115e11d564 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 29 Sep 2020 20:05:25 +0000 Subject: [PATCH 270/280] Bump Wiretap to master for 1.34 and 1.35 compatibility --- config/MezaCoreExtensions.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index 9fa5f5f4a..3f982753c 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -259,9 +259,9 @@ list: # - name: CopyWatchers # repo: https://github.com/jamesmontalvo3/MediaWiki-CopyWatchers.git # version: tags/0.10.0 -# - name: Wiretap -# repo: https://github.com/enterprisemediawiki/Wiretap.git -# version: tags/0.1.0 + - name: Wiretap + repo: https://github.com/enterprisemediawiki/Wiretap.git + version: master - name: ApprovedRevs repo: https://github.com/wikimedia/mediawiki-extensions-ApprovedRevs.git version: master From be1c719c458ce87e45f6f2bdb879d193ba3e075f Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Wed, 30 Sep 2020 15:45:47 +0000 Subject: [PATCH 271/280] add enableSemantics condition to setupStore.php --- src/roles/mediawiki/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index 438e9519b..cfb6b1b29 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -486,6 +486,7 @@ shell: WIKI={{ item }} php {{ m_mediawiki }}/extensions/SemanticMediaWiki/maintenance/setupStore.php with_items: "{{ list_of_wikis }}" run_once: true + when: "{{ enableSemantics | default(false,true) }}" # - name: Verify metastore index upgraded From bf2a753d632e93671d88f196a19ff0964ab8d9d5 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 8 Oct 2020 17:38:54 -0400 Subject: [PATCH 272/280] Enterprise security teams are demanding that 3DES ciphers be disabled - per Rich Evans --- src/roles/haproxy/templates/haproxy.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index bfb539768..539ed65df 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -17,7 +17,7 @@ global # Default ciphers to use on SSL-enabled listening sockets. # For more information, see ciphers(1SSL). This list is from: # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ - ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS + ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:!RSA+3DES:!aNULL:!MD5:!DSS ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 From 1ccc89f7e3ade8e9d8781009b459826ea24c5b24 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 15 Oct 2020 00:47:49 -0400 Subject: [PATCH 273/280] Update README with project rename --- README.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d13f6d17e..35a78b5d9 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,16 @@ -# meza - -[![Build Status](https://travis-ci.org/enterprisemediawiki/meza.svg?branch=master)](https://travis-ci.org/enterprisemediawiki/meza) -[![Code Climate](https://codeclimate.com/github/enterprisemediawiki/meza/badges/gpa.svg)](https://codeclimate.com/github/enterprisemediawiki/meza) - - +# QualityBox Setup an enterprise MediaWiki server with **simple commands**. Put all components on a single monolithic server or split them out over many. Run a solitary master database or have replicas. Deploy to multiple environments. Run backups. Do it all using the `meza` command. Run `meza --help` for more info. -## Why meza? +> Note: This project is a fork of the original Meza project developed at NASA. We still use the same 'meza' command internally but are gradually updating the system to be consistent with the QualityBox (or qb) brand. + +## Why QualityBox? -Standard MediaWiki is easy to install, but increasingly its newer and better features are contained within extensions that are more complicated. Additionally, they may be particularly difficult to install on Enterprise Linux derivatives. This project aims to make these features (VisualEditor, CirrusSearch, etc) easy to *install, backup, reconfigure, and maintain* in a robust and well-tested way. +Standard MediaWiki is easy to install, but increasingly its newer and better features are contained within extensions that are more complicated. Additionally, they may be particularly difficult to install on Enterprise Linux derivatives. This project aims to make these features (VisualEditor, CirrusSearch, etc) easy to *install, backup, reconfigure, and maintain* in a robust and well-tested way. Furthermore, we aim to package all the extra security, tools and configuration that enable scalable PUBLIC wikis to thrive in a hostile Internet. ## Requirements -1. CentOS 7 or RHEL 7 (for now, with Debian support in the works) -2. Minimal install: Attempting to install it on a server with many other things already installed may not work properly due to conflicts. +1. CentOS 7 RHEL 7 or Debian 10 ## Install and usage From 74d50997018e2fbf6b69ee085818b3cdae7dd025 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Thu, 15 Oct 2020 00:49:45 -0400 Subject: [PATCH 274/280] Remove reference to master --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 35a78b5d9..d17e66ff6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # QualityBox -Setup an enterprise MediaWiki server with **simple commands**. Put all components on a single monolithic server or split them out over many. Run a solitary master database or have replicas. Deploy to multiple environments. Run backups. Do it all using the `meza` command. Run `meza --help` for more info. +Setup an enterprise MediaWiki server with **simple commands**. Put all components on a single monolithic server or split them out over many. Run a single database or have replicas. Deploy to multiple environments. Run backups. Do it all using the `meza` command. Run `meza --help` for more info. > Note: This project is a fork of the original Meza project developed at NASA. We still use the same 'meza' command internally but are gradually updating the system to be consistent with the QualityBox (or qb) brand. From c038ff5c703ce7975123ad5e4cfa9001eed98a96 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 25 Oct 2020 09:13:46 -0400 Subject: [PATCH 275/280] Change Apache log format to Combined Including XFF header first. Make GoAccess compatible with new format --- src/roles/apache-php/tasks/main.yml | 14 +++++++++++++- src/roles/apache-php/templates/httpd.conf.j2 | 12 ++++++------ src/roles/goaccess/templates/my.goaccess.conf.j2 | 4 +++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/roles/apache-php/tasks/main.yml b/src/roles/apache-php/tasks/main.yml index 608733d41..d7fd21ed8 100644 --- a/src/roles/apache-php/tasks/main.yml +++ b/src/roles/apache-php/tasks/main.yml @@ -102,8 +102,20 @@ enabled: yes when: docker_skip_tasks is not defined or not docker_skip_tasks +# Make sure that there's a symlink for the access and error logs +# @TODO Test this on CentOS/RHEL and possibly source /etc/httpd/envvars +- name: Add symbolic link to Apache's access log file for easy access + file: + src: "${APACHE_LOG_DIR}/access_log" + dest: "${APACHE_LOG_DIR}/access.log" + state: link - +- name: Add symbolic link to Apache's error log file for easy access + file: + src: "${APACHE_LOG_DIR}/error_log" + dest: "${APACHE_LOG_DIR}/error.log" + state: link + # Might need these for SELinux to be turned back on # - name: Configure SELinux to start mysql on any port # seboolean: name=mysql_connect_any state=true persistent=yes diff --git a/src/roles/apache-php/templates/httpd.conf.j2 b/src/roles/apache-php/templates/httpd.conf.j2 index 43384ed7c..cfd7dd092 100644 --- a/src/roles/apache-php/templates/httpd.conf.j2 +++ b/src/roles/apache-php/templates/httpd.conf.j2 @@ -202,7 +202,7 @@ ServerName https://{{ wiki_app_fqdn }} {% if ansible_os_family == "RedHat" %} ErrorLog "logs/error_log" {% else %} -ErrorLog ${APACHE_LOG_DIR}/error.log +ErrorLog ${APACHE_LOG_DIR}/error_log {% endif %} # @@ -247,19 +247,19 @@ AddOutputFilterByType DEFLATE application/x-javascript # %I - Bytes received, including request and headers, cannot be zero. # %O - Bytes sent, including headers, cannot be zero. - # Logging for normal requests from clients - LogFormat "%{X-Forwarded-For}i %h %l %u %t %D \"%r\" %>s \"%{Referer}i\" \"%{User-Agent}i\" %I %O" proxy + # Logging for normal requests from clients ("combined" format, prefixed with XFF header) + LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" proxy # This is for non-proxied requests, meaning requests made from within the # server itself or by other non-HAProxy nodes (if that's possible) - LogFormat "noproxy %h %l %u %t %D \"%r\" %>s \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combined + LogFormat "noproxy %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" internal SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded {% if ansible_os_family == "RedHat" %} - CustomLog "logs/access_log" combined env=!forwarded + CustomLog "logs/access_log" internal env=!forwarded CustomLog "logs/access_log" proxy env=forwarded {% else %} - CustomLog "${APACHE_LOG_DIR}/access_log" combined env=!forwarded + CustomLog "${APACHE_LOG_DIR}/access_log" internal env=!forwarded CustomLog "${APACHE_LOG_DIR}/access_log" proxy env=forwarded {% endif %} diff --git a/src/roles/goaccess/templates/my.goaccess.conf.j2 b/src/roles/goaccess/templates/my.goaccess.conf.j2 index b705c4d76..8cd90019d 100644 --- a/src/roles/goaccess/templates/my.goaccess.conf.j2 +++ b/src/roles/goaccess/templates/my.goaccess.conf.j2 @@ -56,7 +56,9 @@ date-format %d/%b/%Y # Meza Custom Log format log-format ~h{, } %^ %e [%d:%t %z] %D "%r" %s "%R" "%u" %^ %b -# +# XFF + NCSA Combined Log Format +log-format ~h %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + # The log-format variable followed by a space or \t for # tab-delimited, specifies the log format string. # From c91ee6c2e6b9236130cd1b175fa75f833ed014f6 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 25 Oct 2020 10:38:59 -0400 Subject: [PATCH 276/280] Bugfix: can't use syntax with CentOS/RedHat --- src/roles/elasticsearch/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index dfec774fb..386849f34 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -56,7 +56,7 @@ - name: Ensure elasticsearch installed package: - name: elasticsearch=6* + name: elasticsearch state: present notify: restart elasticsearch From 0c77aae57987ae35d28db6c86abf5f4f4ade0ad5 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Sun, 25 Oct 2020 15:27:23 -0400 Subject: [PATCH 277/280] Fix path for symlinks --- src/roles/apache-php/tasks/main.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/roles/apache-php/tasks/main.yml b/src/roles/apache-php/tasks/main.yml index d7fd21ed8..4140d8d1a 100644 --- a/src/roles/apache-php/tasks/main.yml +++ b/src/roles/apache-php/tasks/main.yml @@ -103,17 +103,16 @@ when: docker_skip_tasks is not defined or not docker_skip_tasks # Make sure that there's a symlink for the access and error logs -# @TODO Test this on CentOS/RHEL and possibly source /etc/httpd/envvars - name: Add symbolic link to Apache's access log file for easy access file: - src: "${APACHE_LOG_DIR}/access_log" - dest: "${APACHE_LOG_DIR}/access.log" + src: "/var/log/{{ service_apache }}/access_log" + dest: "/var/log/{{ service_apache }}/access.log" state: link - name: Add symbolic link to Apache's error log file for easy access file: - src: "${APACHE_LOG_DIR}/error_log" - dest: "${APACHE_LOG_DIR}/error.log" + src: "/var/log/{{ service_apache }}/error_log" + dest: "/var/log/{{ service_apache }}/error.log" state: link # Might need these for SELinux to be turned back on From ec0f9a6651f35a7482479f7623f2cec17b09407e Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 16 Nov 2020 14:28:16 -0500 Subject: [PATCH 278/280] Make composer-merge-plugin work by setting version; fixes issue #28 --- src/roles/composer/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roles/composer/defaults/main.yml b/src/roles/composer/defaults/main.yml index d6f113328..8787dcbda 100644 --- a/src/roles/composer/defaults/main.yml +++ b/src/roles/composer/defaults/main.yml @@ -1,7 +1,7 @@ --- composer_path: /usr/local/bin/composer composer_keep_updated: true -composer_version: '' +composer_version: '1.10.16' # The directory where global packages will be installed. composer_home_path: '~/.composer' From 0c2227e1908e7d9e472cff2a0e751d25060f1c53 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Mon, 16 Nov 2020 14:29:07 -0500 Subject: [PATCH 279/280] Make HAProxy work with no IPv6; fixes issue #18 --- src/roles/haproxy/templates/haproxy.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index 539ed65df..8fe7464d4 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -46,7 +46,7 @@ defaults frontend www-http bind *:80 - {% if ansible_default_ipv6.address | length > 0 -%} + {% if ansible_default_ipv6 | length > 0 -%} bind {{ ansible_default_ipv6.address }}:80 {%- endif %} @@ -55,7 +55,7 @@ frontend www-http frontend www-https bind *:443 ssl crt /etc/haproxy/certs/ - {% if ansible_default_ipv6.address | length > 0 -%} + {% if ansible_default_ipv6 | length > 0 -%} # Create an AAAA record in DNS using the IP below to IPv6 enable your wiki bind {{ ansible_default_ipv6.address }}:443 ssl crt /etc/haproxy/certs/ {%- endif %} From e61ab45253d2142a9cd3c4bd0e3c871f60d2e308 Mon Sep 17 00:00:00 2001 From: Greg Rundlett Date: Tue, 3 Nov 2020 23:55:15 +0000 Subject: [PATCH 280/280] Added file checks for Apache log symbolic links --- src/roles/apache-php/tasks/main.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/roles/apache-php/tasks/main.yml b/src/roles/apache-php/tasks/main.yml index 4140d8d1a..ffabf0ac7 100644 --- a/src/roles/apache-php/tasks/main.yml +++ b/src/roles/apache-php/tasks/main.yml @@ -102,18 +102,41 @@ enabled: yes when: docker_skip_tasks is not defined or not docker_skip_tasks +# check if a symbolic link needs to be created +# If the file exists (and is regular) then we should not create a symlink +- name: Check whether we need a symbolic link to the access log + stat: + path: "/var/log/{{ service_apache }}/access.log" + register: accesslog_details + +- debug: + msg: "access.log is NOT a symbolic link" + when: accesslog_details.stat.islnk is not defined + +- name: Check whether we need a symbolic link to the error log + stat: + path: "/var/log/{{ service_apache }}/error.log" + register: errorlog_details + +- debug: + msg: "error.log is NOT a symbolic link" + when: errorlog_details.stat.islnk is not defined + # Make sure that there's a symlink for the access and error logs +# but if the file/directory/link already exists, leave it alone - name: Add symbolic link to Apache's access log file for easy access file: src: "/var/log/{{ service_apache }}/access_log" dest: "/var/log/{{ service_apache }}/access.log" state: link + when: not accesslog_details.stat.exists - name: Add symbolic link to Apache's error log file for easy access file: src: "/var/log/{{ service_apache }}/error_log" dest: "/var/log/{{ service_apache }}/error.log" state: link + when: not errorlog_details.stat.exists # Might need these for SELinux to be turned back on # - name: Configure SELinux to start mysql on any port