diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..86c5bfad5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +--- +"on": + # Trigger the workflow on pushes to these branches + push: + branches: + - master + - 34.x +jobs: + one: + # ubuntu-18.04 + # runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + + # GitHub Runners include a LOT of software by default + # https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md + - name: Uninstall MySQL so we can use MariaDB + run: sudo apt-get remove --purge mysql-server + + - name: Make target directory for sources + run: mkdir -p /opt/meza + + - name: Copy code + run: sudo cp -r ${{ github.workspace }} /opt/ + + - name: Install prerequisites + run: sudo apt install -y git ansible + + # - name: Clone Meza + # run: sudo git clone https://github.com/freephile/meza /opt/meza + + - name: Run "getmeza with sudo -H to avoid cache warnings" + run: sudo -H bash /opt/meza/src/scripts/getmeza.sh + + - name: Setup the monolith environment + run: sudo meza setup env monolith --fqdn="127.0.0.1" --db_pass=1234 --private_net_zone=public + + # The --no-firewall option does not exist + - name: Deploy Meza + run: sudo meza deploy monolith -vvv + + - name: Print hosts file as a rudimentary "test" + run: cat /opt/conf-meza/secret/monolith/hosts diff --git a/README.md b/README.md index d13f6d17e..d17e66ff6 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,16 @@ -# meza +# QualityBox -[![Build Status](https://travis-ci.org/enterprisemediawiki/meza.svg?branch=master)](https://travis-ci.org/enterprisemediawiki/meza) -[![Code Climate](https://codeclimate.com/github/enterprisemediawiki/meza/badges/gpa.svg)](https://codeclimate.com/github/enterprisemediawiki/meza) +Setup an enterprise MediaWiki server with **simple commands**. Put all components on a single monolithic server or split them out over many. Run a single database or have replicas. Deploy to multiple environments. Run backups. Do it all using the `meza` command. Run `meza --help` for more info. - +> Note: This project is a fork of the original Meza project developed at NASA. We still use the same 'meza' command internally but are gradually updating the system to be consistent with the QualityBox (or qb) brand. -Setup an enterprise MediaWiki server with **simple commands**. Put all components on a single monolithic server or split them out over many. Run a solitary master database or have replicas. Deploy to multiple environments. Run backups. Do it all using the `meza` command. Run `meza --help` for more info. +## Why QualityBox? -## Why meza? - -Standard MediaWiki is easy to install, but increasingly its newer and better features are contained within extensions that are more complicated. Additionally, they may be particularly difficult to install on Enterprise Linux derivatives. This project aims to make these features (VisualEditor, CirrusSearch, etc) easy to *install, backup, reconfigure, and maintain* in a robust and well-tested way. +Standard MediaWiki is easy to install, but increasingly its newer and better features are contained within extensions that are more complicated. Additionally, they may be particularly difficult to install on Enterprise Linux derivatives. This project aims to make these features (VisualEditor, CirrusSearch, etc) easy to *install, backup, reconfigure, and maintain* in a robust and well-tested way. Furthermore, we aim to package all the extra security, tools and configuration that enable scalable PUBLIC wikis to thrive in a hostile Internet. ## Requirements -1. CentOS 7 or RHEL 7 (for now, with Debian support in the works) -2. Minimal install: Attempting to install it on a server with many other things already installed may not work properly due to conflicts. +1. CentOS 7 RHEL 7 or Debian 10 ## Install and usage diff --git a/Vagrantfile b/Vagrantfile index 8e59b9e46..a4b1477eb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,7 +28,7 @@ elsif configuration.key?("box_os") box_os = configuration["box_os"] if box_os == "debian" - baseBox = "debian/contrib-stretch64" + baseBox = "debian/contrib-buster64" elsif box_os == "centos" baseBox = "bento/centos-7.4" else diff --git a/config/Debian.yml b/config/Debian.yml index 0a8ff42de..c7cef43d4 100644 --- a/config/Debian.yml +++ b/config/Debian.yml @@ -7,46 +7,11 @@ package_firewall: ufw package_openssh_client: openssh-client package_cron: cron package_apache: apache2 - -# RedHat install has a lot of php/apache dependencies, but these may be carry-overs -# from when Meza installed Apache and PHP from source. They probably are not needed -# anymore. package_php_apache_deps: [] - # Did not immediately find a replacement package in Debian for these RH packages - # - zlib-devel - # - sqlite-devel - # - bzip2-devel - # - pcre-devel - # - libXpm-devel - # - gmp-devel - # - t1lib-devel - # - aspell-devel - # - libvpx-devel - # - readline-devel - # - libmcrypt-devel - # - pam-devel - # - m4 - # - xz-libs - # These are either named the same in RE vs Debian, or were easily found - # - libcurl4-openssl-dev - # - libssl-dev - # - libxml2-dev - # - libicu-dev - # - libjpeg-dev - # - libpng-dev - # - libfreetype6-dev - # - libtidy-dev - # - sendmail - # - sendmail-cf - # - mariadb-common - # These are required to install PHP from source. - # - libxpm-dev - # - libmysqlclient-dev - # - libpq-dev - # - libldap2-dev - # - libxslt-dev - # - libldb-dev +# Use Tokyo Cabinet for on-disk support if you know your distro supports it +# override with goaccess-tcb +package_goaccess: goaccess package_java: "default-jre" package_python3: python3 package_python3_pip: python3-pip diff --git a/config/MezaCoreExtensions.yml b/config/MezaCoreExtensions.yml index ba5e6de7a..3f982753c 100644 --- a/config/MezaCoreExtensions.yml +++ b/config/MezaCoreExtensions.yml @@ -3,7 +3,7 @@ list: - name: Semantic MediaWiki composer: "mediawiki/semantic-media-wiki" - version: "3.0.0" + version: "~3.1" config: | // Enable Semantic MediaWiki semantics enableSemantics( $wikiId ); @@ -23,11 +23,11 @@ list: // created them, and thus they effectively don't replicate). Picking // DB_SLAVE since temporary tables are effectively a read-action since // they are used only for making more efficient queries. - $smwgLocalConnectionConf['mw.db.queryengine'] = [ 'read' => DB_SLAVE, 'write' => DB_SLAVE ]; + $smwgLocalConnectionConf['mw.db.queryengine'] = [ 'read' => DB_REPLICA, 'write' => DB_REPLICA ]; - name: Semantic Result Formats composer: "mediawiki/semantic-result-formats" - version: "3.0.0" + version: "~3.2" config: | // In SRF 3.0+ you need to do this, too: wfLoadExtension( 'SemanticResultFormats' ); @@ -38,7 +38,7 @@ list: // $srfgFormats[] = 'googlebar'; // $srfgFormats[] = 'googlepie'; - // Disabled until the proper dependencies are added (PHPExcel I think) + // Disabled until the proper dependencies are added (the phpspreadsheet library from phpoffice) // $srfgFormats[] = 'excel'; // Enables the "filtered" format. Where do we use this? @@ -46,10 +46,14 @@ list: // Disabled due to some issue on FOD wikis. Confirm, reenable if possible // $srfgFormats[] = 'exhibit'; - + + # Allows for the display of more than one SMW inline query in one results display set. - name: SemanticCompoundQueries composer: "mediawiki/semantic-compound-queries" - version: "1.2.0" + version: "~2.1" + config: | + wfLoadExtension( 'SemanticCompoundQueries' ); + - name: Scribunto repo: https://github.com/wikimedia/mediawiki-extensions-Scribunto.git version: "{{ mediawiki_default_branch }}" @@ -59,22 +63,22 @@ list: $wgScribuntoUseCodeEditor = true; - name: "Semantic Scribunto" composer: "mediawiki/semantic-scribunto" - version: 2.0.0 + version: ~2.1 config: | wfLoadExtension( 'SemanticScribunto' ); - name: SubPageList composer: "mediawiki/sub-page-list" - version: "1.5.0" + version: "1.6.1" - name: Maps composer: "mediawiki/maps" - version: "6.0.3" + version: "~7.0" config: | // In Maps 6.0+ you need to also load the extension wfLoadExtension( 'Maps' ); - name: DisplayTitle repo: https://github.com/wikimedia/mediawiki-extensions-DisplayTitle.git - version: tags/1.2 + version: "{{ mediawiki_default_branch }}" # @@ -89,7 +93,7 @@ list: $wgPFEnableStringFunctions = true; - name: ExternalData repo: https://github.com/wikimedia/mediawiki-extensions-ExternalData.git - version: "{{ mediawiki_default_branch }}" + version: "master" - name: LabeledSectionTransclusion repo: https://github.com/wikimedia/mediawiki-extensions-LabeledSectionTransclusion.git version: "{{ mediawiki_default_branch }}" @@ -106,8 +110,10 @@ list: version: "{{ mediawiki_default_branch }}" - name: PageForms repo: https://github.com/wikimedia/mediawiki-extensions-PageForms.git - # commit includes spreadsheet sorting which didn't make PageForms 4.4.1 - version: "730390a31a56c001af83948af1eefc5174abbe06" + version: "master" + config: | + // If enabled all "red links" will bring up a form chooser + $wgPageFormsLinkAllRedLinksToForms = false; - name: DismissableSiteNotice repo: https://github.com/wikimedia/mediawiki-extensions-DismissableSiteNotice.git version: "{{ mediawiki_default_branch }}" @@ -130,10 +136,10 @@ list: version: "{{ mediawiki_default_branch }}" - name: InputBox repo: https://github.com/wikimedia/mediawiki-extensions-InputBox.git - version: "{{ mediawiki_default_branch }}" + version: "master" - name: ReplaceText repo: https://github.com/wikimedia/mediawiki-extensions-ReplaceText.git - version: tags/1.4 + version: "master" - name: Interwiki repo: https://github.com/wikimedia/mediawiki-extensions-Interwiki.git version: "{{ mediawiki_default_branch }}" @@ -222,11 +228,11 @@ list: $wgPageShowWatchingUsers = true; - name: SemanticInternalObjects repo: https://github.com/wikimedia/mediawiki-extensions-SemanticInternalObjects.git - version: "{{ mediawiki_default_branch }}" + version: "master" legacy_load: true - name: SemanticDrilldown repo: https://github.com/wikimedia/mediawiki-extensions-SemanticDrilldown.git - version: "{{ mediawiki_default_branch }}" + version: "master" legacy_load: true - name: Arrays repo: https://github.com/wikimedia/mediawiki-extensions-Arrays.git @@ -238,8 +244,7 @@ list: legacy_load: true - name: AdminLinks repo: https://github.com/wikimedia/mediawiki-extensions-AdminLinks.git - version: "{{ mediawiki_default_branch }}" - legacy_load: true + version: "master" config: | $wgGroupPermissions['sysop']['adminlinks'] = true; - name: BatchUserRights @@ -251,16 +256,15 @@ list: config: | $wgHeaderTabsEditTabLink = false; $wgHeaderTabsRenderSingleTab = true; - - name: CopyWatchers - repo: https://github.com/jamesmontalvo3/MediaWiki-CopyWatchers.git - version: tags/0.10.0 +# - name: CopyWatchers +# repo: https://github.com/jamesmontalvo3/MediaWiki-CopyWatchers.git +# version: tags/0.10.0 - name: Wiretap repo: https://github.com/enterprisemediawiki/Wiretap.git - version: tags/0.1.0 + version: master - name: ApprovedRevs repo: https://github.com/wikimedia/mediawiki-extensions-ApprovedRevs.git - # Use this commit until a release tag for v1.0 is created - version: tags/1.0 + version: master config: | $egApprovedRevsAutomaticApprovals = false; - name: ImagesLoaded @@ -269,11 +273,11 @@ list: - name: MasonryMainPage repo: https://github.com/enterprisemediawiki/MasonryMainPage.git version: tags/0.3.0 - - name: WatchAnalytics - repo: https://github.com/enterprisemediawiki/WatchAnalytics.git - version: tags/3.1.1 - config: | - $egPendingReviewsEmphasizeDays = 10; // makes Pending Reviews shake after X days +# - name: WatchAnalytics +# repo: https://github.com/enterprisemediawiki/WatchAnalytics.git +# version: tags/3.1.1 +# config: | +# $egPendingReviewsEmphasizeDays = 10; // makes Pending Reviews shake after X days - name: Variables repo: https://github.com/wikimedia/mediawiki-extensions-Variables.git version: "{{ mediawiki_default_branch }}" @@ -328,7 +332,7 @@ list: $wgApiFrameOptions = 'SAMEORIGIN'; // Use UploadWizard by default in navigation bar - $wgUploadNavigationUrl = "$wgScriptPath/index.php/Special:UploadWizard"; + // $wgUploadNavigationUrl = "$wgScriptPath/index.php/Special:UploadWizard"; $wgUploadWizardConfig = array( 'debug' => false, 'autoCategory' => 'Uploaded with UploadWizard', @@ -369,7 +373,7 @@ list: - name: DataTransfer repo: https://github.com/wikimedia/mediawiki-extensions-DataTransfer.git - version: "{{ mediawiki_default_branch }}" + version: "master" - name: PageImporter repo: https://github.com/enterprisemediawiki/PageImporter.git version: tags/0.1.0 diff --git a/config/RedHat.yml b/config/RedHat.yml index 77b8be08e..9367a9fb3 100644 --- a/config/RedHat.yml +++ b/config/RedHat.yml @@ -34,6 +34,7 @@ package_php_apache_deps: - m4 - xz-libs - mariadb-libs +package_goaccess: goaccess package_java: java-1.8.0-openjdk package_python3: python35u package_python3_pip: python35u-pip diff --git a/config/defaults.yml b/config/defaults.yml index f956b11df..422197551 100644 --- a/config/defaults.yml +++ b/config/defaults.yml @@ -9,6 +9,10 @@ m_meza: /opt/meza # production being as performant and secure as possible # development being less secure and providing optional features useful for dev m_use_production_settings: True +# default (False) will fail on local mediawiki extension modifcations +# Overriding can be useful to add to public.yml for development environments +m_ignore_local_mods: False +m_use_subdomains: false # config dir m_config_core: /opt/meza/config @@ -101,6 +105,11 @@ wiki_id_redirects: {} m_force_debug: false +# if you enable certbot, make sure you add m_httpd_server_admin in your public.yml +enable_certbot: false + +enable_goaccess: false + enable_wiki_emails: true enable_haproxy_stats: false @@ -151,23 +160,23 @@ m_meza_sudeoers_file: "/etc/sudoers.d/meza-ansible" # # Version of MediaWiki core -mediawiki_version: "REL1_32" +mediawiki_version: "REL1_34" # Branch to use on many extensions extensions and skins -mediawiki_default_branch: "REL1_32" +mediawiki_default_branch: "REL1_34" # PHP version -php_ius_version: "php71u" -php_debian_version: "7.1" +php_ius_version: "php72u" +php_debian_version: "7.2" # Parsoid version # e6b708 is commit on 16-OCT-2018, the day the REL1_32 branch was made -m_parsoid_version: "e6b708b3d7ff50a314d021295dce874574aa3e9e" +m_parsoid_version: "master" # MediaWiki 1.27 and earlier require ElasticSearch 1.6 # MediaWiki 1.28 and higher require ElasticSearch 2.x # MediaWiki 1.29 and higher require ElasticSearch 5.x -elasticsearch_major_version: "5.x" +elasticsearch_major_version: "6.x" # Whether or not to install Lua/luasandbox m_install_lua: true @@ -199,9 +208,6 @@ packages_php_debian: - php-pear - php-memcached - # No PHP 7.2 version of this (just like with IUS) - - php{{ php_debian_version }}-mcrypt - # No apt package corresponding to: # "{{ php_ius_version }}-process" # "{{ php_ius_version }}-pdo" diff --git a/src/roles/apache-php/tasks/main.yml b/src/roles/apache-php/tasks/main.yml index 608733d41..ffabf0ac7 100644 --- a/src/roles/apache-php/tasks/main.yml +++ b/src/roles/apache-php/tasks/main.yml @@ -102,8 +102,42 @@ enabled: yes when: docker_skip_tasks is not defined or not docker_skip_tasks +# check if a symbolic link needs to be created +# If the file exists (and is regular) then we should not create a symlink +- name: Check whether we need a symbolic link to the access log + stat: + path: "/var/log/{{ service_apache }}/access.log" + register: accesslog_details + +- debug: + msg: "access.log is NOT a symbolic link" + when: accesslog_details.stat.islnk is not defined + +- name: Check whether we need a symbolic link to the error log + stat: + path: "/var/log/{{ service_apache }}/error.log" + register: errorlog_details + +- debug: + msg: "error.log is NOT a symbolic link" + when: errorlog_details.stat.islnk is not defined + +# Make sure that there's a symlink for the access and error logs +# but if the file/directory/link already exists, leave it alone +- name: Add symbolic link to Apache's access log file for easy access + file: + src: "/var/log/{{ service_apache }}/access_log" + dest: "/var/log/{{ service_apache }}/access.log" + state: link + when: not accesslog_details.stat.exists - +- name: Add symbolic link to Apache's error log file for easy access + file: + src: "/var/log/{{ service_apache }}/error_log" + dest: "/var/log/{{ service_apache }}/error.log" + state: link + when: not errorlog_details.stat.exists + # Might need these for SELinux to be turned back on # - name: Configure SELinux to start mysql on any port # seboolean: name=mysql_connect_any state=true persistent=yes diff --git a/src/roles/apache-php/tasks/php-redhat.yml b/src/roles/apache-php/tasks/php-redhat.yml index fc164dfba..9f8e037c8 100644 --- a/src/roles/apache-php/tasks/php-redhat.yml +++ b/src/roles/apache-php/tasks/php-redhat.yml @@ -1,46 +1,16 @@ --- -- name: Install IUS (CentOS) repo. +# IUS requires EPEL, but we already have it installed from getMeza.sh +- name: Install IUS repo. yum: - name: "https://centos{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" - when: ansible_distribution == "CentOS" + name: + - "https://repo.ius.io/ius-release-el{{ ansible_distribution_major_version }}.rpm" +# - "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" + when: (ansible_distribution == "CentOS") or (ansible_distribution == "RedHat") -- name: Install IUS (RHEL) repo. - yum: - name: "https://rhel{{ ansible_distribution_major_version }}.iuscommunity.org/ius-release.rpm" - when: ansible_distribution == "RedHat" - -- name: Import IUS Community Project GPG key - rpm_key: - key: http://dl.iuscommunity.org/pub/ius/IUS-COMMUNITY-GPG-KEY - state: present - - -- name: Ensure PHP 5.6 packages removed - yum: - name: - - php56u - - php56u-cli - - php56u-common - - php56u-devel - - php56u-gd - - php56u-pecl-memcache - - php56u-pspell - - php56u-snmp - - php56u-xml - - php56u-xmlrpc - - php56u-mysqlnd - - php56u-pdo - - php56u-odbc - - php56u-pear - - php56u-pecl-jsonc - - php56u-process - - php56u-bcmath - - php56u-intl - - php56u-opcache - - php56u-soap - - php56u-mbstring - - php56u-mcrypt - - php56u-mssql +- name: Remove PHP 5.x packages + package: + lock_timeout: 180 # wait up to 3 minutes for a lock ansible/ansible#57189 + name: "php5*" state: absent # Check if the desired version of PHP is installed. If it is not, ensure any @@ -81,13 +51,14 @@ - "{{ php_ius_version }}-opcache" - "{{ php_ius_version }}-soap" - "{{ php_ius_version }}-mbstring" + - "{{ php_ius_version }}-ldap" # php56u has memcache and memcached; php7Xu only has memcached # legacy Meza used php56u-pecl-memcache - "{{ php_ius_version }}-pecl-memcached" # Available for php56u, php70u, and php71u. NOT for php72u. - - "{{ php_ius_version }}-mcrypt" + # - "php-mcrypt" # Available for php56u and php70u. NOT php71u or php72u # - "{{ php_ius_version }}-pear" diff --git a/src/roles/apache-php/templates/httpd.conf.j2 b/src/roles/apache-php/templates/httpd.conf.j2 index 4cc8b1bc4..cfd7dd092 100644 --- a/src/roles/apache-php/templates/httpd.conf.j2 +++ b/src/roles/apache-php/templates/httpd.conf.j2 @@ -29,7 +29,9 @@ # least PidFile. # ServerRoot "{{ path_apache_server_root }}" - +{% if ansible_os_family == "Debian" %} +PidFile "/var/run/apache2.pid" +{% endif %} # # Mutex: Allows you to set the mutex mechanism and mutex file directory # for individual mutexes, or change the global defaults @@ -200,7 +202,7 @@ ServerName https://{{ wiki_app_fqdn }} {% if ansible_os_family == "RedHat" %} ErrorLog "logs/error_log" {% else %} -ErrorLog ${APACHE_LOG_DIR}/error.log +ErrorLog ${APACHE_LOG_DIR}/error_log {% endif %} # @@ -210,6 +212,19 @@ ErrorLog ${APACHE_LOG_DIR}/error.log # LogLevel warn + +## Add compression +AddOutputFilterByType DEFLATE text/plain +AddOutputFilterByType DEFLATE text/html +AddOutputFilterByType DEFLATE text/xml +AddOutputFilterByType DEFLATE text/css +AddOutputFilterByType DEFLATE application/xml +AddOutputFilterByType DEFLATE application/xhtml+xml +AddOutputFilterByType DEFLATE application/rss+xml +AddOutputFilterByType DEFLATE application/javascript +AddOutputFilterByType DEFLATE application/x-javascript + + # Logging format: @@ -232,19 +247,19 @@ LogLevel warn # %I - Bytes received, including request and headers, cannot be zero. # %O - Bytes sent, including headers, cannot be zero. - # Logging for normal requests from clients - LogFormat "%{X-Forwarded-For}i %h %l %u %t %D \"%r\" %>s \"%{Referer}i\" \"%{User-Agent}i\" %I %O" proxy + # Logging for normal requests from clients ("combined" format, prefixed with XFF header) + LogFormat "%{X-Forwarded-For}i %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" proxy # This is for non-proxied requests, meaning requests made from within the # server itself or by other non-HAProxy nodes (if that's possible) - LogFormat "noproxy %h %l %u %t %D \"%r\" %>s \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combined + LogFormat "noproxy %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" internal SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded {% if ansible_os_family == "RedHat" %} - CustomLog "logs/access_log" combined env=!forwarded + CustomLog "logs/access_log" internal env=!forwarded CustomLog "logs/access_log" proxy env=forwarded {% else %} - CustomLog "${APACHE_LOG_DIR}/access_log" combined env=!forwarded + CustomLog "${APACHE_LOG_DIR}/access_log" internal env=!forwarded CustomLog "${APACHE_LOG_DIR}/access_log" proxy env=forwarded {% endif %} @@ -418,6 +433,28 @@ SSLRandomSeed connect builtin #RequestHeader unset DNT env=bad_DNT # + +# number of server processes to start (default 5) +StartServers 6 + +# minimum number of server processes in reserve for new requests +MinSpareServers 3 + + # maximum number of idle server processes before they are reaped +MaxSpareServers 6 + +# maximum value of clients for the lifetime of the server before it is reaped to prevent mem leaks +ServerLimit 20000 + +# MaxRequestWorkers was called MaxClients before version 2.3.13. The old name is still supported. +# translates into the maximum number of child processes that will be launched to serve requests on prefork +# the limit on the number of simultaneous requests that will be served; others queued. +MaxRequestWorkers 1000 + +# Server 2.3.9 and later. The old name MaxRequestsPerChild is still supported. +# number of connections that an individual child server will handle during its life +MaxConnectionsPerChild 1000 + # main handling via http Listen 8080 diff --git a/src/roles/apache-php/templates/php.ini.j2 b/src/roles/apache-php/templates/php.ini.j2 index 75e4cf43f..f3bade6d3 100644 --- a/src/roles/apache-php/templates/php.ini.j2 +++ b/src/roles/apache-php/templates/php.ini.j2 @@ -1815,6 +1815,8 @@ soap.wsdl_cache_limit = 5 ;sysvshm.init_mem = 10000 [ldap] +; Enable ldap extension module +extension=ldap.so ; Sets the maximum number of open links or -1 for unlimited. ldap.max_links = -1 diff --git a/src/roles/base-config-scripts/templates/config.php.j2 b/src/roles/base-config-scripts/templates/config.php.j2 index bc3050912..4223c3ce4 100644 --- a/src/roles/base-config-scripts/templates/config.php.j2 +++ b/src/roles/base-config-scripts/templates/config.php.j2 @@ -78,7 +78,7 @@ $backups_environment = '{{ backups_environment }}'; {% if wiki_backup_downloaders is defined %} # Users allowed to download specific wikis $wiki_backup_downloaders = array(); -{% for wiki, users in wiki_backup_downloaders.iteritems() %} +{% for wiki, users in wiki_backup_downloaders.items() %} $wiki_backup_downloaders['{{ wiki }}'] = array( {% for user in users %} '{{ user }}', @@ -109,7 +109,7 @@ $all_backup_downloaders = array( # CUSTOM __public__ DEPLOY VARIABLES # These should only come from public.yml # -{% for key, value in public_deploy_vars.iteritems() %} +{% for key, value in public_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -121,7 +121,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -159,7 +159,7 @@ $all_backup_downloaders = array( # CUSTOM __{{ env }} environment__ DEPLOY VARIABLES # These should only come from env/{{ env }}.yml # -{% for key, value in env_deploy_vars.iteritems() %} +{% for key, value in env_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -171,7 +171,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -208,7 +208,7 @@ $all_backup_downloaders = array( # CUSTOM __secret__ DEPLOY VARIABLES # These should only come from secret.yml # -{% for key, value in secret_deploy_vars.iteritems() %} +{% for key, value in secret_deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -220,7 +220,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; @@ -260,7 +260,7 @@ $all_backup_downloaders = array( # FIXME: remove deploy_vars in lieu of secret_deploy_vars when production wikis # are updated. # -{% for key, value in deploy_vars.iteritems() %} +{% for key, value in deploy_vars.items() %} {% if value is number -%} ${{ key }} = {{ value }}; @@ -272,7 +272,7 @@ $all_backup_downloaders = array( ${{ key }} = []; {%- if value is mapping -%} - {%- for subkey, subvalue in value.iteritems() %} + {%- for subkey, subvalue in value.items() %} {% if subvalue is number -%} ${{ key }}['{{ subkey }}'] = {{ subvalue }}; diff --git a/src/roles/base-config-scripts/templates/config.sh.j2 b/src/roles/base-config-scripts/templates/config.sh.j2 index fa874fafe..1041d310e 100644 --- a/src/roles/base-config-scripts/templates/config.sh.j2 +++ b/src/roles/base-config-scripts/templates/config.sh.j2 @@ -67,7 +67,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __public__ DEPLOY VARIABLES # These should only come from public.yml # -{% for key, value in public_deploy_vars.iteritems() %} +{% for key, value in public_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -84,7 +84,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __{{ env }} environment__ DEPLOY VARIABLES # These should only come from env/{{ env }}.yml # -{% for key, value in env_deploy_vars.iteritems() %} +{% for key, value in env_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -101,7 +101,7 @@ m_i18n="{{ m_i18n }}" # CUSTOM __secret__ DEPLOY VARIABLES # These should only come from secret.yml # -{% for key, value in secret_deploy_vars.iteritems() %} +{% for key, value in secret_deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} @@ -120,7 +120,7 @@ m_i18n="{{ m_i18n }}" # FIXME: remove deploy_vars in lieu of secret_deploy_vars when production wikis # are updated. # -{% for key, value in deploy_vars.iteritems() %} +{% for key, value in deploy_vars.items() %} {% if value is string %} {{ key }}="{{ value }}" {% elif value is iterable %} diff --git a/src/roles/base/tasks/main.yml b/src/roles/base/tasks/main.yml index 5cfc4681b..b49b1e08d 100644 --- a/src/roles/base/tasks/main.yml +++ b/src/roles/base/tasks/main.yml @@ -55,6 +55,12 @@ tags: - latest +- name: Install dependency for apt + package: + name: python-apt + state: present + when: ansible_os_family == "Debian" + - name: Ensure apt cache updated (Debian only) apt: update_cache: yes @@ -122,7 +128,7 @@ - latest - name: ensure libselinux-python installed prior to SELinux (RedHat/CentOS only) - yum: name=libselinux-python state=installed + yum: name=libselinux-python state=present when: ansible_os_family == "RedHat" tags: - latest @@ -225,11 +231,6 @@ # chkconfig ntpd on # Activate service # ntpdate pool.ntp.org # Synchronize the system clock with 0.pool.ntp.org server # service ntpd start # Start service -# FIXME: this is duplicated from above, right? No reason it needs to be here twice? -- name: Install NTP - package: name=ntp state=installed - tags: - - latest - name: Ensure NTP is running and enabled as configured. service: diff --git a/src/roles/base/templates/sshd_config.j2 b/src/roles/base/templates/sshd_config.j2 index a2ac12d5a..db889992d 100644 --- a/src/roles/base/templates/sshd_config.j2 +++ b/src/roles/base/templates/sshd_config.j2 @@ -143,7 +143,12 @@ AcceptEnv LC_IDENTIFICATION LC_ALL LANGUAGE AcceptEnv XMODIFIERS # override default of no subsystems +{% if ansible_os_family == "RedHat" %} Subsystem sftp /usr/libexec/openssh/sftp-server +{% endif %} +{% if ansible_os_family == "Debian" %} +Subsystem sftp /usr/lib/openssh/sftp-server +{% endif %} # Example of overriding settings on a per-user basis #Match User anoncvs diff --git a/src/roles/certbot/tasks/main.yml b/src/roles/certbot/tasks/main.yml new file mode 100644 index 000000000..f075ec957 --- /dev/null +++ b/src/roles/certbot/tasks/main.yml @@ -0,0 +1,71 @@ +--- +# Role to install Let's Encrypt's Certbot, install certificate(s) and automate renewals +# Since certbot installations for Debian/Ubuntu/RHEL/CentOS all setup system.d timers or cron jobs, +# We rely on the system to renew certificates instead of renewing them through ansible +# (We don't even have to setup a cron job.) We *do* have to setup a hook to concatenate certs +# on renewal. If you need to manage multiple domains then you should expand the certificate manually. + +- assert: + that: + - "m_httpd_server_admin is defined" + - "m_httpd_server_admin != 'admin@example.com'" + fail_msg: > + "To use certbot, you must supply a valid email for 'm_httpd_server_admin' in your public.yml + For example, add a line like this to /opt/conf-meza/public/public.yml + m_httpd_server_admin: me@gmail.com" + +- name: Ensure firewall port 54321 OPEN when certbot ENABLED + include_role: + name: firewall_port + vars: + firewall_action: open + firewall_port: 54321 + firewall_protocol: tcp + firewall_zone: "{{m_private_networking_zone|default('public')}}" + +- name: Remove Meza cert files when certbot ENABLED + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/haproxy/certs/meza.crt + - /etc/haproxy/certs/meza.key + +- name: Ensure Certbot installed + package: + name: certbot + state: present + +# We will only generate a certificate when the certs directory is empty +- name: Check for existing cert(s) + find: + paths: /etc/haproxy/certs + patterns: '*.pem' + excludes: 'meza.pem' + register: certs + +- debug: + msg: "Found existing certificate at {{ item.path }}" + with_items: "{{ certs.files }}" + +# There is no notify, because the script is used in cert generation +# By placing the script in the renewal-hooks hierarchy, it's run +# automatically by certbot renew (but not by certonly) +- name: Write the deploy hook script + template: + src: concat.pem.sh.j2 + dest: /etc/letsencrypt/renewal-hooks/deploy/concat.pem.sh + owner: meza-ansible + group: wheel + mode: u=rwx,g=rwx,o=rx + +- name: Generate SSL Certificate + shell: > + certbot certonly --non-interactive --email {{ m_httpd_server_admin }} + --preferred-challenges http --standalone --agree-tos --renew-by-default + --webroot-path {{ m_htdocs }} -d {{ wiki_app_fqdn }} --http-01-port=54321 + --deploy-hook /etc/letsencrypt/renewal-hooks/deploy/concat.pem.sh + delegate_to: localhost + run_once: True + when: certs == "" + diff --git a/src/roles/certbot/templates/concat.pem.sh.j2 b/src/roles/certbot/templates/concat.pem.sh.j2 new file mode 100644 index 000000000..e8bf351e7 --- /dev/null +++ b/src/roles/certbot/templates/concat.pem.sh.j2 @@ -0,0 +1,16 @@ +#!/bin/bash + +# $RENEWED_DOMAINS will contain a space-delimited list of renewed +# certificate domains (for example, "example.com www.example.com" +# loop through a dynamic list of directories in 'live' +# for SITE in $(find /etc/letsencrypt/live -mindepth 1 -maxdepth 1 -type d -exec basename {} \;) +# $RENEWED_LINEAGE will contain the live subdirectory +for SITE in $RENEWED_DOMAINS +do + # move to correct let's encrypt directory + cd $RENEWED_LINEAGE + # cat files to make combined .pem for haproxy + cat fullchain.pem privkey.pem > /etc/haproxy/certs/$SITE.pem +done +# reload haproxy +# systemctl reload haproxy diff --git a/src/roles/composer/defaults/main.yml b/src/roles/composer/defaults/main.yml index d6f113328..8787dcbda 100644 --- a/src/roles/composer/defaults/main.yml +++ b/src/roles/composer/defaults/main.yml @@ -1,7 +1,7 @@ --- composer_path: /usr/local/bin/composer composer_keep_updated: true -composer_version: '' +composer_version: '1.10.16' # The directory where global packages will be installed. composer_home_path: '~/.composer' diff --git a/src/roles/database/tasks/replication.yml b/src/roles/database/tasks/replication.yml index 1a3f0722b..23c41a8ec 100644 --- a/src/roles/database/tasks/replication.yml +++ b/src/roles/database/tasks/replication.yml @@ -72,7 +72,7 @@ delegate_to: "{{ mysql_replication_master }}" register: master when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - debug: { var: master } @@ -88,14 +88,14 @@ register: mysql_content_databases delegate_to: "{{ mysql_replication_master }}" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: export dump file on master - shell: "mysqldump --databases {{ mysql_content_databases.stdout }} | gzip > {{ m_tmp }}/mysqldump-onmaster.sql.gz" + shell: "mysqldump --single-transaction --quick --databases {{ mysql_content_databases.stdout }} | gzip > {{ m_tmp }}/mysqldump-onmaster.sql.gz" delegate_to: "{{ mysql_replication_master }}" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave @@ -114,14 +114,14 @@ flat: yes delegate_to: "{{ mysql_replication_master }}" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: put dump file copy: src: "{{ m_tmp }}/mysqldump-oncontrol.sql.gz" dest: "{{ m_tmp }}/mysqldump-onslave.sql.gz" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave # @@ -133,7 +133,7 @@ name: all target: "{{ m_tmp }}/mysqldump-onslave.sql.gz" when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave # @@ -142,7 +142,7 @@ - mysql_replication: mode: stopslave when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: Configure replication on the slave. mysql_replication: @@ -154,10 +154,10 @@ master_log_pos: "{{ master.Position }}" failed_when: False when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave - name: Start replication. mysql_replication: mode=startslave when: - - slave_needs_configuration + - slave_needs_configuration|bool - role_is_valid_slave diff --git a/src/roles/database/tasks/setup-Debian.yml b/src/roles/database/tasks/setup-Debian.yml index 904f87ccd..3ff770383 100644 --- a/src/roles/database/tasks/setup-Debian.yml +++ b/src/roles/database/tasks/setup-Debian.yml @@ -8,7 +8,9 @@ when: mysql_initially_installed_debian.stat.exists == false - name: Ensure MySQL Python libraries are installed. - apt: "name=python-mysqldb state=installed" + apt: + name: python-mysqldb + state: present - name: Ensure MySQL packages are installed. apt: diff --git a/src/roles/elasticsearch/tasks/main.yml b/src/roles/elasticsearch/tasks/main.yml index 3f7e18249..386849f34 100644 --- a/src/roles/elasticsearch/tasks/main.yml +++ b/src/roles/elasticsearch/tasks/main.yml @@ -10,11 +10,10 @@ when: ansible_os_family == 'RedHat' - name: Ensure Java 1.8.0 OpenJDK is installed - yum: + package: name: "{{ package_java }}" state: present - # Environment setup. - name: Set JAVA_HOME if configured. template: @@ -28,9 +27,10 @@ key: https://packages.elastic.co/GPG-KEY-elasticsearch state: present when: ansible_os_family == 'RedHat' + - name: Add Elasticsearch repository. template: - src: elasticsearch5.repo.j2 + src: elasticsearch.repo.j2 dest: /etc/yum.repos.d/elasticsearch.repo mode: 0644 when: ansible_os_family == 'RedHat' @@ -58,6 +58,7 @@ package: name: elasticsearch state: present + notify: restart elasticsearch # Need to perform this check so `lineinfile` doesn't run in Docker. /etc/hosts # is special in Docker. @@ -166,33 +167,4 @@ do_elasticsearch_upgrade: True when: (es_version_found|int != es_version_desired|int) or (force_do_elasticsearch_upgrade is defined and force_do_elasticsearch_upgrade) -# -# Upgrade Elasticsearch if required -# -- name: Temporarily set Elasticsearch yum repo to point to 2.x - template: - src: elasticsearch2.repo.j2 - dest: /etc/yum.repos.d/elasticsearch.repo - mode: 0644 - when: do_elasticsearch_upgrade and es_version_found|int == 1 - -- name: Include upgrade tasks for ES 1.6 --> 2.x - include: es_upgrade.yml - when: do_elasticsearch_upgrade and es_version_found|int == 1 - -- name: Do reindex if switching for ES 1.6 --> 2.x - include: es_reindex.yml - when: do_elasticsearch_upgrade and es_version_found|int == 1 - -- name: Reset Elasticsearch yum repo to point to 5.x - template: - src: elasticsearch5.repo.j2 - dest: /etc/yum.repos.d/elasticsearch.repo - mode: 0644 - when: do_elasticsearch_upgrade and es_version_found|int == 1 - -# Uncomment this when doing upgrade to 5.x -- name: Include upgrade tasks for ES 2.x --> 5.x - include: es_upgrade.yml - when: do_elasticsearch_upgrade diff --git a/src/roles/elasticsearch/templates/elasticsearch.repo.j2 b/src/roles/elasticsearch/templates/elasticsearch.repo.j2 new file mode 100644 index 000000000..bff9f52ac --- /dev/null +++ b/src/roles/elasticsearch/templates/elasticsearch.repo.j2 @@ -0,0 +1,13 @@ +[elasticsearch] +name=Elasticsearch repository +baseurl=https://artifacts.elastic.co/packages/{{elasticsearch_major_version}}/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=1 + +{% if elasticsearch_proxy_host is defined and elasticsearch_proxy_port is defined %} +proxy=http://{{ elasticsearch_proxy_host }}:{{elasticsearch_proxy_port}} +{% endif %} + +autorefresh=1 +type=rpm-md diff --git a/src/roles/elasticsearch/templates/elasticsearch.yml.j2 b/src/roles/elasticsearch/templates/elasticsearch.yml.j2 index a4db72733..b5bd3d389 100644 --- a/src/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/src/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,25 +1,4 @@ -##################### Elasticsearch Configuration Example ##################### - -# This file contains an overview of various configuration settings, -# targeted at operations staff. Application developers should -# consult the guide at . -# -# The installation procedure is covered at -# . -# -# Elasticsearch comes with reasonable defaults for most settings, -# so you can try it out without bothering with configuration. -# -# Most of the time, these defaults are just fine for running a production -# cluster. If you're fine-tuning your cluster, or wondering about the -# effect of certain configuration option, please _do ask_ on the -# mailing list or IRC channel [http://elasticsearch.org/community]. - -# Any element in the configuration can be replaced with environment variables -# by placing them in ${...} notation. For example: -# -#node.rack: ${RACK_ENV_VAR} - +##################### Elasticsearch Configuration ##################### # For information on supported formats and syntax for the config file, see # @@ -39,375 +18,20 @@ cluster.name: "{{ elasticsearch_cluster_name }}" # node.name: "{{ elasticsearch_node_name }}" -# Every node can be configured to allow or deny being eligible as the master, -# and to allow or deny to store the data. -# -# Allow this node to be eligible as a master node (enabled by default): -# -#node.master: true -# -# Allow this node to store data (enabled by default): -# -#node.data: true - -# You can exploit these settings to design advanced cluster topologies. -# -# 1. You want this node to never become a master node, only to hold data. -# This will be the "workhorse" of your cluster. -# -#node.master: false -#node.data: true -# -# 2. You want this node to only serve as a master: to not store any data and -# to have free resources. This will be the "coordinator" of your cluster. -# -#node.master: true -#node.data: false -# -# 3. You want this node to be neither master nor data node, but -# to act as a "search load balancer" (fetching data from nodes, -# aggregating results, etc.) -# -#node.master: false -#node.data: false - -# Use the Cluster Health API [http://localhost:9200/_cluster/health], the -# Node Info API [http://localhost:9200/_nodes] or GUI tools -# such as , -# , and -# to inspect the cluster state. - -# A node can have generic attributes associated with it, which can later be used -# for customized shard allocation filtering, or allocation awareness. An attribute -# is a simple key value pair, similar to node.key: value, here is an example: -# -#node.rack: rack314 - -# By default, multiple nodes are allowed to start from the same installation location -# to disable it, set the following: -#node.max_local_storage_nodes: 1 - - -#################################### Index #################################### - -# You can set a number of options (such as shard/replica options, mapping -# or analyzer definitions, translog settings, ...) for indices globally, -# in this file. -# -# Note, that it makes more sense to configure index settings specifically for -# a certain index, either when creating it or by using the index templates API. -# -# See and -# -# for more information. - -# Set the number of shards (splits) of an index (5 by default): -# -#index.number_of_shards: 5 - -# Set the number of replicas (additional copies) of an index (1 by default): -# -#index.number_of_replicas: 1 - -# Note, that for development on a local machine, with small indices, it usually -# makes sense to "disable" the distributed features: -# -#index.number_of_shards: 1 -#index.number_of_replicas: 0 - -# These settings directly affect the performance of index and search operations -# in your cluster. Assuming you have enough machines to hold shards and -# replicas, the rule of thumb is: -# -# 1. Having more *shards* enhances the _indexing_ performance and allows to -# _distribute_ a big index across machines. -# 2. Having more *replicas* enhances the _search_ performance and improves the -# cluster _availability_. -# -# The "number_of_shards" is a one-time setting for an index. -# -# The "number_of_replicas" can be increased or decreased anytime, -# by using the Index Update Settings API. -# -# Elasticsearch takes care about load balancing, relocating, gathering the -# results from nodes, etc. Experiment with different settings to fine-tune -# your setup. - -# Use the Index Status API () to inspect -# the index status. #################################### Paths #################################### # Path to directory containing configuration (this file and logging.yml): -# -path.conf: /etc/elasticsearch +# deprecated +# path.conf: /etc/elasticsearch # Path to directory where to store index data allocated for this node. # path.data: {{ m_meza_data }}/elasticsearch/data -# -# Can optionally include more than one location, causing data to be striped across -# the locations (a la RAID 0) on a file level, favouring locations with most free -# space on creation. For example: -# -#path.data: /path/to/data1,/path/to/data2 - -# Path to temporary files: -# -# Appears to have been removed in Elasticsearch 0.90 -# path.work: {{ m_meza_data }}/elasticsearch/work -# Added 5.x: Required for script.inline and/or script.stored (formerly script.indexed) -path.scripts: {{ m_meza_data }}/elasticsearch/scripts # Path to log files: # path.logs: {{ m_meza_data }}/elasticsearch/log -# Path to where plugins are installed: -# -# Removed in Elasticsearch 2.x or 5.x -# path.plugins: {{ m_meza_data }}/elasticsearch/plugins - - -#################################### Plugin ################################### - -# If a plugin listed here is not installed for current node, the node will not start. -# -#plugin.mandatory: mapper-attachments,lang-groovy - - -################################### Memory #################################### - -# Elasticsearch performs poorly when JVM starts swapping: you should ensure that -# it _never_ swaps. -# -# Set this property to true to lock the memory: -# -#bootstrap.mlockall: true - -# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set -# to the same value, and that the machine has enough memory to allocate -# for Elasticsearch, leaving enough memory for the operating system itself. -# -# You should also make sure that the Elasticsearch process is allowed to lock -# the memory, eg. by using `ulimit -l unlimited`. - - -############################## Network And HTTP ############################### - -# Elasticsearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node -# communication. (the range means that if the port is busy, it will automatically -# try the next port). - -# Set the bind address specifically (IPv4 or IPv6): -# -#network.bind_host: 192.168.0.1 - -# Set the address other nodes will use to communicate with this node. If not -# set, it is automatically derived. It must point to an actual IP address. -# -#network.publish_host: 192.168.0.1 - -# Set both 'bind_host' and 'publish_host': -# -#network.host: 192.168.0.1 - -# Set a custom port for the node to node communication (9300 by default): -# -#transport.tcp.port: 9300 - -# Enable compression for all communication between nodes (disabled by default): -# -#transport.tcp.compress: true - -# Set a custom port to listen for HTTP traffic: -# -#http.port: 9200 - -# Set a custom allowed content length: -# -#http.max_content_length: 100mb - -# Disable HTTP completely: -# -#http.enabled: false - - -################################### Gateway ################################### - -# The gateway allows for persisting the cluster state between full cluster -# restarts. Every change to the state (such as adding an index) will be stored -# in the gateway, and when the cluster starts up for the first time, -# it will read its state from the gateway. - -# There are several types of gateway implementations. For more information, see -# . - -# The default gateway type is the "local" gateway (recommended): -# -#gateway.type: local - -# Settings below control how and when to start the initial recovery process on -# a full cluster restart (to reuse as much local data as possible when using shared -# gateway). - -# Allow recovery process after N nodes in a cluster are up: -# -#gateway.recover_after_nodes: 1 - -# Set the timeout to initiate the recovery process, once the N nodes -# from previous setting are up (accepts time value): -# -#gateway.recover_after_time: 5m - -# Set how many nodes are expected in this cluster. Once these N nodes -# are up (and recover_after_nodes is met), begin recovery process immediately -# (without waiting for recover_after_time to expire): -# -#gateway.expected_nodes: 2 - - -############################# Recovery Throttling ############################# - -# These settings allow to control the process of shards allocation between -# nodes during initial recovery, replica allocation, rebalancing, -# or when adding and removing nodes. - -# Set the number of concurrent recoveries happening on a node: -# -# 1. During the initial recovery -# -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -# -# 2. During adding/removing nodes, rebalancing, etc -# -#cluster.routing.allocation.node_concurrent_recoveries: 2 - -# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): -# -#indices.recovery.max_bytes_per_sec: 20mb - -# Set to limit the number of open concurrent streams when -# recovering a shard from a peer: -# -#indices.recovery.concurrent_streams: 5 - - -################################## Discovery ################################## - -# Discovery infrastructure ensures nodes can be found within a cluster -# and master node is elected. Multicast discovery is the default. - -# Set to ensure a node sees N other master eligible nodes to be considered -# operational within the cluster. This should be set to a quorum/majority of -# the master-eligible nodes in the cluster. -# -#discovery.zen.minimum_master_nodes: 1 - -# Set the time to wait for ping responses from other nodes when discovering. -# Set this option to a higher value on a slow or congested network -# to minimize discovery failures: -# -#discovery.zen.ping.timeout: 3s - -# For more information, see -# - -# Unicast discovery allows to explicitly control which nodes will be used -# to discover the cluster. It can be used when multicast is not present, -# or to restrict the cluster communication-wise. -# -# 1. Disable multicast discovery (enabled by default): -# -#discovery.zen.ping.multicast.enabled: false -# -# 2. Configure an initial list of master nodes in the cluster -# to perform discovery when new nodes (master or data) are started: -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2:port"] - -# EC2 discovery allows to use AWS EC2 API in order to perform discovery. -# -# You have to install the cloud-aws plugin for enabling the EC2 discovery. -# -# For more information, see -# -# -# See -# for a step-by-step tutorial. - -# GCE discovery allows to use Google Compute Engine API in order to perform discovery. -# -# You have to install the cloud-gce plugin for enabling the GCE discovery. -# -# For more information, see . - -# Azure discovery allows to use Azure API in order to perform discovery. -# -# You have to install the cloud-azure plugin for enabling the Azure discovery. -# -# For more information, see . - -################################## Slow Log ################################## - -# Shard level query and fetch threshold logging. - -#index.search.slowlog.threshold.query.warn: 10s -#index.search.slowlog.threshold.query.info: 5s -#index.search.slowlog.threshold.query.debug: 2s -#index.search.slowlog.threshold.query.trace: 500ms - -#index.search.slowlog.threshold.fetch.warn: 1s -#index.search.slowlog.threshold.fetch.info: 800ms -#index.search.slowlog.threshold.fetch.debug: 500ms -#index.search.slowlog.threshold.fetch.trace: 200ms - -#index.indexing.slowlog.threshold.index.warn: 10s -#index.indexing.slowlog.threshold.index.info: 5s -#index.indexing.slowlog.threshold.index.debug: 2s -#index.indexing.slowlog.threshold.index.trace: 500ms - -################################## GC Logging ################################ - -#monitor.jvm.gc.young.warn: 1000ms -#monitor.jvm.gc.young.info: 700ms -#monitor.jvm.gc.young.debug: 400ms - -#monitor.jvm.gc.old.warn: 10s -#monitor.jvm.gc.old.info: 5s -#monitor.jvm.gc.old.debug: 2s - -################################## Security ################################ - -# Uncomment if you want to enable JSONP as a valid return transport on the -# http server. With this enabled, it may pose a security risk, so disabling -# it unless you need it is recommended (it is disabled by default). -# -#http.jsonp.enable: true - - -{% if elasticsearch_major_version == "1.6" %} -# FIXME #611: Remove this option after upgrading beyond MW 1.27 -# Enable dynamic scripting per https://www.mediawiki.org/wiki/Thread:Extension_talk:CirrusSearch/error_using_ElasticSearch -script.disable_dynamic: false -{% else %} -# Enable dynamic scripting per: -# MW docs saying dynamic scripting required -# https://www.mediawiki.org/wiki/Thread:Extension_talk:CirrusSearch/error_using_ElasticSearch -# -# Elasticsearch docs for v2.4 -# https://www.elastic.co/guide/en/elasticsearch/reference/2.4/modules-scripting.html#enable-dynamic-scripting -script.inline: true - -{% if elasticsearch_major_version == "2.x" %} -script.indexed: true -{% else %} -# script.indexed has renamed to script.stored in 5.x -# ref: https://www.elastic.co/guide/en/elasticsearch/reference/5.0/breaking_50_settings_changes.html#_script_mode_settings -script.stored: true -{% endif %} - -{% endif %} diff --git a/src/roles/elasticsearch/templates/java_home.sh.j2 b/src/roles/elasticsearch/templates/java_home.sh.j2 index 12de4aa02..6c25eb869 100644 --- a/src/roles/elasticsearch/templates/java_home.sh.j2 +++ b/src/roles/elasticsearch/templates/java_home.sh.j2 @@ -1 +1 @@ -export JAVA_HOME=/usr/bin +export JAVA_HOME=/usr/share/elasticsearch diff --git a/src/roles/goaccess/tasks/main.yml b/src/roles/goaccess/tasks/main.yml new file mode 100644 index 000000000..9d82cec79 --- /dev/null +++ b/src/roles/goaccess/tasks/main.yml @@ -0,0 +1,58 @@ +- name: Ensure firewall port 7890 OPEN when GoAccess ENABLED + include_role: + name: firewall_port + vars: + firewall_action: open + firewall_port: 7890 + firewall_protocol: tcp + firewall_zone: "{{m_private_networking_zone|default('public')}}" + when: + - (docker_skip_tasks is not defined or not docker_skip_tasks) + +- name: Add GoAccess apt key (Debian). + apt_key: + url: https://deb.goaccess.io/gnugpg.key + state: present + when: + - ansible_distribution == "Debian" + +- name: Add GoAccess repo (Debian). + apt_repository: + repo: "deb http://deb.goaccess.io/ {{ ansible_distribution_release }} main" + state: present + register: goaccess_repo + when: + - ansible_distribution == "Debian" + +- name: Update apt caches after repo is added (Debian). + apt: + update_cache: true + when: + - goaccess_repo.changed + - (ansible_distribution == "Debian") + +- name: Ensure GoAccess webserver analyzer installed + package: + name: "{{ package_goaccess }}" + state: present + +- name: Ensure GoAccess config directory exists + file: + state: directory + path: /etc/goaccess + owner: root + group: root + mode: 0775 + +- name: Ensure /etc/goaccess/my.goaccess.conf in place + template: + src: my.goaccess.conf.j2 + dest: /etc/goaccess/my.goaccess.conf + +- name: Make apache own htdocs/public_html directory + file: + state: directory + path: "{{ m_htdocs }}/public_html" + owner: "{{ user_apache }}" + group: "{{ group_apache }}" + mode: 0775 diff --git a/src/roles/goaccess/templates/my.goaccess.conf.j2 b/src/roles/goaccess/templates/my.goaccess.conf.j2 new file mode 100644 index 000000000..8cd90019d --- /dev/null +++ b/src/roles/goaccess/templates/my.goaccess.conf.j2 @@ -0,0 +1,771 @@ +###################################### +# Time Format Options (required) +###################################### +# +# The hour (24-hour clock) [00,23]; leading zeros are permitted but not required. +# The minute [00,59]; leading zeros are permitted but not required. +# The seconds [00,60]; leading zeros are permitted but not required. +# See `man strftime` for more details +# +# The following time format works with any of the +# Apache/NGINX's log formats below. +# +time-format %H:%M:%S +# +# Google Cloud Storage or +# The time in microseconds since the Unix epoch. +# +#time-format %f + +# Squid native log format +# +#time-format %s + +###################################### +# Date Format Options (required) +###################################### +# +# The date-format variable followed by a space, specifies +# the log format date containing any combination of regular +# characters and special format specifiers. They all begin with a +# percentage (%) sign. See `man strftime` +# +# The following date format works with any of the +# Apache/NGINX's log formats below. +# +date-format %d/%b/%Y +# +# AWS | Amazon CloudFront (Download Distribution) +# AWS | Elastic Load Balancing +# W3C (IIS) +# +#date-format %Y-%m-%d +# +# Google Cloud Storage or +# The time in microseconds since the Unix epoch. +# +#date-format %f + +# Squid native log format +# +#date-format %s + +###################################### +# Log Format Options (required) +###################################### +# Meza Custom Log format +log-format ~h{, } %^ %e [%d:%t %z] %D "%r" %s "%R" "%u" %^ %b + +# XFF + NCSA Combined Log Format +log-format ~h %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# The log-format variable followed by a space or \t for +# tab-delimited, specifies the log format string. +# +# NOTE: If the time/date is a timestamp in seconds or microseconds +# %x must be used instead of %d & %t to represent the date & time. + +# NCSA Combined Log Format +#log-format %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# NCSA Combined Log Format with Virtual Host +#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# Common Log Format (CLF) +#log-format %h %^[%d:%t %^] "%r" %s %b + +# Common Log Format (CLF) with Virtual Host +#log-format %v:%^ %h %^[%d:%t %^] "%r" %s %b + +# W3C +#log-format %d %t %h %^ %^ %^ %^ %r %^ %s %b %^ %^ %u %R + +# Squid native log format +#log-format %^ %^ %^ %v %^: %x.%^ %~%L %h %^/%s %b %m %U + +# AWS | Amazon CloudFront (Download Distribution) +#log-format %d\t%t\t%^\t%b\t%h\t%m\t%^\t%r\t%s\t%R\t%u\t%^ + +# Google Cloud Storage +#log-format "%x","%h",%^,%^,"%m","%U","%s",%^,"%b","%D",%^,"%R","%u" + +# AWS | Elastic Load Balancing +#log-format %dT%t.%^ %^ %h:%^ %^ %T %^ %^ %^ %s %^ %b "%r" "%u" + +# AWSS3 | Amazon Simple Storage Service (S3) +#log-format %^[%d:%t %^] %h %^"%r" %s %^ %b %^ %L %^ "%R" "%u" + +# Virtualmin Log Format with Virtual Host +#log-format %h %^ %v %^[%d:%t %^] "%r" %s %b "%R" "%u" + +# In addition to specifying the raw log/date/time formats, for +# simplicity, any of the following predefined log format names can be +# supplied to the log/date/time-format variables. GoAccess can also +# handle one predefined name in one variable and another predefined +# name in another variable. +# +#log-format COMBINED +#log-format VCOMBINED +#log-format COMMON +#log-format VCOMMON +#log-format W3C +#log-format SQUID +#log-format CLOUDFRONT +#log-format CLOUDSTORAGE +#log-format AWSELB +#log-format AWSS3 + +###################################### +# UI Options +###################################### + +# Choose among color schemes +# 1 : Monochrome +# 2 : Green +# 3 : Monokai (if 256-colors supported) +# +#color-scheme 3 + +# Prompt log/date configuration window on program start. +# +config-dialog false + +# Color highlight active panel. +# +hl-header true + +# Specify a custom CSS file in the HTML report. +# +#html-custom-css /path/file.css + +# Specify a custom JS file in the HTML report. +# +#html-custom-js /path/file.js + +# Set default HTML preferences. +# +# NOTE: A valid JSON object is required. +# DO NOT USE A MULTILINE JSON OBJECT. +# The parser will only parse the value next to `html-prefs` (single line) +# It allows the ability to customize each panel plot. See example below. +# +#html-prefs {"theme":"bright","perPage":5,"layout":"horizontal","showTables":true,"visitors":{"plot":{"chartType":"bar"}}} + +# Set HTML report page title and header. +# +html-report-title QualityBox Real-time Web Stats + +# Format JSON output using tabs and newlines. +# +json-pretty-print false + +# Turn off colored output. This is the default output on +# terminals that do not support colors. +# true : for no color output +# false : use color-scheme +# +no-color false + +# Don't write column names in the terminal output. By default, it displays +# column names for each available metric in every panel. +# +no-column-names false + +# Disable summary metrics on the CSV output. +# +no-csv-summary false + +# Disable progress metrics. +# +no-progress false + +# Disable scrolling through panels on TAB. +# +no-tab-scroll false + +# Disable progress metrics and parsing spinner. +# +#no-parsing-spinner true + +# Do not show the last updated field displayed in the HTML generated report. +# +#no-html-last-updated true + +# Enable mouse support on main dashboard. +# +with-mouse true + +# Maximum number of items to show per panel. +# Note: Only the CSV and JSON outputs allow a maximum greater than the +# default value of 366. +# +#max-items 366 + +# Custom colors for the terminal output +# Tailor GoAccess to suit your own tastes. +# +# Color Syntax: +# DEFINITION space/tab colorFG#:colorBG# [[attributes,] PANEL] +# +# FG# = foreground color number [-1...255] (-1 = default terminal color) +# BG# = background color number [-1...255] (-1 = default terminal color) +# +# Optionally: +# +# It is possible to apply color attributes, such as: +# bold,underline,normal,reverse,blink. +# Multiple attributes are comma separated +# +# If desired, it is possible to apply custom colors per panel, that is, a +# metric in the REQUESTS panel can be of color A, while the same metric in the +# BROWSERS panel can be of color B. +# +# The following is a 256 color scheme (hybrid palette) +# +#color COLOR_MTRC_HITS color110:color-1 +#color COLOR_MTRC_VISITORS color173:color-1 +#color COLOR_MTRC_DATA color221:color-1 +#color COLOR_MTRC_BW color167:color-1 +#color COLOR_MTRC_AVGTS color143:color-1 +#color COLOR_MTRC_CUMTS color247:color-1 +#color COLOR_MTRC_MAXTS color186:color-1 +#color COLOR_MTRC_PROT color109:color-1 +#color COLOR_MTRC_MTHD color139:color-1 +#color COLOR_MTRC_HITS_PERC color186:color-1 +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISITORS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 OS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 BROWSERS +#color COLOR_MTRC_HITS_PERC_MAX color139:color-1 VISIT_TIMES +#color COLOR_MTRC_VISITORS_PERC color186:color-1 +#color COLOR_MTRC_VISITORS_PERC_MAX color139:color-1 +#color COLOR_PANEL_COLS color243:color-1 +#color COLOR_BARS color250:color-1 +#color COLOR_ERROR color231:color167 +#color COLOR_SELECTED color7:color167 +#color COLOR_PANEL_ACTIVE color7:color237 +#color COLOR_PANEL_HEADER color250:color235 +#color COLOR_PANEL_DESC color242:color-1 +#color COLOR_OVERALL_LBLS color243:color-1 +#color COLOR_OVERALL_VALS color167:color-1 +#color COLOR_OVERALL_PATH color186:color-1 +#color COLOR_ACTIVE_LABEL color139:color235 bold underline +#color COLOR_BG color250:color-1 +#color COLOR_DEFAULT color243:color-1 +#color COLOR_PROGRESS color7:color110 + +###################################### +# Server Options +###################################### + +# Specify IP address to bind server to. +# +#addr 0.0.0.0 + +# Run GoAccess as daemon (if --real-time-html enabled). +# +#daemonize false + +# Ensure clients send the specified origin header upon the WebSocket +# handshake. +# +#origin http://example.org + +# The port to which the connection is being attempted to connect. +# By default GoAccess' WebSocket server listens on port 7890 +# See man page or http://gwsocket.io for details. +# +#port 7890 + +# Write the PID to a file when used along the daemonize option. +# +#pid-file /var/run/goaccess.pid + +# Enable real-time HTML output. +# +#real-time-html true + +# Path to TLS/SSL certificate. +# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL. +# +#ssl-cert /path/ssl/domain.crt + +# Path to TLS/SSL private key. +# Note that ssl-cert and ssl-key need to be used to enable TLS/SSL. +# +#ssl-key /path/ssl/domain.key + +# URL to which the WebSocket server responds. This is the URL supplied +# to the WebSocket constructor on the client side. +# +# Optionally, it is possible to specify the WebSocket URI scheme, such as ws:// +# or wss:// for unencrypted and encrypted connections. +# e.g., ws-url wss://goaccess.io +# +# If GoAccess is running behind a proxy, you could set the client side +# to connect to a different port by specifying the host followed by a +# colon and the port. +# e.g., ws-url goaccess.io:9999 +# +# By default, it will attempt to connect to localhost. If GoAccess is +# running on a remote server, the host of the remote server should be +# specified here. Also, make sure it is a valid host and NOT an http +# address. +# +#ws-url goaccess.io + +# Path to read named pipe (FIFO). +# +#fifo-in /tmp/wspipein.fifo + +# Path to write named pipe (FIFO). +# +#fifo-in /tmp/wspipeout.fifo + +###################################### +# File Options +###################################### + +# Specify the path to the input log file. If set, it will take +# priority over -f from the command line. +# +#log-file /var/log/apache2/access.log + +# Send all debug messages to the specified file. +# +#debug-file debug.log + +# Specify a custom configuration file to use. If set, it will take +# priority over the global configuration file (if any). +# +#config-file + +# Log invalid requests to the specified file. +# +#invalid-requests + +# Do not load the global configuration file. +# +#no-global-config false + +###################################### +# Parse Options +###################################### + +# Enable a list of user-agents by host. For faster parsing, do not +# enable this flag. +# +agent-list false + +# Enable IP resolver on HTML|JSON|CSV output. +# +with-output-resolver false + +# Exclude an IPv4 or IPv6 from being counted. +# Ranges can be included as well using a dash in between +# the IPs (start-end). +# +#exclude-ip 127.0.0.1 +#exclude-ip 192.168.0.1-192.168.0.100 +#exclude-ip ::1 +#exclude-ip 0:0:0:0:0:ffff:808:804-0:0:0:0:0:ffff:808:808 + +# Include HTTP request method if found. This will create a +# request key containing the request method + the actual request. +# +# [default: yes] +# +http-method yes + +# Include HTTP request protocol if found. This will create a +# request key containing the request protocol + the actual request. +# +# [default: yes] +# +http-protocol yes + +# Write output to stdout given one of the following files and the +# corresponding extension for the output format: +# +# /path/file.csv - Comma-separated values (CSV) +# /path/file.json - JSON (JavaScript Object Notation) +# /path/file.html - HTML +# +# output /path/file.html + +# Ignore request's query string. +# i.e., www.google.com/page.htm?query => www.google.com/page.htm +# +# Note: Removing the query string can greatly decrease memory +# consumption, especially on timestamped requests. +# +no-query-string true + +# Disable IP resolver on terminal output. +# +no-term-resolver false + +# Treat non-standard status code 444 as 404. +# +444-as-404 false + +# Add 4xx client errors to the unique visitors count. +# +4xx-to-unique-count false + +# Store accumulated processing time from parsing day-by-day logs. +# Only if configured with --enable-tcb=btree +# +#accumulated-time false + +# IP address anonymization +# The IP anonymization option sets the last octet of IPv4 user IP addresses and +# the last 80 bits of IPv6 addresses to zeros. +# e.g., 192.168.20.100 => 192.168.20.0 +# e.g., 2a03:2880:2110:df07:face:b00c::1 => 2a03:2880:2110:df07:: +# +#anonymize-ip false + +# Include static files that contain a query string in the static files +# panel. +# e.g., /fonts/fontawesome-webfont.woff?v=4.0.3 +# +all-static-files true + +# Include an additional delimited list of browsers/crawlers/feeds etc. +# See config/browsers.list for an example or +# https://raw.githubusercontent.com/allinurl/goaccess/master/config/browsers.list +# +#browsers-file + +# Date specificity. Possible values: `date` (default), or `hr`. +# +#date-spec hr + +# Decode double-encoded values. +# +double-decode false + +# Enable parsing/displaying the given panel. +# +#enable-panel VISITORS +#enable-panel REQUESTS +#enable-panel REQUESTS_STATIC +#enable-panel NOT_FOUND +#enable-panel HOSTS +#enable-panel OS +#enable-panel BROWSERS +#enable-panel VISIT_TIMES +#enable-panel VIRTUAL_HOSTS +#enable-panel REFERRERS +#enable-panel REFERRING_SITES +#enable-panel KEYPHRASES +#enable-panel STATUS_CODES +#enable-panel REMOTE_USER +#enable-panel GEO_LOCATION + +# Hide a referer but still count it. Wild cards are allowed. i.e., *.bing.com +# +#hide-referer *.google.com +#hide-referer bing.com + +# Hour specificity. Possible values: `hr` (default), or `min` (tenth +# of a minute). +# +#hour-spec min + +# Ignore crawlers from being counted. +# This will ignore robots listed under browsers.c +# Note that it will count them towards the total +# number of requests, but excluded from any of the panels. +# +ignore-crawlers false + +# Parse and display crawlers only. +# This will ignore robots listed under browsers.c +# Note that it will count them towards the total +# number of requests, but excluded from any of the panels. +# +crawlers-only false + +# Ignore static file requests. +# req : Only ignore request from valid requests +# panels : Ignore request from panels. +# Note that it will count them towards the total number of requests +# ignore-statics req + +# Ignore parsing and displaying the given panel. +# +#ignore-panel VISITORS +#ignore-panel REQUESTS +#ignore-panel REQUESTS_STATIC +#ignore-panel NOT_FOUND +#ignore-panel HOSTS +#ignore-panel OS +#ignore-panel BROWSERS +#ignore-panel VISIT_TIMES +#ignore-panel VIRTUAL_HOSTS +ignore-panel REFERRERS +#ignore-panel REFERRING_SITES +ignore-panel KEYPHRASES +#ignore-panel STATUS_CODES +#ignore-panel REMOTE_USER +#ignore-panel GEO_LOCATION + +# Ignore referers from being counted. +# This supports wild cards. For instance, +# '*' matches 0 or more characters (including spaces) +# '?' matches exactly one character +# +#ignore-referer *.domain.com +#ignore-referer ww?.domain.* + +# Ignore parsing and displaying one or multiple status code(s) +# +#ignore-status 400 +#ignore-status 502 + +# Number of lines from the access log to test against the provided +# log/date/time format. By default, the parser is set to test 10 +# lines. If set to 0, the parser won't test any lines and will parse +# the whole access log. +# +#num-tests 10 + +# Parse log and exit without outputting data. +# +#process-and-exit false + +# Display real OS names. e.g, Windows XP, Snow Leopard. +# +real-os true + +# Sort panel on initial load. +# Sort options are separated by comma. +# Options are in the form: PANEL,METRIC,ORDER +# +# Available metrics: +# BY_HITS - Sort by hits +# BY_VISITORS - Sort by unique visitors +# BY_DATA - Sort by data +# BY_BW - Sort by bandwidth +# BY_AVGTS - Sort by average time served +# BY_CUMTS - Sort by cumulative time served +# BY_MAXTS - Sort by maximum time served +# BY_PROT - Sort by http protocol +# BY_MTHD - Sort by http method +# Available orders: +# ASC +# DESC +# +#sort-panel VISITORS,BY_DATA,ASC +#sort-panel REQUESTS,BY_HITS,ASC +#sort-panel REQUESTS_STATIC,BY_HITS,ASC +#sort-panel NOT_FOUND,BY_HITS,ASC +#sort-panel HOSTS,BY_HITS,ASC +#sort-panel OS,BY_HITS,ASC +#sort-panel BROWSERS,BY_HITS,ASC +#sort-panel VISIT_TIMES,BY_DATA,DESC +#sort-panel VIRTUAL_HOSTS,BY_HITS,ASC +#sort-panel REFERRERS,BY_HITS,ASC +#sort-panel REFERRING_SITES,BY_HITS,ASC +#sort-panel KEYPHRASES,BY_HITS,ASC +#sort-panel STATUS_CODES,BY_HITS,ASC +#sort-panel REMOTE_USER,BY_HITS,ASC +#sort-panel GEO_LOCATION,BY_HITS,ASC + +# Consider the following extensions as static files +# The actual '.' is required and extensions are case sensitive +# For a full list, uncomment the less common static extensions below. +# +static-file .css +static-file .js +static-file .jpg +static-file .png +static-file .gif +static-file .ico +static-file .jpeg +static-file .pdf +static-file .csv +static-file .mpeg +static-file .mpg +static-file .swf +static-file .woff +static-file .woff2 +static-file .xls +static-file .xlsx +static-file .doc +static-file .docx +static-file .ppt +static-file .pptx +static-file .txt +static-file .zip +static-file .ogg +static-file .mp3 +static-file .mp4 +static-file .exe +static-file .iso +static-file .gz +static-file .rar +static-file .svg +static-file .bmp +static-file .tar +static-file .tgz +static-file .tiff +static-file .tif +static-file .ttf +static-file .flv +static-file .less +static-file .ac3 +static-file .avi +static-file .bz2 +static-file .class +static-file .cue +static-file .dae +static-file .dat +static-file .dts +static-file .ejs +static-file .eot +static-file .eps +static-file .img +static-file .jar +static-file .map +static-file .mid +static-file .midi +static-file .ogv +static-file .webm +static-file .mkv +static-file .odp +static-file .ods +static-file .odt +static-file .otf +static-file .pict +static-file .pls +static-file .ps +static-file .qt +static-file .rm +static-file .svgz +static-file .wav +static-file .webp + +###################################### +# GeoIP Options +# Only if configured with --enable-geoip +###################################### + +# Standard GeoIP database for less memory usage. +# +#std-geoip false + +# Specify path to GeoIP database file. i.e., GeoLiteCity.dat +# .dat file needs to be downloaded from maxmind.com. +# +# For IPv4 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz +# gunzip GeoLiteCity.dat.gz +# +# For IPv6 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz +# gunzip GeoLiteCityv6.dat.gz +# +# For IPv6 Country database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz +# gunzip GeoIPv6.dat.gz +# +# For GeoIP2 City database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz +# gunzip GeoLite2-City.mmdb.gz +# +# For GeoIP2 Country database: +# wget -N http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz +# gunzip GeoLite2-Country.mmdb.gz +# +# Note: `geoip-city-data` is an alias of `geoip-database` +# +#geoip-database /usr/local/share/GeoIP/GeoLiteCity.dat + +###################################### +# Tokyo Cabinet Options +# Only if configured with --enable-tcb=btree +###################################### + +# GoAccess has the ability to process logs incrementally through the on-disk +# B+Tree database. +# +# It works in the following way: +# - A data set must be persisted first with --keep-db-files, then the same data +# set can be loaded with --load-from-disk. +# - If new data is passed (piped or through a log file), it will append it to +# the original data set. +# - To preserve the data at all times, --keep-db-files must be used. +# - If --load-from-disk is used without --keep-db-files, database files will be +# deleted upon closing the program. + +# On-disk B+ Tree +# Persist parsed data into disk. This should be set to +# the first dataset prior to use `load-from-disk`. +# Setting it to false will delete all database files +# when exiting the program. +# keep-db-files true + +# On-disk B+ Tree +# Load previously stored data from disk. +# Database files need to exist. See `keep-db-files`. +# load-from-disk true + +# On-disk B+ Tree +# Path where the on-disk database files are stored. +# The default value is the /tmp/ directory +# Note the trailing forward-slash. +# +#db-path /tmp/ + +# On-disk B+ Tree +# Set the size in bytes of the extra mapped memory. +# The default value is 0. +# +#xmmap 0 + +# On-disk B+ Tree +# Max number of leaf nodes to be cached. +# Specifies the maximum number of leaf nodes to be cached. +# If it is not more than 0, the default value is specified. +# The default value is 1024. +# +#cache-lcnum 1024 + +# On-disk B+ Tree +# Specifies the maximum number of non-leaf nodes to be cached. +# If it is not more than 0, the default value is specified. +# The default value is 512. +# +#cache-ncnum 512 + +# On-disk B+ Tree +# Specifies the number of members in each leaf page. +# If it is not more than 0, the default value is specified. +# The default value is 128. +# +#tune-lmemb 128 + +# On-disk B+ Tree +# Specifies the number of members in each non-leaf page. +# If it is not more than 0, the default value is specified. +# The default value is 256. +# +#tune-nmemb 256 + +# On-disk B+ Tree +# Specifies the number of elements of the bucket array. +# If it is not more than 0, the default value is specified. +# The default value is 32749. +# Suggested size of the bucket array is about from 1 to 4 +# times of the number of all pages to be stored. +# +#tune-bnum 32749 + +# On-disk B+ Tree +# Specifies that each page is compressed with ZLIB|BZ2 encoding. +# Disabled by default. +# +#compression zlib diff --git a/src/roles/haproxy/tasks/main.yml b/src/roles/haproxy/tasks/main.yml index 66ff9c659..f82ecf354 100644 --- a/src/roles/haproxy/tasks/main.yml +++ b/src/roles/haproxy/tasks/main.yml @@ -1,11 +1,4 @@ --- -# FIXME #748: Eventually add the ability to get SSL cert from letsencrypt -# ref: https://www.digitalocean.com/community/tutorials/how-to-secure-haproxy-with-let-s-encrypt-on-centos-7 -# Other refs: -# https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Load_Balancer_Administration/install_haproxy_example1.html -# https://gist.github.com/thisismitch/7c91e9b2b63f837a0c4b -# https://www.unixmen.com/configure-high-available-load-balancer-haproxy-keepalived/ - - name: Set fact if this load balancer will handle external connections (80/443) set_fact: @@ -75,10 +68,10 @@ # owner/group/mode? # -# 1. If cert/key don't exist ON CONTROLLER, generate self-signed ON CONTROLLER -# 2. Ensure cert and key in /etc/haproxy/certs -# 3. Ensure cert and key assembled into pem file at /etc/haproxy/certs/meza.pem +# 1. If cert/key don't exist ON CONTROLLER, install certbot and create certs ON CONTROLLER +# 2. Ensure certs assembled in /etc/haproxy/certs # + - name: Check if secret config on CONTROLLER has SSL keys stat: path: "{{ m_local_secret }}/{{ env }}/ssl/meza.key" @@ -96,7 +89,7 @@ delegate_to: localhost run_once: True -# FIXME: Better would be "if not exists setup letsencrypt" +# This is for Vagrant setups / those with no domain - name: If not exists, create self-signed SSL cert on CONTROLLER command: | openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 \ @@ -137,30 +130,6 @@ delegate_to: localhost run_once: True -# - name: Ensure cert and key on load balancers -# copy: -# src: "{{ m_local_secret }}/{{ env }}/ssl/{{ item }}" -# dest: "/etc/haproxy/certs/{{ item }}" -# with_items: -# - meza.key -# - meza.crt -# notify: -# - restart haproxy - -- name: Ensure SSL cert on load balancers - copy: - content: "{{ decrypted_cert.stdout }}" - dest: "/etc/haproxy/certs/meza.crt" - notify: - - restart haproxy - -- name: Ensure SSL key on load balancers - copy: - content: "{{ decrypted_key.stdout }}" - dest: "/etc/haproxy/certs/meza.key" - notify: - - restart haproxy - - name: Ensure cert and key assembled into into pem file copy: content: | @@ -170,23 +139,83 @@ notify: - restart haproxy -# - name: Ensure cert and key assembled into into pem file -# assemble: -# src: "/opt/conf-meza/secret/{{ env }}/ssl" -# dest: /etc/haproxy/certs/meza.pem -# notify: -# - restart haproxy +- name: Check if localhost has files other than *.pem in haproxy certs + find: + paths: /etc/haproxy/certs + file_type: file + patterns: '*.*' + excludes: '*.pem' + register: haproxy_certs_directory + +- debug: + var: haproxy_certs_directory + +# Assert that HAProxy certs dir is clean (only pem files) +- assert: + that: + - "haproxy_certs_directory is defined" + - "haproxy_certs_directory.matched == 0" + fail_msg: > + "The HAProxy certificate directory must contain ONLY *.pem files. + These are compiled (concatenated) for you from sources. + If you want to backup the sources, see /etc/letsencrypt/live + or /opt/conf-meza/secret/{{ env }}/ssl/ + Please REMOVE any non pem files from /etc/haproxy/certs and redeploy." + + +- name: Use Certbot for SSL + include_role: + name: certbot + when: + - inventory_hostname in groups['load-balancers'] + - enable_certbot + - (docker_skip_tasks is not defined or not docker_skip_tasks) + tags: certbot + +- name: Install GoAccess for Real-time Web Analytics + include_role: + name: goaccess + when: + - inventory_hostname in groups['load-balancers'] + - enable_goaccess + tags: goaccess -- name: Ensure haproxy certs have secure permissions +- name: Ensure firewall port 7890 CLOSED when GoAccess DISABLED + include_role: + name: firewall_port + vars: + firewall_action: close + firewall_port: 7890 + firewall_protocol: tcp + firewall_zone: "{{m_private_networking_zone|default('public')}}" + when: + - not enable_goaccess + - (docker_skip_tasks is not defined or not docker_skip_tasks) + +- name: Ensure haproxy cert directory has secure permissions file: path: /etc/haproxy/certs state: directory - recurse: yes + recurse: no + owner: root + group: root + mode: 0700 + +- name: Find haproxy cert(s) + find: + paths: /etc/haproxy/certs + file_type: file + patterns: "*.pem" + register: certlist + +- name: Ensure haproxy cert(s) have secure permissions + file: + path: "{{ item.path }}" + state: file owner: root group: root mode: 0600 - notify: - - restart haproxy + with_items: "{{ certlist.files }}" - name: write the haproxy config file template: @@ -195,15 +224,6 @@ notify: - restart haproxy -# - name: Ensure haproxy socket directory exists -# file: -# path: /run/haproxy -# state: directory -# owner: root -# group: root -# mode: 0660 - - - name: Ensure error files directory in place file: path: /etc/haproxy/errors @@ -353,23 +373,46 @@ # state: enabled # zone: public +## Accept incoming log messages over UDP +# https://books.google.com/books?id=C5vcDgAAQBAJ&pg=PA313&lpg=PA313#v=onepage&q&f=false -- name: Uncomment '$ModLoad imudp' in /etc/rsyslog.conf +- name: Uncomment '$ModLoad imudp' in /etc/rsyslog.conf (RedHat) replace: dest: /etc/rsyslog.conf regexp: '^#\$ModLoad imudp' replace: '$ModLoad imudp' + when: ansible_os_family == "RedHat" notify: - restart rsyslog -- name: Uncomment '$UDPServerRun 514' in /etc/rsyslog.conf +- name: Uncomment '$UDPServerRun 514' in /etc/rsyslog.conf (RedHat) replace: dest: /etc/rsyslog.conf regexp: '^#\$UDPServerRun 514' replace: '$UDPServerRun 514' + when: ansible_os_family == "RedHat" notify: - restart rsyslog +- name: Uncomment '$ModLoad imudp' in /etc/rsyslog.conf (Debian) + replace: + dest: /etc/rsyslog.conf + regexp: '^#module\(load="imudp"\)' + replace: 'module(load="imudp")' + when: ansible_os_family == "Debian" + notify: + - restart rsyslog + +- name: Uncomment '$UDPServerRun 514' in /etc/rsyslog.conf (Debian) + replace: + dest: /etc/rsyslog.conf + regexp: '^#input\(type="imudp" port="514"\)' + replace: 'input(type="imudp" port="514")' + when: ansible_os_family == "Debian" + notify: + - restart rsyslog + + - name: Ensure /etc/rsyslog.d/haproxy.conf configured template: src: haproxy.rsyslog.conf.j2 @@ -377,6 +420,19 @@ notify: - restart rsyslog +# @todo remove me +- name: Check haproxy status + command: sudo systemctl status haproxy.service + register: haproxy_status + +# @todo remove me +- debug: + var: haproxy_status + verbosity: 3 + +# @todo remove me +- name: Dump haproxy config + command: sudo cat /etc/haproxy/haproxy.cfg - name: ensure haproxy is running (and enable it at boot) service: @@ -384,3 +440,4 @@ state: started enabled: yes when: docker_skip_tasks is not defined or not docker_skip_tasks + diff --git a/src/roles/haproxy/templates/haproxy.cfg.j2 b/src/roles/haproxy/templates/haproxy.cfg.j2 index 11fa057e5..8fe7464d4 100644 --- a/src/roles/haproxy/templates/haproxy.cfg.j2 +++ b/src/roles/haproxy/templates/haproxy.cfg.j2 @@ -1,6 +1,6 @@ global log 127.0.0.1 local2 - # need logging for letsencrypt if setup ??? + # need logging for letsencrypt ??? chroot /var/lib/haproxy stats socket /run/haproxy-admin.sock mode 660 level admin stats timeout 30s @@ -17,25 +17,10 @@ global # Default ciphers to use on SSL-enabled listening sockets. # For more information, see ciphers(1SSL). This list is from: # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ - ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS - ssl-default-bind-options no-sslv3 + ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:!RSA+3DES:!aNULL:!MD5:!DSS + ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 - # Apache httpd.conf settings regarding SSL which we should verify if they - # need to be included somehow here. - # - # SSLProtocol all -SSLv3 -TLSv1 - # SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 - # SSLHonorCipherOrder on - # SSLCompression off - # #SSLSessionTickets off # not available until httpd 2.4.11 - - # # OCSP Stapling - # SSLUseStapling on - # SSLStaplingResponderTimeout 5 - # SSLStaplingReturnResponderErrors off - # SSLStaplingCache shmcb:/var/run/ocsp(128000) - defaults log global mode http @@ -53,7 +38,7 @@ defaults # errorfile 504 /etc/haproxy/errors/504.http option forwardfor option http-server-close - + timeout tunnel 1h # timeout to use with WebSocket and CONNECT @@ -61,15 +46,30 @@ defaults frontend www-http bind *:80 - reqadd X-Forwarded-Proto:\ http + {% if ansible_default_ipv6 | length > 0 -%} + bind {{ ansible_default_ipv6.address }}:80 + {%- endif %} + + http-request set-header X-Forwarded-Proto http default_backend www-backend frontend www-https - bind *:443 ssl crt /etc/haproxy/certs/meza.pem - reqadd X-Forwarded-Proto:\ https - # Keep letsencrypt stuff here for now. Probably add it back later. - # acl letsencrypt-acl path_beg /.well-known/acme-challenge/ - # use_backend letsencrypt-backend if letsencrypt-acl + bind *:443 ssl crt /etc/haproxy/certs/ + {% if ansible_default_ipv6 | length > 0 -%} + # Create an AAAA record in DNS using the IP below to IPv6 enable your wiki + bind {{ ansible_default_ipv6.address }}:443 ssl crt /etc/haproxy/certs/ + {%- endif %} + + http-request set-header X-Forwarded-Proto https + {% if enable_goaccess %} + # goaccess + # Add config for websocket server GoAccess + acl is_websocket hdr(Upgrade) -i WebSocket + use_backend ws if is_websocket + {% endif %} + # certbot + acl letsencrypt-acl path_beg /.well-known/acme-challenge/ + use_backend letsencrypt-backend if letsencrypt-acl default_backend www-backend backend www-backend @@ -91,12 +91,12 @@ backend www-backend # @FIXME loop w/ index and tie all # app servers together with controller as registry {% if m_install_netdata %} - frontend netdata - bind *:20000 ssl crt {{ ssl_certificate_file }} - mode http - default_backend netdata-back - backend netdata-back - server nd1 127.0.0.1:19999 +frontend netdata + bind *:20000 ssl crt /etc/haproxy/certs/ + mode http + default_backend netdata-back +backend netdata-back + server nd1 127.0.0.1:19999 {% endif %} {% endif %} @@ -141,7 +141,7 @@ listen mediawiki-internal {% if enable_haproxy_stats %} listen stats - bind *:1936 ssl crt /etc/haproxy/certs/meza.pem + bind *:1936 ssl crt /etc/haproxy/certs/ stats enable stats hide-version stats realm Haproxy\ Statistics @@ -150,5 +150,16 @@ listen stats stats refresh 30s {% endif %} -# backend letsencrypt-backend -# server letsencrypt 127.0.0.1:54321 +{% if enable_goaccess %} +backend ws + mode http + option forwardfor + option http-server-close + option forceclose + no option httpclose + server ws1 127.0.0.1:7890 ssl verify none + +{% endif %} + +backend letsencrypt-backend + server letsencrypt 127.0.0.1:54321 diff --git a/src/roles/htdocs/templates/.htaccess.j2 b/src/roles/htdocs/templates/.htaccess.j2 index fcebf0db0..e0296fb24 100644 --- a/src/roles/htdocs/templates/.htaccess.j2 +++ b/src/roles/htdocs/templates/.htaccess.j2 @@ -15,6 +15,16 @@ # Allow access to root index.php RewriteRule ^index.php(.*) - [L] + # This page can bring the wiki to it's knees + RewriteCond %{REQUEST_URI} ^.*Special:MostLinkedPages$ [NC] + RewriteRule ^.*$ - [R=404,L] + + # Allow certbot access + RewriteRule ^.well-known(.*) - [L] + + # Allow access to public_html + RewriteRule ^public_html(.*) - [L] + # Allow access to /wikis directory (where all wiki content and # and settings are located) RewriteRule ^wikis(?:/|$)(.*)$ - [L] @@ -30,10 +40,22 @@ RewriteRule ^BackupDownload(?:/|$)(.*)$ - [L] {% endif %} - # Taken from MediaWiki.org [[Extension:Simple Farm]] - # - # Redirect virtual wiki path to physical wiki path. There - # can be no wiki accessible using this path. - RewriteRule ^(?!mediawiki(?:/|$))[^/]+(?:/(.*))?$ mediawiki/$1 - + # If the request is not for a valid directory, file, link + RewriteCond %{REQUEST_FILENAME} !-d + RewriteCond %{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_FILENAME} !-l + {% if m_mediawiki_prefix_dir is defined %} + RewriteRule ^/?(?:wiki)?/(de|en|es|fr|it|ja|ko|pt|ru|sv|zh)/(.*)?$ %{DOCUMENT_ROOT}/wiki/mediawiki/ [env=WIKI:$1,L] + {% else %} + RewriteRule ^/?([^/]+)/(.*)?$ %{DOCUMENT_ROOT}/mediawiki/ [env=WIKI:$1,L] + {% endif %} + + + +Require all denied + + + +Require all denied + diff --git a/src/roles/htdocs/templates/robots.txt.j2 b/src/roles/htdocs/templates/robots.txt.j2 index 613944545..691f8b474 100644 --- a/src/roles/htdocs/templates/robots.txt.j2 +++ b/src/roles/htdocs/templates/robots.txt.j2 @@ -16,7 +16,7 @@ {% if m_robots_rules == 'nocrawl' %} User-agent: * -Disallow / +Disallow: / {% else %} @@ -167,7 +167,6 @@ Allow: /w/api.php?action=mobileview& Allow: /w/load.php? Allow: /api/rest_v1/?doc # Disallow indexing of non-article content -Disallow: /w/ Disallow: /api/ Disallow: /trap/ # diff --git a/src/roles/init-controller-config/templates/public.yml.j2 b/src/roles/init-controller-config/templates/public.yml.j2 index 11070b969..e680c059a 100644 --- a/src/roles/init-controller-config/templates/public.yml.j2 +++ b/src/roles/init-controller-config/templates/public.yml.j2 @@ -7,6 +7,10 @@ # This variable tells certain aspects of meza to be better suited for production # or for development servers. m_use_production_settings: True +# default (False) will fail on local mediawiki extension modifcations +# Overriding can be useful to add to public.yml for development environments +m_ignore_local_mods: False + # Set a default authentication method for all wikis that don't specify one # FIXME #763: List types, and descriptions diff --git a/src/roles/mediawiki/tasks/main.yml b/src/roles/mediawiki/tasks/main.yml index a7eaea45e..cfb6b1b29 100644 --- a/src/roles/mediawiki/tasks/main.yml +++ b/src/roles/mediawiki/tasks/main.yml @@ -91,6 +91,28 @@ tags: - mediawiki-core +# create symlink to core for short urls +# tag it latest, so that the symlink gets created whenever core is downloaded +- name: Create symlink to core, to enable short urls + file: + src: "{{ m_mediawiki }}" + dest: "{{ m_htdocs }}/w" + state: link + tags: + - latest + +# create symlink to wiki for short urls +# tag it latest, so that the symlink gets created whenever core is downloaded +- name: Link wikis to core, to enable short urls + file: + src: "{{ m_htdocs }}/w" + dest: "{{ m_htdocs }}/{{ item }}" + state: link + with_items: + - "{{ list_of_wikis }}" + tags: + - latest + # # EXTENSIONS AND SKINS # @@ -129,6 +151,7 @@ - git-extensions - git-core-extensions - latest + ignore_errors: "{{ m_ignore_local_mods }}" - name: Set variable holding list of local extensions include_vars: @@ -147,6 +170,7 @@ tags: - skins - core-skins + ignore_errors: "{{ m_ignore_local_mods }}" - name: Ensure local meza extensions installed (non-Composer) become: yes @@ -163,6 +187,7 @@ - git-extensions - git-local-extensions - latest + ignore_errors: "{{ m_ignore_local_mods }}" - name: Ensure local meza skins installed (non-Composer) git: @@ -174,6 +199,7 @@ tags: - skins - local-skins + ignore_errors: "{{ m_ignore_local_mods }}" # File holding extension loading and config for core and local extensions # as well as core and local skins @@ -193,11 +219,12 @@ owner: meza-ansible group: wheel -- name: Run composer install on MediaWiki for dependencies +# install doesn't appear to do extensions +- name: Run composer update on MediaWiki for extensions become: yes become_user: "meza-ansible" composer: - command: install + command: update working_dir: "{{ m_mediawiki }}" no_dev: no # FIXME #317: need ignore_errors because composer throws an error when running as root. @@ -207,12 +234,11 @@ - latest - skins -# install doesn't appear to do extensions -- name: Run composer update on MediaWiki for extensions +- name: Run composer install on MediaWiki for dependencies become: yes become_user: "meza-ansible" composer: - command: update + command: install working_dir: "{{ m_mediawiki }}" no_dev: no # FIXME #317: need ignore_errors because composer throws an error when running as root. @@ -456,10 +482,16 @@ # es_do_upgrade: False # when: not es_do_upgrade_stat.stat.exists - -- name: Verify metastore index upgraded - shell: WIKI={{ list_of_wikis[0] }} php /opt/htdocs/mediawiki/extensions/CirrusSearch/maintenance/metastore.php --upgrade +- name: "Ensure SMW storage setup" + shell: WIKI={{ item }} php {{ m_mediawiki }}/extensions/SemanticMediaWiki/maintenance/setupStore.php + with_items: "{{ list_of_wikis }}" run_once: true + when: "{{ enableSemantics | default(false,true) }}" + + +# - name: Verify metastore index upgraded +# shell: WIKI={{ list_of_wikis[0] }} php /opt/htdocs/mediawiki/extensions/CirrusSearch/maintenance/metastore.php --upgrade +# run_once: true # Wikis are totally built at this point, but SMW and search need rebuilding # FIXME #811: Will this work when controller is not an app server? @@ -517,25 +549,18 @@ tags: - latest -- name: "Ensure python3 symlink in place" - file: - # dest = symlink, src = dir linked to - src: "/usr/bin/python3.5" - dest: "/usr/bin/python3" - state: link - owner: root - group: root - mode: 0755 - when: ansible_os_family == 'RedHat' -- name: "Ensure pip3 symlink in place" - file: - # dest = symlink, src = dir linked to - src: "/usr/bin/pip3.5" - dest: "/usr/bin/pip3" - state: link - owner: root - group: root - mode: 0755 - when: ansible_os_family == 'RedHat' +- name: "Check for existence of python3" + stat: + path: /usr/bin/python3 + register: python3 + +- debug: { var: python3 } + +- name: "Check for existence of pip3" + stat: + path: /usr/bin/pip3 + register: pip3 + +- debug: { var: pip3 } diff --git a/src/roles/mediawiki/templates/LocalSettings.php.j2 b/src/roles/mediawiki/templates/LocalSettings.php.j2 index e52e5f7d3..7610ee95c 100644 --- a/src/roles/mediawiki/templates/LocalSettings.php.j2 +++ b/src/roles/mediawiki/templates/LocalSettings.php.j2 @@ -43,23 +43,47 @@ require "{{ m_deploy }}/samlLocalSettings.php"; require '/opt/.deploy-meza/config.php'; -if( $wgCommandLineMode ) { - - $mezaWikiEnvVarName='WIKI'; - - // get $wikiId from environment variable - $wikiId = getenv( $mezaWikiEnvVarName ); - +/* + * Fix ENV vars getting prepended with 'REDIRECT_' by Apache + */ +function fixApacheEnv () { + foreach ($_ENV as $key => $value) { + if (substr($key, 0, 9) === 'REDIRECT_') { + $_ENV[str_replace('REDIRECT_', '', $key)] = $value; + putenv(str_replace('REDIRECT_', '', $key) . '=' . $value); + } + } } -else { - - // get $wikiId from URI - $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); - $wikiId = strtolower( $uriParts[1] ); // URI has leading slash, so $uriParts[0] is empty string - +fixApacheEnv(); + +$mezaWikiEnvVarName='WIKI'; + +$wikiId = getenv( $mezaWikiEnvVarName ); + +if (empty($wikiId)) { + + {% if m_use_subdomains %} + // get $wikiId from subdomain + // FIXME make regex a config variable + // FIXME add 'ignore' list like 'www' that should not be a wiki + if ( preg_match( '%([a-z]+)\.([a-z]+)\..{2,4}[\d]*$%im', $_SERVER['HTTP_HOST'], $matches ) ) { + $wikiId = $matches[1]; + } + + {% else %} + + // get $wikiId from URI path + $uriParts = explode( '/', $_SERVER['REQUEST_URI'] ); + if ($uriParts[1] == 'wiki') { + $wikiId = strtolower( $uriParts[2] ); + } elseif ( in_array( $uriParts[1], ['de','en','es','fr','it','ja','ko','pt','sv','ru','zh'] ) ) { + $wikiId = $uriParts[1]; + } else { + $wikiId = strtolower($uriParts[1]); // default + } + {% endif %} } - {% if wiki_id_redirects is defined and wiki_id_redirects|length > 0 %} // array point wiki IDs to redirect from and to @@ -90,11 +114,9 @@ if ( isset( $wikiIdRedirects[ $wikiId ] ) ) { $wikis = array_slice( scandir( "$m_htdocs/wikis" ), 2 ); -if ( ! in_array( $wikiId, $wikis ) ) { - +if ( ! in_array( $wikiId, $wikis ) && empty ( $wikiId ) ) { // handle invalid wiki die( "No sir, I ain't heard'a no wiki that goes by the name \"$wikiId\"\n" ); - } {% if meza_auth_type is defined %} @@ -245,6 +267,18 @@ else { * **/ + if (isset($_SERVER["HTTP_HOST"])) { + define('WIKI_HOST', 'https://' . $_SERVER["HTTP_HOST"]); + } else { // probably commandline invocation + $fullhost = shell_exec("hostname -f"); + if ( strstr( $fullhost, 'qualitybox' ) ) { + define('WIKI_HOST', 'https://beta.familysearch.org'); + } else { + define('WIKI_HOST', 'https://www.familysearch.org'); + } + } + + // ref: https://www.mediawiki.org/wiki/Manual:$wgServer // From section #Autodetection: // "When $wgServer is not set, the default value is calculated @@ -254,10 +288,29 @@ else { // Depending on proxy setup (particularly for Varnish/Squid caching) may need // to set $wgInternalServer: // ref: https://www.mediawiki.org/wiki/Manual:$wgInternalServer -$wgServer = 'https://{{ wiki_app_fqdn }}'; +if ( ! $wgCommandLineMode ) { + // $wgServer for subdomain mode + $wgServer = "https://" . $_SERVER['HTTP_HOST'] ; +} else { + $wgServer = 'https://{{ wiki_app_fqdn }}'; +} + // https://www.mediawiki.org/wiki/Manual:$wgScriptPath +// https://www.mediawiki.org/wiki/Manual:$wgLogo +// https://www.mediawiki.org/wiki/Manual:$wgFavicon +{% if m_mediawiki_prefix_dir is defined %} +$mPrefix = "{{ m_mediawiki_prefix_dir|replace('/', '') }}"; +$wgScriptPath = "/$mPrefix/$wikiId"; +$wgArticlePath = "/$mPrefix/$wikiId/$1"; +$wgLogo = "/$mPrefix/wikis/$wikiId/config/logo.png"; +$wgFavicon = "/$mPrefix/wikis/$wikiId/config/favicon.ico"; +{% else %} $wgScriptPath = "/$wikiId"; +$wgArticlePath = "/$wikiId/$1"; +$wgLogo = "/wikis/$wikiId/config/logo.png"; +$wgFavicon = "/wikis/$wikiId/config/favicon.ico"; +{% endif %} // https://www.mediawiki.org/wiki/Manual:$wgUploadPath $wgUploadPath = "$wgScriptPath/img_auth.php"; @@ -265,12 +318,6 @@ $wgUploadPath = "$wgScriptPath/img_auth.php"; // https://www.mediawiki.org/wiki/Manual:$wgUploadDirectory $wgUploadDirectory = "{{ m_uploads_dir }}/$wikiId"; -// https://www.mediawiki.org/wiki/Manual:$wgLogo -$wgLogo = "/wikis/$wikiId/config/logo.png"; - -// https://www.mediawiki.org/wiki/Manual:$wgFavicon -$wgFavicon = "/wikis/$wikiId/config/favicon.ico"; - // https://www.mediawiki.org/wiki/Manual:$wgMetaNamespace $wgMetaNamespace = str_replace( ' ', '_', $wgSitename ); @@ -487,7 +534,7 @@ $wgMaxUploadSize = 1024*1024*100; // 100 MB $wgUseImageMagick = true; {% if ansible_os_family == "RedHat" %} -$wgImageMagickConvertCommand = '/usr/local/bin/convert'; +$wgImageMagickConvertCommand = '/usr/bin/convert'; {% else %} $wgImageMagickConvertCommand = 'convert'; {% endif %} @@ -645,7 +692,7 @@ $wgULSGeoService = false; $wgNamespacesWithSubpages[NS_MAIN] = true; -$wgUseRCPatrol = false; +$wgUseRCPatrol = {{ wgUseRCPatrol|default(false) }}; diff --git a/src/roles/meza-log/templates/server-performance.sh.j2 b/src/roles/meza-log/templates/server-performance.sh.j2 index e8c7e8f40..b1a3df18b 100644 --- a/src/roles/meza-log/templates/server-performance.sh.j2 +++ b/src/roles/meza-log/templates/server-performance.sh.j2 @@ -48,7 +48,7 @@ cd /opt/htdocs/wikis for d in */ do wiki_id=${d%/} - moreJobs=$(WIKI=$wiki_id php /opt/htdocs/mediawiki/maintenance/showJobs.php) + moreJobs=$(WIKI=$wiki_id php {{ m_mediawiki }}/maintenance/showJobs.php) jobs=$(($jobs+$moreJobs)) done diff --git a/src/roles/parsoid-settings/templates/config.yaml.j2 b/src/roles/parsoid-settings/templates/config.yaml.j2 index 5ae98adf7..f3b5a8f0a 100644 --- a/src/roles/parsoid-settings/templates/config.yaml.j2 +++ b/src/roles/parsoid-settings/templates/config.yaml.j2 @@ -47,6 +47,7 @@ services: {% if groups['app-servers']|length|int == 1 and groups['parsoid-servers']|length|int == 1 and groups['app-servers'][0] == groups['parsoid-servers'][0] -%} uri: 'http://127.0.0.1:8080/{{ wiki }}/api.php' +# uri: 'https://{{wiki_app_fqdn}}/wiki/{{ wiki }}/api.php' # for FamilySearch {%- elif 'load-balancers' not in groups or groups['load-balancers']|length|int == 0 -%} diff --git a/src/roles/remote-mysqldump/tasks/main.yml b/src/roles/remote-mysqldump/tasks/main.yml index 9a96a2594..d9196f6e3 100644 --- a/src/roles/remote-mysqldump/tasks/main.yml +++ b/src/roles/remote-mysqldump/tasks/main.yml @@ -96,7 +96,7 @@ -i /root/meza-ansible-id_rsa -o UserKnownHostsFile=/root/meza-ansible-known_hosts {{ remote_server_ssh_user }}@{{ remote_server }} - "mysqldump + "mysqldump --single-transaction --quick {{ user_option }} {{ password_option }} {{ dump_database }} @@ -106,7 +106,7 @@ - name: remote_server == target_server ({{ target_server }}); run mysqldump locally set_fact: mysqldump_command: > - mysqldump + mysqldump --single-transaction --quick {{ user_option }} {{ password_option }} {{ dump_database }} diff --git a/src/roles/saml/templates/samlLocalSettings.php.j2 b/src/roles/saml/templates/samlLocalSettings.php.j2 index 7ef8fe3ab..41659455f 100644 --- a/src/roles/saml/templates/samlLocalSettings.php.j2 +++ b/src/roles/saml/templates/samlLocalSettings.php.j2 @@ -37,7 +37,7 @@ if ( isset( $_SERVER['HTTP_X_SKIP_SAML'] ) ) { // FIXME #822: The indenting below will be heinous when Ansible does its templating {% if allow_skip_saml_users is defined -%} $wgMezaAllowSkipSamlUsers = array(); - {% for user, ipaddrs in allow_skip_saml_users.iteritems() -%} + {% for user, ipaddrs in allow_skip_saml_users.items() -%} $wgMezaAllowSkipSamlUsers['{{ user }}'] = array( {%- for ipaddr in ipaddrs -%}'{{ ipaddr }}',{%- endfor -%} ); diff --git a/src/scripts/meza.py b/src/scripts/meza.py index 41f9a4316..c663435dd 100755 --- a/src/scripts/meza.py +++ b/src/scripts/meza.py @@ -751,7 +751,7 @@ def playbook_cmd ( playbook, env=False, more_extra_vars=False ): extra_vars = {} if more_extra_vars: - for varname, value in more_extra_vars.iteritems(): + for varname, value in more_extra_vars.items(): extra_vars[varname] = value if len(extra_vars) > 0: