From c25c490677bcba2c2e9fee90db10594bd43f6982 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Mon, 5 Nov 2018 23:18:54 +0100 Subject: Importing Bitraf's mediawiki o Some improvements, reusing old backup scripts. --- ansible/roles/apache/defaults/main.yml | 2 + ansible/roles/apache/handlers/main.yml | 11 ++ ansible/roles/apache/tasks/main.yml | 32 ++++ ansible/roles/mediawiki/defaults/main.yml | 2 + ansible/roles/mediawiki/files/bin/mw-dump-backup | 9 + ansible/roles/mediawiki/files/bin/mw-dump-db | 12 ++ .../files/systemd/mediawiki-jobqueue.service | 13 ++ .../files/systemd/mediawiki-jobqueue.timer | 9 + ansible/roles/mediawiki/handlers/main.yml | 10 ++ ansible/roles/mediawiki/tasks/main.yml | 177 +++++++++++++++++++ ansible/roles/mw-backend/files/bin/mw-dump-backup | 9 - ansible/roles/mw-backend/files/bin/mw-dump-db | 12 -- ansible/roles/mw-backend/files/bin/mw-make-backup | 5 - .../etc/apache2/sites-enabled/000-default.conf | 60 ------- .../files/etc/systemd/system/mw-jobqueue.service | 12 -- .../files/etc/systemd/system/mw-jobqueue.timer | 9 - ansible/roles/mw-backend/handlers/main.yml | 13 -- ansible/roles/mw-backend/tasks/main.yml | 164 ------------------ .../templates/etc/mediawiki/LocalSettings.php.j2 | 188 --------------------- 19 files changed, 277 insertions(+), 472 deletions(-) create mode 100644 ansible/roles/apache/defaults/main.yml create mode 100644 ansible/roles/apache/handlers/main.yml create mode 100644 ansible/roles/apache/tasks/main.yml create mode 100644 ansible/roles/mediawiki/defaults/main.yml create mode 100644 ansible/roles/mediawiki/files/bin/mw-dump-backup create mode 100644 ansible/roles/mediawiki/files/bin/mw-dump-db create mode 100644 ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.service create mode 100644 ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.timer create mode 100644 ansible/roles/mediawiki/handlers/main.yml create mode 100644 ansible/roles/mediawiki/tasks/main.yml delete mode 100644 ansible/roles/mw-backend/files/bin/mw-dump-backup delete mode 100644 ansible/roles/mw-backend/files/bin/mw-dump-db delete mode 100644 ansible/roles/mw-backend/files/bin/mw-make-backup delete mode 100644 ansible/roles/mw-backend/files/etc/apache2/sites-enabled/000-default.conf delete mode 100644 ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.service delete mode 100644 ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.timer delete mode 100644 ansible/roles/mw-backend/handlers/main.yml delete mode 100644 ansible/roles/mw-backend/tasks/main.yml delete mode 100644 ansible/roles/mw-backend/templates/etc/mediawiki/LocalSettings.php.j2 (limited to 'ansible/roles') diff --git a/ansible/roles/apache/defaults/main.yml b/ansible/roles/apache/defaults/main.yml new file mode 100644 index 0000000..e28958f --- /dev/null +++ b/ansible/roles/apache/defaults/main.yml @@ -0,0 +1,2 @@ +apache__template: +apache__copy: diff --git a/ansible/roles/apache/handlers/main.yml b/ansible/roles/apache/handlers/main.yml new file mode 100644 index 0000000..9a5a629 --- /dev/null +++ b/ansible/roles/apache/handlers/main.yml @@ -0,0 +1,11 @@ +- name: systemctl reload apache + become: yes + systemd: + name: apache2 + state: reloaded + +- name: systemctl restart apache + become: yes + systemd: + name: apache2 + state: restarted diff --git a/ansible/roles/apache/tasks/main.yml b/ansible/roles/apache/tasks/main.yml new file mode 100644 index 0000000..2bb953d --- /dev/null +++ b/ansible/roles/apache/tasks/main.yml @@ -0,0 +1,32 @@ +- become: yes + tags: + - apache + block: + - name: packages + vars: + items: + - apache2 + apt: + name: "{{ items }}" + install_recommends: no + +- become: yes + tags: + - apache + - apache-config + block: + - name: "fill /etc/apache2" + with_items: "{{ apache__copy }}" + when: item is defined and item + notify: systemctl reload apache + copy: + dest: "/etc/apache2/{{ item }}" + src: "{{ item }}" + + - name: "fill /etc/apache2" + with_items: "{{ apache__template }}" + when: item is defined and item + notify: systemctl reload apache + template: + dest: "/etc/apache2/{{ item|replace(apache__template_prefix, '') }}" + src: "{{ item }}" diff --git a/ansible/roles/mediawiki/defaults/main.yml b/ansible/roles/mediawiki/defaults/main.yml new file mode 100644 index 0000000..5858563 --- /dev/null +++ b/ansible/roles/mediawiki/defaults/main.yml @@ -0,0 +1,2 @@ +mediawiki__enable_local_composer: no +mediawiki__force_update: no diff --git a/ansible/roles/mediawiki/files/bin/mw-dump-backup b/ansible/roles/mediawiki/files/bin/mw-dump-backup new file mode 100644 index 0000000..eb98d7d --- /dev/null +++ b/ansible/roles/mediawiki/files/bin/mw-dump-backup @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail + +exec sudo -u www-data php /usr/share/mediawiki/maintenance/dumpBackup.php \ + --full \ + --include-files \ + --uploads \ + > /opt/mediawiki/backup/content-dump.xml diff --git a/ansible/roles/mediawiki/files/bin/mw-dump-db b/ansible/roles/mediawiki/files/bin/mw-dump-db new file mode 100644 index 0000000..5a6699a --- /dev/null +++ b/ansible/roles/mediawiki/files/bin/mw-dump-db @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +export PGHOST=$(sed -n 's,^.wgDBserver.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) +export PGUDATABASE=$(sed -n 's,^.wgDBname.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) +export PGUSER=$(sed -n 's,^.wgDBuser.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) +export PGPASSWORD=$(sed -n 's,^.wgDBpassword.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) +export PGPORT=$(sed -n 's,^.wgDBport.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) + +rm -rf /opt/mediawiki/backup/db +exec pg_dump -b --format directory -f /opt/mediawiki/backup/db diff --git a/ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.service b/ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.service new file mode 100644 index 0000000..a27891e --- /dev/null +++ b/ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.service @@ -0,0 +1,13 @@ +[Unit] +Description=MediaWiki job runner + +[Service] +WorkingDirectory=/opt/mediawiki/mediawiki/maintenance +# 900 is 15 minutes, this task is triggered every 15 minutes +ExecStart=/usr/bin/php runJobs.php --conf ../LocalSettings.php --maxtime=800 --wait +User=www-data +Group=www-data +StandardOutput=journal + +Nice=10 +OOMScoreAdjust=500 diff --git a/ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.timer b/ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.timer new file mode 100644 index 0000000..1d12500 --- /dev/null +++ b/ansible/roles/mediawiki/files/systemd/mediawiki-jobqueue.timer @@ -0,0 +1,9 @@ +[Unit] +Description=MediaWiki job runner timer + +[Timer] +OnBootSec=15min +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/mediawiki/handlers/main.yml b/ansible/roles/mediawiki/handlers/main.yml new file mode 100644 index 0000000..8735482 --- /dev/null +++ b/ansible/roles/mediawiki/handlers/main.yml @@ -0,0 +1,10 @@ +- name: systemd daemon-reload + become: yes + systemd: + daemon_reload: true + +- name: service restart php-fpm + become: yes + service: + name: php7.0-fpm + state: restarted diff --git a/ansible/roles/mediawiki/tasks/main.yml b/ansible/roles/mediawiki/tasks/main.yml new file mode 100644 index 0000000..6d1850b --- /dev/null +++ b/ansible/roles/mediawiki/tasks/main.yml @@ -0,0 +1,177 @@ +- become: yes + tags: + - mediawiki + - mediawiki-packages + block: + - name: packages + vars: + items: + - composer + - php-apcu + - php-gd + - php-intl + - php-mbstring + - php-pgsql + - php-xml + - php-zip + apt: + name: "{{ items }}" + install_recommends: no + +- become: yes + tags: + - mediawiki + - mediawiki-php + block: + - name: increase maximum file size (for uploads) to 20M + notify: service restart php-fpm + ini_file: + path: /etc/php/7.0/fpm/php.ini + section: PHP + option: "{{ item.option }}" + value: "{{ item.value }}" + with_items: + - option: upload_max_filesize + value: 20M + - option: post_max_size + value: 20M + +- become: yes + tags: + - mediawiki + - mediawiki-clone + block: + - name: mkdir /opt/mediawiki + file: + path: /opt/mediawiki + state: directory + + - name: mkdir /opt/mediawiki/mediawiki + file: + path: /opt/mediawiki/mediawiki + state: directory + owner: www-data + + - name: git clone + become: yes + become_user: www-data + vars: + ansible_ssh_pipelining: true + git: + repo: https://github.com/bitraf/mediawiki + dest: /opt/mediawiki/mediawiki + version: "{{ mediawiki__version }}" + update: "{{ 'yes' if mediawiki__force_update else 'no' }}" + force: "{{ 'yes' if mediawiki__force_update else 'no' }}" + +- become: yes + tags: + - mediawiki + - mediawiki-composer + block: + - name: mkdirs + with_items: + - /var/www/.ansible + - /var/www/.composer + file: + path: "{{ item }}" + state: directory + group: www-data + owner: www-data + + - name: Create composer.local.json + when: mediawiki__composer_require is defined + vars: + content: | + {{ '{ "require": {' }} + {% for k, v in mediawiki__composer_require.items() %} + "{{ k }}": "{{ v }}"{% if not loop.last %},{% endif %} + {% endfor %} + {{ '} }' }} + copy: + dest: /opt/mediawiki/mediawiki/composer.local.json + content: "{{ content | to_nice_json }}" + + - name: run composer + become: yes + become_user: www-data + vars: + ansible_ssh_pipelining: true + composer: + command: update + working_dir: /opt/mediawiki/mediawiki + +- name: "Extensions" + become: yes + tags: + - mediawiki + - mediawiki-extensions + with_items: "{{ mediawiki__extensions }}" + git: + repo: "{{ item.repo }}" + dest: "/opt/mediawiki/mediawiki/extensions/{{ item.dest }}" + version: "{{ item.version }}" + +- become: yes + tags: + - mediawiki + - mediawiki-backup + block: + # TODO: install systemd timer and service to run these + - name: mkdir + file: + state: directory + path: "/opt/mediawiki/backup" + - copy: + src: "{{ item }}" + dest: "/{{ item }}" + mode: a+rx + with_items: + - bin/mw-dump-backup + - bin/mw-dump-db + +- become: yes + tags: + - mediawiki + - mediawiki-config + block: + - file: + path: /etc/mediawiki + state: directory + - file: + dest: /opt/mediawiki/mediawiki/LocalSettings.php + src: /etc/mediawiki/LocalSettings.php + state: link + force: yes + + - name: mediawiki systemd timer + notify: systemd daemon-reload + copy: + src: "systemd/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + with_items: + - mediawiki-jobqueue.service + - mediawiki-jobqueue.timer + +- meta: flush_handlers + +- become: yes + tags: + - mediawiki + - mediawiki-config + block: + - systemd: + enabled: true + state: started + name: mediawiki-jobqueue.timer + +- become: yes + tags: + - mediawiki + - mediawiki-config + - update-password + block: + - name: LocalSettings.php + template: + dest: /etc/mediawiki + src: "{{ mediawiki__local_settings }}" diff --git a/ansible/roles/mw-backend/files/bin/mw-dump-backup b/ansible/roles/mw-backend/files/bin/mw-dump-backup deleted file mode 100644 index 0e21eb2..0000000 --- a/ansible/roles/mw-backend/files/bin/mw-dump-backup +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -exec php /usr/share/mediawiki/maintenance/dumpBackup.php \ - --full \ - --include-files \ - --uploads \ - > /opt/mediawiki/backup/content-dump.xml diff --git a/ansible/roles/mw-backend/files/bin/mw-dump-db b/ansible/roles/mw-backend/files/bin/mw-dump-db deleted file mode 100644 index 5a6699a..0000000 --- a/ansible/roles/mw-backend/files/bin/mw-dump-db +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -export PGHOST=$(sed -n 's,^.wgDBserver.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) -export PGUDATABASE=$(sed -n 's,^.wgDBname.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) -export PGUSER=$(sed -n 's,^.wgDBuser.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) -export PGPASSWORD=$(sed -n 's,^.wgDBpassword.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) -export PGPORT=$(sed -n 's,^.wgDBport.*"\([^"]*\)".*,\1,p' /etc/mediawiki/LocalSettings.php) - -rm -rf /opt/mediawiki/backup/db -exec pg_dump -b --format directory -f /opt/mediawiki/backup/db diff --git a/ansible/roles/mw-backend/files/bin/mw-make-backup b/ansible/roles/mw-backend/files/bin/mw-make-backup deleted file mode 100644 index e99b909..0000000 --- a/ansible/roles/mw-backend/files/bin/mw-make-backup +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -exec borg create /opt/mediawiki/borg::mediawiki-{now:%Y%m%d-%H%M%S} /opt/mediawiki/backup diff --git a/ansible/roles/mw-backend/files/etc/apache2/sites-enabled/000-default.conf b/ansible/roles/mw-backend/files/etc/apache2/sites-enabled/000-default.conf deleted file mode 100644 index 0af172a..0000000 --- a/ansible/roles/mw-backend/files/etc/apache2/sites-enabled/000-default.conf +++ /dev/null @@ -1,60 +0,0 @@ -# Based on /etc/apache2/conf-available/mediawiki.conf - - - ServerName mw.trygvis.io - - ServerAdmin webmaster@trygvis.io - DocumentRoot /var/www/mw.trygvis.io - - ErrorLog ${APACHE_LOG_DIR}/error.log - CustomLog ${APACHE_LOG_DIR}/access.log combined - - Alias /w /var/lib/mediawiki - Alias /wiki /var/lib/mediawiki/index.php - RedirectMatch permanent ^/$ /wiki/Main_Page - - - Options +FollowSymLinks - AllowOverride All - = 2.3> - Require all granted - - - order allow,deny - allow from all - - - -# some directories must be protected - - Options -FollowSymLinks - AllowOverride None - - php_admin_flag engine off - - - php_admin_flag engine off - - - - Options -FollowSymLinks - AllowOverride None - - php_admin_flag engine off - - - php_admin_flag engine off - - - - Options -FollowSymLinks - AllowOverride None - - php_admin_flag engine off - - - php_admin_flag engine off - - - -# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.service b/ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.service deleted file mode 100644 index a944443..0000000 --- a/ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=MediaWiki Job runner - -[Service] -WorkingDirectory=/var/lib/mediawiki/maintenance -ExecStart=/usr/bin/php runJobs.php --maxjobs 100 -User=www-data -Group=www-data -StandardOutput=journal - -Nice=10 -OOMScoreAdjust=500 diff --git a/ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.timer b/ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.timer deleted file mode 100644 index 1d12500..0000000 --- a/ansible/roles/mw-backend/files/etc/systemd/system/mw-jobqueue.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=MediaWiki job runner timer - -[Timer] -OnBootSec=15min -OnUnitActiveSec=10m - -[Install] -WantedBy=timers.target diff --git a/ansible/roles/mw-backend/handlers/main.yml b/ansible/roles/mw-backend/handlers/main.yml deleted file mode 100644 index a8bccb6..0000000 --- a/ansible/roles/mw-backend/handlers/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: update apt cache - apt: - update_cache: yes - -- name: reload apache - service: - name: apache2 - state: reloaded - -- name: systemd daemon-reload - systemd: - daemon_reload: true diff --git a/ansible/roles/mw-backend/tasks/main.yml b/ansible/roles/mw-backend/tasks/main.yml deleted file mode 100644 index f5d3e67..0000000 --- a/ansible/roles/mw-backend/tasks/main.yml +++ /dev/null @@ -1,164 +0,0 @@ ---- -- name: apt setup - tags: - - mw-backend - - packages - block: - - file: - dest: /etc/apt/apt.conf.d/99force-ipv4 - state: absent - - name: configure debian repositories - notify: update apt cache - copy: - dest: /etc/apt/sources.list - content: | - deb http://httpredir.debian.org/debian/ stretch main contrib non-free - deb http://security.debian.org/debian-security stretch/updates main contrib non-free - deb http://httpredir.debian.org/debian/ stretch-updates main contrib non-free - deb http://httpredir.debian.org/debian/ stretch-backports main contrib non-free - - - meta: flush_handlers - -# TODO: Remove, use unix-machine instead - - name: packages - apt: - name: "{{ item }}" - install_recommends: no - with_items: - - git - - etckeeper - -- name: system setup - tags: - - mw-backend - - packages - block: - - name: misc packages - apt: - name: "{{ item }}" - install_recommends: no - with_items: - - iputils-ping - - vim-nox - - host - - less - - file: - dest: /var/www/.ansible - state: directory - owner: www-data - mode: u=rwX,go= - -- name: Mediawiki - tags: - - mw-backend - - mediawiki - block: - - name: packages - notify: reload apache - apt: - name: "{{ item }}" - install_recommends: no - with_items: - - git - - php-pgsql - - php-intl - - php-gd - - php-apcu - - composer - - php-zip - - mediawiki # TODO: this should be replaced with composer - - name: /var/www/mw.trygvis.io - tags: apache-config - file: - state: directory - path: "/var/www/mw.trygvis.io" - owner: root - group: root - - name: apache config - notify: reload apache - tags: apache-config - copy: - src: etc/apache2/sites-enabled/000-default.conf - dest: /etc/apache2/sites-enabled/000-default.conf - - name: mediawiki config - template: - src: etc/mediawiki/LocalSettings.php.j2 - dest: /etc/mediawiki/LocalSettings.php - - name: mediawiki job systemd service - notify: systemd daemon-reload - copy: - src: etc/systemd/system/mw-jobqueue.service - dest: /etc/systemd/system/mw-jobqueue.service - - name: mediawiki job systemd timer - notify: systemd daemon-reload - copy: - src: etc/systemd/system/mw-jobqueue.timer - dest: /etc/systemd/system/mw-jobqueue.timer - - systemd: - enabled: true - state: started - name: mw-jobqueue.timer - -- name: Mediawiki extensions - tags: mw-extensions - block: - - name: directories - file: - state: directory - path: /opt/mediawiki/extensions - owner: www-data - group: root - - - name: run composer install scribunto - with_items: "{{ mediawiki_extensions }}" - become: yes - become_user: www-data - composer: - command: require - arguments: "{{ item.package }} {{ item.version }}" - working_dir: /opt/mediawiki/extensions - - - name: symlink extensions into mediawiki - with_items: "{{ mediawiki_extensions }}" - file: - state: link - dest: "/var/lib/mediawiki/extensions/{{ item.extension }}" - src: "/opt/mediawiki/extensions/extensions/{{ item.extension }}" - - - name: symlink maintenance into smw - file: - state: link - dest: "/opt/mediawiki/extensions/maintenance" - src: "/var/lib/mediawiki/maintenance" - owner: root - group: root - -- name: Mediawiki Backup - tags: - - mw-backend - - mediawiki-backup - block: - - name: packages - apt: - name: "{{ item }}" - install_recommends: no - with_items: - - borgbackup - - name: mkdir - file: - state: directory - path: "/opt/mediawiki/backup" - - - name: borgbackup init - shell: borgbackup init -e none /opt/mediawiki/borg - args: - creates: /opt/mediawiki/borg - - - copy: - src: "{{ item }}" - dest: "/{{ item }}" - mode: a+rx - with_items: - - bin/mw-dump-backup - - bin/mw-dump-db - - bin/mw-make-backup diff --git a/ansible/roles/mw-backend/templates/etc/mediawiki/LocalSettings.php.j2 b/ansible/roles/mw-backend/templates/etc/mediawiki/LocalSettings.php.j2 deleted file mode 100644 index 450dfab..0000000 --- a/ansible/roles/mw-backend/templates/etc/mediawiki/LocalSettings.php.j2 +++ /dev/null @@ -1,188 +0,0 @@ - "/w/resources/assets/debian/poweredby_debian_1x.png", - "url" => "https://www.debian.org/", - "alt" => "Powered by Debian", - "srcset" => - "/w/resources/assets/debian/poweredby_debian_1_5x.png 1.5x, " . - "/w/resources/assets/debian/poweredby_debian_2x.png 2x", -]; -# End Debian specific generated settings -# Add more configuration options below. - -$wgUseCategoryBrowser = true; - -// Define constants for my additional namespaces. -define("NS_CHIP", 3000); // This MUST be even. -define("NS_CHIP_TALK", 3001); // This MUST be the following odd integer. -define("NS_BOARD", 3002); // This MUST be even. -define("NS_BOARD_TALK", 3003); // This MUST be the following odd integer. - -enableSemantics( 'https://mw.trygvis.io/wiki/', true ); - -$smwgShowFactbox = SMW_FACTBOX_NONEMPTY; - -// Add "Chip" namespace -$wgExtraNamespaces[NS_CHIP] = "Chip"; -$wgExtraNamespaces[NS_CHIP_TALK] = "Chip_talk"; // Note underscores in the namespace name. -$wgContentNamespaces[] = NS_CHIP; -$smwgNamespacesWithSemanticLinks[NS_CHIP] = true; - -$wgExtraNamespaces[NS_BOARD] = "Board"; -$wgExtraNamespaces[NS_BOARD_TALK] = "Board_talk"; // Note underscores in the namespace name. -$wgContentNamespaces[] = NS_BOARD; -$smwgNamespacesWithSemanticLinks[NS_BOARD] = true; - -# https://www.mediawiki.org/wiki/Manual:$wgSMTP -$wgSMTP = array('host' => '10.0.3.1', 'auth' => false); -- cgit v1.2.3