From d77a2af7acee55457f4cab5f3acc8e3060564196 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Wed, 20 Jan 2021 10:29:19 +0100 Subject: Minio + wal-g --- .gitignore | 4 ++ .settings.sh | 6 +++ ansible/inventory-terraform | 8 ++++ ansible/knot.yml | 47 +++++++++++++++++++++++ ansible/terraform-to-ansible-inventory.py | 2 +- bin/mc | 19 ++++++++++ terraform-minio/root.tf | 2 +- terraform/main.tf | 12 ++++-- terraform/minio/user.tf | 24 ------------ terraform/modules/minio-pg-backup/main.tf | 63 +++++++++++++++++++++++++++++++ terraform/modules/minio-pg-backup/vars.tf | 3 ++ 11 files changed, 160 insertions(+), 30 deletions(-) create mode 100755 ansible/inventory-terraform create mode 100755 bin/mc create mode 100644 terraform/modules/minio-pg-backup/main.tf create mode 100644 terraform/modules/minio-pg-backup/vars.tf diff --git a/.gitignore b/.gitignore index e99d1cd..8372094 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ secrets/ .terraform .vault +bin/.mc + +terraform-*/*state* +terraform/*state* diff --git a/.settings.sh b/.settings.sh index b61e32a..6ccc1b7 100644 --- a/.settings.sh +++ b/.settings.sh @@ -14,4 +14,10 @@ else source <(cd $basedir/ansible; ansible-vault view ../.vault) fi +if [[ -d $basedir/bin/.mc ]] +then + echo "Loading completions for mc" + complete -C $basedir/bin/mc mc +fi + alias terraform="ANSIBLE_VAULT_PASS=\$($(pwd)/ansible/.vault-password) $basedir/bin/terraform" diff --git a/ansible/inventory-terraform b/ansible/inventory-terraform new file mode 100755 index 0000000..6eeba30 --- /dev/null +++ b/ansible/inventory-terraform @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +basedir=$(dirname $0) + +(cd "$basedir/../terraform" && terraform output -json) |\ + "$basedir/env/bin/python" "$basedir/terraform-to-ansible-inventory.py" diff --git a/ansible/knot.yml b/ansible/knot.yml index 9bd7632..796bdc1 100644 --- a/ansible/knot.yml +++ b/ansible/knot.yml @@ -22,3 +22,50 @@ - role: knot-misc tags: knot-misc become: true + tasks: + - tags: pg-backup + vars: + wal_g: /etc/postgresql/wal-g.env + wal_g_bin: /var/lib/postgresql/wal-g + block: + - name: "mkdir {{ wal_g }}" + become: yes + file: + path: "{{ wal_g }}" + state: directory + mode: ug=rx,o= + owner: root + group: postgres + + - name: Configure environment + become: yes + copy: + dest: "{{ wal_g }}/{{ item.file }}" + content: "{{ item.content }}" + owner: root + group: postgres + mode: g=r,u=r,o= + loop: + - {file: "AWS_ACCESS_KEY_ID", content: "{{ pg_backup_knot.sender.access_key }}"} + - {file: "AWS_ENDPOINT", content: "https://minio.trygvis.io"} + - {file: "AWS_REGION", content: "us-east-1"} + - {file: "AWS_S3_FORCE_PATH_STYLE", content: "true"} + - {file: "AWS_SECRET_ACCESS_KEY", content: "{{ pg_backup_knot.sender.secret_key }}"} + - {file: "WALG_S3_PREFIX", content: "s3://{{ pg_backup_knot.bucket.name }}"} + - {file: "PGHOST", content: "/var/run/postgresql"} + + - name: /etc/postgresql/13/main/wal-g.conf + become: yes + copy: + dest: /etc/postgresql/13/main/wal-g.conf + content: | + archive_mode = yes + archive_command = '/usr/bin/envdir {{ wal_g }} {{ wal_g_bin }} wal-push %p' + archive_timeout = 60 + + - name: /etc/postgresql/13/main/postgresql.conf + become: yes + lineinfile: + path: /etc/postgresql/13/main/postgresql.conf + regexp: wal-g.conf + line: "include = 'wal-g.conf'" diff --git a/ansible/terraform-to-ansible-inventory.py b/ansible/terraform-to-ansible-inventory.py index 25b402b..6e2e4a9 100644 --- a/ansible/terraform-to-ansible-inventory.py +++ b/ansible/terraform-to-ansible-inventory.py @@ -10,4 +10,4 @@ for k, v in blob.items(): new[k] = v["value"] new = {"all": {"vars": new}} -json.dump(new, fp=sys.stdout) +json.dump(new, fp=sys.stdout, indent=2) diff --git a/bin/mc b/bin/mc new file mode 100755 index 0000000..0f84a63 --- /dev/null +++ b/bin/mc @@ -0,0 +1,19 @@ +#!/bin/bash + +set -euo pipefail + +v=RELEASE.2021-01-05T05-03-58Z +basedir=$(dirname $0) + +mc="$basedir/.mc/mc.$v" + +mkdir -p "$(dirname $mc)" + +if [[ ! -x $mc ]] +then + wget -O "$mc" https://dl.min.io/client/mc/release/linux-amd64/archive/mc.$v + chmod +x "$mc" + ln -s mc.$v "$basedir/.mc/mc" +fi + +exec "$basedir/.mc/mc" "${@}" diff --git a/terraform-minio/root.tf b/terraform-minio/root.tf index 9751f27..dcf4be4 100644 --- a/terraform-minio/root.tf +++ b/terraform-minio/root.tf @@ -1,6 +1,6 @@ resource "minio_s3_bucket" "terraform" { bucket = "terraform" - acl = "public-read-write" + acl = "none" } resource "minio_iam_policy" "terraform-access" { diff --git a/terraform/main.tf b/terraform/main.tf index 71db6a8..74dc140 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -74,10 +74,14 @@ module "dns" { source = "./dns" } -module "minio" { - source = "./minio" +module "pg-backup-knot" { + source = "./modules/minio-pg-backup" + id = "knot" } -output "secret" { - value = module.minio.secret +output "pg_backup_knot" { + value = { + sender: module.pg-backup-knot.sender, + bucket: module.pg-backup-knot.bucket, + } } diff --git a/terraform/minio/user.tf b/terraform/minio/user.tf index b0148a7..e69de29 100644 --- a/terraform/minio/user.tf +++ b/terraform/minio/user.tf @@ -1,24 +0,0 @@ -resource "minio_iam_user" "knot-postgresql-sender" { - name = "knot-postgresql-sender" -# update_secret = true -} - -output "secret" { - value = minio_iam_user.knot-postgresql-sender.secret -} - -resource "minio_s3_bucket" "knot-postgresql" { - bucket = "knot-postgresql" - acl = "public" -} - -# resource "minio_iam_group_membership" "developer" { -# name = "tf-testing-group-membership" -# -# users = [ -# minio_iam_user.user_one.name, -# minio_iam_user.user_two.name, -# ] -# -# group = minio_iam_group.developer.name -# } diff --git a/terraform/modules/minio-pg-backup/main.tf b/terraform/modules/minio-pg-backup/main.tf new file mode 100644 index 0000000..f9e774a --- /dev/null +++ b/terraform/modules/minio-pg-backup/main.tf @@ -0,0 +1,63 @@ +terraform { + required_providers { + minio = { + source = "tidalf/minio" + version = "1.1.1" + } + } +} + +resource "minio_iam_user" "sender" { + name = "pg-backup-${var.id}-sender" +# update_secret = true +} + +resource "minio_s3_bucket" "bucket" { + bucket = "pg-backup-${var.id}" + acl = "public" +} + +resource "minio_iam_policy" "sender" { + name = minio_iam_user.sender.id + policy= <