Merge branch 'master' of trygvis.io:git/infra

20 files changed, 216 insertions, 211 deletions

diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg
index 4712f76..e7dde87 100644
--- a/ansible/ansible.cfg
+++ b/ansible/ansible.cfg
@@ -1,12 +1,12 @@
 [defaults]
 become_method = sudo
 connection_plugins = ./connection_plugins
-inventory = ./inventory
+inventory = ./inventory,./inventory-terraform
 nocows = True
 stdout_callback = debug
 vault_password_file = ./.vault-password
 roles_path = roles:thirdparty
 retry_files_enabled = False
 
-strategy_plugins = env/lib/python3.8/site-packages/ansible_mitogen/plugins/strategy
+strategy_plugins = env/lib/python3.9/site-packages/ansible_mitogen/plugins/strategy
 strategy = mitogen_linear
diff --git a/ansible/minio-policies.yml b/ansible/minio-policies.yml
deleted file mode 100644
index 536314c..0000000
--- a/ansible/minio-policies.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-- hosts: localhost
-  tasks:
-    - command: mc admin user list --json "{{ minio_config }}"
-      register: cmd_users
-    # - debug: var=cmd_users.stdout
-
-- hosts: localhost
-  tasks:
-    - with_items: "{{ backup_policies }}"
-      include_tasks: minio/backup-policy.yml
-      vars:
-        hostname: "{{ item }}"
-        host: "{{ minio_users['backup-' + item] }}"
-
-- hosts: localhost
-  vars:
-    registered_minio_users: "{{ cmd_users.stdout_lines | map('from_json') | list }}"
-    present_users: "{{ minio_users | dict2items | json_query('[] | [?value.state == `present`]') | items2dict }}"
-    absent_users: "{{ minio_users | dict2items | json_query('[] | [?value.state == `absent`]') | items2dict }}"
-  tasks:
-    - name: all present users
-      debug:
-        msg: "{{ present_users | join(', ') }}"
-      when: false
-
-    - name: all absent users
-      debug:
-        msg: "{{ absent_users | join(', ') }}"
-      when: false
-
-    - name: all minio users
-      debug:
-        msg: "{{ registered_minio_users }}"
-      when: false
-
-    - name: Adding user to Minio
-      command: "mc admin user add {{ minio_config }} {{ item }} {{ user.secret }} {{ user.policy }}"
-      when: user_count == "0"
-      vars:
-        user: "{{ minio_users[item] }}"
-        user_count: "{{ registered_minio_users | json_query('[] | [?accessKey == `' + item + '`]') | length }}"
-      with_items: "{{ present_users }}"
-
-    - name: Removing user from Minio
-      command: "mc admin user remove {{ minio_config }} {{ item }}"
-      when: user_count
-      vars:
-        user_count: "{{ registered_minio_users | json_query('[] | [?accessKey == `' + item + '`]') | length }}"
-      with_items: "{{ absent_users }}"
diff --git a/ansible/minio/backup-policy.yml b/ansible/minio/backup-policy.yml
deleted file mode 100644
index e0b8376..0000000
--- a/ansible/minio/backup-policy.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-- register: policy
-  when: host.state == 'present'
-  local_action:
-    module: copy
-    dest: minio/policies/backup-{{ hostname }}.json
-    content: |
-      {
-        "Version": "2012-10-17",
-        "Statement": [
-          {
-            "Action": [
-              "s3:ListBucket"
-            ],
-            "Effect": "Allow",
-            "Resource": [
-              "arn:aws:s3:::backup-{{ hostname }}/*"
-            ],
-            "Sid": ""
-          },
-          {
-            "Action": [
-              "s3:GetObject",
-              "s3:DeleteObject",
-              "s3:PutObject"
-            ],
-            "Effect": "Allow",
-            "Resource": [
-              "arn:aws:s3:::backup-{{ hostname }}/*"
-            ],
-            "Sid": ""
-          }
-        ]
-      }
-
-- name: Registering policy
-  when: policy.changed
-  command: mc admin policy add {{ minio_config }} backup-{{ hostname }} minio/policies/backup-{{ hostname }}.json
-
-- name: checking if bucked exists
-  command: mc ls --json "{{ minio_config }}"
-  register: cmd_ls
-  failed_when: false
-
-#- debug: var=foo
-#  vars:  
-#    foo: "{{ cmd_ls.stdout_lines | map('from_json') | list }}"
-
-#- debug: var=foo
-#  vars:  
-#    foo: "{{ cmd_ls.stdout_lines | map('from_json') | list | json_query('[?key==`backup-' + hostname + '/`]') }}"
-
-- name: Creating backup bucket
-  vars:
-    len: "{{ cmd_ls.stdout_lines | map('from_json') | list | json_query('[?key==`backup-' + hostname + '/`]') | length }}"
-  when: len == "0"
-  command: mc mb {{ minio_config }}/backup-{{ hostname }}
-
-- name: Removing policy file
-  when: host.state != 'present'
-  register: removed
-  file:
-    path: minio/policies/backup-{{ hostname }}.json
-    state: absent
-
-- name: Unregistering policy
-  when: removed.changed
-  command: mc admin policy remove {{ minio_config }} backup-{{ hostname }}
diff --git a/ansible/minio/group_vars/all/vault.yml b/ansible/minio/group_vars/all/vault.yml
new file mode 100644
index 0000000..f8c5f3c
--- /dev/null
+++ b/ansible/minio/group_vars/all/vault.yml
@@ -0,0 +1,13 @@
+$ANSIBLE_VAULT;1.1;AES256
+37316439376635346334323665326364636264623536646662346333333831356233386266326565
+6666613663303766373933346233323831333065353266630a363062333237323736636138643563
+39613864326262323138326236633163616366363635306335323331663636313332383538343434
+3364623632383033380a303332666165393031333237333533616233353936353337633266386336
+39363066396362343531373138353562626430626435386361653036313330363037326139663666
+34646530386537613162373931373462653463336136643232343261653961653434363631613964
+36373239393436366133663065343930343064623336323364333437626132326134653336623135
+62303930623135303933343634666439643935643966323937303266313463346538613163646532
+62353336323132376339616230636637636530353537363064666361303138633664343462613161
+61653566343537636162376463323731343236656637363631333262386631363666323136303165
+66366336326666653266363538653937333535643262316566653365316663393962366364663738
+37613136333634303330
diff --git a/ansible/minio/minio.yml b/ansible/minio/minio.yml
new file mode 100644
index 0000000..d4687a6
--- /dev/null
+++ b/ansible/minio/minio.yml
@@ -0,0 +1,19 @@
+- hosts:
+    - birgitte
+  vars:
+    minio_zfs: "pool1/minio/data"
+    minio_data: "/{{ minio_zfs }}"
+    minio_version: RELEASE.2020-12-29T23-29-29Z
+  tasks:
+    - name: ZFS for minio
+      become: yes
+      zfs:
+        name: "{{ minio_zfs }}"
+        state: present
+
+    - import_role:
+        name: docker-service
+      tags: docker-service
+      vars:
+        service: minio
+        template: templates/docker-compose.yml
diff --git a/ansible/minio/policies/backup-conflatorio.json b/ansible/minio/policies/backup-conflatorio.json
deleted file mode 100644
index 97ea158..0000000
--- a/ansible/minio/policies/backup-conflatorio.json
+++ /dev/null
@@ -1 +0,0 @@
-{"Version": "2012-10-17", "Statement": [{"Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::backup-conflatorio/*"], "Effect": "Allow", "Sid": ""}, {"Action": ["s3:GetObject", "s3:DeleteObject", "s3:PutObject"], "Resource": ["arn:aws:s3:::backup-conflatorio/*"], "Effect": "Allow", "Sid": ""}]}
\ No newline at end of file
diff --git a/ansible/minio/policies/backup-fuckaduck.json b/ansible/minio/policies/backup-fuckaduck.json
deleted file mode 100644
index 0f25369..0000000
--- a/ansible/minio/policies/backup-fuckaduck.json
+++ /dev/null
@@ -1 +0,0 @@
-{"Version": "2012-10-17", "Statement": [{"Action": ["s3:ListBucket"], "Resource": ["arn:aws:s3:::backup-fuckaduck/*"], "Effect": "Allow", "Sid": ""}, {"Action": ["s3:GetObject", "s3:DeleteObject", "s3:PutObject"], "Resource": ["arn:aws:s3:::backup-fuckaduck/*"], "Effect": "Allow", "Sid": ""}]}
\ No newline at end of file
diff --git a/ansible/minio/templates/docker-compose.yml b/ansible/minio/templates/docker-compose.yml
new file mode 100644
index 0000000..4377d0b
--- /dev/null
+++ b/ansible/minio/templates/docker-compose.yml
@@ -0,0 +1,17 @@
+version: "3"
+services:
+  minio:
+    image: minio/minio:{{ minio_version }}
+    environment:
+      # It seems like minio want to replace access_key/secret_key with root_, but it doesn't work yet.
+      MINIO_ROOT_USER: {{ MINIO_ROOT_USER }}
+      MINIO_ROOT_PASSWORD: {{ MINIO_ROOT_PASSWORD }}
+      MINIO_ACCESS_KEY: {{ MINIO_ROOT_USER }}
+      MINIO_SECRET_KEY: {{ MINIO_ROOT_PASSWORD }}
+    command:
+      - server
+      - /data
+    ports:
+      - "9000:9000"
+    volumes:
+      - {{ minio_data }}:/data
diff --git a/ansible/minio/vars.yml b/ansible/minio/vars.yml
deleted file mode 100644
index 67f65f6..0000000
--- a/ansible/minio/vars.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-38623132333131643666333832396131366536303864616161386562613735383938643566663639
-6562383332623834623538313262323765353666313562640a303538383939376231366537613433
-65333766303731323661366437313132333332373130386637306537613332653264383330313931
-6131303363386639650a363963323031626565306366313961353632656362346538316161313662
-31636562323135323733303266303364616139333663663334343766303163613265643066663039
-33613030373636636637666164316438623864316363383534633832613338393965646135643166
-35313361643334646363346539393464396131373230376632336133383362353135616131643961
-65646361303735656432343263663332333736346636646633376463316338316331316564643835
-36623030306538613536393463343763363062626465383637386662653239386265663932376131
-37376432353866343738383331353065613066616431393666326135363130663734303237303864
-39323065663935373863643530333938383931393234646164633334376362323263383932623834
-61613236656238666465393337333361623131633031353137316366303564313364663737346562
-65646530303965633561363539626234643337313733643231363764303731613030386565346163
-33323765363533633564303064396533396536323265323537373136656438373039363664336236
-39393366353930383531366630343034303935393231643538343964643232663538386337666130
-66346433656237663738356563343264363636333662366332373533643535323335356166393531
-61396336396362346461333236646138376365623964336138343431336564303864626639666330
-65613039666262303761306631303539663534343032376164393732616465663961356364636138
-35633134323639386630316166613431323463353535336531353866633065393162313561623936
-63623930613162373765643639313966376231643136333639363563356434346461653066626331
-66653965386664623431313738343462363533356631636665623361343261666437363833623335
-35343434336261353533666132353032626235633864653361613266373035363062356139393261
-30336362373562313436623665343964613161366630323365613438313963303263646136626130
-63626562666331333331623236366532393965343366383330396138666365623135356336626232
-64373234626339313537653362646566643762386534393034663436633864343935633539353935
-36376661336333653065306534643534666565656539353732623063346538616365383733646135
-38633662356231393137363532346130363163373365346634383130353136383031626361306230
-62663336323438346361393531303563646436643962653361343330386334623032346132383263
-38303262303162323137646631313430396634666534373261326330623235626538373861393731
-37303838636565666133396534663562376335346563336334643964346539613266646266633866
-62643263623639666163623763386265386337323435363761666164353466333333376132366166
-34373135326130383839313561393933646236623830356232636162373465346266356230303132
-65306638313737633564373938313564313436333061636536643765323031323763633131303131
-39633236656362386266633831333762366230336231613363373332396139373864646437636436
-61373666373664366133366563643731386138643463313436393239626563386633336632656231
-31616530646337393161623735636239356137646539356633353933366230643366316332663833
-37306531353837326664323430316635393333353366643165393964326431663366356535646431
-3535373634326239356561356562386466363636393238643766
diff --git a/ansible/requirements.txt b/ansible/requirements.txt
index 4375bc0..42251bf 100644
--- a/ansible/requirements.txt
+++ b/ansible/requirements.txt
@@ -1 +1,2 @@
 mitogen==0.2.9
+pyyaml==5.3.1
diff --git a/ansible/terraform-to-ansible-inventory.py b/ansible/terraform-to-ansible-inventory.py
new file mode 100644
index 0000000..25b402b
--- /dev/null
+++ b/ansible/terraform-to-ansible-inventory.py
@@ -0,0 +1,13 @@
+from collections.abc import Iterable
+import os
+import sys
+import json
+
+j = blob = json.load(sys.stdin)
+
+new = {}
+for k, v in blob.items():
+  new[k] = v["value"]
+
+new = {"all": {"vars": new}}
+json.dump(new, fp=sys.stdout)
diff --git a/terraform/Makefile b/terraform/Makefile
index c26c670..bd6278c 100644
--- a/terraform/Makefile
+++ b/terraform/Makefile
@@ -4,11 +4,7 @@ terraform_unzip=.terraform/unzip/$(terraform_version)/
 terraform_zip=.terraform/zip/terraform_$(terraform_version)_linux_amd64.zip
 terraform_bin=.terraform/bin/terraform
 
-ansiblevault_version=2.0.1
-ansiblevault_url=https://github.com/MeilleursAgents/terraform-provider-ansiblevault/releases/download/v$(ansiblevault_version)/terraform-provider-ansiblevault_linux_amd64_v$(ansiblevault_version)
-ansiblevault_path=terraform.d/plugins/linux_amd64/terraform-provider-ansiblevault_v$(ansiblevault_version)_x4
-
-all: $(terraform_bin) $(ansiblevault_path) setup
+all: $(terraform_bin) setup
 
 $(terraform_bin): $(terraform_zip)
 	rm -rf $(dir $(terraform_unzip))
@@ -21,21 +17,3 @@ $(terraform_bin): $(terraform_zip)
 $(terraform_zip):
 	mkdir -p $(dir $@)
 	curl -L -o "$@" $(terraform_url)
-
-$(ansiblevault_path): terraform.d
-	mkdir -p $(dir $@)
-	curl -L -o "$@" $(ansiblevault_url)
-	chmod +x $(@)
-
-terraform.d:
-	mkdir $@
-
-MAIN=$(patsubst %/main.tf,%,$(wildcard */main.tf))
-setup: $(patsubst %,%/terraform.d,$(MAIN))
-.PHONY: setup
-
-%/terraform.d: terraform.d
-	ln -s ../terraform.d $@
-
-.terraform/plugins/linux_amd64:
-	mkdir -p $@
diff --git a/terraform/dns/main.tf b/terraform/dns/main.tf
index d80fb70..e476f03 100644
--- a/terraform/dns/main.tf
+++ b/terraform/dns/main.tf
@@ -1,21 +1,8 @@
 terraform {
-  backend "local" {
-    path = "../state/dns"
+  required_providers {
+    linode = {
+      version = "~> 1.13"
+      source = "linode/linode"
+    }
   }
 }
-
-provider "linode" {
-  version = "~> 1.13"
-
-  token = data.ansiblevault_path.linode_token.value
-}
-
-provider "ansiblevault" {
-  version     = "~> 2.2"
-  root_folder = "../../ansible"
-}
-
-data "ansiblevault_path" "linode_token" {
-  path = "group_vars/all/linode-dns.yml"
-  key  = "linode_token_v4"
-}
diff --git a/terraform/dns/trygvis.tf b/terraform/dns/trygvis.tf
index 659d56a..531661f 100644
--- a/terraform/dns/trygvis.tf
+++ b/terraform/dns/trygvis.tf
@@ -117,3 +117,10 @@ resource "linode_domain_record" "unifi" {
   record_type = "CNAME"
   target      = "vs.trygvis.io"
 }
+
+resource "linode_domain_record" "minio" {
+  domain_id   = linode_domain.root.id
+  name        = "minio"
+  record_type = "CNAME"
+  target      = "vs.trygvis.io"
+}
diff --git a/terraform/dns/versions.tf b/terraform/dns/versions.tf
deleted file mode 100644
index f98850f..0000000
--- a/terraform/dns/versions.tf
+++ /dev/null
@@ -1,11 +0,0 @@
-terraform {
-  required_providers {
-    linode = {
-      source = "linode/linode"
-    }
-    ansiblevault = {
-      source = "MeilleursAgents/ansiblevault"
-    }
-  }
-  required_version = ">= 0.13"
-}
diff --git a/terraform/main.tf b/terraform/main.tf
new file mode 100644
index 0000000..853a87d
--- /dev/null
+++ b/terraform/main.tf
@@ -0,0 +1,75 @@
+terraform {
+  required_version = ">= 0.13"
+
+  backend "local" {
+    path = "../state/dns"
+  }
+
+  required_providers {
+    linode = {
+      version = "~> 1.13"
+      source = "linode/linode"
+    }
+
+    ansiblevault = {
+      version     = "~> 2.2"
+      source = "MeilleursAgents/ansiblevault"
+    }
+
+    minio = {
+      source = "tidalf/minio"
+      version = "1.1.1"
+    }
+  }
+}
+
+provider "ansiblevault" {
+  root_folder = "../ansible"
+}
+
+#################################################
+# Linode
+
+data "ansiblevault_path" "linode_token" {
+  path = "group_vars/all/linode-dns.yml"
+  key  = "linode_token_v4"
+}
+
+provider "linode" {
+  token = data.ansiblevault_path.linode_token.value
+}
+
+#################################################
+# Minio
+
+data "ansiblevault_path" "minio_access_key" {
+  path = "minio/group_vars/all/vault.yml"
+  key  = "MINIO_ROOT_USER"
+}
+
+data "ansiblevault_path" "minio_secret_key" {
+  path = "minio/group_vars/all/vault.yml"
+  key  = "MINIO_ROOT_PASSWORD"
+}
+
+provider "minio" {
+  minio_server = "minio.trygvis.io:443"
+  minio_ssl = "true"
+  minio_access_key = data.ansiblevault_path.minio_access_key.value
+  minio_secret_key = data.ansiblevault_path.minio_secret_key.value
+}
+
+#################################################
+# Modules
+
+module "dns" {
+  source = "./dns"
+}
+
+module "minio" {
+  source = "./minio"
+}
+
+output "secret" {
+  value = module.minio.secret
+}
diff --git a/terraform/minio/.settings.sh b/terraform/minio/.settings.sh
new file mode 100644
index 0000000..1e4fba4
--- /dev/null
+++ b/terraform/minio/.settings.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+alias terraform="$(pwd)/.terraform/bin/terraform"
diff --git a/terraform/minio/.terraform.lock.hcl b/terraform/minio/.terraform.lock.hcl
new file mode 100755
index 0000000..324bd44
--- /dev/null
+++ b/terraform/minio/.terraform.lock.hcl
@@ -0,0 +1,21 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/tidalf/minio" {
+  version     = "1.1.1"
+  constraints = "1.1.1"
+  hashes = [
+    "h1:tP7RCiSUSutKCO7VLoupvInov9wXTSWtLCRrM5amggE=",
+    "zh:09b2f987e3991d489bba39310400e2241457e638201d23c9730195fe782cf449",
+    "zh:1b64279f5695c5b598c1eb48db9a9954bfcf41ccd84062c7603ca3360d8a0f3f",
+    "zh:1df8894f48051c6a672df21187dcdb9ee4b61b05c7aeaea19ee13f4ab6975003",
+    "zh:376b15cda30f7ff2c014e77728bff5d5a6be7150eaa6deb0a4d1b14c4b9bf5d8",
+    "zh:55ff772c833f9b2895fbb951a52515bd171a9ed150ef3acf7d47a8d616753285",
+    "zh:a3348818aaead45f9783c098b97018801ca8d98a22525dde566354eb0e325c5a",
+    "zh:b395547203e05d199a54a8a917845d7bec81a02df586ed267fedfc5b5fa43e74",
+    "zh:bf1b69c2de4310caf4865729e8d97683b7d277dafd037149cf81c870516eb94a",
+    "zh:cb9c40dc351d62c5032cd555787b64b3abd4f47af519ac20b92110c4f1cee45a",
+    "zh:e76ab684b061569a82b8cf5fdef4dc40f7cb9446be2253fc91792f3d78fcdd48",
+    "zh:f15fc7466ee8f35ad87da34229d64cd449a9d181699e6bb72411f46fb29f941a",
+  ]
+}
diff --git a/terraform/minio/main.tf b/terraform/minio/main.tf
new file mode 100644
index 0000000..245b5ad
--- /dev/null
+++ b/terraform/minio/main.tf
@@ -0,0 +1,15 @@
+terraform {
+#  required_providers {
+#    minio = {
+#      source = "aminueza/minio"
+#      version = ">= 1.0.0"
+#    }
+#  }
+
+  required_providers {
+    minio = {
+      source = "tidalf/minio"
+      version = "1.1.1"
+    }
+  }
+}
diff --git a/terraform/minio/user.tf b/terraform/minio/user.tf
new file mode 100644
index 0000000..b0148a7
--- /dev/null
+++ b/terraform/minio/user.tf
@@ -0,0 +1,24 @@
+resource "minio_iam_user" "knot-postgresql-sender" {
+  name = "knot-postgresql-sender"
+#  update_secret = true
+}
+
+output "secret" {
+  value = minio_iam_user.knot-postgresql-sender.secret
+}
+
+resource "minio_s3_bucket" "knot-postgresql" {
+  bucket = "knot-postgresql"
+  acl    = "public"
+}
+
+# resource "minio_iam_group_membership" "developer" {
+#     name = "tf-testing-group-membership"
+# 
+#   users = [
+#     minio_iam_user.user_one.name,
+#     minio_iam_user.user_two.name,
+#   ]
+# 
+#   group = minio_iam_group.developer.name
+# }