From 023b447ea78380bdec965db04b3ba3621c4d3a5b Mon Sep 17 00:00:00 2001
From: Trygve Laugstøl <trygvis@inamo.no>
Date: Wed, 30 Dec 2020 17:33:04 +0100
Subject: Organizing code.

---
 py/acme/rai/__main__.py  | 259 +++++++++++++++++++++++++++++++++++++++++++++++
 py/acme/rai/acme.py      |  39 +++++++
 py/acme/rai/dba.py       |  13 +++
 py/acme/rai/machine.py   |  22 ++++
 py/acme/rai/terraform.py |  42 ++++++++
 py/acme/rai/utils.py     |  59 +++++++++++
 6 files changed, 434 insertions(+)
 create mode 100644 py/acme/rai/__main__.py
 create mode 100644 py/acme/rai/acme.py
 create mode 100644 py/acme/rai/dba.py
 create mode 100644 py/acme/rai/machine.py
 create mode 100644 py/acme/rai/terraform.py
 create mode 100644 py/acme/rai/utils.py

(limited to 'py/acme/rai')

diff --git a/py/acme/rai/__main__.py b/py/acme/rai/__main__.py
new file mode 100644
index 0000000..9319ec5
--- /dev/null
+++ b/py/acme/rai/__main__.py
@@ -0,0 +1,259 @@
+from durable.engine import MessageObservedException
+from durable.lang import *
+import shutil
+import os
+import os.path
+import jinja2
+
+from .utils import *
+
+from . import acme
+from . import dba
+from . import machine
+from . import terraform
+
+with ruleset("phase-1"):
+    acme.Acme.declare_rules()
+    acme.AcmeOperations.declare_rules()
+    machine.Machine.declare_rules()
+    terraform.Terraform.declare_rules()
+
+    @when_all(+s.exception)
+    def second(c):
+        print("Processing failed!")
+        print(c.s["exception"])
+        c.s.exception = None
+
+    @when_all(pri(1000), (m.type == 'machine'))
+    def defaultMachine(c):
+        pass
+
+    @when_all(pri(1000), (m.type == 'dba-container'))
+    def dba_container(c):
+        pass
+        # print(f"dba-container: {c.m}")
+
+    @when_all(pri(900), (m.type == 'dba-container') & (m.image == "statera") & -m.ports)
+    def addPortsToStatera(c):
+        c.retract_fact(c.m)
+        c.m.ports = [8090]
+        c.assert_fact(c.m)
+
+    @when_all(pri(900), (m.type == 'dba-container') & (m.image == "statera-console") & -m.ports)
+    def addPortsToStateraConsole(c):
+        c.retract_fact(c.m)
+        c.m.ports = [80]
+        c.assert_fact(c.m)
+
+    # The none() part doesn't work as is, but it is worked around with the try/except block.
+    @when_all(
+        (c.container << m.type == "dba-container"),
+        none((m.type == "dba-cluster") & (m.key == c.container.cluster)),
+    )
+    def dbCluster(c):
+        cluster = c.container.cluster
+        try:
+            c.assert_fact(dba.cluster(cluster))
+            # print(f"NEW CLUSTER: c.container={c.container}")
+        except MessageObservedException:
+            pass
+
+#    @when_all(pri(40),
+#        (c.container << m.type == "dba-container"),
+#        (c.cluster << (m.type == "dba-cluster") & (m.key == c.container.cluster)),
+#        )
+#    def dbCluster(c):
+#        print("dba-cluster: CATCH ALL")
+#        print(f"c.container: {c.container}")
+#        print(f"c.cluster:   {c.cluster}")
+#        pass
+
+m1 = machine.Machine.make("acme-1")
+m2 = machine.Machine.make("acme-2")
+m3 = machine.Machine.make("acme-3")
+
+acmeCi = acme.Acme.make("ci", "development")
+acmeProduction = acme.Acme.make("production", "master")
+acmeOps = acme.AcmeOperations.make()
+
+x = assert_fact("phase-1", acmeCi); print(f"x: {x}")
+x = assert_fact("phase-1", acmeProduction); print(f"x: {x}")
+x = assert_fact("phase-1", acmeOps); print(f"x: {x}")
+x = assert_fact("phase-1", m1); print(f"x: {x}")
+x = assert_fact("phase-1", m2); print(f"x: {x}")
+x = assert_fact("phase-1", m3); print(f"x: {x}")
+
+if False:
+    print("Facts:")
+    for f in get_facts("phase-1"):
+        print(f"fact: {f}")
+
+    print("dba-clusters:")
+    for f in [f for f in get_facts("phase-1") if f["type"] == "dba-cluster"]:
+        cluster_name = f["key"]
+
+        del f["key"]
+        print(f" cluster:")
+        print(f"  key:  {cluster_name}")
+        print(f"  json: {f}")
+
+        print("  dba-containers:")
+        for f in [f for f in get_facts("phase-1") if f.get("cluster") == cluster_name and f["type"] == "dba-container"]:
+            del f["cluster"]
+            del f["type"]
+            print(f"   container: {f}")
+
+write_facts("phase-1")
+
+with ruleset("phase-2"):
+
+    @when_all(+s.exception)
+    def second(c):
+        print("Processing failed!")
+        print(c.s["exception"])
+        c.s.exception = None
+
+#    @when_all(pri(100), m.type == "dba-cluster")
+#    def container(c):
+#        print(f"default: cluster: {c.m}")
+
+    @when_all((m.type == "dba-container") & -m.ports_classified)
+    def mark_public_containers(c):
+#        print(f"marking container.. {c.m}")
+        item = c.m
+        c.retract_fact(item)
+
+#        print(f"marking container.. {c.m.name}, ports={c.m.ports}")
+        item["public_ports"] = len(item.ports or []) > 0
+        item["ports_classified"] = True
+#        print(f"marking container.. {item}")
+        c.assert_fact(item)
+
+    @when_all(pri(50),
+        c.cluster << (m.type == "dba-cluster"),
+#        c.container << ((m.type == "dba-container") & (m.cluster == c.cluster.key) & m.ports.anyItem(item > 0))
+        c.container << ((m.type == "dba-container") & +m.ports_classified & (m.public_ports > 0))
+        )
+    def container(c):
+        pass
+        # print(f"public container")
+        # print(f"  cluster:   {c.cluster}")
+        # print(f"  container: {c.container}")
+
+    @when_all(((m.type == "dba-container") & (+m.ports_classified) & (m.public_ports == 0)))
+    def container(c):
+        pass
+        # print(f"private container: {c.m}")
+
+print("PHASE 2")
+
+for f in [f for f in get_facts("phase-1") if f["type"] in ("dba-cluster", "dba-container")]:
+    x = assert_fact("phase-2", f); print(f"x: {x}")
+
+write_facts("phase-2")
+
+# Prepare
+if os.path.isdir("gen"):
+    shutil.rmtree("gen")
+os.mkdir("gen")
+os.mkdir("gen/platform")
+os.mkdir("gen/platform/terraform")
+os.mkdir("gen/platform/ansible")
+os.mkdir("gen/dns")
+
+print("PHASE 3: Generating stuff")
+
+file_loader = jinja2.FileSystemLoader("j2")
+j2 = jinja2.Environment(loader=file_loader)
+
+# TODO: merge the dns files into the platform tf files as they are
+# one-to-one now.
+with ruleset("phase-3"):
+    @when_all(m.type == "meta")
+    def ignoreMeta(c):
+        print(f"ignoring {c.m}")
+        pass
+
+    @when_all(m.type == "meta")
+    def ignoreMeta(c):
+        print(f"ignoring 2 {c.m}")
+        pass
+
+    @when_all(
+        pri(1000),
+        (m.type == "terraform-machine"),
+        none(m.done == "platform/terraform/main.tf"),
+    )
+    def mainTf(c):
+        print(f"WRITING gen/platform/terraform/main.tf")
+        with open(f"gen/platform/terraform/main.tf", "w") as f:
+            f.write("""
+terraform {
+  required_providers {
+    scaleway = {
+      source = "scaleway/scaleway"
+    }
+  }
+}
+""".strip())
+            f.write("\n")
+        c.assert_fact({"type": "meta", "done": "platform/terraform/main.tf"})
+
+        machines = []
+        for f in c.get_facts():
+            if f.get("type") != "terraform-machine":
+                continue
+            machines.append(f)
+            print(f"machine: {f}")
+
+        template = j2.get_template("terraform-machine-outputs.j2")
+        with open(f"gen/platform/terraform/outputs.tf", "w") as f:
+            s = template.render(**{"machines": machines})
+            f.write(s.strip())
+            f.write("\n")
+
+    @when_all((m.type == "terraform-machine"))
+    def ansibleMachine(c):
+        template = j2.get_template("platform-ansible.j2")
+        with open(f"gen/platform/ansible/{c.m.key}.yml", "w") as f:
+            s = template.render(**{"m": c.m})
+            f.write(s.strip())
+            f.write("\n")
+
+    @when_all((m.type == "terraform-machine"))
+    def terraformMachine(c):
+        template = j2.get_template("terraform-machine.j2")
+        with open(f"gen/platform/terraform/{c.m.key}.tf", "w") as f:
+            s = template.render(**{"m": c.m})
+            f.write(s.strip())
+            f.write("\n")
+
+    @when_all((m.type == "terraform-record-set"))
+    def terraformRecordSet(c):
+        template = j2.get_template("terraform-record-set.j2")
+        with open(f"gen/dns/{c.m.key}.tf", "w") as f:
+            s = template.render(**{"m": c.m})
+            f.write(s.strip())
+            f.write("\n")
+
+    @when_all(
+        (m.type == "terraform-record-set"),
+        none(m.done == "dns/inputs.tf"),
+    )
+    def mainTf(c):
+        print("WRITING dns/inputs.tf")
+        with open(f"gen/dns/inputs.tf", "w") as f:
+            f.write("""
+variable "addresses" {
+  type    = map(string)
+}
+""".strip())
+            f.write("\n")
+        c.assert_fact({"type": "meta", "done": "dns/inputs.tf"})
+
+facts = [f for f in get_facts("phase-1") if f["type"] in ("terraform-record-set", "terraform-machine")]
+#for f in facts:
+#    x = assert_fact("phase-3", f); print(f"x: {x}")
+x = assert_facts("phase-3", facts); print(f"x: {x}")
+
+write_facts("phase-3")
diff --git a/py/acme/rai/acme.py b/py/acme/rai/acme.py
new file mode 100644
index 0000000..e75d492
--- /dev/null
+++ b/py/acme/rai/acme.py
@@ -0,0 +1,39 @@
+from durable.lang import *
+
+from . import dba
+
+class Acme:
+    @staticmethod
+    def make(env: str, tag: str):
+        return {"type": "acme-application", "key": env, "env": env, "tag": tag}
+
+    @staticmethod
+    def declare_rules():
+        @when_all((m.type == "acme-application"))
+        def acmeApp(c):
+            cluster = f"acme-{c.m.env}"
+            tag = f"{c.m.tag}"
+            # c.assert_fact(dba.cluster(cluster))
+            for f in [
+                dba.container(cluster, "app", "statera", "statera", tag),
+                dba.container(cluster, "app", "statera-console", "statera-console", tag),
+                dba.container(cluster, "app", "4tune-web", "4tune-web", tag),
+                dba.container(cluster, "app", "4tune-api", "4tune-api", tag),
+                dba.container(cluster, "db", "pdb", "postgresql", "13"),
+                dba.container(cluster, "db", "mdb", "mongodb", "3.2"),
+                ]:
+                c.assert_fact(f)
+
+class AcmeOperations:
+    @staticmethod
+    def make():
+        return {"type": "acme-ops", "key": "acme-ops"}
+
+    @staticmethod
+    def declare_rules():
+        @when_all((m.type == "acme-ops"))
+        def acmeOps(c):
+            cluster = "acme-ops"
+            c.assert_fact(dba.cluster(cluster))
+            c.assert_fact(dba.container(cluster, "app", "pdb", "postgresql", "11"))
+            c.assert_fact(dba.container(cluster, "app", "n8n", "n8n", "0.84.1"))
diff --git a/py/acme/rai/dba.py b/py/acme/rai/dba.py
new file mode 100644
index 0000000..04d10bf
--- /dev/null
+++ b/py/acme/rai/dba.py
@@ -0,0 +1,13 @@
+from durable.lang import *
+
+def cluster(key: str):
+    return {"type": "dba-cluster", "key": key}
+
+def container(cluster: str, machineRole: str, name: str, image: str, tag: str):
+    return {"type": "dba-container",
+        "key": f"{cluster}/{name}",
+        "cluster": cluster,
+        "machineRole": machineRole,
+        "name": name,
+        "image": image,
+        "tag": tag}
diff --git a/py/acme/rai/machine.py b/py/acme/rai/machine.py
new file mode 100644
index 0000000..5b5da22
--- /dev/null
+++ b/py/acme/rai/machine.py
@@ -0,0 +1,22 @@
+from durable.lang import *
+
+class Dns:
+    @staticmethod
+    def a(key: str, fqdn: str):
+        return {"type": "dns-entry",
+            "key": key,
+            "fqdn": fqdn,
+            "rrType": "A"
+        }
+
+class Machine:
+    @staticmethod
+    def make(name: str):
+        return {"type": "machine", "key": name, "name": name}
+
+    @staticmethod
+    def declare_rules():
+        @when_all((m.type == "machine"))
+        def acmeApp(c):
+            c.assert_fact(Dns.a(c.m.key, f"{c.m.name}.machine.acme.corp"))
+
diff --git a/py/acme/rai/terraform.py b/py/acme/rai/terraform.py
new file mode 100644
index 0000000..c84a7e1
--- /dev/null
+++ b/py/acme/rai/terraform.py
@@ -0,0 +1,42 @@
+from durable.lang import *
+
+class Terraform:
+    @staticmethod
+    def recordSetForMachine(machineKey: str, terraformId: str, fqdn: str):
+        return {"type": "terraform-record-set",
+            "key": machineKey,
+            "terraformId": terraformId,
+            "fqdn": fqdn,
+            "rrType": "A",
+            "rrData": f"scaleway_instance_ip.{machineKey}.address",
+        }
+
+    @staticmethod
+    def machine(key: str):
+        return {"type": "terraform-machine",
+            "key": key,
+        }
+
+    @staticmethod
+    def declare_rules():
+        @when_all(
+            c.dns << (m.type == "dns-entry"),
+            c.machine << ((m.type == "terraform-machine") & (m.key == c.dns.key)),
+        )
+        def onDnsEntry(c):
+            print("yooooooooooooooooo")
+            terraformId = c.dns.fqdn.replace(".", "_")
+            c.assert_fact(Terraform.recordSetForMachine(c.machine.key, terraformId, c.dns.fqdn))
+
+        @when_all((m.type == "terraform-record-set"))
+        def defaultTerraformRecordSet(c):
+            pass
+
+        @when_all((m.type == "machine"))
+        def onDnsEntry(c):
+            print(f"matched machine: {c.m}")
+            c.assert_fact(Terraform.machine(c.m.key))
+
+        @when_all((m.type == "terraform-machine"))
+        def defaultTerraformMachine(c):
+            print(f"matched terraform-machine: {c.m}")
diff --git a/py/acme/rai/utils.py b/py/acme/rai/utils.py
new file mode 100644
index 0000000..bc47d8f
--- /dev/null
+++ b/py/acme/rai/utils.py
@@ -0,0 +1,59 @@
+import os
+import os.path
+import yaml
+from durable.lang import *
+
+def write_facts(ruleset: set):
+    facts = get_facts(ruleset)
+
+    types = set((f.get("type") for f in facts))
+
+    print(f"types: {types}")
+
+    out_dir = "out"
+    if not os.path.exists(out_dir):
+        os.mkdir(out_dir)
+
+    basedir = os.path.join(out_dir, ruleset)
+    if os.path.exists(basedir):
+        for f in os.listdir(basedir):
+            p = os.path.join(basedir, f)
+            if os.path.isdir(p):
+                for f2 in os.listdir(p):
+                    os.remove(os.path.join(p, f2))
+                os.rmdir(p)
+            else:
+                os.remove(p)
+        os.rmdir(basedir)
+    os.mkdir(basedir)
+
+    for t in types:
+        typedir = os.path.join(basedir, t)
+        os.mkdir(typedir)
+
+        fs = []
+        for fact in facts:
+            if fact["type"] != t:
+                continue
+
+            if fact["type"] == "meta":
+                continue
+
+            if fact.get("key") is None:
+                raise Exception(f"Bad fact: no 'key' {fact}")
+
+            try:
+                del fact["sid"]
+            except KeyError:
+                pass
+
+            fs.append(fact)
+
+        i = 0
+        for fact in sorted(fs, key=lambda f: f["key"]):
+            key = fact["key"]
+            path = os.path.join(typedir, f"{key.replace('/', '_')}.yaml")
+            with open(path, "w") as f:
+                s = yaml.dump(fact)
+                f.write(s)
+            i = i + 1
-- 
cgit v1.2.3