From 866eafabd7398fd1485e7aa285345a7dcea480d0 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Wed, 30 Dec 2020 20:48:51 +0100 Subject: Moving everything under durable_rules-test. --- durable_rules-test/py/acme/rai/__main__.py | 273 ++++++++++++++++++++++++++++ durable_rules-test/py/acme/rai/acme.py | 39 ++++ durable_rules-test/py/acme/rai/dba.py | 13 ++ durable_rules-test/py/acme/rai/machine.py | 22 +++ durable_rules-test/py/acme/rai/terraform.py | 41 +++++ durable_rules-test/py/acme/rai/utils.py | 59 ++++++ 6 files changed, 447 insertions(+) create mode 100644 durable_rules-test/py/acme/rai/__main__.py create mode 100644 durable_rules-test/py/acme/rai/acme.py create mode 100644 durable_rules-test/py/acme/rai/dba.py create mode 100644 durable_rules-test/py/acme/rai/machine.py create mode 100644 durable_rules-test/py/acme/rai/terraform.py create mode 100644 durable_rules-test/py/acme/rai/utils.py (limited to 'durable_rules-test/py/acme') diff --git a/durable_rules-test/py/acme/rai/__main__.py b/durable_rules-test/py/acme/rai/__main__.py new file mode 100644 index 0000000..b2664fa --- /dev/null +++ b/durable_rules-test/py/acme/rai/__main__.py @@ -0,0 +1,273 @@ +from durable.engine import MessageObservedException +from durable.lang import * +import shutil +import os +import os.path +import jinja2 + +from .utils import * + +from . import acme +from . import dba +from . import machine +from . import terraform + +with ruleset("phase-1"): + acme.Acme.declare_rules() + acme.AcmeOperations.declare_rules() + machine.Machine.declare_rules() + terraform.Terraform.declare_rules() + + @when_all(+s.exception) + def second(c): + print("Processing failed!") + print(c.s["exception"]) + c.s.exception = None + + @when_all(pri(1000), (m.type == 'machine')) + def defaultMachine(c): + pass + + @when_all(pri(1000), (m.type == 'dba-container')) + def dba_container(c): + pass + # print(f"dba-container: {c.m}") + + @when_all(pri(900), (m.type == 'dba-container') & (m.image == "statera") & -m.ports) + def addPortsToStatera(c): + c.retract_fact(c.m) + c.m.ports = [8090] + c.assert_fact(c.m) + + @when_all(pri(900), (m.type == 'dba-container') & (m.image == "statera-console") & -m.ports) + def addPortsToStateraConsole(c): + c.retract_fact(c.m) + c.m.ports = [80] + c.assert_fact(c.m) + + # The none() part doesn't work as is, but it is worked around with the try/except block. + @when_all( + (c.container << m.type == "dba-container"), + none((m.type == "dba-cluster") & (m.key == c.container.cluster)), + ) + def dbCluster(c): + cluster = c.container.cluster + try: + c.assert_fact(dba.cluster(cluster)) + # print(f"NEW CLUSTER: c.container={c.container}") + except MessageObservedException: + pass + +# @when_all(pri(40), +# (c.container << m.type == "dba-container"), +# (c.cluster << (m.type == "dba-cluster") & (m.key == c.container.cluster)), +# ) +# def dbCluster(c): +# print("dba-cluster: CATCH ALL") +# print(f"c.container: {c.container}") +# print(f"c.cluster: {c.cluster}") +# pass + +m1 = machine.Machine.make("acme-1") +m2 = machine.Machine.make("acme-2") +m3 = machine.Machine.make("acme-3") + +acmeCi = acme.Acme.make("ci", "development") +acmeProduction = acme.Acme.make("production", "master") +acmeOps = acme.AcmeOperations.make() + +x = assert_fact("phase-1", acmeCi); print(f"x: {x}") +x = assert_fact("phase-1", acmeProduction); print(f"x: {x}") +x = assert_fact("phase-1", acmeOps); print(f"x: {x}") +x = assert_fact("phase-1", m1); print(f"x: {x}") +x = assert_fact("phase-1", m2); print(f"x: {x}") +x = assert_fact("phase-1", m3); print(f"x: {x}") + +if False: + print("Facts:") + for f in get_facts("phase-1"): + print(f"fact: {f}") + + print("dba-clusters:") + for f in [f for f in get_facts("phase-1") if f["type"] == "dba-cluster"]: + cluster_name = f["key"] + + del f["key"] + print(f" cluster:") + print(f" key: {cluster_name}") + print(f" json: {f}") + + print(" dba-containers:") + for f in [f for f in get_facts("phase-1") if f.get("cluster") == cluster_name and f["type"] == "dba-container"]: + del f["cluster"] + del f["type"] + print(f" container: {f}") + +write_facts("phase-1") + +with ruleset("phase-2"): + + @when_all(+s.exception) + def second(c): + print("Processing failed!") + print(c.s["exception"]) + c.s.exception = None + +# @when_all(pri(100), m.type == "dba-cluster") +# def container(c): +# print(f"default: cluster: {c.m}") + + @when_all((m.type == "dba-container") & -m.ports_classified) + def mark_public_containers(c): +# print(f"marking container.. {c.m}") + item = c.m + c.retract_fact(item) + +# print(f"marking container.. {c.m.name}, ports={c.m.ports}") + item["public_ports"] = len(item.ports or []) > 0 + item["ports_classified"] = True +# print(f"marking container.. {item}") + c.assert_fact(item) + + @when_all(pri(50), + c.cluster << (m.type == "dba-cluster"), +# c.container << ((m.type == "dba-container") & (m.cluster == c.cluster.key) & m.ports.anyItem(item > 0)) + c.container << ((m.type == "dba-container") & +m.ports_classified & (m.public_ports > 0)) + ) + def container(c): + pass + # print(f"public container") + # print(f" cluster: {c.cluster}") + # print(f" container: {c.container}") + + @when_all(((m.type == "dba-container") & (+m.ports_classified) & (m.public_ports == 0))) + def container(c): + pass + # print(f"private container: {c.m}") + +print("PHASE 2") + +for f in [f for f in get_facts("phase-1") if f["type"] in ("dba-cluster", "dba-container")]: + x = assert_fact("phase-2", f); print(f"x: {x}") + +write_facts("phase-2") + +# Prepare +if os.path.isdir("gen"): + shutil.rmtree("gen") +os.mkdir("gen") +os.mkdir("gen/platform") +os.mkdir("gen/platform/terraform") +os.mkdir("gen/platform/ansible") +os.mkdir("gen/dns") + +print("PHASE 3: Generating stuff") + +file_loader = jinja2.FileSystemLoader("j2") +j2 = jinja2.Environment(loader=file_loader) + +# TODO: merge the dns files into the platform tf files as they are +# one-to-one now. +with ruleset("phase-3"): + @when_all(m.type == "meta") + def ignoreMeta(c): + print(f"ignoring {c.m}") + pass + + @when_all( + pri(-1), + (m.type == "terraform-machine"), + none(m.done == "platform/terraform/main.tf"), + ) + def mainTf(c): + path = f"gen/platform/terraform/main.tf" + print(f"WRITING {path}") + with open(path, "w") as f: + f.write(""" +terraform { + required_providers { + scaleway = { + source = "scaleway/scaleway" + } + } +} +""".strip()) + f.write("\n") + + machines = [] + for f in c.get_facts(): + if f.get("type") != "terraform-machine": + continue + machines.append(f) +# print(f"machine: {f}") + + template = j2.get_template("terraform-machine-outputs.j2") + path = f"gen/platform/terraform/outputs.tf" + print(f"WRITING {path}") + with open(path, "w") as f: + s = template.render(**{"machines": machines}) + f.write(s.strip()) + f.write("\n") + + try: + c.assert_fact({"type": "meta", "done": path}) + except MessageObservedException: + pass + + @when_all((m.type == "terraform-machine")) + def ansibleMachine(c): + template = j2.get_template("platform-ansible.j2") + path = f"gen/platform/ansible/{c.m.key}.yml" + print(f"WRITING {path}") + with open(path, "w") as f: + s = template.render(**{"m": c.m}) + f.write(s.strip()) + f.write("\n") + + @when_all((m.type == "terraform-machine")) + def terraformMachine(c): + template = j2.get_template("terraform-machine.j2") + path = f"gen/platform/terraform/{c.m.key}.tf" + print(f"WRITING {path}") + with open(path, "w") as f: + s = template.render(**{"m": c.m}) + f.write(s.strip()) + f.write("\n") + + @when_all((m.type == "terraform-record-set")) + def terraformRecordSet(c): + template = j2.get_template("terraform-record-set.j2") + path = f"gen/dns/{c.m.key}.tf" + print(f"WRITING {path}") + with open(path, "w") as f: + s = template.render(**{"m": c.m}) + f.write(s.strip()) + f.write("\n") + + @when_all( + pri(-1), + c.x << (m.type == "terraform-record-set"), + none(m.done == "gen/dns/inputs.tf"), +# c.x << (m.done == "gen/dns/inputs.tf"), + ) + def mainTf(c): + path = f"gen/dns/inputs.tf" + print(f"WRITING {path}") + with open(path, "w") as f: + f.write(""" +variable "addresses" { + type = map(string) +} +""".strip()) + f.write("\n") + try: + c.assert_fact({"type": "meta", "done": path}) + except MessageObservedException: + pass + +facts = [f for f in get_facts("phase-1") if f["type"] in ("terraform-record-set", "terraform-machine")] +#for f in facts: +# x = assert_fact("phase-3", f); print(f"x: {x}") +x = assert_facts("phase-3", facts); print(f"x: {x}") + +write_facts("phase-3") diff --git a/durable_rules-test/py/acme/rai/acme.py b/durable_rules-test/py/acme/rai/acme.py new file mode 100644 index 0000000..e75d492 --- /dev/null +++ b/durable_rules-test/py/acme/rai/acme.py @@ -0,0 +1,39 @@ +from durable.lang import * + +from . import dba + +class Acme: + @staticmethod + def make(env: str, tag: str): + return {"type": "acme-application", "key": env, "env": env, "tag": tag} + + @staticmethod + def declare_rules(): + @when_all((m.type == "acme-application")) + def acmeApp(c): + cluster = f"acme-{c.m.env}" + tag = f"{c.m.tag}" + # c.assert_fact(dba.cluster(cluster)) + for f in [ + dba.container(cluster, "app", "statera", "statera", tag), + dba.container(cluster, "app", "statera-console", "statera-console", tag), + dba.container(cluster, "app", "4tune-web", "4tune-web", tag), + dba.container(cluster, "app", "4tune-api", "4tune-api", tag), + dba.container(cluster, "db", "pdb", "postgresql", "13"), + dba.container(cluster, "db", "mdb", "mongodb", "3.2"), + ]: + c.assert_fact(f) + +class AcmeOperations: + @staticmethod + def make(): + return {"type": "acme-ops", "key": "acme-ops"} + + @staticmethod + def declare_rules(): + @when_all((m.type == "acme-ops")) + def acmeOps(c): + cluster = "acme-ops" + c.assert_fact(dba.cluster(cluster)) + c.assert_fact(dba.container(cluster, "app", "pdb", "postgresql", "11")) + c.assert_fact(dba.container(cluster, "app", "n8n", "n8n", "0.84.1")) diff --git a/durable_rules-test/py/acme/rai/dba.py b/durable_rules-test/py/acme/rai/dba.py new file mode 100644 index 0000000..04d10bf --- /dev/null +++ b/durable_rules-test/py/acme/rai/dba.py @@ -0,0 +1,13 @@ +from durable.lang import * + +def cluster(key: str): + return {"type": "dba-cluster", "key": key} + +def container(cluster: str, machineRole: str, name: str, image: str, tag: str): + return {"type": "dba-container", + "key": f"{cluster}/{name}", + "cluster": cluster, + "machineRole": machineRole, + "name": name, + "image": image, + "tag": tag} diff --git a/durable_rules-test/py/acme/rai/machine.py b/durable_rules-test/py/acme/rai/machine.py new file mode 100644 index 0000000..5b5da22 --- /dev/null +++ b/durable_rules-test/py/acme/rai/machine.py @@ -0,0 +1,22 @@ +from durable.lang import * + +class Dns: + @staticmethod + def a(key: str, fqdn: str): + return {"type": "dns-entry", + "key": key, + "fqdn": fqdn, + "rrType": "A" + } + +class Machine: + @staticmethod + def make(name: str): + return {"type": "machine", "key": name, "name": name} + + @staticmethod + def declare_rules(): + @when_all((m.type == "machine")) + def acmeApp(c): + c.assert_fact(Dns.a(c.m.key, f"{c.m.name}.machine.acme.corp")) + diff --git a/durable_rules-test/py/acme/rai/terraform.py b/durable_rules-test/py/acme/rai/terraform.py new file mode 100644 index 0000000..6feedd1 --- /dev/null +++ b/durable_rules-test/py/acme/rai/terraform.py @@ -0,0 +1,41 @@ +from durable.lang import * + +class Terraform: + @staticmethod + def recordSetForMachine(machineKey: str, terraformId: str, fqdn: str): + return {"type": "terraform-record-set", + "key": machineKey, + "terraformId": terraformId, + "fqdn": fqdn, + "rrType": "A", + "rrData": f"scaleway_instance_ip.{machineKey}.address", + } + + @staticmethod + def machine(key: str): + return {"type": "terraform-machine", + "key": key, + } + + @staticmethod + def declare_rules(): + @when_all( + c.dns << (m.type == "dns-entry"), + c.machine << ((m.type == "terraform-machine") & (m.key == c.dns.key)), + ) + def onDnsEntry(c): + terraformId = c.dns.fqdn.replace(".", "_") + c.assert_fact(Terraform.recordSetForMachine(c.machine.key, terraformId, c.dns.fqdn)) + + @when_all((m.type == "terraform-record-set")) + def defaultTerraformRecordSet(c): + pass + + @when_all((m.type == "machine")) + def onDnsEntry(c): + print(f"matched machine: {c.m}") + c.assert_fact(Terraform.machine(c.m.key)) + + @when_all((m.type == "terraform-machine")) + def defaultTerraformMachine(c): + print(f"matched terraform-machine: {c.m}") diff --git a/durable_rules-test/py/acme/rai/utils.py b/durable_rules-test/py/acme/rai/utils.py new file mode 100644 index 0000000..bc47d8f --- /dev/null +++ b/durable_rules-test/py/acme/rai/utils.py @@ -0,0 +1,59 @@ +import os +import os.path +import yaml +from durable.lang import * + +def write_facts(ruleset: set): + facts = get_facts(ruleset) + + types = set((f.get("type") for f in facts)) + + print(f"types: {types}") + + out_dir = "out" + if not os.path.exists(out_dir): + os.mkdir(out_dir) + + basedir = os.path.join(out_dir, ruleset) + if os.path.exists(basedir): + for f in os.listdir(basedir): + p = os.path.join(basedir, f) + if os.path.isdir(p): + for f2 in os.listdir(p): + os.remove(os.path.join(p, f2)) + os.rmdir(p) + else: + os.remove(p) + os.rmdir(basedir) + os.mkdir(basedir) + + for t in types: + typedir = os.path.join(basedir, t) + os.mkdir(typedir) + + fs = [] + for fact in facts: + if fact["type"] != t: + continue + + if fact["type"] == "meta": + continue + + if fact.get("key") is None: + raise Exception(f"Bad fact: no 'key' {fact}") + + try: + del fact["sid"] + except KeyError: + pass + + fs.append(fact) + + i = 0 + for fact in sorted(fs, key=lambda f: f["key"]): + key = fact["key"] + path = os.path.join(typedir, f"{key.replace('/', '_')}.yaml") + with open(path, "w") as f: + s = yaml.dump(fact) + f.write(s) + i = i + 1 -- cgit v1.2.3