From 1182531ed31ccff36e26121ae0cc5b3243669317 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Tue, 2 Feb 2021 17:05:02 +0100 Subject: Creating engine.yaml and engine.ninja generator. --- .gitignore | 4 + example/engine.ninja | 56 ++++++++ example/engine.yaml | 17 +++ .../main/java/io/trygvis/rules/engine/Engine.java | 45 ++++-- .../rules/engine/JinjavaTemplateEngine.java | 9 +- .../main/java/io/trygvis/rules/engine/Main.java | 3 +- .../io/trygvis/rules/engine/cli/EngineFile.java | 18 +++ .../io/trygvis/rules/engine/cli/NinjaCommand.java | 156 +++++++++++++++++++++ .../io/trygvis/rules/engine/cli/RunCommand.java | 13 +- .../rules/engine/AcmeWireguardTestMain.java | 2 +- .../io/trygvis/rules/engine/NinjaTestMain.java | 16 +++ module/ri-module-parent/pom.xml | 1 + pom.xml | 32 +++-- 13 files changed, 344 insertions(+), 28 deletions(-) create mode 100644 example/engine.ninja create mode 100644 example/engine.yaml create mode 100644 module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/EngineFile.java create mode 100644 module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/NinjaCommand.java create mode 100644 module/ri-engine/src/test/java/io/trygvis/rules/engine/NinjaTestMain.java diff --git a/.gitignore b/.gitignore index 93a2aa4..b9363a9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ target *.log .idea + +.classpath +.settings +.project diff --git a/example/engine.ninja b/example/engine.ninja new file mode 100644 index 0000000..3e6c6f3 --- /dev/null +++ b/example/engine.ninja @@ -0,0 +1,56 @@ +# Generated +# +### engine.ninja + +rule engine-yaml-to-ninja + command = engine ninja + +build engine.ninja: engine.yaml + +### engine.png + +rule ninja-to-dot + command = ninja -t graph > $out + +rule dot-to-png + command = dot -Tpng < $in > $out + +build engine.dot: ninja-to-dot build.ninja engine.ninja + +build engine.png: dot-to-png engine.dot + +# Jobs + +rule acme + command=engine run $name $inputs $output_state $agenda_group $modules + +build db/acme.yaml: acme + name=--name acme + output_state=--output-state db/acme.yaml + agenda_group= + modules=$ + --module=foo $ + --module=bar + +rule acme-apps + command=engine run $name $inputs $output_state $agenda_group $modules + +build db/acme-apps.yaml: acme-apps db/acme.yaml + name=--name acme-apps + inputs=$ + --input=db/acme.yaml + output_state=--output-state db/acme-apps.yaml + agenda_group= + +rule acme-wireguard + command=engine run $name $inputs $output_state $agenda_group $modules + +build db/acme-wireguard.yaml: acme-wireguard db/acme.yaml + name=--name acme-wireguard + inputs=$ + --input=db/acme.yaml + output_state=--output-state db/acme-wireguard.yaml + agenda_group= + modules=$ + --module=foo $ + --module=$$MODULE_HOME/bar diff --git a/example/engine.yaml b/example/engine.yaml new file mode 100644 index 0000000..e4216ea --- /dev/null +++ b/example/engine.yaml @@ -0,0 +1,17 @@ +dbDir: db +jobs: + - name: acme + modules: + - foo + - bar + + - name: acme-apps + inputs: + - acme + + - name: acme-wireguard + inputs: + - acme + modules: + - foo + - $MODULE_HOME/bar diff --git a/module/ri-engine/src/main/java/io/trygvis/rules/engine/Engine.java b/module/ri-engine/src/main/java/io/trygvis/rules/engine/Engine.java index ef6b4b1..46a3302 100644 --- a/module/ri-engine/src/main/java/io/trygvis/rules/engine/Engine.java +++ b/module/ri-engine/src/main/java/io/trygvis/rules/engine/Engine.java @@ -1,6 +1,7 @@ package io.trygvis.rules.engine; import org.drools.core.audit.WorkingMemoryConsoleLogger; +import org.drools.core.base.MapGlobalResolver; import org.drools.reflective.classloader.ProjectClassLoader; import org.kie.api.KieServices; import org.kie.api.event.rule.AgendaEventListener; @@ -11,6 +12,7 @@ import org.kie.api.runtime.KieSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.File; import java.io.FileNotFoundException; @@ -24,20 +26,25 @@ import java.util.List; import static io.trygvis.rules.engine.TemplateEngine.TemplateLoader; public class Engine implements Closeable { + @SuppressWarnings("FieldCanBeLocal") private final Logger logger = LoggerFactory.getLogger(getClass()); + public final String name; + @Nullable + public final File output; public final DbIo io; public final KieSession session; - public Engine(String name, File[] databases, File output, String[] agendaGroups, File[] modules) throws IOException { + public Engine(String name, File[] databases, @Nullable File output, String[] agendaGroups, File[] modules) + throws IOException { + this.name = name; + this.output = output; + logger.info("Getting KieServices"); var services = KieServices.Factory.get(); - logger.info("services = {}", services); - var kieRepository = services.getRepository(); - logger.info("kieRepository.getDefaultReleaseId() = {}", kieRepository.getDefaultReleaseId()); KieContainer container; TemplateLoader templateLoader; @@ -85,12 +92,7 @@ public class Engine implements Closeable { session.addEventListener((AgendaEventListener) l); session.addEventListener((RuleRuntimeEventListener) l); - var te = session.getGlobals().get("te"); - try { - session.setGlobal("te", new JinjavaTemplateEngine(templateLoader, output)); - } catch (java.lang.RuntimeException ignore) { - // This happens if the rules doesn't need the template engine. - } + session.getGlobals().setDelegate(new EngineGlobalResolver(templateLoader)); logger.info("Loading data"); io = new DbIo(container, kieBase); @@ -146,4 +148,27 @@ public class Engine implements Closeable { } } } + + private class EngineGlobalResolver extends MapGlobalResolver { + private final TemplateLoader templateLoader; + + public EngineGlobalResolver() { + templateLoader = null; + } + + public EngineGlobalResolver(TemplateLoader templateLoader) { + this.templateLoader = templateLoader; + } + + @Override + public Object resolveGlobal(String identifier) { + if ("te".equals(identifier)) { + if (output == null) { + throw new IllegalArgumentException("An instance of the TemplateEngine is required, but this job is not configured with a output directory."); + } + return new JinjavaTemplateEngine(templateLoader, output); + } + return super.resolveGlobal(identifier); + } + } } diff --git a/module/ri-engine/src/main/java/io/trygvis/rules/engine/JinjavaTemplateEngine.java b/module/ri-engine/src/main/java/io/trygvis/rules/engine/JinjavaTemplateEngine.java index 286029e..ba23089 100644 --- a/module/ri-engine/src/main/java/io/trygvis/rules/engine/JinjavaTemplateEngine.java +++ b/module/ri-engine/src/main/java/io/trygvis/rules/engine/JinjavaTemplateEngine.java @@ -3,22 +3,29 @@ package io.trygvis.rules.engine; import ch.qos.logback.core.util.FileUtil; import com.hubspot.jinjava.Jinjava; import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.util.Map; +import java.util.Objects; /** * TODO: cache templates. */ public class JinjavaTemplateEngine implements TemplateEngine { + private final Logger logger = LoggerFactory.getLogger(getClass()); + private final Jinjava jinjava = new Jinjava(); private final TemplateLoader loader; private final File basedir; public JinjavaTemplateEngine(TemplateLoader templateLoader, File basedir) { + Objects.requireNonNull(templateLoader); + Objects.requireNonNull(basedir); this.loader = templateLoader; this.basedir = basedir; } @@ -26,7 +33,7 @@ public class JinjavaTemplateEngine implements TemplateEngine { @Override public void clean() { try { - System.out.println("Cleaning gen!"); + logger.info("Removing output directory: {}", basedir); FileUtils.deleteDirectory(basedir); } catch (IOException e) { throw new RuntimeException(e); diff --git a/module/ri-engine/src/main/java/io/trygvis/rules/engine/Main.java b/module/ri-engine/src/main/java/io/trygvis/rules/engine/Main.java index 4f06091..dc9ed02 100644 --- a/module/ri-engine/src/main/java/io/trygvis/rules/engine/Main.java +++ b/module/ri-engine/src/main/java/io/trygvis/rules/engine/Main.java @@ -1,13 +1,14 @@ package io.trygvis.rules.engine; import io.trygvis.rules.engine.cli.DatabaseCommand; +import io.trygvis.rules.engine.cli.NinjaCommand; import io.trygvis.rules.engine.cli.RunCommand; import picocli.CommandLine; import picocli.CommandLine.Command; @Command( name = "engine", - subcommands = {RunCommand.class, DatabaseCommand.class}, + subcommands = {RunCommand.class, DatabaseCommand.class, NinjaCommand.class}, mixinStandardHelpOptions = true, version = "UNSPECIFIED") class Main { diff --git a/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/EngineFile.java b/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/EngineFile.java new file mode 100644 index 0000000..8da0e39 --- /dev/null +++ b/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/EngineFile.java @@ -0,0 +1,18 @@ +package io.trygvis.rules.engine.cli; + +import java.util.ArrayList; +import java.util.List; + +public class EngineFile { + public String dbDir; + public List jobs; + + public static class Job { + public String name; + public List inputs = new ArrayList<>(); + public List outputIncludes = new ArrayList<>(); + public String generatedOutput; + public List agendaGroups = new ArrayList<>(); + public List modules = new ArrayList<>(); + } +} diff --git a/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/NinjaCommand.java b/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/NinjaCommand.java new file mode 100644 index 0000000..ef5ed1d --- /dev/null +++ b/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/NinjaCommand.java @@ -0,0 +1,156 @@ +package io.trygvis.rules.engine.cli; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +import static picocli.CommandLine.Command; + +@Command(name = "ninja") +public class NinjaCommand implements Callable { + + public File basedir = null; + + public Path basepath; + + @Override + public Integer call() throws Exception { + basepath = Objects.requireNonNullElseGet(basedir, () -> new File("").getAbsoluteFile()).toPath(); + + var factory = new YAMLFactory(); + factory.enable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID); + factory.enable(YAMLGenerator.Feature.USE_NATIVE_OBJECT_ID); + var mapper = new ObjectMapper(factory); + mapper.enable(MapperFeature.AUTO_DETECT_FIELDS); + + var f = mapper.readValue(new File(basedir, "engine.yaml"), EngineFile.class); + + Path dbDir; + if (StringUtils.trimToNull(f.dbDir) == null) { + System.err.println("Missing required field: dbDir"); + return 1; + } else { + dbDir = Path.of(f.dbDir); + } + + var buf = new StringWriter(); + var out = new PrintWriter(buf); + + out.println("# Generated"); + out.println("#"); + out.println("### engine.ninja"); + out.println(""); + out.println("rule engine-yaml-to-ninja"); + out.println(" command = engine ninja"); + out.println(""); + out.println("build engine.ninja: engine-yaml-to-ninja engine.yaml"); + out.println(""); + out.println("### engine.png"); + out.println(""); + out.println("rule ninja-to-dot"); + out.println(" command = ninja -t graph > $out"); + out.println(""); + out.println("rule dot-to-png"); + out.println(" command = dot -Tpng < $in > $out"); + out.println(""); + out.println("build engine.dot: ninja-to-dot build.ninja engine.ninja"); + out.println(""); + out.println("build engine.png: dot-to-png engine.dot"); + out.println(""); + out.println("# Jobs"); + + for (var job : f.jobs) { + out.println(""); + + out.println("rule %s".formatted(job.name)); + out.println(" command=engine run $name $inputs $output_state $output_includes $generated_output $agenda_groups $modules"); + out.println(); + var dependencies = job.inputs.stream() + .map(s -> dbDir.resolve(s + ".yaml").toString()) + .collect(Collectors.joining(" ")); + + var outputState = dbDir.resolve(job.name + ".yaml"); + + var generated = List.of(outputState).stream() + .map(Path::toString) + .collect(Collectors.joining(" ")); + + out.println("build %s: %s %s".formatted(generated, job.name, dependencies)); + out.println(" name=--name %s".formatted(job.name)); + + if (!job.inputs.isEmpty()) { + var is = job.inputs.stream() + .map(s -> "--input=" + dbDir.resolve(s + ".yaml")) + .collect(Collectors.joining(" $\n ", "\n ", "")); + + out.println(" inputs=$%s".formatted(is)); + } + + out.println(" output_state=--output-state %s".formatted(outputState)); + if (!job.outputIncludes.isEmpty()) { + var str = job.outputIncludes.stream() + .map(s -> "--output-include=" + s) + .collect(Collectors.joining(" $\n ", "\n ", "")); + + out.println(" output_includes=$%s".formatted(str)); + } + + if (job.generatedOutput != null) { + out.println(" generated_output=--generated-output %s".formatted(fixPath(job.generatedOutput))); + } + + if (!job.agendaGroups.isEmpty()) { + var ag = job.agendaGroups.stream() + .map(s -> "--agenda-group=" + s) + .collect(Collectors.joining(" $\n ", "\n ", "")); + out.println(" agenda_groups=%s".formatted(ag)); + } + + if (!job.modules.isEmpty()) { + var ms = job.modules.stream() + .map(this::fixPath) + .map(s -> "--module=" + s) + .collect(Collectors.joining(" $\n ", "\n ", "")); + + out.println(" modules=$%s".formatted(ms)); + } + } + + var ninjaFile = new File(basedir, "engine.ninja"); + try (var writer = new FileWriter(ninjaFile)) { + writer.write(buf.toString()); + } + + return 0; + } + + private String fixPath(String s) { + if (s.startsWith("$MODULE_HOME/")) { + s = "$" + s; + } + + var p = Path.of(s); + + if (p.isAbsolute()) { + s = basepath.relativize(Path.of(s)).toString(); + } + + if (s.contains("*")) { + s = "$$(echo " + s + ")"; + } + + return s; + } +} diff --git a/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/RunCommand.java b/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/RunCommand.java index 1f3e5ed..35f30cd 100644 --- a/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/RunCommand.java +++ b/module/ri-engine/src/main/java/io/trygvis/rules/engine/cli/RunCommand.java @@ -20,17 +20,16 @@ public class RunCommand implements Callable { @Option(names = {"--output-state"}) public File outputState; - @Option(names = {"--include"}, split = ",", arity = "1..*") - public String[] includes; + @Option(names = {"--output-include"}, split = ",", arity = "1..*") + public String[] outputIncludes; @Option(names = {"--generated-output"}) public File generatedOutput; - @Option(names = {"--agenda-groups"}) + @Option(names = {"--agenda-group"}) public String[] agendaGroups; - // TODO: Remove --modules - @Option(names = {"--modules", "--module"}, split = ",", arity = "1..*") + @Option(names = {"--module"}, split = ",", arity = "1..*") public File[] module; @Override @@ -43,14 +42,14 @@ public class RunCommand implements Callable { try (var engine = new Engine(name, input, generatedOutput, agendaGroups, module)) { engine.io.dump(outputState, engine.session.getFactHandles(), (Object o) -> { - if (includes == null || includes.length == 0) { + if (outputIncludes == null || outputIncludes.length == 0) { return true; } var name = o.getClass().getName(); var simpleName = o.getClass().getSimpleName(); - for (var i : includes) { + for (var i : outputIncludes) { var ok = false; if (i.startsWith("*")) { i = i.substring(1); diff --git a/module/ri-engine/src/test/java/io/trygvis/rules/engine/AcmeWireguardTestMain.java b/module/ri-engine/src/test/java/io/trygvis/rules/engine/AcmeWireguardTestMain.java index 4d88e39..cf6b1c8 100644 --- a/module/ri-engine/src/test/java/io/trygvis/rules/engine/AcmeWireguardTestMain.java +++ b/module/ri-engine/src/test/java/io/trygvis/rules/engine/AcmeWireguardTestMain.java @@ -14,7 +14,7 @@ class AcmeWireguardTestMain { c.outputState = new File("out/acme/wireguard.yaml"); c.agendaGroups = new String[]{"init", "generate"}; c.generatedOutput = new File("acme-wireguard"); - c.includes = new String[]{ + c.outputIncludes = new String[]{ "Wg*", "Machine", "DnsEntry", diff --git a/module/ri-engine/src/test/java/io/trygvis/rules/engine/NinjaTestMain.java b/module/ri-engine/src/test/java/io/trygvis/rules/engine/NinjaTestMain.java new file mode 100644 index 0000000..d62eec1 --- /dev/null +++ b/module/ri-engine/src/test/java/io/trygvis/rules/engine/NinjaTestMain.java @@ -0,0 +1,16 @@ +package io.trygvis.rules.engine; + +import io.trygvis.rules.engine.cli.NinjaCommand; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class NinjaTestMain { + public static void main(String[] args) throws Exception { + var c = new NinjaCommand(); + c.basedir = new File("example"); + + assertEquals(0, c.call()); + } +} diff --git a/module/ri-module-parent/pom.xml b/module/ri-module-parent/pom.xml index 5f56fda..d593ca8 100644 --- a/module/ri-module-parent/pom.xml +++ b/module/ri-module-parent/pom.xml @@ -38,6 +38,7 @@ + ${project.artifactId} org.kie diff --git a/pom.xml b/pom.xml index ee39434..ab5682c 100644 --- a/pom.xml +++ b/pom.xml @@ -70,14 +70,30 @@ - - module/acme - module/acme-planner - module/ri-base - module/ri-engine - module/ri-module-parent - module/ri-wireguard - + + + engine + + true + + + module/acme + module/acme-planner + module/ri-engine + + + + modules + + true + + + module/ri-base + module/ri-module-parent + module/ri-wireguard + + + install -- cgit v1.2.3