o Storing the raw bytes with the parsed data from Jenkins.

4 files changed, 62 insertions, 36 deletions

diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java
index c9e83d4..600ef33 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java
@@ -18,6 +18,7 @@ import java.net.*;
 import java.util.*;
 import java.util.List;
 
+import static fj.P.p;
 import static fj.data.Option.*;
 import static io.trygvis.esper.testing.util.HttpClient.inputStreamOnly;
 import static java.lang.System.currentTimeMillis;
@@ -68,20 +69,18 @@ public class JenkinsClient {
         return URI.create(u + "/api/xml");
     }
 
-    public Option<List<JenkinsEntryXml>> fetchRss(URI url) {
+    public Option<P2<List<JenkinsEntryXml>, byte[]>> fetchRss(URI url) {
         return jenkinsEntryXmlClient.fetch(url);
     }
 
-    public JenkinsXml fetchJobs(URI url) {
-        Option<Document> d = xmlHttpClient.fetch(url);
+    public Option<P2<JenkinsXml, byte[]>> fetchJobs(URI url) {
+        Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url);
 
         if (d.isNone()) {
-            Option<String> n = Option.none();
-
-            return new JenkinsXml(n, n, n, Collections.<JenkinsXml.JobXml>emptyList());
+            return none();
         }
 
-        Element root = d.some().getRootElement();
+        Element root = d.some()._1().getRootElement();
 
         List<JenkinsXml.JobXml> jobs = new ArrayList<>();
         for (Element job : root.getChildren("job")) {
@@ -96,48 +95,48 @@ public class JenkinsClient {
             jobs.add(new JenkinsXml.JobXml(name, u, color));
         }
 
-        return new JenkinsXml(
-            Option.fromNull(root.getChildText("nodeName")),
-            Option.fromNull(root.getChildText("nodeDescription")),
-            Option.fromNull(root.getChildText("description")), jobs);
+        return some(p(new JenkinsXml(
+                Option.fromNull(root.getChildText("nodeName")),
+                Option.fromNull(root.getChildText("nodeDescription")),
+                Option.fromNull(root.getChildText("description")), jobs), d.some()._2()));
     }
 
-    public Option<JenkinsJobXml> fetchJob(URI url) {
-        Option<Document> d = xmlHttpClient.fetch(url);
+    public Option<P2<JenkinsJobXml, byte[]>> fetchJob(URI url) {
+        Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url);
 
         if (d.isNone()) {
             return Option.none();
         }
 
-        Element root = d.some().getRootElement();
+        Element root = d.some()._1().getRootElement();
 
         String name = root.getName();
 
         switch (name) {
             case "freeStyleProject":
-                return some(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root));
+                return some(p(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root), d.some()._2()));
             case "mavenModuleSet":
-                return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE_SET, root));
+                return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE_SET, root), d.some()._2()));
             case "mavenModule":
-                return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE, root));
+                return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE, root), d.some()._2()));
             case "matrixProject":
-                return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root));
+                return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root), d.some()._2()));
             case "matrixConfiguration":
-                return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX_CONFIGURATION, root));
+                return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX_CONFIGURATION, root), d.some()._2()));
             default:
                 logger.warn("Unknown project type: " + name);
                 return Option.none();
         }
     }
 
-    public Option<JenkinsBuildXml> fetchBuild(URI url) {
-        Option<Document> d = xmlHttpClient.fetch(url);
+    public Option<P2<JenkinsBuildXml, byte[]>> fetchBuild(URI url) {
+        final Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url);
 
         if (d.isNone()) {
             return Option.none();
         }
 
-        Element root = d.some().getRootElement();
+        Element root = d.some()._1().getRootElement();
 
         String name = root.getName();
 
@@ -148,7 +147,11 @@ public class JenkinsClient {
             case "mavenModuleSetBuild":
             case "mavenBuild":
             case "freeStyleBuild":
-                return JenkinsBuildXml.parse(root);
+                return JenkinsBuildXml.parse(root).map(new F<JenkinsBuildXml, P2<JenkinsBuildXml, byte[]>>() {
+                    public P2<JenkinsBuildXml, byte[]> f(JenkinsBuildXml x) {
+                        return p(x, d.some()._2());
+                    }
+                });
             default:
                 logger.warn("Unknown build type: " + name);
                 return Option.none();
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java
index 7199bd8..5ca449e 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java
@@ -1,5 +1,6 @@
 package io.trygvis.esper.testing.jenkins;
 
+import fj.*;
 import fj.data.*;
 import static fj.data.Option.*;
 import static io.trygvis.esper.testing.jenkins.JenkinsClient.apiXml;
@@ -23,7 +24,7 @@ public class JenkinsJob implements Closeable {
         this.client = client;
         this.url = apiXml(url);
 
-        long initialDelay = (long) Math.random() + 1;
+        long initialDelay = (long) (Math.random() + 1);
         long period = (long) (Math.random() * 100d) + 1;
         future = executorService.scheduleAtFixedRate(new Runnable() {
             public void run() {
@@ -48,7 +49,7 @@ public class JenkinsJob implements Closeable {
         try {
             logger.info("Updating " + name);
             long start = currentTimeMillis();
-            latestStatus = client.fetchJob(url);
+            latestStatus = client.fetchJob(url).map(P2.<JenkinsJobXml, byte[]>__1());
             long end = currentTimeMillis();
             logger.info("Updated " + name + " in " + (end - start) + "ms");
         } catch (Throwable e) {
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java
index cf95c4e..32728a6 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java
@@ -1,6 +1,8 @@
 package io.trygvis.esper.testing.jenkins;
 
+import fj.*;
 import fj.data.*;
+import io.trygvis.esper.testing.core.db.*;
 import io.trygvis.esper.testing.util.object.*;
 import io.trygvis.esper.testing.util.sql.*;
 import org.slf4j.*;
@@ -30,13 +32,18 @@ public class JenkinsServerActor implements TransactionalActor {
         long start = currentTimeMillis();
 
         JenkinsDao dao = new JenkinsDao(c);
-        Option<List<JenkinsEntryXml>> option = client.fetchRss(URI.create(server.url.toASCIIString() + "/rssAll"));
+        FileDao fileDao = new FileDao(c);
+
+        URI rssUrl = URI.create(server.url.toASCIIString() + "/rssAll");
+        Option<P2<List<JenkinsEntryXml>, byte[]>> option = client.fetchRss(rssUrl);
 
         if (option.isNone()) {
             return;
         }
 
-        List<JenkinsEntryXml> list = option.some();
+        fileDao.store(rssUrl, "application/xml", option.some()._2());
+
+        List<JenkinsEntryXml> list = option.some()._1();
 
         logger.info("Got " + list.size() + " entries.");
 
@@ -54,18 +61,23 @@ public class JenkinsServerActor implements TransactionalActor {
 
             logger.debug("Build: " + entry.id + ", fetching info");
 
-            Option<JenkinsBuildXml> buildXmlOption = client.fetchBuild(apiXml(entry.url));
+            URI buildUrl = apiXml(entry.url);
+
+            Option<P2<JenkinsBuildXml, byte[]>> buildXmlOption = client.fetchBuild(buildUrl);
 
             if (buildXmlOption.isNone()) {
                 continue;
             }
 
-            JenkinsBuildXml build = buildXmlOption.some();
+            JenkinsBuildXml build = buildXmlOption.some()._1();
 
-            if(build.result.isNone()) {
+            if (build.result.isNone()) {
                 logger.debug("Not done building, <result> is not available.");
                 continue;
             }
+
+            fileDao.store(buildUrl, "application/xml", buildXmlOption.some()._2());
+
             String result = build.result.some();
 
             // -----------------------------------------------------------------------
@@ -74,11 +86,11 @@ public class JenkinsServerActor implements TransactionalActor {
 
             Set<UUID> users = new HashSet<>();
 
-            if(build.changeSet.isSome()) {
+            if (build.changeSet.isSome()) {
                 JenkinsBuildXml.ChangeSetXml changeSetXml = build.changeSet.some();
 
                 for (JenkinsBuildXml.ChangeSetItemXml item : changeSetXml.items) {
-                    if(item.author.isNone()) {
+                    if (item.author.isNone()) {
                         continue;
                     }
 
@@ -86,7 +98,7 @@ public class JenkinsServerActor implements TransactionalActor {
 
                     UUID uuid = authors.get(url);
 
-                    if(uuid == null) {
+                    if (uuid == null) {
                         SqlOption<JenkinsUserDto> userO = dao.selectUserByAbsoluteUrl(server.uuid, url);
                         if (userO.isNone()) {
                             logger.info("New user: {}", url);
@@ -117,13 +129,17 @@ public class JenkinsServerActor implements TransactionalActor {
             } else {
                 logger.info("New job: {}, fetching info", jobUrl);
 
-                Option<JenkinsJobXml> jobXmlOption = client.fetchJob(apiXml(jobUrl));
+                URI uri = apiXml(jobUrl);
+
+                Option<P2<JenkinsJobXml, byte[]>> jobXmlOption = client.fetchJob(uri);
 
                 if (jobXmlOption.isNone()) {
                     continue;
                 }
 
-                JenkinsJobXml xml = jobXmlOption.some();
+                fileDao.store(uri, "application/xml", jobXmlOption.some()._2());
+
+                JenkinsJobXml xml = jobXmlOption.some()._1();
 
                 job = dao.insertJob(server.uuid, xml.url, xml.type, xml.displayName);
 
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java
index 9f48191..e21f336 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java
@@ -83,7 +83,13 @@ public class JenkinsServerOld implements Closeable {
 
     private void doWork() {
         try {
-            JenkinsXml xml = client.fetchJobs(url);
+            Option<P2<JenkinsXml, byte[]>> o = client.fetchJobs(url);
+
+            if(o.isNone()) {
+                return;
+            }
+
+            JenkinsXml xml = o.some()._1();
 
             List<URI> jobUris = new ArrayList<>(xml.jobs.size());
             for (JenkinsXml.JobXml job : xml.jobs) {