diff options
12 files changed, 140 insertions, 48 deletions
@@ -178,6 +178,10 @@ <name>jenkins-build-poller</name> <mainClass>io.trygvis.esper.testing.core.jenkins.JenkinsBuildPoller</mainClass> </program> + <program> + <name>unbreakable-poller</name> + <mainClass>io.trygvis.esper.testing.core.badge.UnbreakablePoller</mainClass> + </program> </programs> </configuration> </execution> diff --git a/src/main/java/io/trygvis/esper/testing/core/db/FileDao.java b/src/main/java/io/trygvis/esper/testing/core/db/FileDao.java new file mode 100644 index 0000000..0878f63 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/core/db/FileDao.java @@ -0,0 +1,29 @@ +package io.trygvis.esper.testing.core.db; + +import java.io.*; +import java.net.*; +import java.sql.*; +import java.util.*; + +import static java.lang.System.*; + +public class FileDao { + private final Connection c; + + public FileDao(Connection c) { + this.c = c; + } + + public void store(URI url, String contentType, byte[] data) throws SQLException { + try (PreparedStatement s = c.prepareStatement("INSERT INTO file(uuid, created_date, url, content_type, data) VALUES(?, ?, ?, ?, ?)")) { + UUID uuid = UUID.randomUUID(); + int i = 1; + s.setString(i++, uuid.toString()); + s.setTimestamp(i++, new Timestamp(currentTimeMillis())); + s.setString(i++, url.toASCIIString()); + s.setString(i++, contentType); + s.setBinaryStream(i, new ByteArrayInputStream(data), data.length); + s.executeUpdate(); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java index fff84fe..0071b1f 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -77,13 +77,15 @@ public class GitoriousClient { Set<GitoriousProjectXml> all = new HashSet<>(); while (true) { - Option<List<GitoriousProjectXml>> option = http.fetch(new URI(projectsUri + "?page=" + page)); + Option<P2<List<GitoriousProjectXml>, byte[]>> option = http.fetch(new URI(projectsUri + "?page=" + page)); if (option.isNone()) { return all; } - List<GitoriousProjectXml> list = option.some(); + List<GitoriousProjectXml> list = option.some()._1(); + + // TODO: store data // This indicates the last page. if (list.size() == 0) { diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java index c9e83d4..600ef33 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java @@ -18,6 +18,7 @@ import java.net.*; import java.util.*; import java.util.List; +import static fj.P.p; import static fj.data.Option.*; import static io.trygvis.esper.testing.util.HttpClient.inputStreamOnly; import static java.lang.System.currentTimeMillis; @@ -68,20 +69,18 @@ public class JenkinsClient { return URI.create(u + "/api/xml"); } - public Option<List<JenkinsEntryXml>> fetchRss(URI url) { + public Option<P2<List<JenkinsEntryXml>, byte[]>> fetchRss(URI url) { return jenkinsEntryXmlClient.fetch(url); } - public JenkinsXml fetchJobs(URI url) { - Option<Document> d = xmlHttpClient.fetch(url); + public Option<P2<JenkinsXml, byte[]>> fetchJobs(URI url) { + Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url); if (d.isNone()) { - Option<String> n = Option.none(); - - return new JenkinsXml(n, n, n, Collections.<JenkinsXml.JobXml>emptyList()); + return none(); } - Element root = d.some().getRootElement(); + Element root = d.some()._1().getRootElement(); List<JenkinsXml.JobXml> jobs = new ArrayList<>(); for (Element job : root.getChildren("job")) { @@ -96,48 +95,48 @@ public class JenkinsClient { jobs.add(new JenkinsXml.JobXml(name, u, color)); } - return new JenkinsXml( - Option.fromNull(root.getChildText("nodeName")), - Option.fromNull(root.getChildText("nodeDescription")), - Option.fromNull(root.getChildText("description")), jobs); + return some(p(new JenkinsXml( + Option.fromNull(root.getChildText("nodeName")), + Option.fromNull(root.getChildText("nodeDescription")), + Option.fromNull(root.getChildText("description")), jobs), d.some()._2())); } - public Option<JenkinsJobXml> fetchJob(URI url) { - Option<Document> d = xmlHttpClient.fetch(url); + public Option<P2<JenkinsJobXml, byte[]>> fetchJob(URI url) { + Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url); if (d.isNone()) { return Option.none(); } - Element root = d.some().getRootElement(); + Element root = d.some()._1().getRootElement(); String name = root.getName(); switch (name) { case "freeStyleProject": - return some(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root)); + return some(p(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root), d.some()._2())); case "mavenModuleSet": - return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE_SET, root)); + return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE_SET, root), d.some()._2())); case "mavenModule": - return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE, root)); + return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE, root), d.some()._2())); case "matrixProject": - return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root)); + return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root), d.some()._2())); case "matrixConfiguration": - return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX_CONFIGURATION, root)); + return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX_CONFIGURATION, root), d.some()._2())); default: logger.warn("Unknown project type: " + name); return Option.none(); } } - public Option<JenkinsBuildXml> fetchBuild(URI url) { - Option<Document> d = xmlHttpClient.fetch(url); + public Option<P2<JenkinsBuildXml, byte[]>> fetchBuild(URI url) { + final Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url); if (d.isNone()) { return Option.none(); } - Element root = d.some().getRootElement(); + Element root = d.some()._1().getRootElement(); String name = root.getName(); @@ -148,7 +147,11 @@ public class JenkinsClient { case "mavenModuleSetBuild": case "mavenBuild": case "freeStyleBuild": - return JenkinsBuildXml.parse(root); + return JenkinsBuildXml.parse(root).map(new F<JenkinsBuildXml, P2<JenkinsBuildXml, byte[]>>() { + public P2<JenkinsBuildXml, byte[]> f(JenkinsBuildXml x) { + return p(x, d.some()._2()); + } + }); default: logger.warn("Unknown build type: " + name); return Option.none(); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java index 7199bd8..5ca449e 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java @@ -1,5 +1,6 @@ package io.trygvis.esper.testing.jenkins; +import fj.*; import fj.data.*; import static fj.data.Option.*; import static io.trygvis.esper.testing.jenkins.JenkinsClient.apiXml; @@ -23,7 +24,7 @@ public class JenkinsJob implements Closeable { this.client = client; this.url = apiXml(url); - long initialDelay = (long) Math.random() + 1; + long initialDelay = (long) (Math.random() + 1); long period = (long) (Math.random() * 100d) + 1; future = executorService.scheduleAtFixedRate(new Runnable() { public void run() { @@ -48,7 +49,7 @@ public class JenkinsJob implements Closeable { try { logger.info("Updating " + name); long start = currentTimeMillis(); - latestStatus = client.fetchJob(url); + latestStatus = client.fetchJob(url).map(P2.<JenkinsJobXml, byte[]>__1()); long end = currentTimeMillis(); logger.info("Updated " + name + " in " + (end - start) + "ms"); } catch (Throwable e) { diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java index cf95c4e..32728a6 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java @@ -1,6 +1,8 @@ package io.trygvis.esper.testing.jenkins; +import fj.*; import fj.data.*; +import io.trygvis.esper.testing.core.db.*; import io.trygvis.esper.testing.util.object.*; import io.trygvis.esper.testing.util.sql.*; import org.slf4j.*; @@ -30,13 +32,18 @@ public class JenkinsServerActor implements TransactionalActor { long start = currentTimeMillis(); JenkinsDao dao = new JenkinsDao(c); - Option<List<JenkinsEntryXml>> option = client.fetchRss(URI.create(server.url.toASCIIString() + "/rssAll")); + FileDao fileDao = new FileDao(c); + + URI rssUrl = URI.create(server.url.toASCIIString() + "/rssAll"); + Option<P2<List<JenkinsEntryXml>, byte[]>> option = client.fetchRss(rssUrl); if (option.isNone()) { return; } - List<JenkinsEntryXml> list = option.some(); + fileDao.store(rssUrl, "application/xml", option.some()._2()); + + List<JenkinsEntryXml> list = option.some()._1(); logger.info("Got " + list.size() + " entries."); @@ -54,18 +61,23 @@ public class JenkinsServerActor implements TransactionalActor { logger.debug("Build: " + entry.id + ", fetching info"); - Option<JenkinsBuildXml> buildXmlOption = client.fetchBuild(apiXml(entry.url)); + URI buildUrl = apiXml(entry.url); + + Option<P2<JenkinsBuildXml, byte[]>> buildXmlOption = client.fetchBuild(buildUrl); if (buildXmlOption.isNone()) { continue; } - JenkinsBuildXml build = buildXmlOption.some(); + JenkinsBuildXml build = buildXmlOption.some()._1(); - if(build.result.isNone()) { + if (build.result.isNone()) { logger.debug("Not done building, <result> is not available."); continue; } + + fileDao.store(buildUrl, "application/xml", buildXmlOption.some()._2()); + String result = build.result.some(); // ----------------------------------------------------------------------- @@ -74,11 +86,11 @@ public class JenkinsServerActor implements TransactionalActor { Set<UUID> users = new HashSet<>(); - if(build.changeSet.isSome()) { + if (build.changeSet.isSome()) { JenkinsBuildXml.ChangeSetXml changeSetXml = build.changeSet.some(); for (JenkinsBuildXml.ChangeSetItemXml item : changeSetXml.items) { - if(item.author.isNone()) { + if (item.author.isNone()) { continue; } @@ -86,7 +98,7 @@ public class JenkinsServerActor implements TransactionalActor { UUID uuid = authors.get(url); - if(uuid == null) { + if (uuid == null) { SqlOption<JenkinsUserDto> userO = dao.selectUserByAbsoluteUrl(server.uuid, url); if (userO.isNone()) { logger.info("New user: {}", url); @@ -117,13 +129,17 @@ public class JenkinsServerActor implements TransactionalActor { } else { logger.info("New job: {}, fetching info", jobUrl); - Option<JenkinsJobXml> jobXmlOption = client.fetchJob(apiXml(jobUrl)); + URI uri = apiXml(jobUrl); + + Option<P2<JenkinsJobXml, byte[]>> jobXmlOption = client.fetchJob(uri); if (jobXmlOption.isNone()) { continue; } - JenkinsJobXml xml = jobXmlOption.some(); + fileDao.store(uri, "application/xml", jobXmlOption.some()._2()); + + JenkinsJobXml xml = jobXmlOption.some()._1(); job = dao.insertJob(server.uuid, xml.url, xml.type, xml.displayName); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java index 9f48191..e21f336 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java @@ -83,7 +83,13 @@ public class JenkinsServerOld implements Closeable { private void doWork() { try { - JenkinsXml xml = client.fetchJobs(url); + Option<P2<JenkinsXml, byte[]>> o = client.fetchJobs(url); + + if(o.isNone()) { + return; + } + + JenkinsXml xml = o.some()._1(); List<URI> jobUris = new ArrayList<>(xml.jobs.size()); for (JenkinsXml.JobXml job : xml.jobs) { diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java index 9900bfc..fe0db33 100644 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java @@ -1,5 +1,6 @@ package io.trygvis.esper.testing.nexus; +import fj.*; import fj.data.*; import static fj.data.Option.*; import static io.trygvis.esper.testing.nexus.SearchNGResponseParser.*; @@ -45,13 +46,15 @@ public class NexusClient { System.out.println("args = " + args); URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline + args); - Option<Document> d = xmlHttpClient.fetch(uri); + Option<P2<Document,byte[]>> d = xmlHttpClient.fetch(uri); if (d.isNone()) { return new NexusFeed(Collections.<HasNexusEvent>emptyList()); } - Document document = d.some(); + Document document = d.some()._1(); + + // TODO: store data return NexusFeedParser.parseDocument(document); } diff --git a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java index a194cd1..69c74fe 100644 --- a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java +++ b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java @@ -3,14 +3,15 @@ package io.trygvis.esper.testing.util; import fj.*; import fj.data.*; import io.trygvis.esper.testing.*; +import org.apache.commons.io.*; import org.apache.http.conn.scheme.*; import org.apache.http.impl.client.*; import org.apache.http.impl.conn.tsccm.*; import org.apache.http.params.*; import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; +import org.codehaus.httpcache4j.payload.*; import org.codehaus.httpcache4j.resolver.*; -import org.h2.util.*; import org.slf4j.*; import java.io.*; @@ -39,13 +40,13 @@ public class HttpClient<A> { try { return f.f(inputStream); } finally { - IOUtils.closeSilently(inputStream); + IOUtils.closeQuietly(inputStream); } } }; } - public Option<A> fetch(URI uri) { + public Option<P2<A, byte[]>> fetch(URI uri) { HTTPResponse response = null; try { @@ -57,10 +58,25 @@ public class HttpClient<A> { return none(); } - return f.f(response); + final ByteArrayOutputStream buffer = new ByteArrayOutputStream(1024 * 1024); + + IOUtils.copy(response.getPayload().getInputStream(), buffer); + + final byte[] bytes = buffer.toByteArray(); + + response = new HTTPResponse(new ByteArrayPayload(new ByteArrayInputStream(bytes), response.getPayload().getMimeType()), response.getStatusLine(), response.getHeaders()); + + return f.f(response).map(new F<A, P2<A, byte[]>>() { + public P2<A, byte[]> f(A a) { + return P.p(a, bytes); + } + }); } catch (HTTPException e) { logger.warn("Error while fetching: " + uri, e); return none(); + } catch (IOException e) { + logger.warn("Error while fetching: " + uri, e); + return none(); } finally { if (response != null) { try { diff --git a/src/main/resources/ddl-core.sql b/src/main/resources/ddl-core.sql index a244236..376ca9f 100644 --- a/src/main/resources/ddl-core.sql +++ b/src/main/resources/ddl-core.sql @@ -8,6 +8,16 @@ DROP TABLE IF EXISTS person_badge; DROP TABLE IF EXISTS person_jenkins_user; DROP TABLE IF EXISTS person; DROP TABLE IF EXISTS table_poller_status; +DROP TABLE IF EXISTS file; + +CREATE TABLE file ( + uuid CHAR(36) NOT NULL, + created_date TIMESTAMP NOT NULL, + url VARCHAR(1000) NOT NULL, + content_type VARCHAR(100) NOT NULL, + data BYTEA, + CONSTRAINT pk_file PRIMARY KEY (uuid) +); CREATE TABLE table_poller_status ( poller_name VARCHAR(100) NOT NULL, diff --git a/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java b/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java index 9cc47ad..f03086b 100644 --- a/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java +++ b/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java @@ -7,6 +7,7 @@ import java.sql.ResultSet; import java.util.List; import java.util.UUID; +import fj.*; import org.apache.abdera.Abdera; import org.codehaus.httpcache4j.cache.HTTPCache; @@ -48,14 +49,14 @@ public class CreateMissingMavenModuleJobsApp { URI url = URI.create(u); - Option<JenkinsJobXml> xmlOption = client.fetchJob(apiXml(url)); + Option<P2<JenkinsJobXml,byte[]>> xmlOption = client.fetchJob(apiXml(url)); if(xmlOption.isNone()) { System.out.println("None"); continue; } - JenkinsJobXml jobXml = xmlOption.some(); + JenkinsJobXml jobXml = xmlOption.some()._1(); if(dao.selectJobByUrl(jobXml.url).isSome()) { System.out.println("Duplicate"); diff --git a/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java b/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java index 46060a9..ffd3212 100644 --- a/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java +++ b/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java @@ -5,6 +5,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.List; +import fj.*; import org.apache.abdera.Abdera; import org.codehaus.httpcache4j.cache.HTTPCache; @@ -40,14 +41,14 @@ public class SetJobTypeApp { for (JenkinsJobDto jobDto : jobs) { System.out.println("job = " + jobDto.url); - Option<JenkinsJobXml> xmlOption = jenkinsClient.fetchJob(apiXml(jobDto.url)); + Option<P2<JenkinsJobXml,byte[]>> xmlOption = jenkinsClient.fetchJob(apiXml(jobDto.url)); if(xmlOption.isNone()) { System.out.println("None"); continue; } - JenkinsJobXml jobXml = xmlOption.some(); + JenkinsJobXml jobXml = xmlOption.some()._1(); System.out.println("jobXml.type = " + jobXml.type); |