aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorTrygve Laugstøl <trygvis@inamo.no>2012-12-23 00:21:57 +0100
committerTrygve Laugstøl <trygvis@inamo.no>2012-12-23 00:21:57 +0100
commiteaa0a2a0828037227312970668eba7bf951b4049 (patch)
treeeced0d4f6c1b53abe0b2af527ecba3bd493684cd /src
parent78e7933c0af1d11d9476c5ee213c64ad87066f55 (diff)
downloadesper-testing-eaa0a2a0828037227312970668eba7bf951b4049.tar.gz
esper-testing-eaa0a2a0828037227312970668eba7bf951b4049.tar.bz2
esper-testing-eaa0a2a0828037227312970668eba7bf951b4049.tar.xz
esper-testing-eaa0a2a0828037227312970668eba7bf951b4049.zip
o Storing the raw bytes with the parsed data from Jenkins.
Diffstat (limited to 'src')
-rw-r--r--src/main/java/io/trygvis/esper/testing/core/db/FileDao.java29
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java6
-rw-r--r--src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java49
-rw-r--r--src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java5
-rw-r--r--src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java36
-rw-r--r--src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java8
-rw-r--r--src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java7
-rw-r--r--src/main/java/io/trygvis/esper/testing/util/HttpClient.java24
-rw-r--r--src/main/resources/ddl-core.sql10
-rw-r--r--src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java5
-rw-r--r--src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java5
11 files changed, 136 insertions, 48 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/core/db/FileDao.java b/src/main/java/io/trygvis/esper/testing/core/db/FileDao.java
new file mode 100644
index 0000000..0878f63
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/core/db/FileDao.java
@@ -0,0 +1,29 @@
+package io.trygvis.esper.testing.core.db;
+
+import java.io.*;
+import java.net.*;
+import java.sql.*;
+import java.util.*;
+
+import static java.lang.System.*;
+
+public class FileDao {
+ private final Connection c;
+
+ public FileDao(Connection c) {
+ this.c = c;
+ }
+
+ public void store(URI url, String contentType, byte[] data) throws SQLException {
+ try (PreparedStatement s = c.prepareStatement("INSERT INTO file(uuid, created_date, url, content_type, data) VALUES(?, ?, ?, ?, ?)")) {
+ UUID uuid = UUID.randomUUID();
+ int i = 1;
+ s.setString(i++, uuid.toString());
+ s.setTimestamp(i++, new Timestamp(currentTimeMillis()));
+ s.setString(i++, url.toASCIIString());
+ s.setString(i++, contentType);
+ s.setBinaryStream(i, new ByteArrayInputStream(data), data.length);
+ s.executeUpdate();
+ }
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
index fff84fe..0071b1f 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -77,13 +77,15 @@ public class GitoriousClient {
Set<GitoriousProjectXml> all = new HashSet<>();
while (true) {
- Option<List<GitoriousProjectXml>> option = http.fetch(new URI(projectsUri + "?page=" + page));
+ Option<P2<List<GitoriousProjectXml>, byte[]>> option = http.fetch(new URI(projectsUri + "?page=" + page));
if (option.isNone()) {
return all;
}
- List<GitoriousProjectXml> list = option.some();
+ List<GitoriousProjectXml> list = option.some()._1();
+
+ // TODO: store data
// This indicates the last page.
if (list.size() == 0) {
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java
index c9e83d4..600ef33 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java
@@ -18,6 +18,7 @@ import java.net.*;
import java.util.*;
import java.util.List;
+import static fj.P.p;
import static fj.data.Option.*;
import static io.trygvis.esper.testing.util.HttpClient.inputStreamOnly;
import static java.lang.System.currentTimeMillis;
@@ -68,20 +69,18 @@ public class JenkinsClient {
return URI.create(u + "/api/xml");
}
- public Option<List<JenkinsEntryXml>> fetchRss(URI url) {
+ public Option<P2<List<JenkinsEntryXml>, byte[]>> fetchRss(URI url) {
return jenkinsEntryXmlClient.fetch(url);
}
- public JenkinsXml fetchJobs(URI url) {
- Option<Document> d = xmlHttpClient.fetch(url);
+ public Option<P2<JenkinsXml, byte[]>> fetchJobs(URI url) {
+ Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url);
if (d.isNone()) {
- Option<String> n = Option.none();
-
- return new JenkinsXml(n, n, n, Collections.<JenkinsXml.JobXml>emptyList());
+ return none();
}
- Element root = d.some().getRootElement();
+ Element root = d.some()._1().getRootElement();
List<JenkinsXml.JobXml> jobs = new ArrayList<>();
for (Element job : root.getChildren("job")) {
@@ -96,48 +95,48 @@ public class JenkinsClient {
jobs.add(new JenkinsXml.JobXml(name, u, color));
}
- return new JenkinsXml(
- Option.fromNull(root.getChildText("nodeName")),
- Option.fromNull(root.getChildText("nodeDescription")),
- Option.fromNull(root.getChildText("description")), jobs);
+ return some(p(new JenkinsXml(
+ Option.fromNull(root.getChildText("nodeName")),
+ Option.fromNull(root.getChildText("nodeDescription")),
+ Option.fromNull(root.getChildText("description")), jobs), d.some()._2()));
}
- public Option<JenkinsJobXml> fetchJob(URI url) {
- Option<Document> d = xmlHttpClient.fetch(url);
+ public Option<P2<JenkinsJobXml, byte[]>> fetchJob(URI url) {
+ Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url);
if (d.isNone()) {
return Option.none();
}
- Element root = d.some().getRootElement();
+ Element root = d.some()._1().getRootElement();
String name = root.getName();
switch (name) {
case "freeStyleProject":
- return some(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root));
+ return some(p(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root), d.some()._2()));
case "mavenModuleSet":
- return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE_SET, root));
+ return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE_SET, root), d.some()._2()));
case "mavenModule":
- return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE, root));
+ return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN_MODULE, root), d.some()._2()));
case "matrixProject":
- return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root));
+ return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root), d.some()._2()));
case "matrixConfiguration":
- return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX_CONFIGURATION, root));
+ return some(p(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX_CONFIGURATION, root), d.some()._2()));
default:
logger.warn("Unknown project type: " + name);
return Option.none();
}
}
- public Option<JenkinsBuildXml> fetchBuild(URI url) {
- Option<Document> d = xmlHttpClient.fetch(url);
+ public Option<P2<JenkinsBuildXml, byte[]>> fetchBuild(URI url) {
+ final Option<P2<Document, byte[]>> d = xmlHttpClient.fetch(url);
if (d.isNone()) {
return Option.none();
}
- Element root = d.some().getRootElement();
+ Element root = d.some()._1().getRootElement();
String name = root.getName();
@@ -148,7 +147,11 @@ public class JenkinsClient {
case "mavenModuleSetBuild":
case "mavenBuild":
case "freeStyleBuild":
- return JenkinsBuildXml.parse(root);
+ return JenkinsBuildXml.parse(root).map(new F<JenkinsBuildXml, P2<JenkinsBuildXml, byte[]>>() {
+ public P2<JenkinsBuildXml, byte[]> f(JenkinsBuildXml x) {
+ return p(x, d.some()._2());
+ }
+ });
default:
logger.warn("Unknown build type: " + name);
return Option.none();
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java
index 7199bd8..5ca449e 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java
@@ -1,5 +1,6 @@
package io.trygvis.esper.testing.jenkins;
+import fj.*;
import fj.data.*;
import static fj.data.Option.*;
import static io.trygvis.esper.testing.jenkins.JenkinsClient.apiXml;
@@ -23,7 +24,7 @@ public class JenkinsJob implements Closeable {
this.client = client;
this.url = apiXml(url);
- long initialDelay = (long) Math.random() + 1;
+ long initialDelay = (long) (Math.random() + 1);
long period = (long) (Math.random() * 100d) + 1;
future = executorService.scheduleAtFixedRate(new Runnable() {
public void run() {
@@ -48,7 +49,7 @@ public class JenkinsJob implements Closeable {
try {
logger.info("Updating " + name);
long start = currentTimeMillis();
- latestStatus = client.fetchJob(url);
+ latestStatus = client.fetchJob(url).map(P2.<JenkinsJobXml, byte[]>__1());
long end = currentTimeMillis();
logger.info("Updated " + name + " in " + (end - start) + "ms");
} catch (Throwable e) {
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java
index cf95c4e..32728a6 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerActor.java
@@ -1,6 +1,8 @@
package io.trygvis.esper.testing.jenkins;
+import fj.*;
import fj.data.*;
+import io.trygvis.esper.testing.core.db.*;
import io.trygvis.esper.testing.util.object.*;
import io.trygvis.esper.testing.util.sql.*;
import org.slf4j.*;
@@ -30,13 +32,18 @@ public class JenkinsServerActor implements TransactionalActor {
long start = currentTimeMillis();
JenkinsDao dao = new JenkinsDao(c);
- Option<List<JenkinsEntryXml>> option = client.fetchRss(URI.create(server.url.toASCIIString() + "/rssAll"));
+ FileDao fileDao = new FileDao(c);
+
+ URI rssUrl = URI.create(server.url.toASCIIString() + "/rssAll");
+ Option<P2<List<JenkinsEntryXml>, byte[]>> option = client.fetchRss(rssUrl);
if (option.isNone()) {
return;
}
- List<JenkinsEntryXml> list = option.some();
+ fileDao.store(rssUrl, "application/xml", option.some()._2());
+
+ List<JenkinsEntryXml> list = option.some()._1();
logger.info("Got " + list.size() + " entries.");
@@ -54,18 +61,23 @@ public class JenkinsServerActor implements TransactionalActor {
logger.debug("Build: " + entry.id + ", fetching info");
- Option<JenkinsBuildXml> buildXmlOption = client.fetchBuild(apiXml(entry.url));
+ URI buildUrl = apiXml(entry.url);
+
+ Option<P2<JenkinsBuildXml, byte[]>> buildXmlOption = client.fetchBuild(buildUrl);
if (buildXmlOption.isNone()) {
continue;
}
- JenkinsBuildXml build = buildXmlOption.some();
+ JenkinsBuildXml build = buildXmlOption.some()._1();
- if(build.result.isNone()) {
+ if (build.result.isNone()) {
logger.debug("Not done building, <result> is not available.");
continue;
}
+
+ fileDao.store(buildUrl, "application/xml", buildXmlOption.some()._2());
+
String result = build.result.some();
// -----------------------------------------------------------------------
@@ -74,11 +86,11 @@ public class JenkinsServerActor implements TransactionalActor {
Set<UUID> users = new HashSet<>();
- if(build.changeSet.isSome()) {
+ if (build.changeSet.isSome()) {
JenkinsBuildXml.ChangeSetXml changeSetXml = build.changeSet.some();
for (JenkinsBuildXml.ChangeSetItemXml item : changeSetXml.items) {
- if(item.author.isNone()) {
+ if (item.author.isNone()) {
continue;
}
@@ -86,7 +98,7 @@ public class JenkinsServerActor implements TransactionalActor {
UUID uuid = authors.get(url);
- if(uuid == null) {
+ if (uuid == null) {
SqlOption<JenkinsUserDto> userO = dao.selectUserByAbsoluteUrl(server.uuid, url);
if (userO.isNone()) {
logger.info("New user: {}", url);
@@ -117,13 +129,17 @@ public class JenkinsServerActor implements TransactionalActor {
} else {
logger.info("New job: {}, fetching info", jobUrl);
- Option<JenkinsJobXml> jobXmlOption = client.fetchJob(apiXml(jobUrl));
+ URI uri = apiXml(jobUrl);
+
+ Option<P2<JenkinsJobXml, byte[]>> jobXmlOption = client.fetchJob(uri);
if (jobXmlOption.isNone()) {
continue;
}
- JenkinsJobXml xml = jobXmlOption.some();
+ fileDao.store(uri, "application/xml", jobXmlOption.some()._2());
+
+ JenkinsJobXml xml = jobXmlOption.some()._1();
job = dao.insertJob(server.uuid, xml.url, xml.type, xml.displayName);
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java
index 9f48191..e21f336 100644
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java
@@ -83,7 +83,13 @@ public class JenkinsServerOld implements Closeable {
private void doWork() {
try {
- JenkinsXml xml = client.fetchJobs(url);
+ Option<P2<JenkinsXml, byte[]>> o = client.fetchJobs(url);
+
+ if(o.isNone()) {
+ return;
+ }
+
+ JenkinsXml xml = o.some()._1();
List<URI> jobUris = new ArrayList<>(xml.jobs.size());
for (JenkinsXml.JobXml job : xml.jobs) {
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
index 9900bfc..fe0db33 100644
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
@@ -1,5 +1,6 @@
package io.trygvis.esper.testing.nexus;
+import fj.*;
import fj.data.*;
import static fj.data.Option.*;
import static io.trygvis.esper.testing.nexus.SearchNGResponseParser.*;
@@ -45,13 +46,15 @@ public class NexusClient {
System.out.println("args = " + args);
URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline + args);
- Option<Document> d = xmlHttpClient.fetch(uri);
+ Option<P2<Document,byte[]>> d = xmlHttpClient.fetch(uri);
if (d.isNone()) {
return new NexusFeed(Collections.<HasNexusEvent>emptyList());
}
- Document document = d.some();
+ Document document = d.some()._1();
+
+ // TODO: store data
return NexusFeedParser.parseDocument(document);
}
diff --git a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java
index a194cd1..69c74fe 100644
--- a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java
+++ b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java
@@ -3,14 +3,15 @@ package io.trygvis.esper.testing.util;
import fj.*;
import fj.data.*;
import io.trygvis.esper.testing.*;
+import org.apache.commons.io.*;
import org.apache.http.conn.scheme.*;
import org.apache.http.impl.client.*;
import org.apache.http.impl.conn.tsccm.*;
import org.apache.http.params.*;
import org.codehaus.httpcache4j.*;
import org.codehaus.httpcache4j.cache.*;
+import org.codehaus.httpcache4j.payload.*;
import org.codehaus.httpcache4j.resolver.*;
-import org.h2.util.*;
import org.slf4j.*;
import java.io.*;
@@ -39,13 +40,13 @@ public class HttpClient<A> {
try {
return f.f(inputStream);
} finally {
- IOUtils.closeSilently(inputStream);
+ IOUtils.closeQuietly(inputStream);
}
}
};
}
- public Option<A> fetch(URI uri) {
+ public Option<P2<A, byte[]>> fetch(URI uri) {
HTTPResponse response = null;
try {
@@ -57,10 +58,25 @@ public class HttpClient<A> {
return none();
}
- return f.f(response);
+ final ByteArrayOutputStream buffer = new ByteArrayOutputStream(1024 * 1024);
+
+ IOUtils.copy(response.getPayload().getInputStream(), buffer);
+
+ final byte[] bytes = buffer.toByteArray();
+
+ response = new HTTPResponse(new ByteArrayPayload(new ByteArrayInputStream(bytes), response.getPayload().getMimeType()), response.getStatusLine(), response.getHeaders());
+
+ return f.f(response).map(new F<A, P2<A, byte[]>>() {
+ public P2<A, byte[]> f(A a) {
+ return P.p(a, bytes);
+ }
+ });
} catch (HTTPException e) {
logger.warn("Error while fetching: " + uri, e);
return none();
+ } catch (IOException e) {
+ logger.warn("Error while fetching: " + uri, e);
+ return none();
} finally {
if (response != null) {
try {
diff --git a/src/main/resources/ddl-core.sql b/src/main/resources/ddl-core.sql
index a244236..376ca9f 100644
--- a/src/main/resources/ddl-core.sql
+++ b/src/main/resources/ddl-core.sql
@@ -8,6 +8,16 @@ DROP TABLE IF EXISTS person_badge;
DROP TABLE IF EXISTS person_jenkins_user;
DROP TABLE IF EXISTS person;
DROP TABLE IF EXISTS table_poller_status;
+DROP TABLE IF EXISTS file;
+
+CREATE TABLE file (
+ uuid CHAR(36) NOT NULL,
+ created_date TIMESTAMP NOT NULL,
+ url VARCHAR(1000) NOT NULL,
+ content_type VARCHAR(100) NOT NULL,
+ data BYTEA,
+ CONSTRAINT pk_file PRIMARY KEY (uuid)
+);
CREATE TABLE table_poller_status (
poller_name VARCHAR(100) NOT NULL,
diff --git a/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java b/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java
index 9cc47ad..f03086b 100644
--- a/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java
+++ b/src/test/java/io/trygvis/esper/testing/jenkins/CreateMissingMavenModuleJobsApp.java
@@ -7,6 +7,7 @@ import java.sql.ResultSet;
import java.util.List;
import java.util.UUID;
+import fj.*;
import org.apache.abdera.Abdera;
import org.codehaus.httpcache4j.cache.HTTPCache;
@@ -48,14 +49,14 @@ public class CreateMissingMavenModuleJobsApp {
URI url = URI.create(u);
- Option<JenkinsJobXml> xmlOption = client.fetchJob(apiXml(url));
+ Option<P2<JenkinsJobXml,byte[]>> xmlOption = client.fetchJob(apiXml(url));
if(xmlOption.isNone()) {
System.out.println("None");
continue;
}
- JenkinsJobXml jobXml = xmlOption.some();
+ JenkinsJobXml jobXml = xmlOption.some()._1();
if(dao.selectJobByUrl(jobXml.url).isSome()) {
System.out.println("Duplicate");
diff --git a/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java b/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java
index 46060a9..ffd3212 100644
--- a/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java
+++ b/src/test/java/io/trygvis/esper/testing/jenkins/SetJobTypeApp.java
@@ -5,6 +5,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.List;
+import fj.*;
import org.apache.abdera.Abdera;
import org.codehaus.httpcache4j.cache.HTTPCache;
@@ -40,14 +41,14 @@ public class SetJobTypeApp {
for (JenkinsJobDto jobDto : jobs) {
System.out.println("job = " + jobDto.url);
- Option<JenkinsJobXml> xmlOption = jenkinsClient.fetchJob(apiXml(jobDto.url));
+ Option<P2<JenkinsJobXml,byte[]>> xmlOption = jenkinsClient.fetchJob(apiXml(jobDto.url));
if(xmlOption.isNone()) {
System.out.println("None");
continue;
}
- JenkinsJobXml jobXml = xmlOption.some();
+ JenkinsJobXml jobXml = xmlOption.some()._1();
System.out.println("jobXml.type = " + jobXml.type);