From 5d56a80555ebe5a419dae3b1b849ffa78449b410 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Sat, 8 Dec 2012 00:52:19 +0100 Subject: o Storing Jenkins builds discovered from the feed. --- src/main/java/io/trygvis/esper/testing/Util.java | 10 ++ .../esper/testing/jenkins/JenkinsClient.java | 64 +++++++++- .../trygvis/esper/testing/jenkins/JenkinsDao.java | 132 +++++++++++++++++++-- .../esper/testing/jenkins/JenkinsImporter.java | 6 +- .../esper/testing/jenkins/JenkinsServer.java | 104 ++++++---------- .../esper/testing/jenkins/JenkinsServer2.java | 36 ------ .../esper/testing/jenkins/JenkinsServerOld.java | 99 ++++++++++++++++ 7 files changed, 329 insertions(+), 122 deletions(-) delete mode 100644 src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java create mode 100644 src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java (limited to 'src/main/java/io/trygvis/esper/testing') diff --git a/src/main/java/io/trygvis/esper/testing/Util.java b/src/main/java/io/trygvis/esper/testing/Util.java index 49d6a37..57288d5 100644 --- a/src/main/java/io/trygvis/esper/testing/Util.java +++ b/src/main/java/io/trygvis/esper/testing/Util.java @@ -18,6 +18,16 @@ public class Util { } }; + public static F> parseLong = new F>() { + public Option f(String s) { + try { + return some(Long.parseLong(s)); + } catch (NumberFormatException e) { + return none(); + } + } + }; + public static F> parseUri = new F>() { public Option f(String s) { try { diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java index 8276eaf..40d7618 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java @@ -12,10 +12,8 @@ import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.jdom2.Document; import org.jdom2.Element; -import org.jdom2.*; import org.joda.time.DateTime; -import javax.xml.stream.*; import java.io.*; import java.net.*; import java.util.*; @@ -56,7 +54,7 @@ public class JenkinsClient { return jenkinsEntryXmlClient.fetch(uri); } - public JenkinsXml fetchJobs(URI uri) throws XMLStreamException, JDOMException, IOException { + public JenkinsXml fetchJobs(URI uri) throws IOException { Option d = xmlHttpClient.fetch(uri); if (d.isNone()) { @@ -86,7 +84,7 @@ public class JenkinsClient { Option.fromNull(root.getChildText("description")), jobs); } - public Option fetchJob(URI uri) throws IOException, JDOMException, XMLStreamException { + public Option fetchJob(URI uri) throws IOException { Option d = xmlHttpClient.fetch(uri); if (d.isNone()) { @@ -107,6 +105,29 @@ public class JenkinsClient { return Option.none(); } } + + public Option fetchBuild(URI uri) throws IOException { + Option d = xmlHttpClient.fetch(uri); + + if (d.isNone()) { + return Option.none(); + } + + Element root = d.some().getRootElement(); + + String name = root.getName(); + + switch (name) { + case "matrixBuild": + case "matrixRun": + case "mavenModuleSetBuild": + case "freeStyleBuild": + return JenkinsBuildXml.parse(root); + default: + System.out.println("Unknown build type: " + name); + return Option.none(); + } + } } class JenkinsEntryXml { @@ -149,7 +170,7 @@ class JenkinsJobEntryXml { class JenkinsJobXml { enum JenkinsJobType { - MAVEN, FREE_STYLE + MAVEN, FREE_STYLE, MATRIX } public final JenkinsJobType type; @@ -221,3 +242,36 @@ class JenkinsJobXml { child(root, "lastUnsuccessfulBuild").bind(BuildXml.buildXml)); } } + +class JenkinsBuildXml { + + public final URI uri; + public final int number; + public final String result; + public final int duration; + public final long timestamp; + + JenkinsBuildXml(URI uri, int number, String result, int duration, long timestamp) { + this.uri = uri; + this.number = number; + this.result = result; + this.duration = duration; + this.timestamp = timestamp; + } + + public static Option parse(Element root) { + + Option uri = childText(root, "url").bind(Util.parseUri); + Option number = childText(root, "number").bind(Util.parseInt); + Option result = childText(root, "result"); + Option duration = childText(root, "duration").bind(Util.parseInt); + Option timestamp = childText(root, "timestamp").bind(Util.parseLong); + + if(uri.isNone() || number.isNone() || result.isNone() || duration.isNone() || timestamp.isNone()) { + System.out.println("Missing required fields."); + return none(); + } + + return some(new JenkinsBuildXml(uri.some(), number.some(), result.some(), duration.some(), timestamp.some())); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java index 757902a..05ed567 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java @@ -1,23 +1,59 @@ package io.trygvis.esper.testing.jenkins; +import fj.data.*; +import org.joda.time.*; + import java.net.URI; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import static fj.data.Option.none; +import static fj.data.Option.some; +import static java.lang.System.currentTimeMillis; + public class JenkinsDao { private final Connection c; - private static final String JENKINS_SERVER = "uuid, url"; + + private static final String JENKINS_SERVER = "uuid, created_date, url"; + + private static final String JENKINS_BUILD = "uuid, created_date, entry_id, url, result, number, duration, timestamp"; public JenkinsDao(Connection c) { this.c = c; } + private JenkinsServerDto jenkinsServer(ResultSet rs) throws SQLException { + int i = 1; + return new JenkinsServerDto( + UUID.fromString(rs.getString(i++)), + new DateTime(rs.getTimestamp(i++).getTime()), + URI.create(rs.getString(i))); + } + +// private JenkinsEventDto jenkinsEvent(ResultSet rs) throws SQLException { +// int i = 1; +// return new JenkinsEventDto( +// UUID.fromString(rs.getString(i++)), +// new DateTime(rs.getTimestamp(i++).getTime()), +// URI.create(rs.getString(i))); +// } + + private JenkinsBuildDto jenkinsBuild(ResultSet rs) throws SQLException { + int i = 1; + return new JenkinsBuildDto( + UUID.fromString(rs.getString(i++)), + new DateTime(rs.getTimestamp(i++).getTime()), + rs.getString(i++), + URI.create(rs.getString(i++)), + rs.getString(i++), + rs.getInt(i++), + rs.getInt(i++), + new DateTime(rs.getTimestamp(i).getTime())); + } + public List selectServer() throws SQLException { try (PreparedStatement s = c.prepareStatement("SELECT " + JENKINS_SERVER + " FROM jenkins_server")) { ResultSet rs = s.executeQuery(); @@ -28,21 +64,49 @@ public class JenkinsDao { } return servers; } + } + + public Option selectBuildByEntryId(String id) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT " + JENKINS_BUILD + " FROM jenkins_build WHERE entry_id=?")) { + int i = 1; + s.setString(i, id); + ResultSet rs = s.executeQuery(); + + if (!rs.next()) { + return none(); + } + return some(jenkinsBuild(rs)); + } } - private JenkinsServerDto jenkinsServer(ResultSet rs) throws SQLException { - int i = 1; - return new JenkinsServerDto(UUID.fromString(rs.getString(i++)), URI.create(rs.getString(i))); + public UUID insertBuild(String entryId, URI uri, String result, int number, int duration, long timestamp) throws SQLException { + try (PreparedStatement s = c.prepareStatement("INSERT INTO jenkins_build(" + JENKINS_BUILD + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)")) { + UUID uuid = UUID.randomUUID(); + int i = 1; + s.setString(i++, uuid.toString()); + s.setTimestamp(i++, new Timestamp(currentTimeMillis())); + s.setString(i++, entryId); + s.setString(i++, uri.toASCIIString()); + s.setString(i++, result); + s.setInt(i++, number); + s.setInt(i++, duration); + s.setTimestamp(i, new Timestamp(timestamp)); + s.executeUpdate(); + + return uuid; + } } } class JenkinsServerDto { public final UUID uuid; + public final DateTime created_date; public final URI uri; - JenkinsServerDto(UUID uuid, URI uri) { + JenkinsServerDto(UUID uuid, DateTime created_date, URI uri) { this.uuid = uuid; + this.created_date = created_date; this.uri = uri; } @@ -61,3 +125,53 @@ class JenkinsServerDto { return uuid.hashCode(); } } + +class JenkinsJobDto { + public final UUID uuid; + public final DateTime created_date; + public final URI uri; + public final String title; + + JenkinsJobDto(UUID uuid, DateTime created_date, URI uri, String title) { + this.uuid = uuid; + this.created_date = created_date; + this.uri = uri; + this.title = title; + } +} + +class JenkinsBuildDto { + public final UUID uuid; + public final DateTime created_date; + public final String entryId; + public final URI uri; + public final String result; + public final int number; + public final int duration; + public final DateTime timestamp; + + JenkinsBuildDto(UUID uuid, DateTime created_date, String entryId, URI uri, String result, int number, int duration, DateTime timestamp) { + this.uuid = uuid; + this.created_date = created_date; + this.entryId = entryId; + this.uri = uri; + this.result = result; + this.number = number; + this.duration = duration; + this.timestamp = timestamp; + } +} + +class JenkinsEventDto { + public final UUID uuid; + public final DateTime created_date; + public final String id; + public final DateTime timestamp; + + JenkinsEventDto(UUID uuid, DateTime created_date, String id, DateTime timestamp) { + this.uuid = uuid; + this.created_date = created_date; + this.id = id; + this.timestamp = timestamp; + } +} diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java index 6fec157..c273d8a 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java @@ -26,10 +26,10 @@ public class JenkinsImporter { HashSet servers = new HashSet<>(); - ObjectManager> serverManager = new ObjectManager<>("JenkinsServer", servers, new ObjectFactory>() { - public ActorRef create(JenkinsServerDto server) { + ObjectManager> serverManager = new ObjectManager<>("JenkinsServerOld", servers, new ObjectFactory>() { + public ActorRef create(JenkinsServerDto server) { String name = "Jenkins: " + server.uri; - return threadedActor(name, config.jenkinsUpdateInterval, boneCp, name, new JenkinsServer2(jenkinsClient, server.uri)); + return threadedActor(name, config.jenkinsUpdateInterval, boneCp, name, new JenkinsServer(jenkinsClient, server.uri)); } }); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java index 49b8995..9d12456 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java @@ -1,99 +1,65 @@ package io.trygvis.esper.testing.jenkins; -import fj.*; import fj.data.*; -import static fj.data.Option.*; import io.trygvis.esper.testing.object.*; -import org.codehaus.httpcache4j.util.*; -import org.joda.time.*; -import java.io.*; import java.net.*; +import java.sql.*; import java.util.*; -import java.util.HashSet; import java.util.List; -import java.util.concurrent.*; - -public class JenkinsServer implements Closeable { +public class JenkinsServer implements TransactionalActor { private final JenkinsClient client; public final URI uri; - private final ObjectManager jobManager; - private boolean shouldRun = true; - private final Thread thread; + public JenkinsServer(JenkinsClient client, URI uri) { + this.client = client; + this.uri = uri; + } - private Option> jenkins = none(); + public void act(Connection c) throws Exception { + JenkinsDao dao = new JenkinsDao(c); + Option> option = client.fetchRss(URI.create(uri.toASCIIString() + "/rssAll")); - public JenkinsServer(final ScheduledExecutorService executorService, final JenkinsClient client, URI uri) { - this.client = client; - this.uri = URIBuilder.fromURI(uri).addRawPath("api/xml").toURI(); + if(option.isNone()) { + return; + } - jobManager = new ObjectManager<>("JenkinsJob", Collections.emptySet(), new ObjectFactory() { - public JenkinsJob create(URI uri) { - return new JenkinsJob(executorService, client, uri); - } - }); + List list = option.some(); - thread = new Thread(new Runnable() { - public void run() { - JenkinsServer.this.run(); - } - }); - thread.setDaemon(true); - thread.start(); - } + System.out.println("Got " + list.size() + " entries."); - public void close() throws IOException { - shouldRun = false; - thread.interrupt(); - while (thread.isAlive()) { - try { - thread.join(); - } catch (InterruptedException e) { - continue; - } - } - } + int i = 0; - private void run() { - while (shouldRun) { - try { - doWork(); - } catch (Exception e) { - e.printStackTrace(System.out); - } + for (JenkinsEntryXml entry : list) { + Option o = dao.selectBuildByEntryId(entry.id); - try { - Thread.sleep(10 * 1000); - } catch (InterruptedException e) { - // ignore + if(o.isSome()) { + System.out.println("Old event: " + entry.id); + continue; } - } - } - public Option> getJenkins() { - return jenkins; - } + System.out.println("New event: " + entry.id + ", fetching build info"); - public Collection getJobs() { - return jobManager.getObjects(); - } + i++; - private void doWork() { - try { - JenkinsXml xml = client.fetchJobs(uri); + Option o2 = client.fetchBuild(URI.create(entry.uri.toASCIIString() + "/api/xml")); - List jobUris = new ArrayList<>(xml.jobs.size()); - for (JenkinsJobEntryXml job : xml.jobs) { - jobUris.add(URI.create(job.url)); + if(o2.isNone()) { + continue; } - this.jenkins = some(P.p(xml, new LocalDateTime())); + JenkinsBuildXml build = o2.some(); + + UUID uuid = dao.insertBuild(entry.id, build.uri, build.result, build.number, build.duration, build.timestamp); - jobManager.update(new HashSet<>(jobUris)); - } catch (Throwable e) { - e.printStackTrace(System.out); + System.out.println("Build inserted: " + uuid + ", i=" + i); + +// if(i == 1) { +// break; +// } } + + System.out.println("Inserted " + i + " new events."); } } diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java deleted file mode 100644 index bad040c..0000000 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java +++ /dev/null @@ -1,36 +0,0 @@ -package io.trygvis.esper.testing.jenkins; - -import fj.data.*; -import io.trygvis.esper.testing.object.*; - -import java.net.*; -import java.sql.*; -import java.util.List; - -public class JenkinsServer2 implements TransactionalActor { - private final JenkinsClient client; - public final URI uri; - - public JenkinsServer2(JenkinsClient client, URI uri) { - this.client = client; - this.uri = uri; - } - - public void act(Connection c) throws Exception { - System.out.println("polling " + uri); - - Option> option = client.fetchRss(URI.create(uri.toASCIIString() + "/rssAll")); - - if(option.isNone()) { - return; - } - - List list = option.some(); - - System.out.println("Got " + list.size() + " entries."); - - for (JenkinsEntryXml entry : list) { - System.out.println("entry.uri = " + entry.uri); - } - } -} diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java new file mode 100644 index 0000000..fe5c090 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java @@ -0,0 +1,99 @@ +package io.trygvis.esper.testing.jenkins; + +import fj.*; +import fj.data.*; +import static fj.data.Option.*; +import io.trygvis.esper.testing.object.*; +import org.codehaus.httpcache4j.util.*; +import org.joda.time.*; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.*; + +public class JenkinsServerOld implements Closeable { + + private final JenkinsClient client; + public final URI uri; + private final ObjectManager jobManager; + + private boolean shouldRun = true; + private final Thread thread; + + private Option> jenkins = none(); + + public JenkinsServerOld(final ScheduledExecutorService executorService, final JenkinsClient client, URI uri) { + this.client = client; + this.uri = URIBuilder.fromURI(uri).addRawPath("api/xml").toURI(); + + jobManager = new ObjectManager<>("JenkinsJob", Collections.emptySet(), new ObjectFactory() { + public JenkinsJob create(URI uri) { + return new JenkinsJob(executorService, client, uri); + } + }); + + thread = new Thread(new Runnable() { + public void run() { + JenkinsServerOld.this.run(); + } + }); + thread.setDaemon(true); + thread.start(); + } + + public void close() throws IOException { + shouldRun = false; + thread.interrupt(); + while (thread.isAlive()) { + try { + thread.join(); + } catch (InterruptedException e) { + continue; + } + } + } + + private void run() { + while (shouldRun) { + try { + doWork(); + } catch (Exception e) { + e.printStackTrace(System.out); + } + + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException e) { + // ignore + } + } + } + + public Option> getJenkins() { + return jenkins; + } + + public Collection getJobs() { + return jobManager.getObjects(); + } + + private void doWork() { + try { + JenkinsXml xml = client.fetchJobs(uri); + + List jobUris = new ArrayList<>(xml.jobs.size()); + for (JenkinsJobEntryXml job : xml.jobs) { + jobUris.add(URI.create(job.url)); + } + + this.jenkins = some(P.p(xml, new LocalDateTime())); + + jobManager.update(new HashSet<>(jobUris)); + } catch (Throwable e) { + e.printStackTrace(System.out); + } + } +} -- cgit v1.2.3