From 953ddf830be3ec5188762c2471d007478e6573a1 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Sat, 8 Dec 2012 13:39:28 +0100 Subject: o Discovering and storing jobs in addition to the build. --- src/main/java/io/trygvis/esper/testing/Config.java | 8 +- .../esper/testing/jenkins/JenkinsClient.java | 62 ++++++++------ .../trygvis/esper/testing/jenkins/JenkinsDao.java | 96 +++++++++++++--------- .../esper/testing/jenkins/JenkinsImporter.java | 4 +- .../trygvis/esper/testing/jenkins/JenkinsJob.java | 12 +-- .../esper/testing/jenkins/JenkinsServer.java | 76 ++++++++++++++--- .../esper/testing/jenkins/JenkinsServerOld.java | 15 ++-- .../io/trygvis/esper/testing/util/HttpClient.java | 9 +- 8 files changed, 182 insertions(+), 100 deletions(-) (limited to 'src/main/java/io') diff --git a/src/main/java/io/trygvis/esper/testing/Config.java b/src/main/java/io/trygvis/esper/testing/Config.java index 4ff034a..225d289 100755 --- a/src/main/java/io/trygvis/esper/testing/Config.java +++ b/src/main/java/io/trygvis/esper/testing/Config.java @@ -1,12 +1,7 @@ package io.trygvis.esper.testing; -import ch.qos.logback.classic.*; -import ch.qos.logback.core.util.*; import com.jolbox.bonecp.*; import fj.data.*; -import static fj.data.Option.*; -import static org.apache.commons.lang.StringUtils.*; - import org.apache.abdera.*; import org.slf4j.*; @@ -15,6 +10,9 @@ import java.sql.*; import java.util.*; import java.util.concurrent.atomic.*; +import static fj.data.Option.*; +import static org.apache.commons.lang.StringUtils.*; + public class Config { public static class GitoriousConfig { public final String url; diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java index 59c4f30..bd7c82b 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java @@ -8,7 +8,6 @@ import io.trygvis.esper.testing.util.*; import org.apache.abdera.*; import org.apache.abdera.model.*; import org.apache.abdera.parser.*; -import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.jdom2.Document; import org.jdom2.Element; @@ -53,12 +52,22 @@ public class JenkinsClient { })); } - public Option> fetchRss(URI uri) throws IOException { - return jenkinsEntryXmlClient.fetch(uri); + public static URI apiXml(URI url) { + String u = url.toASCIIString(); + + if(u.endsWith("/")) { + u = u.substring(0, u.length() - 1); + } + + return URI.create(u + "/api/xml"); + } + + public Option> fetchRss(URI url) { + return jenkinsEntryXmlClient.fetch(url); } - public JenkinsXml fetchJobs(URI uri) throws IOException { - Option d = xmlHttpClient.fetch(uri); + public JenkinsXml fetchJobs(URI url) { + Option d = xmlHttpClient.fetch(url); if (d.isNone()) { Option n = Option.none(); @@ -71,14 +80,14 @@ public class JenkinsClient { List jobs = new ArrayList<>(); for (Element job : root.getChildren("job")) { String name = trimToNull(job.getChildText("name")); - String url = trimToNull(job.getChildText("url")); + String u = trimToNull(job.getChildText("url")); String color = trimToNull(job.getChildText("color")); - if (name == null || url == null || color == null) { + if (name == null || u == null || color == null) { continue; } - jobs.add(new JenkinsJobEntryXml(name, url, color)); + jobs.add(new JenkinsJobEntryXml(name, u, color)); } return new JenkinsXml( @@ -87,8 +96,8 @@ public class JenkinsClient { Option.fromNull(root.getChildText("description")), jobs); } - public Option fetchJob(URI uri) throws IOException { - Option d = xmlHttpClient.fetch(uri); + public Option fetchJob(URI url) { + Option d = xmlHttpClient.fetch(url); if (d.isNone()) { return Option.none(); @@ -100,17 +109,19 @@ public class JenkinsClient { switch (name) { case "freeStyleProject": - return some(JenkinsJobXml.parse(uri, JenkinsJobType.FREE_STYLE, root)); + return some(JenkinsJobXml.parse(url, JenkinsJobType.FREE_STYLE, root)); case "mavenModuleSet": - return some(JenkinsJobXml.parse(uri, JenkinsJobType.MAVEN, root)); + return some(JenkinsJobXml.parse(url, JenkinsJobType.MAVEN, root)); + case "matrixProject": + return some(JenkinsJobXml.parse(url, JenkinsJobType.MATRIX, root)); default: logger.warn("Unknown project type: " + name); return Option.none(); } } - public Option fetchBuild(URI uri) throws IOException { - Option d = xmlHttpClient.fetch(uri); + public Option fetchBuild(URI url) { + Option d = xmlHttpClient.fetch(url); if (d.isNone()) { return Option.none(); @@ -134,14 +145,14 @@ public class JenkinsClient { public static class JenkinsBuildXml { - public final URI uri; + public final URI url; public final int number; public final String result; public final int duration; public final long timestamp; - JenkinsBuildXml(URI uri, int number, String result, int duration, long timestamp) { - this.uri = uri; + JenkinsBuildXml(URI url, int number, String result, int duration, long timestamp) { + this.url = url; this.number = number; this.result = result; this.duration = duration; @@ -149,31 +160,30 @@ public class JenkinsClient { } public static Option parse(Element root) { - - Option uri = childText(root, "url").bind(Util.parseUri); + Option url = childText(root, "url").bind(Util.parseUri); Option number = childText(root, "number").bind(Util.parseInt); Option result = childText(root, "result"); Option duration = childText(root, "duration").bind(Util.parseInt); Option timestamp = childText(root, "timestamp").bind(Util.parseLong); - if(uri.isNone() || number.isNone() || result.isNone() || duration.isNone() || timestamp.isNone()) { + if(url.isNone() || number.isNone() || result.isNone() || duration.isNone() || timestamp.isNone()) { logger.warn("Missing required fields."); return none(); } - return some(new JenkinsBuildXml(uri.some(), number.some(), result.some(), duration.some(), timestamp.some())); + return some(new JenkinsBuildXml(url.some(), number.some(), result.some(), duration.some(), timestamp.some())); } }} class JenkinsEntryXml { public final String id; public final DateTime timestamp; - public final URI uri; + public final URI url; - JenkinsEntryXml(String id, DateTime timestamp, URI uri) { + JenkinsEntryXml(String id, DateTime timestamp, URI url) { this.id = id; this.timestamp = timestamp; - this.uri = uri; + this.url = url; } } @@ -262,12 +272,12 @@ class JenkinsJobXml { } } - public static JenkinsJobXml parse(URI uri, JenkinsJobType type, Element root) { + public static JenkinsJobXml parse(URI url, JenkinsJobType type, Element root) { return new JenkinsJobXml(type, childText(root, "description"), childText(root, "displayName"), childText(root, "name"), - childText(root, "url").bind(Util.parseUri).orSome(uri), + childText(root, "url").bind(Util.parseUri).orSome(url), childText(root, "color"), childText(root, "buildable").bind(Util.parseBoolean).orSome(false), child(root, "lastBuild").bind(BuildXml.buildXml), diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java index 8a7480b..9f3c752 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import static fj.data.Option.fromNull; import static fj.data.Option.none; import static fj.data.Option.some; import static java.lang.System.currentTimeMillis; @@ -19,7 +20,9 @@ public class JenkinsDao { private static final String JENKINS_SERVER = "uuid, created_date, url"; - private static final String JENKINS_BUILD = "uuid, created_date, server, entry_id, url, result, number, duration, timestamp"; + private static final String JENKINS_JOB = "uuid, created_date, server, url, display_name"; + + private static final String JENKINS_BUILD = "uuid, created_date, job, entry_id, url, result, number, duration, timestamp"; public JenkinsDao(Connection c) { this.c = c; @@ -33,13 +36,15 @@ public class JenkinsDao { URI.create(rs.getString(i))); } -// private JenkinsEventDto jenkinsEvent(ResultSet rs) throws SQLException { -// int i = 1; -// return new JenkinsEventDto( -// UUID.fromString(rs.getString(i++)), -// new DateTime(rs.getTimestamp(i++).getTime()), -// URI.create(rs.getString(i))); -// } + private JenkinsJobDto jenkinsJob(ResultSet rs) throws SQLException { + int i = 1; + return new JenkinsJobDto( + UUID.fromString(rs.getString(i++)), + new DateTime(rs.getTimestamp(i++).getTime()), + UUID.fromString(rs.getString(i++)), + URI.create(rs.getString(i++)), + fromNull(rs.getString(i))); + } private JenkinsBuildDto jenkinsBuild(ResultSet rs) throws SQLException { int i = 1; @@ -70,6 +75,34 @@ public class JenkinsDao { } } + public Option selectJobByUrl(URI url) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT " + JENKINS_JOB + " FROM jenkins_job WHERE url=?")) { + s.setString(1, url.toASCIIString()); + ResultSet rs = s.executeQuery(); + + if(!rs.next()) { + return none(); + } + + return some(jenkinsJob(rs)); + } + } + + public UUID insertJob(UUID server, URI url, Option displayName) throws SQLException { + try (PreparedStatement s = c.prepareStatement("INSERT INTO jenkins_job(" + JENKINS_JOB + ") VALUES(?, ?, ?, ?, ?)")) { + UUID uuid = UUID.randomUUID(); + int i = 1; + s.setString(i++, uuid.toString()); + s.setTimestamp(i++, new Timestamp(currentTimeMillis())); + s.setString(i++, server.toString()); + s.setString(i++, url.toASCIIString()); + s.setString(i, displayName.toNull()); + s.executeUpdate(); + + return uuid; + } + } + public Option selectBuildByEntryId(String id) throws SQLException { try (PreparedStatement s = c.prepareStatement("SELECT " + JENKINS_BUILD + " FROM jenkins_build WHERE entry_id=?")) { int i = 1; @@ -84,7 +117,7 @@ public class JenkinsDao { } } - public UUID insertBuild(UUID server, String entryId, URI uri, String result, int number, int duration, long timestamp) throws SQLException { + public UUID insertBuild(UUID server, String entryId, URI url, String result, int number, int duration, long timestamp) throws SQLException { try (PreparedStatement s = c.prepareStatement("INSERT INTO jenkins_build(" + JENKINS_BUILD + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")) { UUID uuid = UUID.randomUUID(); int i = 1; @@ -92,7 +125,7 @@ public class JenkinsDao { s.setTimestamp(i++, new Timestamp(currentTimeMillis())); s.setString(i++, server.toString()); s.setString(i++, entryId); - s.setString(i++, uri.toASCIIString()); + s.setString(i++, url.toASCIIString()); s.setString(i++, result); s.setInt(i++, number); s.setInt(i++, duration); @@ -107,12 +140,12 @@ public class JenkinsDao { class JenkinsServerDto { public final UUID uuid; public final DateTime created_date; - public final URI uri; + public final URI url; - JenkinsServerDto(UUID uuid, DateTime created_date, URI uri) { + JenkinsServerDto(UUID uuid, DateTime created_date, URI url) { this.uuid = uuid; this.created_date = created_date; - this.uri = uri; + this.url = url; } @Override @@ -134,52 +167,39 @@ class JenkinsServerDto { class JenkinsJobDto { public final UUID uuid; public final DateTime created_date; - public final URI uri; - public final String title; + public final UUID server; + public final URI url; + public final Option displayName; - JenkinsJobDto(UUID uuid, DateTime created_date, URI uri, String title) { + JenkinsJobDto(UUID uuid, DateTime created_date, UUID server, URI url, Option displayName) { this.uuid = uuid; this.created_date = created_date; - this.uri = uri; - this.title = title; + this.server = server; + this.url = url; + this.displayName = displayName; } } class JenkinsBuildDto { public final UUID uuid; public final DateTime created_date; - // TODO: should be job - public final UUID server; + public final UUID job; public final String entryId; - public final URI uri; + public final URI url; public final String result; public final int number; public final int duration; public final DateTime timestamp; - JenkinsBuildDto(UUID uuid, DateTime created_date, UUID server, String entryId, URI uri, String result, int number, int duration, DateTime timestamp) { + JenkinsBuildDto(UUID uuid, DateTime created_date, UUID job, String entryId, URI url, String result, int number, int duration, DateTime timestamp) { this.uuid = uuid; this.created_date = created_date; - this.server = server; + this.job = job; this.entryId = entryId; - this.uri = uri; + this.url = url; this.result = result; this.number = number; this.duration = duration; this.timestamp = timestamp; } } - -class JenkinsEventDto { - public final UUID uuid; - public final DateTime created_date; - public final String id; - public final DateTime timestamp; - - JenkinsEventDto(UUID uuid, DateTime created_date, String id, DateTime timestamp) { - this.uuid = uuid; - this.created_date = created_date; - this.id = id; - this.timestamp = timestamp; - } -} diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java index c9a28b2..713982f 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java @@ -28,7 +28,7 @@ public class JenkinsImporter { ObjectManager> serverManager = new ObjectManager<>("JenkinsServerOld", servers, new ObjectFactory>() { public ActorRef create(JenkinsServerDto server) { - String name = "Jenkins: " + server.uri; + String name = "Jenkins: " + server.url; return threadedActor(name, config.jenkinsUpdateInterval, boneCp, name, new JenkinsServer(jenkinsClient, server)); } }); @@ -56,7 +56,7 @@ public class JenkinsImporter { // P2 p = o.some(); // System.out.println("Last update: " + p._2() + ", jobs=" + p._1().jobs.size()); // } else { -// System.out.println("Never updated: url=" + server.uri); +// System.out.println("Never updated: url=" + server.url); // } // } diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java index 7a7f6af..7199bd8 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java @@ -2,8 +2,8 @@ package io.trygvis.esper.testing.jenkins; import fj.data.*; import static fj.data.Option.*; +import static io.trygvis.esper.testing.jenkins.JenkinsClient.apiXml; import static java.lang.System.currentTimeMillis; -import org.codehaus.httpcache4j.util.*; import org.slf4j.*; import java.io.*; @@ -14,14 +14,14 @@ public class JenkinsJob implements Closeable { private final Logger logger = LoggerFactory.getLogger("jenkins.job"); private final JenkinsClient client; - private final URI uri; + private final URI url; private Option latestStatus = none(); private final ScheduledFuture future; - public JenkinsJob(ScheduledExecutorService executorService, JenkinsClient client, URI uri) { + public JenkinsJob(ScheduledExecutorService executorService, JenkinsClient client, URI url) { this.client = client; - this.uri = URIBuilder.fromURI(uri).addRawPath("api/xml").toURI(); + this.url = apiXml(url); long initialDelay = (long) Math.random() + 1; long period = (long) (Math.random() * 100d) + 1; @@ -43,12 +43,12 @@ public class JenkinsJob implements Closeable { private void doWork() { String name = latestStatus.isSome() && latestStatus.some().name.isSome() ? - latestStatus.some().name.some() : uri.toASCIIString(); + latestStatus.some().name.some() : url.toASCIIString(); try { logger.info("Updating " + name); long start = currentTimeMillis(); - latestStatus = client.fetchJob(uri); + latestStatus = client.fetchJob(url); long end = currentTimeMillis(); logger.info("Updated " + name + " in " + (end - start) + "ms"); } catch (Throwable e) { diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java index 92c0569..a913b63 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer.java @@ -9,6 +9,7 @@ import java.sql.*; import java.util.List; import java.util.*; +import static fj.data.Option.*; import static io.trygvis.esper.testing.jenkins.JenkinsClient.*; public class JenkinsServer implements TransactionalActor { @@ -23,9 +24,9 @@ public class JenkinsServer implements TransactionalActor { public void act(Connection c) throws Exception { JenkinsDao dao = new JenkinsDao(c); - Option> option = client.fetchRss(URI.create(server.uri.toASCIIString() + "/rssAll")); + Option> option = client.fetchRss(URI.create(server.url.toASCIIString() + "/rssAll")); - if(option.isNone()) { + if (option.isNone()) { return; } @@ -38,39 +39,88 @@ public class JenkinsServer implements TransactionalActor { for (JenkinsEntryXml entry : list) { Option o = dao.selectBuildByEntryId(entry.id); - if(o.isSome()) { - logger.info("Old event: " + entry.id); + if (o.isSome()) { + logger.debug("Old event: " + entry.id); continue; } - logger.info("New event: " + entry.id + ", fetching build info"); + logger.info("New build: " + entry.id + ", fetching info"); i++; - Option o2 = client.fetchBuild(URI.create(entry.uri.toASCIIString() + "/api/xml")); + Option o2 = client.fetchBuild(apiXml(entry.url)); - if(o2.isNone()) { + if (o2.isNone()) { continue; } JenkinsBuildXml build = o2.some(); + Option job = findJob(dao, server, build); + + if (job.isNone()) { + continue; + } + UUID uuid = dao.insertBuild( - server.uuid, + job.some(), entry.id, - build.uri, + build.url, build.result, build.number, build.duration, build.timestamp); logger.info("Build inserted: " + uuid + ", i=" + i); - -// if(i == 1) { -// break; -// } } logger.info("Inserted " + i + " new events."); } + + private Option findJob(JenkinsDao dao, JenkinsServerDto server, JenkinsBuildXml build) throws SQLException { + URI jobUrl = createJobUrl(build.url.toASCIIString()); + + Option o2 = dao.selectJobByUrl(jobUrl); + + if (o2.isSome()) { + return some(o2.some().uuid); + } + + logger.info("New job: " + jobUrl + ", fetching info"); + + Option o = client.fetchJob(apiXml(jobUrl)); + + if (o.isNone()) { + return none(); + } + + JenkinsJobXml xml = o.some(); + + UUID uuid = dao.insertJob(server.uuid, xml.url, xml.displayName); + + logger.info("New job: " + xml.displayName.orSome(xml.url.toASCIIString()) + ", uuid=" + uuid); + + return some(uuid); + } + + /** + * This sucks, a build should really include the URL to the job. + */ + public static URI createJobUrl(String u) { + if (u.matches(".*/[-_a-zA-Z]*=.*/[0-9]*/")) { + u = u.substring(0, u.lastIndexOf("/")); + u = u.substring(0, u.lastIndexOf("/")); + u = u.substring(0, u.lastIndexOf("/") + 1); + + return URI.create(u); + } + + if (u.endsWith("/")) { + u = u.substring(0, u.length() - 1); + } + int i = u.lastIndexOf("/"); + u = u.substring(0, i); + + return URI.create(u + "/"); + } } diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java index fe5c090..7e1dc9a 100644 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServerOld.java @@ -3,8 +3,9 @@ package io.trygvis.esper.testing.jenkins; import fj.*; import fj.data.*; import static fj.data.Option.*; +import static io.trygvis.esper.testing.jenkins.JenkinsClient.apiXml; + import io.trygvis.esper.testing.object.*; -import org.codehaus.httpcache4j.util.*; import org.joda.time.*; import java.io.*; @@ -17,7 +18,7 @@ import java.util.concurrent.*; public class JenkinsServerOld implements Closeable { private final JenkinsClient client; - public final URI uri; + public final URI url; private final ObjectManager jobManager; private boolean shouldRun = true; @@ -25,13 +26,13 @@ public class JenkinsServerOld implements Closeable { private Option> jenkins = none(); - public JenkinsServerOld(final ScheduledExecutorService executorService, final JenkinsClient client, URI uri) { + public JenkinsServerOld(final ScheduledExecutorService executorService, final JenkinsClient client, URI url) { this.client = client; - this.uri = URIBuilder.fromURI(uri).addRawPath("api/xml").toURI(); + this.url = apiXml(url); jobManager = new ObjectManager<>("JenkinsJob", Collections.emptySet(), new ObjectFactory() { - public JenkinsJob create(URI uri) { - return new JenkinsJob(executorService, client, uri); + public JenkinsJob create(URI url) { + return new JenkinsJob(executorService, client, url); } }); @@ -82,7 +83,7 @@ public class JenkinsServerOld implements Closeable { private void doWork() { try { - JenkinsXml xml = client.fetchJobs(uri); + JenkinsXml xml = client.fetchJobs(url); List jobUris = new ArrayList<>(xml.jobs.size()); for (JenkinsJobEntryXml job : xml.jobs) { diff --git a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java index 79159ac..5ac7d13 100755 --- a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java +++ b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java @@ -16,6 +16,7 @@ import org.slf4j.*; import java.io.*; import java.net.*; +import static fj.data.Option.none; import static java.lang.System.*; public class HttpClient { @@ -44,7 +45,7 @@ public class HttpClient { }; } - public Option fetch(URI uri) throws IOException { + public Option fetch(URI uri) { HTTPResponse response = null; try { @@ -52,12 +53,14 @@ public class HttpClient { int code = response.getStatus().getCode(); if (code != 200) { - throw new IOException("Did not get 200 back, got " + code); + logger.warn("Did not get 200 back, got " + code); + return none(); } return f.f(response); } catch (HTTPException e) { - throw new IOException(e); + logger.warn("Error while fetching: " + uri, e); + return none(); } finally { if (response != null) { try { -- cgit v1.2.3