diff options
14 files changed, 326 insertions, 115 deletions
diff --git a/config.properties.template b/config.properties.template index b552f5e..a8c4b97 100755 --- a/config.properties.template +++ b/config.properties.template @@ -5,6 +5,8 @@ gitorious.projectListUpdateInterval=60 nexus.updateInterval=3600 +jenkins.updateInterval=3600 + database.driver= database.url=jdbc:h2:tcp://127.0.0.1/esper;DB_CLOSE_DELAY=-1 database.username= @@ -38,7 +38,7 @@ <dependency> <groupId>org.jdom</groupId> <artifactId>jdom</artifactId> - <version>2.0.4</version> + <version>2.0.2</version> </dependency> <dependency> <groupId>com.jolbox</groupId> diff --git a/src/main/java/io/trygvis/esper/testing/Config.java b/src/main/java/io/trygvis/esper/testing/Config.java index 31b39dc..95e5a99 100755 --- a/src/main/java/io/trygvis/esper/testing/Config.java +++ b/src/main/java/io/trygvis/esper/testing/Config.java @@ -4,6 +4,8 @@ import com.jolbox.bonecp.*; import fj.data.*; import static fj.data.Option.*; import static org.apache.commons.lang.StringUtils.*; + +import org.apache.abdera.*; import org.slf4j.*; import java.io.*; @@ -40,13 +42,17 @@ public class Config { public final long nexusUpdateInterval; + public final long jenkinsUpdateInterval; + public final String databaseUrl; public final String databaseUsername; public final String databasePassword; - public Config(GitoriousConfig gitorious, long nexusUpdateInterval, String databaseUrl, String databaseUsername, String databasePassword) { + public Config(GitoriousConfig gitorious, long nexusUpdateInterval, long jenkinsUpdateInterval, String databaseUrl, + String databaseUsername, String databasePassword) { this.gitorious = gitorious; this.nexusUpdateInterval = nexusUpdateInterval; + this.jenkinsUpdateInterval = jenkinsUpdateInterval; this.databaseUrl = databaseUrl; this.databaseUsername = databaseUsername; this.databasePassword = databasePassword; @@ -62,6 +68,7 @@ public class Config { return new Config(GitoriousConfig.fromProperties(properties), getProperty(properties, "nexus.updateInterval").bind(parseInt).valueE("Missing required property: nexus.updateInterval") * 1000, + getProperty(properties, "jenkins.updateInterval").bind(parseInt).valueE("Missing required property: jenkins.updateInterval") * 1000, trimToNull(properties.getProperty("database.url")), trimToNull(properties.getProperty("database.username")), trimToNull(properties.getProperty("database.password"))); @@ -79,10 +86,15 @@ public class Config { setUsername(databaseUsername); setPassword(databasePassword); setDefaultAutoCommit(false); + setCloseConnectionWatch(true); setMaxConnectionsPerPartition(10); }}); } + public Abdera createAbdera() { + return new Abdera(); + } + public void addShutdownHook(final Thread t, final AtomicBoolean shouldRun) { Runtime.getRuntime().addShutdownHook(new Thread() { { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java index b7b21d6..9b9eafa 100755 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java @@ -18,8 +18,7 @@ public class GitoriousAtomFeedParser { public final Parser parser; public static final STAXEventReader xmlReader = new STAXEventReader(); - public GitoriousAtomFeedParser() { - Abdera abdera = new Abdera(); + public GitoriousAtomFeedParser(Abdera abdera) { parser = abdera.getParser(); } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java index 986d4c1..fff84fe 100755 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -1,13 +1,10 @@ package io.trygvis.esper.testing.gitorious; -import static java.lang.System.*; - -import fj.data.Option; -import org.apache.abdera.parser.ParseException; +import fj.*; +import fj.data.*; +import io.trygvis.esper.testing.util.*; +import org.apache.abdera.parser.*; import org.apache.commons.io.*; -import static org.apache.commons.lang.StringUtils.*; -import static org.codehaus.httpcache4j.HTTPMethod.*; - import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.dom4j.*; @@ -16,63 +13,85 @@ import org.dom4j.io.*; import javax.xml.stream.*; import java.io.*; import java.net.*; -import java.sql.SQLException; +import java.sql.*; import java.util.*; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static fj.data.Option.*; +import static org.apache.commons.lang.StringUtils.*; public class GitoriousClient { public static final STAXEventReader xmlReader = new STAXEventReader(); public final String baseUrl; - private final HTTPCache http; + private final HttpClient<List<GitoriousProjectXml>> http; private final String projectsUri; - private final GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser(); - - public GitoriousClient(HTTPCache http, String baseUrl) throws URISyntaxException { - this.http = http; - this.baseUrl = new URI(baseUrl).toASCIIString(); - this.projectsUri = baseUrl + "/projects.xml"; - } - - public Set<GitoriousProjectXml> findProjects() throws Exception { - System.out.println("Fetching all projects"); - int page = 1; - - Set<GitoriousProjectXml> all = new HashSet<>(); - while (true) { - System.out.println("Fetching projects, page=" + page); - long start = currentTimeMillis(); - HTTPResponse response = http.execute(new HTTPRequest(new URI(projectsUri + "?page=" + page), GET)); - long end = currentTimeMillis(); - System.out.println("Fetched in " + (end - start) + "ms."); - - if (!response.getStatus().equals(Status.OK)) { - System.out.println("Got non-200 status from server: " + response.getStatus()); - break; - } + private final GitoriousAtomFeedParser parser; + private final F<HTTPResponse, Option<List<GitoriousProjectXml>>> parseDocument = new F<HTTPResponse, Option<List<GitoriousProjectXml>>>() { + @Override + public Option<List<GitoriousProjectXml>> f(HTTPResponse response) { MIMEType mimeType = MIMEType.valueOf(trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type"))); if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) { System.out.println("Unexpected mime type, probably at the end of the list: " + mimeType); - break; + return none(); + } + + byte[] bytes; + + try { + bytes = IOUtils.toByteArray(response.getPayload().getInputStream()); + } catch (IOException e) { + e.printStackTrace(); + return none(); } - byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream()); try { Document doc = xmlReader.readDocument(new ByteArrayInputStream(bytes)); List<GitoriousProjectXml> list = GitoriousProjectXml.projectsFromXml(doc.getRootElement()); - // This indicates the last page. - if (list.size() == 0) { - break; - } - System.out.println("Parsed out " + list.size() + " projects."); - all.addAll(list); + + return some(list); } catch (XMLStreamException e) { System.out.println("Unable to parse XML."); System.out.println(new String(bytes)); + return none(); + } + } + }; + + public GitoriousClient(HTTPCache cache, String baseUrl, GitoriousAtomFeedParser parser) throws URISyntaxException { + this.http = new HttpClient<>(cache, parseDocument); + this.baseUrl = new URI(baseUrl).toASCIIString(); + this.parser = parser; + this.projectsUri = baseUrl + "/projects.xml"; + } + + public Set<GitoriousProjectXml> findProjects() throws Exception { + System.out.println("Fetching all projects"); + int page = 1; + + Set<GitoriousProjectXml> all = new HashSet<>(); + while (true) { + Option<List<GitoriousProjectXml>> option = http.fetch(new URI(projectsUri + "?page=" + page)); + + if (option.isNone()) { + return all; + } + + List<GitoriousProjectXml> list = option.some(); + + // This indicates the last page. + if (list.size() == 0) { + break; } + all.addAll(list); + page++; } @@ -84,19 +103,20 @@ public class GitoriousClient { } public Iterable<GitoriousEvent> fetchGitoriousEvents(GitoriousRepositoryDto repository, Option<Date> lastUpdate) throws SQLException, ParseException { - System.out.println("Fetching " + repository.atomFeed); - - long start = currentTimeMillis(); - HTTPResponse response = http.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET)); - long end = currentTimeMillis(); - System.out.println("Fetched in " + (end - start) + "ms"); - - // Use the server's timestamp - Date responseDate = response.getDate().toDate(); - - System.out.println("responseDate = " + responseDate); - - return parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name); + throw new RuntimeException("re-implement"); +// System.out.println("Fetching " + repository.atomFeed); +// +// long start = currentTimeMillis(); +// HTTPResponse response = http.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET)); +// long end = currentTimeMillis(); +// System.out.println("Fetched in " + (end - start) + "ms"); +// +// // Use the server's timestamp +// Date responseDate = response.getDate().toDate(); +// +// System.out.println("responseDate = " + responseDate); +// +// return parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name); } } @@ -109,7 +129,7 @@ class GitoriousProjectXml implements Comparable<GitoriousProjectXml> { this.repositories = repositories; } - public static GitoriousProjectXml fromXml(Element project) throws URISyntaxException { + public static GitoriousProjectXml fromXml(Element project) { String slug = trimToNull(project.elementText("slug")); if (slug == null) { @@ -144,7 +164,7 @@ class GitoriousProjectXml implements Comparable<GitoriousProjectXml> { return new GitoriousProjectXml(slug, repositoryList); } - public static List<GitoriousProjectXml> projectsFromXml(Element root) throws URISyntaxException { + public static List<GitoriousProjectXml> projectsFromXml(Element root) { List<GitoriousProjectXml> projects = new ArrayList<>(); @SuppressWarnings("unchecked") List<Element> elements = (List<Element>) root.elements("project"); for (Element project : elements) { @@ -189,7 +209,7 @@ class GitoriousRepositoryXml implements Comparable<GitoriousRepositoryXml> { this.name = name; } - public static GitoriousRepositoryXml fromXml(String project, Element element) throws URISyntaxException { + public static GitoriousRepositoryXml fromXml(String project, Element element) { String name = trimToNull(element.elementText("name")); if (name == null) { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java index 9bafc17..0294896 100755 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java @@ -31,8 +31,8 @@ public class GitoriousProjectDiscovery { public GitoriousProjectDiscovery(final Config config) throws Exception { boneCp = config.createBoneCp(); - - gitoriousClient = new GitoriousClient(HttpClient.createHttpCache(config), config.gitorious.url); + GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser(config.createAbdera()); + gitoriousClient = new GitoriousClient(HttpClient.createHttpCache(config), config.gitorious.url, parser); final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java index e31fcfe..8276eaf 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java @@ -3,31 +3,66 @@ package io.trygvis.esper.testing.jenkins; import fj.*; import fj.data.*; import io.trygvis.esper.testing.*; -import static io.trygvis.esper.testing.Util.*; import io.trygvis.esper.testing.jenkins.JenkinsJobXml.*; import io.trygvis.esper.testing.util.*; -import static org.apache.commons.lang.StringUtils.*; +import org.apache.abdera.*; +import org.apache.abdera.model.*; +import org.apache.abdera.parser.*; +import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; +import org.jdom2.Document; +import org.jdom2.Element; import org.jdom2.*; +import org.joda.time.DateTime; +import javax.xml.stream.*; import java.io.*; import java.net.*; import java.util.*; import java.util.List; -import javax.xml.stream.*; + +import static fj.data.Option.*; +import static io.trygvis.esper.testing.Util.*; +import static org.apache.commons.lang.StringUtils.*; public class JenkinsClient { private final XmlHttpClient xmlHttpClient; + private final HttpClient<List<JenkinsEntryXml>> jenkinsEntryXmlClient; + private final Parser parser; - public JenkinsClient(HTTPCache http) { + public JenkinsClient(HTTPCache http, Abdera abdera) { this.xmlHttpClient = new XmlHttpClient(http); + this.parser = abdera.getParser(); + + jenkinsEntryXmlClient = new HttpClient<>(http, new F<HTTPResponse, Option<List<JenkinsEntryXml>>>() { + public Option<List<JenkinsEntryXml>> f(HTTPResponse response) { + Feed feed = (Feed) parser.parse(response.getPayload().getInputStream()).getRoot(); + + List<JenkinsEntryXml> list = new ArrayList<>(); + + for (Entry entry : feed.getEntries()) { + try { + list.add(new JenkinsEntryXml(entry.getIdElement().getText(), new DateTime(entry.getPublished().getTime()), entry.getAlternateLinkResolvedHref().toURI())); + } catch (URISyntaxException ignore) { + } + } + + return some(list); + } + }); + } + + public Option<List<JenkinsEntryXml>> fetchRss(URI uri) throws IOException { + return jenkinsEntryXmlClient.fetch(uri); } public JenkinsXml fetchJobs(URI uri) throws XMLStreamException, JDOMException, IOException { Option<Document> d = xmlHttpClient.fetch(uri); - if(d.isNone()) { - return new JenkinsXml(Option.<String>none(), Option.<String>none(), Option.<String>none(), Collections.<JenkinsJobEntryXml>emptyList()); + if (d.isNone()) { + Option<String> n = Option.none(); + + return new JenkinsXml(n, n, n, Collections.<JenkinsJobEntryXml>emptyList()); } Element root = d.some().getRootElement(); @@ -54,7 +89,7 @@ public class JenkinsClient { public Option<JenkinsJobXml> fetchJob(URI uri) throws IOException, JDOMException, XMLStreamException { Option<Document> d = xmlHttpClient.fetch(uri); - if(d.isNone()) { + if (d.isNone()) { return Option.none(); } @@ -64,9 +99,9 @@ public class JenkinsClient { switch (name) { case "freeStyleProject": - return Option.some(JenkinsJobXml.parse(uri, JenkinsJobType.FREE_STYLE, root)); + return some(JenkinsJobXml.parse(uri, JenkinsJobType.FREE_STYLE, root)); case "mavenModuleSet": - return Option.some(JenkinsJobXml.parse(uri, JenkinsJobType.MAVEN, root)); + return some(JenkinsJobXml.parse(uri, JenkinsJobType.MAVEN, root)); default: System.out.println("Unknown project type: " + name); return Option.none(); @@ -74,6 +109,18 @@ public class JenkinsClient { } } +class JenkinsEntryXml { + public final String id; + public final DateTime timestamp; + public final URI uri; + + JenkinsEntryXml(String id, DateTime timestamp, URI uri) { + this.id = id; + this.timestamp = timestamp; + this.uri = uri; + } +} + class JenkinsXml { public final Option<String> nodeName; public final Option<String> nodeDescription; @@ -145,11 +192,11 @@ class JenkinsJobXml { Option<Integer> number = childText(element, "number").bind(Util.parseInt); Option<URI> url = childText(element, "url").bind(Util.parseUri); - if(number.isNone() || url.isNone()) { + if (number.isNone() || url.isNone()) { return Option.none(); } - return Option.some(new BuildXml(number.some(), url.some())); + return some(new BuildXml(number.some(), url.some())); } }; diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java new file mode 100644 index 0000000..757902a --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsDao.java @@ -0,0 +1,63 @@ +package io.trygvis.esper.testing.jenkins; + +import java.net.URI; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class JenkinsDao { + + private final Connection c; + private static final String JENKINS_SERVER = "uuid, url"; + + public JenkinsDao(Connection c) { + this.c = c; + } + + public List<JenkinsServerDto> selectServer() throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT " + JENKINS_SERVER + " FROM jenkins_server")) { + ResultSet rs = s.executeQuery(); + + List<JenkinsServerDto> servers = new ArrayList<>(); + while (rs.next()) { + servers.add(jenkinsServer(rs)); + } + return servers; + } + + } + + private JenkinsServerDto jenkinsServer(ResultSet rs) throws SQLException { + int i = 1; + return new JenkinsServerDto(UUID.fromString(rs.getString(i++)), URI.create(rs.getString(i))); + } +} + +class JenkinsServerDto { + public final UUID uuid; + public final URI uri; + + JenkinsServerDto(UUID uuid, URI uri) { + this.uuid = uuid; + this.uri = uri; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + JenkinsServerDto that = (JenkinsServerDto) o; + + return uuid.equals(that.uuid); + } + + @Override + public int hashCode() { + return uuid.hashCode(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java index 6be720f..6fec157 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java @@ -1,35 +1,35 @@ package io.trygvis.esper.testing.jenkins; -import fj.*; -import fj.data.*; +import com.jolbox.bonecp.*; import io.trygvis.esper.testing.*; import io.trygvis.esper.testing.object.*; import io.trygvis.esper.testing.util.*; -import static java.lang.Thread.currentThread; +import org.apache.abdera.*; import org.codehaus.httpcache4j.cache.*; -import org.joda.time.*; -import java.net.URI; -import java.util.HashSet; +import java.sql.*; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.*; +import static io.trygvis.esper.testing.object.ObjectUtil.*; +import static java.lang.Thread.*; + public class JenkinsImporter { public static void main(String[] args) throws Exception { - Config config = Config.loadFromDisk(); - + final Config config = Config.loadFromDisk(); + final BoneCPDataSource boneCp = config.createBoneCp(); HTTPCache httpCache = HttpClient.createHttpCache(config); - - final JenkinsClient jenkinsClient = new JenkinsClient(httpCache); - - HashSet<URI> servers = new HashSet<>(); - servers.add(URI.create("https://builds.apache.org")); - + Abdera abdera = config.createAbdera(); + final JenkinsClient jenkinsClient = new JenkinsClient(httpCache, abdera); final ScheduledThreadPoolExecutor executorService = new ScheduledThreadPoolExecutor(5); - ObjectManager<URI, JenkinsServer> serverManager = new ObjectManager<>("JenkinsServer", servers, new ObjectFactory<URI, JenkinsServer>() { - public JenkinsServer create(URI uri) { - return new JenkinsServer(executorService, jenkinsClient, uri); + HashSet<JenkinsServerDto> servers = new HashSet<>(); + + ObjectManager<JenkinsServerDto, ActorRef<JenkinsServer2>> serverManager = new ObjectManager<>("JenkinsServer", servers, new ObjectFactory<JenkinsServerDto, ActorRef<JenkinsServer2>>() { + public ActorRef<JenkinsServer2> create(JenkinsServerDto server) { + String name = "Jenkins: " + server.uri; + return threadedActor(name, config.jenkinsUpdateInterval, boneCp, name, new JenkinsServer2(jenkinsClient, server.uri)); } }); @@ -37,17 +37,29 @@ public class JenkinsImporter { config.addShutdownHook(currentThread(), shouldRun); while (shouldRun.get()) { - for (JenkinsServer server : serverManager.getObjects()) { - Option<P2<JenkinsXml, LocalDateTime>> o = server.getJenkins(); - - if (o.isSome()) { - P2<JenkinsXml, LocalDateTime> p = o.some(); - System.out.println("Last update: " + p._2() + ", jobs=" + p._1().jobs.size()); - } else { - System.out.println("Never updated: url=" + server.uri); + try { + java.util.List<JenkinsServerDto> newKeys; + + try (Connection c = boneCp.getConnection()) { + newKeys = new JenkinsDao(c).selectServer(); } + + serverManager.update(newKeys); + } catch (SQLException e) { + e.printStackTrace(System.out); } +// for (ActorRef<JenkinsServer2> server : serverManager.getObjects()) { +// Option<P2<JenkinsXml, LocalDateTime>> o = server.underlying().getJenkins(); +// +// if (o.isSome()) { +// P2<JenkinsXml, LocalDateTime> p = o.some(); +// System.out.println("Last update: " + p._2() + ", jobs=" + p._1().jobs.size()); +// } else { +// System.out.println("Never updated: url=" + server.uri); +// } +// } + synchronized (shouldRun) { shouldRun.wait(1000); } diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java new file mode 100644 index 0000000..bad040c --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsServer2.java @@ -0,0 +1,36 @@ +package io.trygvis.esper.testing.jenkins; + +import fj.data.*; +import io.trygvis.esper.testing.object.*; + +import java.net.*; +import java.sql.*; +import java.util.List; + +public class JenkinsServer2 implements TransactionalActor { + private final JenkinsClient client; + public final URI uri; + + public JenkinsServer2(JenkinsClient client, URI uri) { + this.client = client; + this.uri = uri; + } + + public void act(Connection c) throws Exception { + System.out.println("polling " + uri); + + Option<List<JenkinsEntryXml>> option = client.fetchRss(URI.create(uri.toASCIIString() + "/rssAll")); + + if(option.isNone()) { + return; + } + + List<JenkinsEntryXml> list = option.some(); + + System.out.println("Got " + list.size() + " entries."); + + for (JenkinsEntryXml entry : list) { + System.out.println("entry.uri = " + entry.uri); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java index ef6f2a9..420a949 100755 --- a/src/main/java/io/trygvis/esper/testing/util/HttpClient.java +++ b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java @@ -3,8 +3,6 @@ package io.trygvis.esper.testing.util; import fj.*; import fj.data.*; import io.trygvis.esper.testing.*; -import static java.lang.System.*; - import org.apache.http.conn.scheme.*; import org.apache.http.impl.client.*; import org.apache.http.impl.conn.tsccm.*; @@ -12,44 +10,59 @@ import org.apache.http.params.*; import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.codehaus.httpcache4j.resolver.*; -import org.jdom2.*; import java.io.*; import java.net.*; -import javax.xml.stream.*; + +import static java.lang.System.*; public class HttpClient<A> { private final HTTPCache http; - private final F<InputStream, Option<A>> f; + private final F<HTTPResponse, Option<A>> f; - public HttpClient(HTTPCache http, F<InputStream, Option<A>> f) { + public HttpClient(HTTPCache http, F<HTTPResponse, Option<A>> f) { this.http = http; this.f = f; } + public static <A> HttpClient<A> httpClient(HTTPCache http, final F<InputStream, Option<A>> f) { + return new HttpClient<>(http, new F<HTTPResponse, Option<A>>() { + @Override + public Option<A> f(HTTPResponse response) { + return f.f(response.getPayload().getInputStream()); + } + }); + } + public Option<A> fetch(URI uri) throws IOException { HTTPResponse response = null; try { + System.out.println("Fetching " + uri); + long start = currentTimeMillis(); response = http.execute(new HTTPRequest(uri)); + long end = currentTimeMillis(); + int code = response.getStatus().getCode(); + System.out.println("Fetched in " + (end - start) + "ms. Status: " + code); - if (response.getStatus().getCode() != 200) { - throw new IOException("Did not get 200 back, got " + response.getStatus().getCode()); + if (code != 200) { + throw new IOException("Did not get 200 back, got " + code); } -// return getDocument(response.getPayload().getInputStream()); - return f.f(response.getPayload().getInputStream()); + return f.f(response); } catch (HTTPException e) { throw new IOException(e); } finally { if (response != null) { - response.consume(); + try { + response.consume(); + } catch (Exception ignore) { + } } } } - public static HTTPCache createHttpCache(Config config) { SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); diff --git a/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java b/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java index 2c2f922..dc3b46f 100755 --- a/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java +++ b/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java @@ -14,7 +14,7 @@ public class XmlHttpClient { public XmlHttpClient(HTTPCache http) { final XmlParser parser = new XmlParser(); - httpClient = new HttpClient<>(http, new F<InputStream, Option<Document>>() { + httpClient = HttpClient.httpClient(http, new F<InputStream, Option<Document>>() { public Option<Document> f(InputStream inputStream) { return parser.parseDocument(inputStream); } diff --git a/src/main/resources/ddl-jenkins.sql b/src/main/resources/ddl-jenkins.sql index 37e82c1..0c669e8 100755 --- a/src/main/resources/ddl-jenkins.sql +++ b/src/main/resources/ddl-jenkins.sql @@ -2,8 +2,14 @@ BEGIN; DROP TABLE IF EXISTS jenkins_server; -CREATE TABLE jenkins_server( - url VARCHAR(1000) NOT NULL PRIMARY KEY +CREATE TABLE jenkins_server ( + uuid CHAR(36) NOT NULL, + url VARCHAR(1000) NOT NULL, + CONSTRAINT pk_jenkins_server PRIMARY KEY (uuid), + CONSTRAINT uq_jenkins_server__url UNIQUE (url) ); +-- INSERT INTO jenkins_server (uuid, url) VALUES ('782a75f6-40a4-11e2-aca6-20cf30557fa0', 'https://builds.apache.org'); +INSERT INTO jenkins_server (uuid, url) VALUES ('4c473c86-40ad-11e2-ae61-20cf30557fa0', 'http://ci.jruby.org'); + COMMIT; diff --git a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java index a6ba2f7..aab523f 100644 --- a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java +++ b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java @@ -2,6 +2,7 @@ package io.trygvis.esper.testing.gitorious; import fj.data.*; import junit.framework.*; +import org.apache.abdera.*; import org.dom4j.*; import java.io.*; @@ -25,7 +26,7 @@ public class TestXmlParsing extends TestCase { } public void testEventParsing() throws Exception { - GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser(); + GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser(new Abdera()); try (InputStream stream = getClass().getResourceAsStream("/gitorious/esper-test-project.atom")) { List<GitoriousEvent> events = parser.parseStream(stream, Option.<Date>none(), "esper-test-project", "esper-test-project"); |