aboutsummaryrefslogtreecommitdiff
path: root/src/main/java/io
diff options
context:
space:
mode:
authorTrygve Laugstøl <trygvis@inamo.no>2012-11-19 18:04:49 +0100
committerTrygve Laugstøl <trygvis@inamo.no>2012-11-19 18:05:27 +0100
commitaf92579129943acac73542f4e05e1c7faeda0994 (patch)
treeefa76694deade25434fdfb359c130b278d6b608f /src/main/java/io
parent389b8373b7f04a5bdb039b9b690daa12a50fb144 (diff)
downloadesper-testing-af92579129943acac73542f4e05e1c7faeda0994.tar.gz
esper-testing-af92579129943acac73542f4e05e1c7faeda0994.tar.bz2
esper-testing-af92579129943acac73542f4e05e1c7faeda0994.tar.xz
esper-testing-af92579129943acac73542f4e05e1c7faeda0994.zip
o Adding logic to stop fetching pages when Gitorious returns non-XML response.
o Adding support for self-signed certificates with https. o Moving client code to GitoriousClient. o Adding Nexus client and an importer that fetches all artifacts. o A nexus client that actually fetches the entire set of artifacts.
Diffstat (limited to 'src/main/java/io')
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/Config.java48
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/Dao.java19
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java232
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java112
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/HttpClient.java82
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java4
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java58
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java77
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java2
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusClient.java75
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusDao.java52
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java24
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusParser.java197
13 files changed, 915 insertions, 67 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/Config.java b/src/main/java/io/trygvis/esper/testing/Config.java
new file mode 100755
index 0000000..d089511
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/Config.java
@@ -0,0 +1,48 @@
+package io.trygvis.esper.testing;
+
+import fj.data.*;
+import org.apache.commons.httpclient.protocol.*;
+import static org.apache.commons.lang.StringUtils.*;
+import org.apache.log4j.*;
+
+import java.io.*;
+import java.util.*;
+
+public class Config {
+ public final String gitoriousUrl;
+ public final Option<String> gitoriousSessionValue;
+ public final String nexusUrl;
+
+ public Config(String gitoriousUrl, Option<String> gitoriousSessionValue, String nexusUrl) {
+ this.gitoriousUrl = gitoriousUrl;
+ this.gitoriousSessionValue = gitoriousSessionValue;
+ this.nexusUrl = nexusUrl;
+ }
+
+ public static Config loadFromDisk() throws IOException {
+ configureLog4j();
+
+ Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory) new EasySSLProtocolSocketFactory(), 443));
+
+ Properties properties = new Properties();
+ try (FileInputStream inputStream = new FileInputStream("config.properties")) {
+ properties.load(inputStream);
+ }
+
+ return new Config(trimToNull(properties.getProperty("gitorious.url")),
+ Option.fromNull(trimToNull(properties.getProperty("gitorious.sessionValue"))),
+ trimToNull(properties.getProperty("nexus.url")));
+ }
+
+ public static void configureLog4j() {
+ Properties properties = new Properties();
+ properties.setProperty("log4j.rootLogger", "DEBUG, A1");
+ properties.setProperty("log4j.logger.httpclient.wire.content", "INFO");
+ properties.setProperty("log4j.logger.httpclient.wire.header", "INFO");
+ properties.setProperty("log4j.logger.org.apache.commons.httpclient", "INFO");
+ properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender");
+ properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
+ properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
+ PropertyConfigurator.configure(properties);
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/Dao.java b/src/main/java/io/trygvis/esper/testing/Dao.java
index 25535ee..79981ad 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/Dao.java
+++ b/src/main/java/io/trygvis/esper/testing/Dao.java
@@ -1,18 +1,29 @@
package io.trygvis.esper.testing;
import fj.*;
+import org.joda.time.*;
import java.sql.*;
+import java.util.*;
public class Dao {
private final Connection c;
+ private final Map<String, PreparedStatement> statements = new HashMap<>();
protected Dao(Connection c) {
this.c = c;
}
protected PreparedStatement prepareStatement(String sql) throws SQLException {
- return c.prepareStatement(sql);
+ PreparedStatement s = statements.get(sql);
+
+ if (s != null) {
+ return s;
+ }
+
+ statements.put(sql, c.prepareStatement(sql));
+
+ return s;
}
public static final F<Timestamp, java.util.Date> timestampToDate = new F<Timestamp, java.util.Date>() {
@@ -21,6 +32,12 @@ public class Dao {
}
};
+ public static final F<Timestamp, LocalDateTime> timestampToLocalDateTime = new F<Timestamp, LocalDateTime>() {
+ public LocalDateTime f(Timestamp timestamp) {
+ return new LocalDateTime(timestamp.getTime());
+ }
+ };
+
public static final F<java.util.Date, Timestamp> dateToTimestamp = new F<java.util.Date, Timestamp>() {
public Timestamp f(java.util.Date date) {
return new Timestamp(date.getTime());
diff --git a/src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java b/src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java
new file mode 100755
index 0000000..c0bf0a5
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java
@@ -0,0 +1,232 @@
+/*
+ * $HeadURL$
+ * $Revision$
+ * $Date$
+ *
+ * ====================================================================
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+package io.trygvis.esper.testing;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+
+import javax.net.SocketFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+
+import org.apache.commons.httpclient.ConnectTimeoutException;
+import org.apache.commons.httpclient.HttpClientError;
+import org.apache.commons.httpclient.params.HttpConnectionParams;
+import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * <p>
+ * EasySSLProtocolSocketFactory can be used to creats SSL {@link Socket}s
+ * that accept self-signed certificates.
+ * </p>
+ * <p>
+ * This socket factory SHOULD NOT be used for productive systems
+ * due to security reasons, unless it is a concious decision and
+ * you are perfectly aware of security implications of accepting
+ * self-signed certificates
+ * </p>
+ *
+ * <p>
+ * Example of using custom protocol socket factory for a specific host:
+ * <pre>
+ * Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
+ *
+ * URI uri = new URI("https://localhost/", true);
+ * // use relative url only
+ * GetMethod httpget = new GetMethod(uri.getPathQuery());
+ * HostConfiguration hc = new HostConfiguration();
+ * hc.setHost(uri.getHost(), uri.getPort(), easyhttps);
+ * HttpClient client = new HttpClient();
+ * client.executeMethod(hc, httpget);
+ * </pre>
+ * </p>
+ * <p>
+ * Example of using custom protocol socket factory per default instead of the standard one:
+ * <pre>
+ * Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
+ * Protocol.registerProtocol("https", easyhttps);
+ *
+ * HttpClient client = new HttpClient();
+ * GetMethod httpget = new GetMethod("https://localhost/");
+ * client.executeMethod(httpget);
+ * </pre>
+ * </p>
+ *
+ * @author <a href="mailto:oleg -at- ural.ru">Oleg Kalnichevski</a>
+ *
+ * <p>
+ * DISCLAIMER: HttpClient developers DO NOT actively support this component.
+ * The component is provided as a reference material, which may be inappropriate
+ * for use without additional customization.
+ * </p>
+ */
+
+public class EasySSLProtocolSocketFactory implements SecureProtocolSocketFactory {
+
+ /** Log object for this class. */
+ private static final Log LOG = LogFactory.getLog(EasySSLProtocolSocketFactory.class);
+
+ private SSLContext sslcontext = null;
+
+ /**
+ * Constructor for EasySSLProtocolSocketFactory.
+ */
+ public EasySSLProtocolSocketFactory() {
+ super();
+ }
+
+ private static SSLContext createEasySSLContext() {
+ try {
+ SSLContext context = SSLContext.getInstance("SSL");
+ context.init(
+ null,
+ new TrustManager[] {new EasyX509TrustManager(null)},
+ null);
+ return context;
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new HttpClientError(e.toString());
+ }
+ }
+
+ private SSLContext getSSLContext() {
+ if (this.sslcontext == null) {
+ this.sslcontext = createEasySSLContext();
+ }
+ return this.sslcontext;
+ }
+
+ /**
+ * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int,java.net.InetAddress,int)
+ */
+ public Socket createSocket(
+ String host,
+ int port,
+ InetAddress clientHost,
+ int clientPort)
+ throws IOException, UnknownHostException {
+
+ return getSSLContext().getSocketFactory().createSocket(
+ host,
+ port,
+ clientHost,
+ clientPort
+ );
+ }
+
+ /**
+ * Attempts to get a new socket connection to the given host within the given time limit.
+ * <p>
+ * To circumvent the limitations of older JREs that do not support connect timeout a
+ * controller thread is executed. The controller thread attempts to create a new socket
+ * within the given limit of time. If socket constructor does not return until the
+ * timeout expires, the controller terminates and throws an {@link ConnectTimeoutException}
+ * </p>
+ *
+ * @param host the host name/IP
+ * @param port the port on the host
+ * @param clientHost the local host name/IP to bind the socket to
+ * @param clientPort the port on the local machine
+ * @param params {@link HttpConnectionParams Http connection parameters}
+ *
+ * @return Socket a new socket
+ *
+ * @throws IOException if an I/O error occurs while creating the socket
+ * @throws UnknownHostException if the IP address of the host cannot be
+ * determined
+ */
+ public Socket createSocket(
+ final String host,
+ final int port,
+ final InetAddress localAddress,
+ final int localPort,
+ final HttpConnectionParams params
+ ) throws IOException, UnknownHostException, ConnectTimeoutException {
+ if (params == null) {
+ throw new IllegalArgumentException("Parameters may not be null");
+ }
+ int timeout = params.getConnectionTimeout();
+ SocketFactory socketfactory = getSSLContext().getSocketFactory();
+ if (timeout == 0) {
+ return socketfactory.createSocket(host, port, localAddress, localPort);
+ } else {
+ Socket socket = socketfactory.createSocket();
+ SocketAddress localaddr = new InetSocketAddress(localAddress, localPort);
+ SocketAddress remoteaddr = new InetSocketAddress(host, port);
+ socket.bind(localaddr);
+ socket.connect(remoteaddr, timeout);
+ return socket;
+ }
+ }
+
+ /**
+ * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int)
+ */
+ public Socket createSocket(String host, int port)
+ throws IOException, UnknownHostException {
+ return getSSLContext().getSocketFactory().createSocket(
+ host,
+ port
+ );
+ }
+
+ /**
+ * @see SecureProtocolSocketFactory#createSocket(java.net.Socket,java.lang.String,int,boolean)
+ */
+ public Socket createSocket(
+ Socket socket,
+ String host,
+ int port,
+ boolean autoClose)
+ throws IOException, UnknownHostException {
+ return getSSLContext().getSocketFactory().createSocket(
+ socket,
+ host,
+ port,
+ autoClose
+ );
+ }
+
+ public boolean equals(Object obj) {
+ return ((obj != null) && obj.getClass().equals(EasySSLProtocolSocketFactory.class));
+ }
+
+ public int hashCode() {
+ return EasySSLProtocolSocketFactory.class.hashCode();
+ }
+
+}
diff --git a/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java b/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java
new file mode 100755
index 0000000..9e35075
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java
@@ -0,0 +1,112 @@
+package io.trygvis.esper.testing;
+
+/*
+ * ====================================================================
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ *
+ */
+
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+/**
+ * <p>
+ * EasyX509TrustManager unlike default {@link X509TrustManager} accepts
+ * self-signed certificates.
+ * </p>
+ * <p>
+ * This trust manager SHOULD NOT be used for productive systems
+ * due to security reasons, unless it is a concious decision and
+ * you are perfectly aware of security implications of accepting
+ * self-signed certificates
+ * </p>
+ *
+ * @author <a href="mailto:adrian.sutton@ephox.com">Adrian Sutton</a>
+ * @author <a href="mailto:oleg@ural.ru">Oleg Kalnichevski</a>
+ *
+ * <p>
+ * DISCLAIMER: HttpClient developers DO NOT actively support this component.
+ * The component is provided as a reference material, which may be inappropriate
+ * for use without additional customization.
+ * </p>
+ */
+
+public class EasyX509TrustManager implements X509TrustManager
+{
+ private X509TrustManager standardTrustManager = null;
+
+ /** Log object for this class. */
+// private static final Log LOG = LogFactory.getLog(EasyX509TrustManager.class);
+
+ /**
+ * Constructor for EasyX509TrustManager.
+ */
+ public EasyX509TrustManager(KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException {
+ super();
+ TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ factory.init(keystore);
+ TrustManager[] trustmanagers = factory.getTrustManagers();
+ if (trustmanagers.length == 0) {
+ throw new NoSuchAlgorithmException("no trust manager found");
+ }
+ this.standardTrustManager = (X509TrustManager)trustmanagers[0];
+ }
+
+ /**
+ * @see javax.net.ssl.X509TrustManager#checkClientTrusted(X509Certificate[],String authType)
+ */
+ public void checkClientTrusted(X509Certificate[] certificates,String authType) throws CertificateException {
+ standardTrustManager.checkClientTrusted(certificates,authType);
+ }
+
+ /**
+ * @see javax.net.ssl.X509TrustManager#checkServerTrusted(X509Certificate[],String authType)
+ */
+ public void checkServerTrusted(X509Certificate[] certificates,String authType) throws CertificateException {
+// if (certificates != null) {
+// System.out.println("Server certificate chain:");
+// for (int i = 0; i < certificates.length; i++) {
+// System.out.println("X509Certificate[" + i + "]=" + certificates[i]);
+// }
+// }
+
+ if ((certificates != null) && (certificates.length == 1)) {
+ certificates[0].checkValidity();
+ } else {
+ standardTrustManager.checkServerTrusted(certificates,authType);
+ }
+ }
+
+ /**
+ * @see javax.net.ssl.X509TrustManager#getAcceptedIssuers()
+ */
+ public X509Certificate[] getAcceptedIssuers() {
+ return this.standardTrustManager.getAcceptedIssuers();
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/HttpClient.java b/src/main/java/io/trygvis/esper/testing/HttpClient.java
new file mode 100755
index 0000000..d9adc8d
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/HttpClient.java
@@ -0,0 +1,82 @@
+package io.trygvis.esper.testing;
+
+import static java.lang.System.*;
+import org.codehaus.httpcache4j.*;
+import org.codehaus.httpcache4j.cache.*;
+import org.codehaus.httpcache4j.client.*;
+import org.codehaus.httpcache4j.resolver.*;
+
+import java.io.*;
+
+public class HttpClient {
+
+ public static HTTPCache createHttpClient(Config config) {
+ return new HTTPCache(new MemoryCacheStorage(), createResponseResolver(config));
+ }
+
+ private static ResponseResolver createResponseResolver(final Config config) {
+ ResponseResolver responseResolver = HTTPClientResponseResolver.createMultithreadedInstance();
+
+ if (config.gitoriousSessionValue.isSome()) {
+ responseResolver = new GitoriousResponseResolver(config.gitoriousSessionValue.some(), responseResolver);
+ }
+
+ responseResolver = new TimingResponseResolver(responseResolver);
+
+ return responseResolver;
+ }
+
+ private static class TimingResponseResolver implements ResponseResolver {
+ private final ResponseResolver r;
+
+ private TimingResponseResolver(ResponseResolver r) {
+ this.r = r;
+ }
+
+ public HTTPResponse resolve(HTTPRequest request) throws IOException {
+ System.out.println(request.getRequestURI() + ": Executing");
+ long start = currentTimeMillis();
+ Status status = null;
+ try {
+ HTTPResponse response = r.resolve(request);
+ status = response.getStatus();
+ return response;
+ } finally {
+ long end = currentTimeMillis();
+
+ String s = request.getRequestURI() + ": Executed in " + (end - start) + "ms, ";
+
+ if (status != null) {
+ s += "response: " + status.getCode() + " " + status.getName();
+ } else {
+ s += "with exception";
+ }
+
+ System.out.println(s);
+ }
+ }
+
+ public void shutdown() {
+ r.shutdown();
+ }
+ }
+
+ private static class GitoriousResponseResolver implements ResponseResolver {
+ private final String gitoriousSessionValue;
+ private final ResponseResolver responseResolver;
+
+ public GitoriousResponseResolver(String gitoriousSessionValue, ResponseResolver responseResolver) {
+ this.gitoriousSessionValue = gitoriousSessionValue;
+ this.responseResolver = responseResolver;
+ }
+
+ public HTTPResponse resolve(HTTPRequest request) throws IOException {
+ request = request.addHeader("Cookie", "_gitorious_sess=" + gitoriousSessionValue);
+ return responseResolver.resolve(request);
+ }
+
+ public void shutdown() {
+ responseResolver.shutdown();
+ }
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java
index 7e0a1b7..b7b21d6 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java
@@ -2,9 +2,9 @@ package io.trygvis.esper.testing.gitorious;
import fj.data.*;
import org.apache.abdera.*;
-import org.apache.abdera.model.*;
import org.apache.abdera.model.Document;
import org.apache.abdera.model.Element;
+import org.apache.abdera.model.*;
import org.apache.abdera.parser.*;
import org.dom4j.*;
import org.dom4j.io.*;
@@ -82,7 +82,7 @@ public class GitoriousAtomFeedParser {
switch (event) {
case "created repository":
case "created branch":
- // This is similar "pushed", but doesn't contain any info on commit IDs or branches
+ // This is similar "pushed", but doesn't contain any info on commit IDs or branches
case "started development of":
return null;
case "pushed":
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
index 892d8d0..f44635f 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -1,9 +1,13 @@
package io.trygvis.esper.testing.gitorious;
import static java.lang.System.*;
+
+import fj.data.Option;
+import org.apache.abdera.parser.ParseException;
import org.apache.commons.io.*;
import static org.apache.commons.lang.StringUtils.*;
import static org.codehaus.httpcache4j.HTTPMethod.*;
+
import org.codehaus.httpcache4j.*;
import org.codehaus.httpcache4j.cache.*;
import org.dom4j.*;
@@ -12,16 +16,18 @@ import org.dom4j.io.*;
import javax.xml.stream.*;
import java.io.*;
import java.net.*;
+import java.sql.SQLException;
import java.util.*;
public class GitoriousClient {
public static final STAXEventReader xmlReader = new STAXEventReader();
- private final HTTPCache httpCache;
public final String baseUrl;
+ private final HTTPCache http;
private final String projectsUri;
+ private final GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser();
- public GitoriousClient(HTTPCache httpCache, String baseUrl) throws URISyntaxException {
- this.httpCache = httpCache;
+ public GitoriousClient(HTTPCache http, String baseUrl) throws URISyntaxException {
+ this.http = http;
this.baseUrl = new URI(baseUrl).toASCIIString();
this.projectsUri = baseUrl + "/projects.xml";
}
@@ -31,13 +37,23 @@ public class GitoriousClient {
int page = 1;
Set<GitoriousProjectXml> all = new HashSet<>();
- while (page <= 10) {
- System.out.println("Fetching projects XML, page=" + page);
+ while (true) {
+ System.out.println("Fetching projects, page=" + page);
long start = currentTimeMillis();
- HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET);
- HTTPResponse response = httpCache.execute(request);
+ HTTPResponse response = http.execute(new HTTPRequest(new URI(projectsUri + "?page=" + page), GET));
long end = currentTimeMillis();
- System.out.println("Fetched XML in " + (end - start) + "ms.");
+ System.out.println("Fetched in " + (end - start) + "ms.");
+
+ if (!response.getStatus().equals(Status.OK)) {
+ System.out.println("Got non-200 status from server: " + response.getStatus());
+ break;
+ }
+
+ MIMEType mimeType = MIMEType.valueOf(trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type")));
+ if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) {
+ System.out.println("Unexpected mime type, probably at the end of the list: " + mimeType);
+ break;
+ }
byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream());
try {
@@ -66,6 +82,22 @@ public class GitoriousClient {
public URI atomFeed(String projectSlug, String repositoryName) {
return URI.create(baseUrl + "/" + projectSlug + "/" + repositoryName + ".atom");
}
+
+ public Iterable<GitoriousEvent> fetchGitoriousEvents(GitoriousRepository repository, Option<Date> lastUpdate) throws SQLException, ParseException {
+ System.out.println("Fetching " + repository.atomFeed);
+
+ long start = currentTimeMillis();
+ HTTPResponse response = http.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET));
+ long end = currentTimeMillis();
+ System.out.println("Fetched in " + (end - start) + "ms");
+
+ // Use the server's timestamp
+ Date responseDate = response.getDate().toDate();
+
+ System.out.println("responseDate = " + responseDate);
+
+ return parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name);
+ }
}
class GitoriousProjectXml implements Comparable<GitoriousProjectXml> {
@@ -138,10 +170,7 @@ class GitoriousProjectXml implements Comparable<GitoriousProjectXml> {
GitoriousProjectXml that = (GitoriousProjectXml) o;
- if (!repositories.equals(that.repositories)) return false;
- if (!slug.equals(that.slug)) return false;
-
- return true;
+ return repositories.equals(that.repositories) && slug.equals(that.slug);
}
public int hashCode() {
@@ -186,10 +215,7 @@ class GitoriousRepositoryXml implements Comparable<GitoriousRepositoryXml> {
GitoriousRepositoryXml that = (GitoriousRepositoryXml) o;
- if (!name.equals(that.name)) return false;
- if (!projectSlug.equals(that.projectSlug)) return false;
-
- return true;
+ return name.equals(that.name) && projectSlug.equals(that.projectSlug);
}
public int hashCode() {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index b4bc683..1e7a7fd 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -6,9 +6,7 @@ import static fj.data.Option.*;
import io.trygvis.esper.testing.*;
import static java.lang.System.*;
import org.apache.abdera.parser.*;
-import org.codehaus.httpcache4j.*;
-import org.codehaus.httpcache4j.cache.*;
-import org.codehaus.httpcache4j.resolver.*;
+import org.apache.commons.httpclient.protocol.*;
import java.io.*;
import java.net.*;
@@ -20,37 +18,34 @@ import java.util.Set;
import java.util.concurrent.*;
public class GitoriousImporter {
- private final GitoriousAtomFeedParser parser;
private final BoneCP boneCp;
private final GitoriousClient gitoriousClient;
- private final HTTPCache httpCache;
public static void main(String[] args) throws Exception {
- Main.configureLog4j();
- new GitoriousImporter(DbMain.JDBC_URL, "esper", "");
+ Config config = Config.loadFromDisk();
+ new GitoriousImporter(config, DbMain.JDBC_URL, "esper", "esper");
}
- public GitoriousImporter(String jdbcUrl, String jdbcUsername, String jdbcPassword) throws Exception {
- parser = new GitoriousAtomFeedParser();
+ public GitoriousImporter(Config config, final String jdbcUrl, final String jdbcUsername, final String jdbcPassword) throws Exception {
+ BoneCPConfig boneCPConfig = new BoneCPConfig(){{
+ setJdbcUrl(jdbcUrl);
+ setUsername(jdbcUsername);
+ setPassword(jdbcPassword);
+ setDefaultAutoCommit(false);
+ setMaxConnectionsPerPartition(10);
+ }};
- BoneCPConfig config = new BoneCPConfig();
- config.setJdbcUrl(jdbcUrl);
- config.setUsername(jdbcUsername);
- config.setPassword(jdbcPassword);
- config.setDefaultAutoCommit(false);
- config.setMaxConnectionsPerPartition(10);
+ boneCp = new BoneCP(boneCPConfig);
- boneCp = new BoneCP(config);
-
- httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
-
- gitoriousClient = new GitoriousClient(httpCache, "http://gitorious.org");
+ gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitoriousUrl);
final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(2);
- boolean projectsUpdateEnabled = false;
+ boolean projectsUpdateEnabled = true;
int projectsUpdateDelay = 0 * 1000;
int projectsUpdateInterval = 60 * 1000;
+
+ boolean repositoriesUpdateEnabled = false;
int repositoriesUpdateDelay = 0;
int repositoriesUpdateInterval = 60 * 1000;
@@ -66,15 +61,17 @@ public class GitoriousImporter {
}, projectsUpdateDelay, projectsUpdateInterval, TimeUnit.MILLISECONDS);
}
- service.scheduleAtFixedRate(new Runnable() {
- public void run() {
- try {
- updateRepositories();
- } catch (Exception e) {
- e.printStackTrace(System.out);
+ if (repositoriesUpdateEnabled) {
+ service.scheduleAtFixedRate(new Runnable() {
+ public void run() {
+ try {
+ updateRepositories();
+ } catch (Exception e) {
+ e.printStackTrace(System.out);
+ }
}
- }
- }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS);
+ }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS);
+ }
}
private void discoverProjects() throws Exception {
@@ -105,8 +102,7 @@ public class GitoriousImporter {
for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) {
boolean found = false;
- for (Iterator<GitoriousRepositoryXml> it = project.repositories.iterator(); it.hasNext(); ) {
- GitoriousRepositoryXml repo = it.next();
+ for (GitoriousRepositoryXml repo : project.repositories) {
if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) {
found = true;
break;
@@ -125,8 +121,7 @@ public class GitoriousImporter {
for (String project : projectDao.selectSlugs()) {
boolean found = false;
- for (Iterator<GitoriousProjectXml> it = projects.iterator(); it.hasNext(); ) {
- GitoriousProjectXml p = it.next();
+ for (GitoriousProjectXml p : projects) {
if (p.slug.equals(project)) {
found = true;
break;
@@ -165,21 +160,9 @@ public class GitoriousImporter {
Option<Date> lastUpdate = repository.lastSuccessfulUpdate;
- System.out.println("Fetching " + repository.atomFeed);
-
- long start = currentTimeMillis();
- HTTPResponse response = httpCache.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET));
- long end = currentTimeMillis();
- System.out.println("Fetched in " + (end - start) + "ms");
-
- // Use the server's timestamp
- Date responseDate = response.getDate().toDate();
-
- System.out.println("responseDate = " + responseDate);
-
- List<GitoriousEvent> events;
+ Iterable<GitoriousEvent> events;
try {
- events = parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name);
+ events = gitoriousClient.fetchGitoriousEvents(repository, lastUpdate);
} catch (ParseException e) {
repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none());
System.out.println("Error parsing " + repository.atomFeed);
diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java
index b7d88dc..247dfe3 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java
@@ -13,7 +13,7 @@ import java.util.concurrent.*;
public class JenkinsImporter {
public static void main(String[] args) throws Exception {
- Main.configureLog4j();
+ Config.configureLog4j();
final JenkinsClient jenkinsClient = new JenkinsClient(http);
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
new file mode 100755
index 0000000..6477a80
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
@@ -0,0 +1,75 @@
+package io.trygvis.esper.testing.nexus;
+
+import fj.data.*;
+import static fj.data.Option.*;
+import static io.trygvis.esper.testing.nexus.NexusParser.*;
+import org.apache.commons.io.*;
+import org.apache.commons.lang.*;
+import static org.codehaus.httpcache4j.HTTPMethod.*;
+import org.codehaus.httpcache4j.*;
+import org.codehaus.httpcache4j.cache.*;
+import org.codehaus.httpcache4j.util.*;
+
+import java.io.*;
+import java.net.*;
+import javax.xml.stream.*;
+
+public class NexusClient {
+ private final HTTPCache http;
+ private final String nexusUrl;
+
+ public NexusClient(HTTPCache http, String nexusUrl) {
+ this.http = http;
+ this.nexusUrl = nexusUrl;
+ }
+
+ public ArtifactSearchResult fetchIndex(String groupId, Option<String> repositoryId) throws IOException {
+ ArtifactSearchResult aggregate = fetchIndexPage(groupId, repositoryId, Option.<Integer>none());
+ ArtifactSearchResult result = aggregate;
+
+ while(result.artifacts.size() > 0) {
+ result = fetchIndexPage(groupId, repositoryId, some(aggregate.artifacts.size()));
+ aggregate = aggregate.append(result);
+ }
+
+ return aggregate;
+ }
+
+ public ArtifactSearchResult fetchIndexPage(String groupId, Option<String> repositoryId, Option<Integer> from) throws IOException {
+ URIBuilder uriBuilder = URIBuilder.fromURI(URI.create(nexusUrl)).
+ addRawPath("/service/local/lucene/search").
+ addParameter("g", groupId + ".*");
+
+ if (repositoryId.isSome()) {
+ uriBuilder = uriBuilder.addParameter("repositoryId", repositoryId.some());
+ }
+
+ if (from.isSome()) {
+ uriBuilder = uriBuilder.addParameter("from", from.some().toString());
+ }
+
+ HTTPResponse response = http.execute(new HTTPRequest(uriBuilder.toURI(), GET));
+
+ int statusCode = response.getStatus().getCode();
+ if (statusCode != 200) {
+ throw new IOException("Got " + statusCode + " from Nexus search, expected 200.");
+ }
+
+ MIMEType mimeType = MIMEType.valueOf(StringUtils.trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type")));
+ if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) {
+ throw new IOException("Unexpected mime type: " + mimeType);
+ }
+
+ byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream());
+
+ try {
+ ArtifactSearchResult result = parseDocument(new ByteArrayInputStream(bytes));
+ System.out.println("Parsed out " + result.artifacts.size() + " artifacts.");
+ return result;
+ } catch (XMLStreamException e) {
+ System.out.println("Unable to parse XML.");
+ System.out.println(new String(bytes));
+ throw new RuntimeException("Unable to parse XML.", e);
+ }
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
new file mode 100755
index 0000000..39d4233
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
@@ -0,0 +1,52 @@
+package io.trygvis.esper.testing.nexus;
+
+import fj.data.*;
+import static fj.data.Option.*;
+import io.trygvis.esper.testing.*;
+import org.joda.time.*;
+
+import java.sql.*;
+
+public class NexusDao extends Dao {
+ protected NexusDao(Connection c) {
+ super(c);
+ }
+
+ public void insertRepository(String repositoryId, LocalDateTime discoveryDate) throws SQLException {
+ PreparedStatement s = prepareStatement("INSERT INTO nexus_repository(id) VALUES(?)");
+ s.setString(1, repositoryId);
+ s.executeUpdate();
+ }
+
+ public Option<NexusRepository> selectRepository(String repositoryId) throws SQLException {
+ PreparedStatement s = prepareStatement("SELECT id, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?");
+ s.setString(1, repositoryId);
+
+ try (ResultSet rs = s.executeQuery()) {
+ if (!rs.next()) {
+ return Option.none();
+ }
+
+ return some(new NexusRepository(
+ rs.getString(1),
+ fromNull(rs.getTimestamp(2)).map(timestampToLocalDateTime),
+ fromNull(rs.getTimestamp(3)).map(timestampToLocalDateTime),
+ fromNull(rs.getTimestamp(4)).map(timestampToLocalDateTime)
+ ));
+ }
+ }
+}
+
+class NexusRepository {
+ public final String repositoryId;
+ public final Option<LocalDateTime> discoveryDate;
+ public final Option<LocalDateTime> lastUpdate;
+ public final Option<LocalDateTime> lastSuccessfulUpdate;
+
+ NexusRepository(String repositoryId, Option<LocalDateTime> discoveryDate, Option<LocalDateTime> lastUpdate, Option<LocalDateTime> lastSuccessfulUpdate) {
+ this.repositoryId = repositoryId;
+ this.discoveryDate = discoveryDate;
+ this.lastUpdate = lastUpdate;
+ this.lastSuccessfulUpdate = lastSuccessfulUpdate;
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
new file mode 100755
index 0000000..896f0e2
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
@@ -0,0 +1,24 @@
+package io.trygvis.esper.testing.nexus;
+
+import com.google.common.collect.*;
+import fj.data.*;
+import io.trygvis.esper.testing.*;
+import org.apache.commons.lang.*;
+
+import java.io.*;
+import java.util.*;
+
+public class NexusImporter {
+ public static void main(String[] args) throws IOException {
+ Config config = Config.loadFromDisk();
+
+ NexusClient client = new NexusClient(HttpClient.createHttpClient(config), config.nexusUrl);
+
+ ArtifactSearchResult result = client.fetchIndex("eu.nets", Option.<String>none());
+ ArrayList<ArtifactXml> artifacts = Lists.newArrayList(result.artifacts);
+ Collections.sort(artifacts);
+ for (ArtifactXml artifact : artifacts) {
+ System.out.println("repo=" + StringUtils.join(artifact.repositories(), ", ") + ", artifact=" + artifact.getId());
+ }
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
new file mode 100755
index 0000000..05dfd5b
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
@@ -0,0 +1,197 @@
+package io.trygvis.esper.testing.nexus;
+
+import com.google.common.base.*;
+import com.google.common.collect.*;
+import fj.*;
+import fj.data.*;
+import static fj.data.Option.fromNull;
+import static org.apache.commons.lang.StringUtils.*;
+import org.dom4j.*;
+import org.dom4j.io.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.xml.stream.*;
+
+public class NexusParser {
+ public static final STAXEventReader xmlReader = new STAXEventReader();
+
+ public static ArtifactSearchResult parseDocument(InputStream is) throws XMLStreamException {
+ Document doc = xmlReader.readDocument(is);
+
+ Option<Integer> totalCount = fromNull(trimToNull(doc.getRootElement().elementText("totalCount"))).
+ bind(Option.parseInt);
+ if (totalCount.isNone()) {
+ throw new RuntimeException("Could not find required element <totalCount>");
+ }
+
+ boolean tooManyResults = "true".equals(trimToNull(doc.getRootElement().elementText("tooManyResults")));
+
+ List<ArtifactXml> list = new ArrayList<>();
+ for (Object o : doc.selectNodes("/searchNGResponse/data/artifact")) {
+ if (!(o instanceof Element)) {
+ continue;
+ }
+
+ Element artifact = (Element) o;
+
+ String groupId = trimToNull(artifact.elementText("groupId"));
+ String artifactId = trimToNull(artifact.elementText("artifactId"));
+ String version = trimToNull(artifact.elementText("version"));
+
+ if (groupId == null || artifactId == null || version == null) {
+ continue;
+ }
+
+ List<ArtifactHits> artifactHitsList = new ArrayList<>();
+
+ @SuppressWarnings("unchecked") List<Element> artifactHits = (List<Element>) artifact.selectNodes("artifactHits/artifactHit");
+ for (Element artifactHit : artifactHits) {
+ String repositoryId = trimToNull(artifactHit.elementText("repositoryId"));
+ if (repositoryId == null) {
+ continue;
+ }
+ List<ArtifactFile> files = new ArrayList<>();
+
+ @SuppressWarnings("unchecked") List<Element> artifactLinks = artifactHit.selectNodes("artifactLinks/artifactLink");
+ for (Element artifactLink : artifactLinks) {
+ Option<String> classifier = Option.fromString(trimToEmpty(artifactLink.elementText("classifier")));
+ String extension = trimToNull(artifactLink.elementText("extension"));
+
+ if (extension == null) {
+ continue;
+ }
+
+ files.add(new ArtifactFile(classifier, extension));
+ }
+
+ artifactHitsList.add(new ArtifactHits(repositoryId, files));
+ }
+
+ list.add(new ArtifactXml(groupId, artifactId, version, artifactHitsList));
+ }
+
+ return new ArtifactSearchResult(totalCount.some(), tooManyResults, list);
+ }
+}
+
+class ArtifactSearchResult {
+ public final int totalCount;
+ public final boolean tooManyResults;
+ public final List<ArtifactXml> artifacts;
+
+ ArtifactSearchResult(int totalCount, boolean tooManyResults, List<ArtifactXml> artifacts) {
+ this.totalCount = totalCount;
+ this.tooManyResults = tooManyResults;
+ this.artifacts = artifacts;
+ }
+
+ public ArtifactSearchResult append(ArtifactSearchResult result) {
+ List<ArtifactXml> list = Lists.newArrayList(artifacts);
+ list.addAll(result.artifacts);
+ return new ArtifactSearchResult(result.totalCount, result.tooManyResults, list);
+ }
+}
+
+class ArtifactXml implements Comparable<ArtifactXml> {
+ public final String groupId;
+ public final String artifactId;
+ public final String version;
+ public final List<ArtifactHits> hits;
+
+ ArtifactXml(String groupId, String artifactId, String version, List<ArtifactHits> hits) {
+ this.groupId = groupId;
+ this.artifactId = artifactId;
+ this.version = version;
+ this.hits = hits;
+ }
+
+ public static Predicate<ArtifactXml> repositoryFilter(final String repositoryId) {
+ return new Predicate<ArtifactXml>() {
+ public boolean apply(ArtifactXml artifact) {
+ return Iterables.any(artifact.hits, new Predicate<ArtifactHits>() {
+ public boolean apply(ArtifactHits hits) {
+ return hits.repositoryId.equals(repositoryId);
+ }
+ });
+ }
+ };
+ }
+
+ public FlatArtifact flatten(String repositoryId) {
+ for (ArtifactHits hit : hits) {
+ if (hit.repositoryId.equals(repositoryId)) {
+ return new FlatArtifact(groupId, artifactId, version, hit.files);
+ }
+ }
+
+ throw new RuntimeException("No hits in repository " + repositoryId);
+ }
+
+ public int compareTo(ArtifactXml o) {
+ int c = groupId.compareTo(o.groupId);
+
+ if(c != 0) {
+ return c;
+ }
+
+ c = artifactId.compareTo(o.artifactId);
+
+ if(c != 0) {
+ return c;
+ }
+
+ return version.compareTo(o.version);
+ }
+
+ public String getId() {
+ return groupId + ":" + artifactId + ":" + version;
+ }
+
+ public Set<String> repositories() {
+ Set<String> repositories = new HashSet<>(10);
+
+ for (ArtifactHits hit : hits) {
+ repositories.add(hit.repositoryId);
+ }
+
+ return repositories;
+ }
+}
+
+class FlatArtifact {
+ public final String groupId;
+ public final String artifactId;
+ public final String version;
+ public final List<ArtifactFile> files;
+
+ FlatArtifact(String groupId, String artifactId, String version, List<ArtifactFile> files) {
+ this.groupId = groupId;
+ this.artifactId = artifactId;
+ this.version = version;
+ this.files = files;
+ }
+}
+
+class ArtifactHits {
+ public final String repositoryId;
+ public final List<ArtifactFile> files;
+
+ ArtifactHits(String repositoryId, List<ArtifactFile> files) {
+ this.repositoryId = repositoryId;
+ this.files = files;
+ }
+}
+
+class ArtifactFile {
+ public final Option<String> classifier;
+ public final String extension;
+
+ ArtifactFile(Option<String> classifier, String extension) {
+ this.classifier = classifier;
+ this.extension = extension;
+ }
+}