From af92579129943acac73542f4e05e1c7faeda0994 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Mon, 19 Nov 2012 18:04:49 +0100 Subject: o Adding logic to stop fetching pages when Gitorious returns non-XML response. o Adding support for self-signed certificates with https. o Moving client code to GitoriousClient. o Adding Nexus client and an importer that fetches all artifacts. o A nexus client that actually fetches the entire set of artifacts. --- src/main/java/io/trygvis/esper/testing/Config.java | 48 +++++ src/main/java/io/trygvis/esper/testing/Dao.java | 19 +- .../testing/EasySSLProtocolSocketFactory.java | 232 +++++++++++++++++++++ .../esper/testing/EasyX509TrustManager.java | 112 ++++++++++ .../java/io/trygvis/esper/testing/HttpClient.java | 82 ++++++++ .../testing/gitorious/GitoriousAtomFeedParser.java | 4 +- .../esper/testing/gitorious/GitoriousClient.java | 58 ++++-- .../esper/testing/gitorious/GitoriousImporter.java | 77 +++---- .../esper/testing/jenkins/JenkinsImporter.java | 2 +- .../trygvis/esper/testing/nexus/NexusClient.java | 75 +++++++ .../io/trygvis/esper/testing/nexus/NexusDao.java | 52 +++++ .../trygvis/esper/testing/nexus/NexusImporter.java | 24 +++ .../trygvis/esper/testing/nexus/NexusParser.java | 197 +++++++++++++++++ 13 files changed, 915 insertions(+), 67 deletions(-) create mode 100755 src/main/java/io/trygvis/esper/testing/Config.java mode change 100644 => 100755 src/main/java/io/trygvis/esper/testing/Dao.java create mode 100755 src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java create mode 100755 src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java create mode 100755 src/main/java/io/trygvis/esper/testing/HttpClient.java mode change 100644 => 100755 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java mode change 100644 => 100755 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java mode change 100644 => 100755 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java mode change 100644 => 100755 src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java (limited to 'src/main/java/io/trygvis/esper/testing') diff --git a/src/main/java/io/trygvis/esper/testing/Config.java b/src/main/java/io/trygvis/esper/testing/Config.java new file mode 100755 index 0000000..d089511 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/Config.java @@ -0,0 +1,48 @@ +package io.trygvis.esper.testing; + +import fj.data.*; +import org.apache.commons.httpclient.protocol.*; +import static org.apache.commons.lang.StringUtils.*; +import org.apache.log4j.*; + +import java.io.*; +import java.util.*; + +public class Config { + public final String gitoriousUrl; + public final Option gitoriousSessionValue; + public final String nexusUrl; + + public Config(String gitoriousUrl, Option gitoriousSessionValue, String nexusUrl) { + this.gitoriousUrl = gitoriousUrl; + this.gitoriousSessionValue = gitoriousSessionValue; + this.nexusUrl = nexusUrl; + } + + public static Config loadFromDisk() throws IOException { + configureLog4j(); + + Protocol.registerProtocol("https", new Protocol("https", (ProtocolSocketFactory) new EasySSLProtocolSocketFactory(), 443)); + + Properties properties = new Properties(); + try (FileInputStream inputStream = new FileInputStream("config.properties")) { + properties.load(inputStream); + } + + return new Config(trimToNull(properties.getProperty("gitorious.url")), + Option.fromNull(trimToNull(properties.getProperty("gitorious.sessionValue"))), + trimToNull(properties.getProperty("nexus.url"))); + } + + public static void configureLog4j() { + Properties properties = new Properties(); + properties.setProperty("log4j.rootLogger", "DEBUG, A1"); + properties.setProperty("log4j.logger.httpclient.wire.content", "INFO"); + properties.setProperty("log4j.logger.httpclient.wire.header", "INFO"); + properties.setProperty("log4j.logger.org.apache.commons.httpclient", "INFO"); + properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender"); + properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout"); + properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); + PropertyConfigurator.configure(properties); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/Dao.java b/src/main/java/io/trygvis/esper/testing/Dao.java old mode 100644 new mode 100755 index 25535ee..79981ad --- a/src/main/java/io/trygvis/esper/testing/Dao.java +++ b/src/main/java/io/trygvis/esper/testing/Dao.java @@ -1,18 +1,29 @@ package io.trygvis.esper.testing; import fj.*; +import org.joda.time.*; import java.sql.*; +import java.util.*; public class Dao { private final Connection c; + private final Map statements = new HashMap<>(); protected Dao(Connection c) { this.c = c; } protected PreparedStatement prepareStatement(String sql) throws SQLException { - return c.prepareStatement(sql); + PreparedStatement s = statements.get(sql); + + if (s != null) { + return s; + } + + statements.put(sql, c.prepareStatement(sql)); + + return s; } public static final F timestampToDate = new F() { @@ -21,6 +32,12 @@ public class Dao { } }; + public static final F timestampToLocalDateTime = new F() { + public LocalDateTime f(Timestamp timestamp) { + return new LocalDateTime(timestamp.getTime()); + } + }; + public static final F dateToTimestamp = new F() { public Timestamp f(java.util.Date date) { return new Timestamp(date.getTime()); diff --git a/src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java b/src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java new file mode 100755 index 0000000..c0bf0a5 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/EasySSLProtocolSocketFactory.java @@ -0,0 +1,232 @@ +/* + * $HeadURL$ + * $Revision$ + * $Date$ + * + * ==================================================================== + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package io.trygvis.esper.testing; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.UnknownHostException; + +import javax.net.SocketFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; + +import org.apache.commons.httpclient.ConnectTimeoutException; +import org.apache.commons.httpclient.HttpClientError; +import org.apache.commons.httpclient.params.HttpConnectionParams; +import org.apache.commons.httpclient.protocol.SecureProtocolSocketFactory; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + *

+ * EasySSLProtocolSocketFactory can be used to creats SSL {@link Socket}s + * that accept self-signed certificates. + *

+ *

+ * This socket factory SHOULD NOT be used for productive systems + * due to security reasons, unless it is a concious decision and + * you are perfectly aware of security implications of accepting + * self-signed certificates + *

+ * + *

+ * Example of using custom protocol socket factory for a specific host: + * 

+ *     Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
+ *
+ *     URI uri = new URI("https://localhost/", true);
+ *     // use relative url only
+ *     GetMethod httpget = new GetMethod(uri.getPathQuery());
+ *     HostConfiguration hc = new HostConfiguration();
+ *     hc.setHost(uri.getHost(), uri.getPort(), easyhttps);
+ *     HttpClient client = new HttpClient();
+ *     client.executeMethod(hc, httpget);
+ *
+ *

+ *

+ * Example of using custom protocol socket factory per default instead of the standard one: + * 

+ *     Protocol easyhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
+ *     Protocol.registerProtocol("https", easyhttps);
+ *
+ *     HttpClient client = new HttpClient();
+ *     GetMethod httpget = new GetMethod("https://localhost/");
+ *     client.executeMethod(httpget);
+ *
+ *

+ * + * @author Oleg Kalnichevski + * + *

+ * DISCLAIMER: HttpClient developers DO NOT actively support this component. + * The component is provided as a reference material, which may be inappropriate + * for use without additional customization. + *

+ */ + +public class EasySSLProtocolSocketFactory implements SecureProtocolSocketFactory { + + /** Log object for this class. */ + private static final Log LOG = LogFactory.getLog(EasySSLProtocolSocketFactory.class); + + private SSLContext sslcontext = null; + + /** + * Constructor for EasySSLProtocolSocketFactory. + */ + public EasySSLProtocolSocketFactory() { + super(); + } + + private static SSLContext createEasySSLContext() { + try { + SSLContext context = SSLContext.getInstance("SSL"); + context.init( + null, + new TrustManager[] {new EasyX509TrustManager(null)}, + null); + return context; + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new HttpClientError(e.toString()); + } + } + + private SSLContext getSSLContext() { + if (this.sslcontext == null) { + this.sslcontext = createEasySSLContext(); + } + return this.sslcontext; + } + + /** + * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int,java.net.InetAddress,int) + */ + public Socket createSocket( + String host, + int port, + InetAddress clientHost, + int clientPort) + throws IOException, UnknownHostException { + + return getSSLContext().getSocketFactory().createSocket( + host, + port, + clientHost, + clientPort + ); + } + + /** + * Attempts to get a new socket connection to the given host within the given time limit. + *

+ * To circumvent the limitations of older JREs that do not support connect timeout a + * controller thread is executed. The controller thread attempts to create a new socket + * within the given limit of time. If socket constructor does not return until the + * timeout expires, the controller terminates and throws an {@link ConnectTimeoutException} + *

+ * + * @param host the host name/IP + * @param port the port on the host + * @param clientHost the local host name/IP to bind the socket to + * @param clientPort the port on the local machine + * @param params {@link HttpConnectionParams Http connection parameters} + * + * @return Socket a new socket + * + * @throws IOException if an I/O error occurs while creating the socket + * @throws UnknownHostException if the IP address of the host cannot be + * determined + */ + public Socket createSocket( + final String host, + final int port, + final InetAddress localAddress, + final int localPort, + final HttpConnectionParams params + ) throws IOException, UnknownHostException, ConnectTimeoutException { + if (params == null) { + throw new IllegalArgumentException("Parameters may not be null"); + } + int timeout = params.getConnectionTimeout(); + SocketFactory socketfactory = getSSLContext().getSocketFactory(); + if (timeout == 0) { + return socketfactory.createSocket(host, port, localAddress, localPort); + } else { + Socket socket = socketfactory.createSocket(); + SocketAddress localaddr = new InetSocketAddress(localAddress, localPort); + SocketAddress remoteaddr = new InetSocketAddress(host, port); + socket.bind(localaddr); + socket.connect(remoteaddr, timeout); + return socket; + } + } + + /** + * @see SecureProtocolSocketFactory#createSocket(java.lang.String,int) + */ + public Socket createSocket(String host, int port) + throws IOException, UnknownHostException { + return getSSLContext().getSocketFactory().createSocket( + host, + port + ); + } + + /** + * @see SecureProtocolSocketFactory#createSocket(java.net.Socket,java.lang.String,int,boolean) + */ + public Socket createSocket( + Socket socket, + String host, + int port, + boolean autoClose) + throws IOException, UnknownHostException { + return getSSLContext().getSocketFactory().createSocket( + socket, + host, + port, + autoClose + ); + } + + public boolean equals(Object obj) { + return ((obj != null) && obj.getClass().equals(EasySSLProtocolSocketFactory.class)); + } + + public int hashCode() { + return EasySSLProtocolSocketFactory.class.hashCode(); + } + +} diff --git a/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java b/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java new file mode 100755 index 0000000..9e35075 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java @@ -0,0 +1,112 @@ +package io.trygvis.esper.testing; + +/* + * ==================================================================== + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +/** + *

+ * EasyX509TrustManager unlike default {@link X509TrustManager} accepts + * self-signed certificates. + *

+ *

+ * This trust manager SHOULD NOT be used for productive systems + * due to security reasons, unless it is a concious decision and + * you are perfectly aware of security implications of accepting + * self-signed certificates + *

+ * + * @author Adrian Sutton + * @author Oleg Kalnichevski + * + *

+ * DISCLAIMER: HttpClient developers DO NOT actively support this component. + * The component is provided as a reference material, which may be inappropriate + * for use without additional customization. + *

+ */ + +public class EasyX509TrustManager implements X509TrustManager +{ + private X509TrustManager standardTrustManager = null; + + /** Log object for this class. */ +// private static final Log LOG = LogFactory.getLog(EasyX509TrustManager.class); + + /** + * Constructor for EasyX509TrustManager. + */ + public EasyX509TrustManager(KeyStore keystore) throws NoSuchAlgorithmException, KeyStoreException { + super(); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(keystore); + TrustManager[] trustmanagers = factory.getTrustManagers(); + if (trustmanagers.length == 0) { + throw new NoSuchAlgorithmException("no trust manager found"); + } + this.standardTrustManager = (X509TrustManager)trustmanagers[0]; + } + + /** + * @see javax.net.ssl.X509TrustManager#checkClientTrusted(X509Certificate[],String authType) + */ + public void checkClientTrusted(X509Certificate[] certificates,String authType) throws CertificateException { + standardTrustManager.checkClientTrusted(certificates,authType); + } + + /** + * @see javax.net.ssl.X509TrustManager#checkServerTrusted(X509Certificate[],String authType) + */ + public void checkServerTrusted(X509Certificate[] certificates,String authType) throws CertificateException { +// if (certificates != null) { +// System.out.println("Server certificate chain:"); +// for (int i = 0; i < certificates.length; i++) { +// System.out.println("X509Certificate[" + i + "]=" + certificates[i]); +// } +// } + + if ((certificates != null) && (certificates.length == 1)) { + certificates[0].checkValidity(); + } else { + standardTrustManager.checkServerTrusted(certificates,authType); + } + } + + /** + * @see javax.net.ssl.X509TrustManager#getAcceptedIssuers() + */ + public X509Certificate[] getAcceptedIssuers() { + return this.standardTrustManager.getAcceptedIssuers(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/HttpClient.java b/src/main/java/io/trygvis/esper/testing/HttpClient.java new file mode 100755 index 0000000..d9adc8d --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/HttpClient.java @@ -0,0 +1,82 @@ +package io.trygvis.esper.testing; + +import static java.lang.System.*; +import org.codehaus.httpcache4j.*; +import org.codehaus.httpcache4j.cache.*; +import org.codehaus.httpcache4j.client.*; +import org.codehaus.httpcache4j.resolver.*; + +import java.io.*; + +public class HttpClient { + + public static HTTPCache createHttpClient(Config config) { + return new HTTPCache(new MemoryCacheStorage(), createResponseResolver(config)); + } + + private static ResponseResolver createResponseResolver(final Config config) { + ResponseResolver responseResolver = HTTPClientResponseResolver.createMultithreadedInstance(); + + if (config.gitoriousSessionValue.isSome()) { + responseResolver = new GitoriousResponseResolver(config.gitoriousSessionValue.some(), responseResolver); + } + + responseResolver = new TimingResponseResolver(responseResolver); + + return responseResolver; + } + + private static class TimingResponseResolver implements ResponseResolver { + private final ResponseResolver r; + + private TimingResponseResolver(ResponseResolver r) { + this.r = r; + } + + public HTTPResponse resolve(HTTPRequest request) throws IOException { + System.out.println(request.getRequestURI() + ": Executing"); + long start = currentTimeMillis(); + Status status = null; + try { + HTTPResponse response = r.resolve(request); + status = response.getStatus(); + return response; + } finally { + long end = currentTimeMillis(); + + String s = request.getRequestURI() + ": Executed in " + (end - start) + "ms, "; + + if (status != null) { + s += "response: " + status.getCode() + " " + status.getName(); + } else { + s += "with exception"; + } + + System.out.println(s); + } + } + + public void shutdown() { + r.shutdown(); + } + } + + private static class GitoriousResponseResolver implements ResponseResolver { + private final String gitoriousSessionValue; + private final ResponseResolver responseResolver; + + public GitoriousResponseResolver(String gitoriousSessionValue, ResponseResolver responseResolver) { + this.gitoriousSessionValue = gitoriousSessionValue; + this.responseResolver = responseResolver; + } + + public HTTPResponse resolve(HTTPRequest request) throws IOException { + request = request.addHeader("Cookie", "_gitorious_sess=" + gitoriousSessionValue); + return responseResolver.resolve(request); + } + + public void shutdown() { + responseResolver.shutdown(); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java old mode 100644 new mode 100755 index 7e0a1b7..b7b21d6 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java @@ -2,9 +2,9 @@ package io.trygvis.esper.testing.gitorious; import fj.data.*; import org.apache.abdera.*; -import org.apache.abdera.model.*; import org.apache.abdera.model.Document; import org.apache.abdera.model.Element; +import org.apache.abdera.model.*; import org.apache.abdera.parser.*; import org.dom4j.*; import org.dom4j.io.*; @@ -82,7 +82,7 @@ public class GitoriousAtomFeedParser { switch (event) { case "created repository": case "created branch": - // This is similar "pushed", but doesn't contain any info on commit IDs or branches + // This is similar "pushed", but doesn't contain any info on commit IDs or branches case "started development of": return null; case "pushed": diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java old mode 100644 new mode 100755 index 892d8d0..f44635f --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -1,9 +1,13 @@ package io.trygvis.esper.testing.gitorious; import static java.lang.System.*; + +import fj.data.Option; +import org.apache.abdera.parser.ParseException; import org.apache.commons.io.*; import static org.apache.commons.lang.StringUtils.*; import static org.codehaus.httpcache4j.HTTPMethod.*; + import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.dom4j.*; @@ -12,16 +16,18 @@ import org.dom4j.io.*; import javax.xml.stream.*; import java.io.*; import java.net.*; +import java.sql.SQLException; import java.util.*; public class GitoriousClient { public static final STAXEventReader xmlReader = new STAXEventReader(); - private final HTTPCache httpCache; public final String baseUrl; + private final HTTPCache http; private final String projectsUri; + private final GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser(); - public GitoriousClient(HTTPCache httpCache, String baseUrl) throws URISyntaxException { - this.httpCache = httpCache; + public GitoriousClient(HTTPCache http, String baseUrl) throws URISyntaxException { + this.http = http; this.baseUrl = new URI(baseUrl).toASCIIString(); this.projectsUri = baseUrl + "/projects.xml"; } @@ -31,13 +37,23 @@ public class GitoriousClient { int page = 1; Set all = new HashSet<>(); - while (page <= 10) { - System.out.println("Fetching projects XML, page=" + page); + while (true) { + System.out.println("Fetching projects, page=" + page); long start = currentTimeMillis(); - HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET); - HTTPResponse response = httpCache.execute(request); + HTTPResponse response = http.execute(new HTTPRequest(new URI(projectsUri + "?page=" + page), GET)); long end = currentTimeMillis(); - System.out.println("Fetched XML in " + (end - start) + "ms."); + System.out.println("Fetched in " + (end - start) + "ms."); + + if (!response.getStatus().equals(Status.OK)) { + System.out.println("Got non-200 status from server: " + response.getStatus()); + break; + } + + MIMEType mimeType = MIMEType.valueOf(trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type"))); + if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) { + System.out.println("Unexpected mime type, probably at the end of the list: " + mimeType); + break; + } byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream()); try { @@ -66,6 +82,22 @@ public class GitoriousClient { public URI atomFeed(String projectSlug, String repositoryName) { return URI.create(baseUrl + "/" + projectSlug + "/" + repositoryName + ".atom"); } + + public Iterable fetchGitoriousEvents(GitoriousRepository repository, Option lastUpdate) throws SQLException, ParseException { + System.out.println("Fetching " + repository.atomFeed); + + long start = currentTimeMillis(); + HTTPResponse response = http.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET)); + long end = currentTimeMillis(); + System.out.println("Fetched in " + (end - start) + "ms"); + + // Use the server's timestamp + Date responseDate = response.getDate().toDate(); + + System.out.println("responseDate = " + responseDate); + + return parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name); + } } class GitoriousProjectXml implements Comparable { @@ -138,10 +170,7 @@ class GitoriousProjectXml implements Comparable { GitoriousProjectXml that = (GitoriousProjectXml) o; - if (!repositories.equals(that.repositories)) return false; - if (!slug.equals(that.slug)) return false; - - return true; + return repositories.equals(that.repositories) && slug.equals(that.slug); } public int hashCode() { @@ -186,10 +215,7 @@ class GitoriousRepositoryXml implements Comparable { GitoriousRepositoryXml that = (GitoriousRepositoryXml) o; - if (!name.equals(that.name)) return false; - if (!projectSlug.equals(that.projectSlug)) return false; - - return true; + return name.equals(that.name) && projectSlug.equals(that.projectSlug); } public int hashCode() { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java old mode 100644 new mode 100755 index b4bc683..1e7a7fd --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java @@ -6,9 +6,7 @@ import static fj.data.Option.*; import io.trygvis.esper.testing.*; import static java.lang.System.*; import org.apache.abdera.parser.*; -import org.codehaus.httpcache4j.*; -import org.codehaus.httpcache4j.cache.*; -import org.codehaus.httpcache4j.resolver.*; +import org.apache.commons.httpclient.protocol.*; import java.io.*; import java.net.*; @@ -20,37 +18,34 @@ import java.util.Set; import java.util.concurrent.*; public class GitoriousImporter { - private final GitoriousAtomFeedParser parser; private final BoneCP boneCp; private final GitoriousClient gitoriousClient; - private final HTTPCache httpCache; public static void main(String[] args) throws Exception { - Main.configureLog4j(); - new GitoriousImporter(DbMain.JDBC_URL, "esper", ""); + Config config = Config.loadFromDisk(); + new GitoriousImporter(config, DbMain.JDBC_URL, "esper", "esper"); } - public GitoriousImporter(String jdbcUrl, String jdbcUsername, String jdbcPassword) throws Exception { - parser = new GitoriousAtomFeedParser(); + public GitoriousImporter(Config config, final String jdbcUrl, final String jdbcUsername, final String jdbcPassword) throws Exception { + BoneCPConfig boneCPConfig = new BoneCPConfig(){{ + setJdbcUrl(jdbcUrl); + setUsername(jdbcUsername); + setPassword(jdbcPassword); + setDefaultAutoCommit(false); + setMaxConnectionsPerPartition(10); + }}; - BoneCPConfig config = new BoneCPConfig(); - config.setJdbcUrl(jdbcUrl); - config.setUsername(jdbcUsername); - config.setPassword(jdbcPassword); - config.setDefaultAutoCommit(false); - config.setMaxConnectionsPerPartition(10); + boneCp = new BoneCP(boneCPConfig); - boneCp = new BoneCP(config); - - httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance()); - - gitoriousClient = new GitoriousClient(httpCache, "http://gitorious.org"); + gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitoriousUrl); final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(2); - boolean projectsUpdateEnabled = false; + boolean projectsUpdateEnabled = true; int projectsUpdateDelay = 0 * 1000; int projectsUpdateInterval = 60 * 1000; + + boolean repositoriesUpdateEnabled = false; int repositoriesUpdateDelay = 0; int repositoriesUpdateInterval = 60 * 1000; @@ -66,15 +61,17 @@ public class GitoriousImporter { }, projectsUpdateDelay, projectsUpdateInterval, TimeUnit.MILLISECONDS); } - service.scheduleAtFixedRate(new Runnable() { - public void run() { - try { - updateRepositories(); - } catch (Exception e) { - e.printStackTrace(System.out); + if (repositoriesUpdateEnabled) { + service.scheduleAtFixedRate(new Runnable() { + public void run() { + try { + updateRepositories(); + } catch (Exception e) { + e.printStackTrace(System.out); + } } - } - }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS); + }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS); + } } private void discoverProjects() throws Exception { @@ -105,8 +102,7 @@ public class GitoriousImporter { for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) { boolean found = false; - for (Iterator it = project.repositories.iterator(); it.hasNext(); ) { - GitoriousRepositoryXml repo = it.next(); + for (GitoriousRepositoryXml repo : project.repositories) { if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) { found = true; break; @@ -125,8 +121,7 @@ public class GitoriousImporter { for (String project : projectDao.selectSlugs()) { boolean found = false; - for (Iterator it = projects.iterator(); it.hasNext(); ) { - GitoriousProjectXml p = it.next(); + for (GitoriousProjectXml p : projects) { if (p.slug.equals(project)) { found = true; break; @@ -165,21 +160,9 @@ public class GitoriousImporter { Option lastUpdate = repository.lastSuccessfulUpdate; - System.out.println("Fetching " + repository.atomFeed); - - long start = currentTimeMillis(); - HTTPResponse response = httpCache.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET)); - long end = currentTimeMillis(); - System.out.println("Fetched in " + (end - start) + "ms"); - - // Use the server's timestamp - Date responseDate = response.getDate().toDate(); - - System.out.println("responseDate = " + responseDate); - - List events; + Iterable events; try { - events = parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name); + events = gitoriousClient.fetchGitoriousEvents(repository, lastUpdate); } catch (ParseException e) { repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.none()); System.out.println("Error parsing " + repository.atomFeed); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java old mode 100644 new mode 100755 index b7d88dc..247dfe3 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java @@ -13,7 +13,7 @@ import java.util.concurrent.*; public class JenkinsImporter { public static void main(String[] args) throws Exception { - Main.configureLog4j(); + Config.configureLog4j(); final JenkinsClient jenkinsClient = new JenkinsClient(http); diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java new file mode 100755 index 0000000..6477a80 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java @@ -0,0 +1,75 @@ +package io.trygvis.esper.testing.nexus; + +import fj.data.*; +import static fj.data.Option.*; +import static io.trygvis.esper.testing.nexus.NexusParser.*; +import org.apache.commons.io.*; +import org.apache.commons.lang.*; +import static org.codehaus.httpcache4j.HTTPMethod.*; +import org.codehaus.httpcache4j.*; +import org.codehaus.httpcache4j.cache.*; +import org.codehaus.httpcache4j.util.*; + +import java.io.*; +import java.net.*; +import javax.xml.stream.*; + +public class NexusClient { + private final HTTPCache http; + private final String nexusUrl; + + public NexusClient(HTTPCache http, String nexusUrl) { + this.http = http; + this.nexusUrl = nexusUrl; + } + + public ArtifactSearchResult fetchIndex(String groupId, Option repositoryId) throws IOException { + ArtifactSearchResult aggregate = fetchIndexPage(groupId, repositoryId, Option.none()); + ArtifactSearchResult result = aggregate; + + while(result.artifacts.size() > 0) { + result = fetchIndexPage(groupId, repositoryId, some(aggregate.artifacts.size())); + aggregate = aggregate.append(result); + } + + return aggregate; + } + + public ArtifactSearchResult fetchIndexPage(String groupId, Option repositoryId, Option from) throws IOException { + URIBuilder uriBuilder = URIBuilder.fromURI(URI.create(nexusUrl)). + addRawPath("/service/local/lucene/search"). + addParameter("g", groupId + ".*"); + + if (repositoryId.isSome()) { + uriBuilder = uriBuilder.addParameter("repositoryId", repositoryId.some()); + } + + if (from.isSome()) { + uriBuilder = uriBuilder.addParameter("from", from.some().toString()); + } + + HTTPResponse response = http.execute(new HTTPRequest(uriBuilder.toURI(), GET)); + + int statusCode = response.getStatus().getCode(); + if (statusCode != 200) { + throw new IOException("Got " + statusCode + " from Nexus search, expected 200."); + } + + MIMEType mimeType = MIMEType.valueOf(StringUtils.trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type"))); + if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) { + throw new IOException("Unexpected mime type: " + mimeType); + } + + byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream()); + + try { + ArtifactSearchResult result = parseDocument(new ByteArrayInputStream(bytes)); + System.out.println("Parsed out " + result.artifacts.size() + " artifacts."); + return result; + } catch (XMLStreamException e) { + System.out.println("Unable to parse XML."); + System.out.println(new String(bytes)); + throw new RuntimeException("Unable to parse XML.", e); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java new file mode 100755 index 0000000..39d4233 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java @@ -0,0 +1,52 @@ +package io.trygvis.esper.testing.nexus; + +import fj.data.*; +import static fj.data.Option.*; +import io.trygvis.esper.testing.*; +import org.joda.time.*; + +import java.sql.*; + +public class NexusDao extends Dao { + protected NexusDao(Connection c) { + super(c); + } + + public void insertRepository(String repositoryId, LocalDateTime discoveryDate) throws SQLException { + PreparedStatement s = prepareStatement("INSERT INTO nexus_repository(id) VALUES(?)"); + s.setString(1, repositoryId); + s.executeUpdate(); + } + + public Option selectRepository(String repositoryId) throws SQLException { + PreparedStatement s = prepareStatement("SELECT id, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?"); + s.setString(1, repositoryId); + + try (ResultSet rs = s.executeQuery()) { + if (!rs.next()) { + return Option.none(); + } + + return some(new NexusRepository( + rs.getString(1), + fromNull(rs.getTimestamp(2)).map(timestampToLocalDateTime), + fromNull(rs.getTimestamp(3)).map(timestampToLocalDateTime), + fromNull(rs.getTimestamp(4)).map(timestampToLocalDateTime) + )); + } + } +} + +class NexusRepository { + public final String repositoryId; + public final Option discoveryDate; + public final Option lastUpdate; + public final Option lastSuccessfulUpdate; + + NexusRepository(String repositoryId, Option discoveryDate, Option lastUpdate, Option lastSuccessfulUpdate) { + this.repositoryId = repositoryId; + this.discoveryDate = discoveryDate; + this.lastUpdate = lastUpdate; + this.lastSuccessfulUpdate = lastSuccessfulUpdate; + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java new file mode 100755 index 0000000..896f0e2 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java @@ -0,0 +1,24 @@ +package io.trygvis.esper.testing.nexus; + +import com.google.common.collect.*; +import fj.data.*; +import io.trygvis.esper.testing.*; +import org.apache.commons.lang.*; + +import java.io.*; +import java.util.*; + +public class NexusImporter { + public static void main(String[] args) throws IOException { + Config config = Config.loadFromDisk(); + + NexusClient client = new NexusClient(HttpClient.createHttpClient(config), config.nexusUrl); + + ArtifactSearchResult result = client.fetchIndex("eu.nets", Option.none()); + ArrayList artifacts = Lists.newArrayList(result.artifacts); + Collections.sort(artifacts); + for (ArtifactXml artifact : artifacts) { + System.out.println("repo=" + StringUtils.join(artifact.repositories(), ", ") + ", artifact=" + artifact.getId()); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java new file mode 100755 index 0000000..05dfd5b --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java @@ -0,0 +1,197 @@ +package io.trygvis.esper.testing.nexus; + +import com.google.common.base.*; +import com.google.common.collect.*; +import fj.*; +import fj.data.*; +import static fj.data.Option.fromNull; +import static org.apache.commons.lang.StringUtils.*; +import org.dom4j.*; +import org.dom4j.io.*; + +import java.io.*; +import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.xml.stream.*; + +public class NexusParser { + public static final STAXEventReader xmlReader = new STAXEventReader(); + + public static ArtifactSearchResult parseDocument(InputStream is) throws XMLStreamException { + Document doc = xmlReader.readDocument(is); + + Option totalCount = fromNull(trimToNull(doc.getRootElement().elementText("totalCount"))). + bind(Option.parseInt); + if (totalCount.isNone()) { + throw new RuntimeException("Could not find required element "); + } + + boolean tooManyResults = "true".equals(trimToNull(doc.getRootElement().elementText("tooManyResults"))); + + List list = new ArrayList<>(); + for (Object o : doc.selectNodes("/searchNGResponse/data/artifact")) { + if (!(o instanceof Element)) { + continue; + } + + Element artifact = (Element) o; + + String groupId = trimToNull(artifact.elementText("groupId")); + String artifactId = trimToNull(artifact.elementText("artifactId")); + String version = trimToNull(artifact.elementText("version")); + + if (groupId == null || artifactId == null || version == null) { + continue; + } + + List artifactHitsList = new ArrayList<>(); + + @SuppressWarnings("unchecked") List artifactHits = (List) artifact.selectNodes("artifactHits/artifactHit"); + for (Element artifactHit : artifactHits) { + String repositoryId = trimToNull(artifactHit.elementText("repositoryId")); + if (repositoryId == null) { + continue; + } + List files = new ArrayList<>(); + + @SuppressWarnings("unchecked") List artifactLinks = artifactHit.selectNodes("artifactLinks/artifactLink"); + for (Element artifactLink : artifactLinks) { + Option classifier = Option.fromString(trimToEmpty(artifactLink.elementText("classifier"))); + String extension = trimToNull(artifactLink.elementText("extension")); + + if (extension == null) { + continue; + } + + files.add(new ArtifactFile(classifier, extension)); + } + + artifactHitsList.add(new ArtifactHits(repositoryId, files)); + } + + list.add(new ArtifactXml(groupId, artifactId, version, artifactHitsList)); + } + + return new ArtifactSearchResult(totalCount.some(), tooManyResults, list); + } +} + +class ArtifactSearchResult { + public final int totalCount; + public final boolean tooManyResults; + public final List artifacts; + + ArtifactSearchResult(int totalCount, boolean tooManyResults, List artifacts) { + this.totalCount = totalCount; + this.tooManyResults = tooManyResults; + this.artifacts = artifacts; + } + + public ArtifactSearchResult append(ArtifactSearchResult result) { + List list = Lists.newArrayList(artifacts); + list.addAll(result.artifacts); + return new ArtifactSearchResult(result.totalCount, result.tooManyResults, list); + } +} + +class ArtifactXml implements Comparable { + public final String groupId; + public final String artifactId; + public final String version; + public final List hits; + + ArtifactXml(String groupId, String artifactId, String version, List hits) { + this.groupId = groupId; + this.artifactId = artifactId; + this.version = version; + this.hits = hits; + } + + public static Predicate repositoryFilter(final String repositoryId) { + return new Predicate() { + public boolean apply(ArtifactXml artifact) { + return Iterables.any(artifact.hits, new Predicate() { + public boolean apply(ArtifactHits hits) { + return hits.repositoryId.equals(repositoryId); + } + }); + } + }; + } + + public FlatArtifact flatten(String repositoryId) { + for (ArtifactHits hit : hits) { + if (hit.repositoryId.equals(repositoryId)) { + return new FlatArtifact(groupId, artifactId, version, hit.files); + } + } + + throw new RuntimeException("No hits in repository " + repositoryId); + } + + public int compareTo(ArtifactXml o) { + int c = groupId.compareTo(o.groupId); + + if(c != 0) { + return c; + } + + c = artifactId.compareTo(o.artifactId); + + if(c != 0) { + return c; + } + + return version.compareTo(o.version); + } + + public String getId() { + return groupId + ":" + artifactId + ":" + version; + } + + public Set repositories() { + Set repositories = new HashSet<>(10); + + for (ArtifactHits hit : hits) { + repositories.add(hit.repositoryId); + } + + return repositories; + } +} + +class FlatArtifact { + public final String groupId; + public final String artifactId; + public final String version; + public final List files; + + FlatArtifact(String groupId, String artifactId, String version, List files) { + this.groupId = groupId; + this.artifactId = artifactId; + this.version = version; + this.files = files; + } +} + +class ArtifactHits { + public final String repositoryId; + public final List files; + + ArtifactHits(String repositoryId, List files) { + this.repositoryId = repositoryId; + this.files = files; + } +} + +class ArtifactFile { + public final Option classifier; + public final String extension; + + ArtifactFile(Option classifier, String extension) { + this.classifier = classifier; + this.extension = extension; + } +} -- cgit v1.2.3