o Adding logic to stop fetching pages when Gitorious returns non-XML response.

o Adding support for self-signed certificates with https.
o Moving client code to GitoriousClient.
o Adding Nexus client and an importer that fetches all artifacts.
o A nexus client that actually fetches the entire set of artifacts.

4 files changed, 348 insertions, 0 deletions

diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
new file mode 100755
index 0000000..6477a80
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
@@ -0,0 +1,75 @@
+package io.trygvis.esper.testing.nexus;
+
+import fj.data.*;
+import static fj.data.Option.*;
+import static io.trygvis.esper.testing.nexus.NexusParser.*;
+import org.apache.commons.io.*;
+import org.apache.commons.lang.*;
+import static org.codehaus.httpcache4j.HTTPMethod.*;
+import org.codehaus.httpcache4j.*;
+import org.codehaus.httpcache4j.cache.*;
+import org.codehaus.httpcache4j.util.*;
+
+import java.io.*;
+import java.net.*;
+import javax.xml.stream.*;
+
+public class NexusClient {
+    private final HTTPCache http;
+    private final String nexusUrl;
+
+    public NexusClient(HTTPCache http, String nexusUrl) {
+        this.http = http;
+        this.nexusUrl = nexusUrl;
+    }
+
+    public ArtifactSearchResult fetchIndex(String groupId, Option<String> repositoryId) throws IOException {
+        ArtifactSearchResult aggregate = fetchIndexPage(groupId, repositoryId, Option.<Integer>none());
+        ArtifactSearchResult result = aggregate;
+
+        while(result.artifacts.size() > 0) {
+            result = fetchIndexPage(groupId, repositoryId, some(aggregate.artifacts.size()));
+            aggregate = aggregate.append(result);
+        }
+
+        return aggregate;
+    }
+
+    public ArtifactSearchResult fetchIndexPage(String groupId, Option<String> repositoryId, Option<Integer> from) throws IOException {
+        URIBuilder uriBuilder = URIBuilder.fromURI(URI.create(nexusUrl)).
+                addRawPath("/service/local/lucene/search").
+                addParameter("g", groupId + ".*");
+
+        if (repositoryId.isSome()) {
+            uriBuilder = uriBuilder.addParameter("repositoryId", repositoryId.some());
+        }
+
+        if (from.isSome()) {
+            uriBuilder = uriBuilder.addParameter("from", from.some().toString());
+        }
+
+        HTTPResponse response = http.execute(new HTTPRequest(uriBuilder.toURI(), GET));
+
+        int statusCode = response.getStatus().getCode();
+        if (statusCode != 200) {
+            throw new IOException("Got " + statusCode + " from Nexus search, expected 200.");
+        }
+
+        MIMEType mimeType = MIMEType.valueOf(StringUtils.trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type")));
+        if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) {
+            throw new IOException("Unexpected mime type: " + mimeType);
+        }
+
+        byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream());
+
+        try {
+            ArtifactSearchResult result = parseDocument(new ByteArrayInputStream(bytes));
+            System.out.println("Parsed out " + result.artifacts.size() + " artifacts.");
+            return result;
+        } catch (XMLStreamException e) {
+            System.out.println("Unable to parse XML.");
+            System.out.println(new String(bytes));
+            throw new RuntimeException("Unable to parse XML.", e);
+        }
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
new file mode 100755
index 0000000..39d4233
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
@@ -0,0 +1,52 @@
+package io.trygvis.esper.testing.nexus;
+
+import fj.data.*;
+import static fj.data.Option.*;
+import io.trygvis.esper.testing.*;
+import org.joda.time.*;
+
+import java.sql.*;
+
+public class NexusDao extends Dao {
+    protected NexusDao(Connection c) {
+        super(c);
+    }
+
+    public void insertRepository(String repositoryId, LocalDateTime discoveryDate) throws SQLException {
+        PreparedStatement s = prepareStatement("INSERT INTO nexus_repository(id) VALUES(?)");
+        s.setString(1, repositoryId);
+        s.executeUpdate();
+    }
+
+    public Option<NexusRepository> selectRepository(String repositoryId) throws SQLException {
+        PreparedStatement s = prepareStatement("SELECT id, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?");
+        s.setString(1, repositoryId);
+
+        try (ResultSet rs = s.executeQuery()) {
+            if (!rs.next()) {
+                return Option.none();
+            }
+
+            return some(new NexusRepository(
+                    rs.getString(1),
+                    fromNull(rs.getTimestamp(2)).map(timestampToLocalDateTime),
+                    fromNull(rs.getTimestamp(3)).map(timestampToLocalDateTime),
+                    fromNull(rs.getTimestamp(4)).map(timestampToLocalDateTime)
+            ));
+        }
+    }
+}
+
+class NexusRepository {
+    public final String repositoryId;
+    public final Option<LocalDateTime> discoveryDate;
+    public final Option<LocalDateTime> lastUpdate;
+    public final Option<LocalDateTime> lastSuccessfulUpdate;
+
+    NexusRepository(String repositoryId, Option<LocalDateTime> discoveryDate, Option<LocalDateTime> lastUpdate, Option<LocalDateTime> lastSuccessfulUpdate) {
+        this.repositoryId = repositoryId;
+        this.discoveryDate = discoveryDate;
+        this.lastUpdate = lastUpdate;
+        this.lastSuccessfulUpdate = lastSuccessfulUpdate;
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
new file mode 100755
index 0000000..896f0e2
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
@@ -0,0 +1,24 @@
+package io.trygvis.esper.testing.nexus;
+
+import com.google.common.collect.*;
+import fj.data.*;
+import io.trygvis.esper.testing.*;
+import org.apache.commons.lang.*;
+
+import java.io.*;
+import java.util.*;
+
+public class NexusImporter {
+    public static void main(String[] args) throws IOException {
+        Config config = Config.loadFromDisk();
+
+        NexusClient client = new NexusClient(HttpClient.createHttpClient(config), config.nexusUrl);
+
+        ArtifactSearchResult result = client.fetchIndex("eu.nets", Option.<String>none());
+        ArrayList<ArtifactXml> artifacts = Lists.newArrayList(result.artifacts);
+        Collections.sort(artifacts);
+        for (ArtifactXml artifact : artifacts) {
+            System.out.println("repo=" + StringUtils.join(artifact.repositories(), ", ") + ", artifact=" + artifact.getId());
+        }
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
new file mode 100755
index 0000000..05dfd5b
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
@@ -0,0 +1,197 @@
+package io.trygvis.esper.testing.nexus;
+
+import com.google.common.base.*;
+import com.google.common.collect.*;
+import fj.*;
+import fj.data.*;
+import static fj.data.Option.fromNull;
+import static org.apache.commons.lang.StringUtils.*;
+import org.dom4j.*;
+import org.dom4j.io.*;
+
+import java.io.*;
+import java.util.*;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import javax.xml.stream.*;
+
+public class NexusParser {
+    public static final STAXEventReader xmlReader = new STAXEventReader();
+
+    public static ArtifactSearchResult parseDocument(InputStream is) throws XMLStreamException {
+        Document doc = xmlReader.readDocument(is);
+
+        Option<Integer> totalCount = fromNull(trimToNull(doc.getRootElement().elementText("totalCount"))).
+                bind(Option.parseInt);
+        if (totalCount.isNone()) {
+            throw new RuntimeException("Could not find required element <totalCount>");
+        }
+
+        boolean tooManyResults = "true".equals(trimToNull(doc.getRootElement().elementText("tooManyResults")));
+
+        List<ArtifactXml> list = new ArrayList<>();
+        for (Object o : doc.selectNodes("/searchNGResponse/data/artifact")) {
+            if (!(o instanceof Element)) {
+                continue;
+            }
+
+            Element artifact = (Element) o;
+
+            String groupId = trimToNull(artifact.elementText("groupId"));
+            String artifactId = trimToNull(artifact.elementText("artifactId"));
+            String version = trimToNull(artifact.elementText("version"));
+
+            if (groupId == null || artifactId == null || version == null) {
+                continue;
+            }
+
+            List<ArtifactHits> artifactHitsList = new ArrayList<>();
+
+            @SuppressWarnings("unchecked") List<Element> artifactHits = (List<Element>) artifact.selectNodes("artifactHits/artifactHit");
+            for (Element artifactHit : artifactHits) {
+                String repositoryId = trimToNull(artifactHit.elementText("repositoryId"));
+                if (repositoryId == null) {
+                    continue;
+                }
+                List<ArtifactFile> files = new ArrayList<>();
+
+                @SuppressWarnings("unchecked") List<Element> artifactLinks = artifactHit.selectNodes("artifactLinks/artifactLink");
+                for (Element artifactLink : artifactLinks) {
+                    Option<String> classifier = Option.fromString(trimToEmpty(artifactLink.elementText("classifier")));
+                    String extension = trimToNull(artifactLink.elementText("extension"));
+
+                    if (extension == null) {
+                        continue;
+                    }
+
+                    files.add(new ArtifactFile(classifier, extension));
+                }
+
+                artifactHitsList.add(new ArtifactHits(repositoryId, files));
+            }
+
+            list.add(new ArtifactXml(groupId, artifactId, version, artifactHitsList));
+        }
+
+        return new ArtifactSearchResult(totalCount.some(), tooManyResults, list);
+    }
+}
+
+class ArtifactSearchResult {
+    public final int totalCount;
+    public final boolean tooManyResults;
+    public final List<ArtifactXml> artifacts;
+
+    ArtifactSearchResult(int totalCount, boolean tooManyResults, List<ArtifactXml> artifacts) {
+        this.totalCount = totalCount;
+        this.tooManyResults = tooManyResults;
+        this.artifacts = artifacts;
+    }
+
+    public ArtifactSearchResult append(ArtifactSearchResult result) {
+        List<ArtifactXml> list = Lists.newArrayList(artifacts);
+        list.addAll(result.artifacts);
+        return new ArtifactSearchResult(result.totalCount, result.tooManyResults, list);
+    }
+}
+
+class ArtifactXml implements Comparable<ArtifactXml> {
+    public final String groupId;
+    public final String artifactId;
+    public final String version;
+    public final List<ArtifactHits> hits;
+
+    ArtifactXml(String groupId, String artifactId, String version, List<ArtifactHits> hits) {
+        this.groupId = groupId;
+        this.artifactId = artifactId;
+        this.version = version;
+        this.hits = hits;
+    }
+
+    public static Predicate<ArtifactXml> repositoryFilter(final String repositoryId) {
+        return new Predicate<ArtifactXml>() {
+            public boolean apply(ArtifactXml artifact) {
+                return Iterables.any(artifact.hits, new Predicate<ArtifactHits>() {
+                    public boolean apply(ArtifactHits hits) {
+                        return hits.repositoryId.equals(repositoryId);
+                    }
+                });
+            }
+        };
+    }
+
+    public FlatArtifact flatten(String repositoryId) {
+        for (ArtifactHits hit : hits) {
+            if (hit.repositoryId.equals(repositoryId)) {
+                return new FlatArtifact(groupId, artifactId, version, hit.files);
+            }
+        }
+
+        throw new RuntimeException("No hits in repository " + repositoryId);
+    }
+
+    public int compareTo(ArtifactXml o) {
+        int c = groupId.compareTo(o.groupId);
+
+        if(c != 0) {
+            return c;
+        }
+
+        c = artifactId.compareTo(o.artifactId);
+
+        if(c != 0) {
+            return c;
+        }
+
+        return version.compareTo(o.version);
+    }
+
+    public String getId() {
+        return groupId + ":" + artifactId + ":" + version;
+    }
+
+    public Set<String> repositories() {
+        Set<String> repositories = new HashSet<>(10);
+
+        for (ArtifactHits hit : hits) {
+            repositories.add(hit.repositoryId);
+        }
+
+        return repositories;
+    }
+}
+
+class FlatArtifact {
+    public final String groupId;
+    public final String artifactId;
+    public final String version;
+    public final List<ArtifactFile> files;
+
+    FlatArtifact(String groupId, String artifactId, String version, List<ArtifactFile> files) {
+        this.groupId = groupId;
+        this.artifactId = artifactId;
+        this.version = version;
+        this.files = files;
+    }
+}
+
+class ArtifactHits {
+    public final String repositoryId;
+    public final List<ArtifactFile> files;
+
+    ArtifactHits(String repositoryId, List<ArtifactFile> files) {
+        this.repositoryId = repositoryId;
+        this.files = files;
+    }
+}
+
+class ArtifactFile {
+    public final Option<String> classifier;
+    public final String extension;
+
+    ArtifactFile(Option<String> classifier, String extension) {
+        this.classifier = classifier;
+        this.extension = extension;
+    }
+}