From af92579129943acac73542f4e05e1c7faeda0994 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Mon, 19 Nov 2012 18:04:49 +0100 Subject: o Adding logic to stop fetching pages when Gitorious returns non-XML response. o Adding support for self-signed certificates with https. o Moving client code to GitoriousClient. o Adding Nexus client and an importer that fetches all artifacts. o A nexus client that actually fetches the entire set of artifacts. --- .../trygvis/esper/testing/nexus/NexusClient.java | 75 ++++++++ .../io/trygvis/esper/testing/nexus/NexusDao.java | 52 ++++++ .../trygvis/esper/testing/nexus/NexusImporter.java | 24 +++ .../trygvis/esper/testing/nexus/NexusParser.java | 197 +++++++++++++++++++++ 4 files changed, 348 insertions(+) create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java create mode 100755 src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java (limited to 'src/main/java/io/trygvis/esper/testing/nexus') diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java new file mode 100755 index 0000000..6477a80 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java @@ -0,0 +1,75 @@ +package io.trygvis.esper.testing.nexus; + +import fj.data.*; +import static fj.data.Option.*; +import static io.trygvis.esper.testing.nexus.NexusParser.*; +import org.apache.commons.io.*; +import org.apache.commons.lang.*; +import static org.codehaus.httpcache4j.HTTPMethod.*; +import org.codehaus.httpcache4j.*; +import org.codehaus.httpcache4j.cache.*; +import org.codehaus.httpcache4j.util.*; + +import java.io.*; +import java.net.*; +import javax.xml.stream.*; + +public class NexusClient { + private final HTTPCache http; + private final String nexusUrl; + + public NexusClient(HTTPCache http, String nexusUrl) { + this.http = http; + this.nexusUrl = nexusUrl; + } + + public ArtifactSearchResult fetchIndex(String groupId, Option repositoryId) throws IOException { + ArtifactSearchResult aggregate = fetchIndexPage(groupId, repositoryId, Option.none()); + ArtifactSearchResult result = aggregate; + + while(result.artifacts.size() > 0) { + result = fetchIndexPage(groupId, repositoryId, some(aggregate.artifacts.size())); + aggregate = aggregate.append(result); + } + + return aggregate; + } + + public ArtifactSearchResult fetchIndexPage(String groupId, Option repositoryId, Option from) throws IOException { + URIBuilder uriBuilder = URIBuilder.fromURI(URI.create(nexusUrl)). + addRawPath("/service/local/lucene/search"). + addParameter("g", groupId + ".*"); + + if (repositoryId.isSome()) { + uriBuilder = uriBuilder.addParameter("repositoryId", repositoryId.some()); + } + + if (from.isSome()) { + uriBuilder = uriBuilder.addParameter("from", from.some().toString()); + } + + HTTPResponse response = http.execute(new HTTPRequest(uriBuilder.toURI(), GET)); + + int statusCode = response.getStatus().getCode(); + if (statusCode != 200) { + throw new IOException("Got " + statusCode + " from Nexus search, expected 200."); + } + + MIMEType mimeType = MIMEType.valueOf(StringUtils.trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type"))); + if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) { + throw new IOException("Unexpected mime type: " + mimeType); + } + + byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream()); + + try { + ArtifactSearchResult result = parseDocument(new ByteArrayInputStream(bytes)); + System.out.println("Parsed out " + result.artifacts.size() + " artifacts."); + return result; + } catch (XMLStreamException e) { + System.out.println("Unable to parse XML."); + System.out.println(new String(bytes)); + throw new RuntimeException("Unable to parse XML.", e); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java new file mode 100755 index 0000000..39d4233 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java @@ -0,0 +1,52 @@ +package io.trygvis.esper.testing.nexus; + +import fj.data.*; +import static fj.data.Option.*; +import io.trygvis.esper.testing.*; +import org.joda.time.*; + +import java.sql.*; + +public class NexusDao extends Dao { + protected NexusDao(Connection c) { + super(c); + } + + public void insertRepository(String repositoryId, LocalDateTime discoveryDate) throws SQLException { + PreparedStatement s = prepareStatement("INSERT INTO nexus_repository(id) VALUES(?)"); + s.setString(1, repositoryId); + s.executeUpdate(); + } + + public Option selectRepository(String repositoryId) throws SQLException { + PreparedStatement s = prepareStatement("SELECT id, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?"); + s.setString(1, repositoryId); + + try (ResultSet rs = s.executeQuery()) { + if (!rs.next()) { + return Option.none(); + } + + return some(new NexusRepository( + rs.getString(1), + fromNull(rs.getTimestamp(2)).map(timestampToLocalDateTime), + fromNull(rs.getTimestamp(3)).map(timestampToLocalDateTime), + fromNull(rs.getTimestamp(4)).map(timestampToLocalDateTime) + )); + } + } +} + +class NexusRepository { + public final String repositoryId; + public final Option discoveryDate; + public final Option lastUpdate; + public final Option lastSuccessfulUpdate; + + NexusRepository(String repositoryId, Option discoveryDate, Option lastUpdate, Option lastSuccessfulUpdate) { + this.repositoryId = repositoryId; + this.discoveryDate = discoveryDate; + this.lastUpdate = lastUpdate; + this.lastSuccessfulUpdate = lastSuccessfulUpdate; + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java new file mode 100755 index 0000000..896f0e2 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java @@ -0,0 +1,24 @@ +package io.trygvis.esper.testing.nexus; + +import com.google.common.collect.*; +import fj.data.*; +import io.trygvis.esper.testing.*; +import org.apache.commons.lang.*; + +import java.io.*; +import java.util.*; + +public class NexusImporter { + public static void main(String[] args) throws IOException { + Config config = Config.loadFromDisk(); + + NexusClient client = new NexusClient(HttpClient.createHttpClient(config), config.nexusUrl); + + ArtifactSearchResult result = client.fetchIndex("eu.nets", Option.none()); + ArrayList artifacts = Lists.newArrayList(result.artifacts); + Collections.sort(artifacts); + for (ArtifactXml artifact : artifacts) { + System.out.println("repo=" + StringUtils.join(artifact.repositories(), ", ") + ", artifact=" + artifact.getId()); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java new file mode 100755 index 0000000..05dfd5b --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java @@ -0,0 +1,197 @@ +package io.trygvis.esper.testing.nexus; + +import com.google.common.base.*; +import com.google.common.collect.*; +import fj.*; +import fj.data.*; +import static fj.data.Option.fromNull; +import static org.apache.commons.lang.StringUtils.*; +import org.dom4j.*; +import org.dom4j.io.*; + +import java.io.*; +import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.xml.stream.*; + +public class NexusParser { + public static final STAXEventReader xmlReader = new STAXEventReader(); + + public static ArtifactSearchResult parseDocument(InputStream is) throws XMLStreamException { + Document doc = xmlReader.readDocument(is); + + Option totalCount = fromNull(trimToNull(doc.getRootElement().elementText("totalCount"))). + bind(Option.parseInt); + if (totalCount.isNone()) { + throw new RuntimeException("Could not find required element "); + } + + boolean tooManyResults = "true".equals(trimToNull(doc.getRootElement().elementText("tooManyResults"))); + + List list = new ArrayList<>(); + for (Object o : doc.selectNodes("/searchNGResponse/data/artifact")) { + if (!(o instanceof Element)) { + continue; + } + + Element artifact = (Element) o; + + String groupId = trimToNull(artifact.elementText("groupId")); + String artifactId = trimToNull(artifact.elementText("artifactId")); + String version = trimToNull(artifact.elementText("version")); + + if (groupId == null || artifactId == null || version == null) { + continue; + } + + List artifactHitsList = new ArrayList<>(); + + @SuppressWarnings("unchecked") List artifactHits = (List) artifact.selectNodes("artifactHits/artifactHit"); + for (Element artifactHit : artifactHits) { + String repositoryId = trimToNull(artifactHit.elementText("repositoryId")); + if (repositoryId == null) { + continue; + } + List files = new ArrayList<>(); + + @SuppressWarnings("unchecked") List artifactLinks = artifactHit.selectNodes("artifactLinks/artifactLink"); + for (Element artifactLink : artifactLinks) { + Option classifier = Option.fromString(trimToEmpty(artifactLink.elementText("classifier"))); + String extension = trimToNull(artifactLink.elementText("extension")); + + if (extension == null) { + continue; + } + + files.add(new ArtifactFile(classifier, extension)); + } + + artifactHitsList.add(new ArtifactHits(repositoryId, files)); + } + + list.add(new ArtifactXml(groupId, artifactId, version, artifactHitsList)); + } + + return new ArtifactSearchResult(totalCount.some(), tooManyResults, list); + } +} + +class ArtifactSearchResult { + public final int totalCount; + public final boolean tooManyResults; + public final List artifacts; + + ArtifactSearchResult(int totalCount, boolean tooManyResults, List artifacts) { + this.totalCount = totalCount; + this.tooManyResults = tooManyResults; + this.artifacts = artifacts; + } + + public ArtifactSearchResult append(ArtifactSearchResult result) { + List list = Lists.newArrayList(artifacts); + list.addAll(result.artifacts); + return new ArtifactSearchResult(result.totalCount, result.tooManyResults, list); + } +} + +class ArtifactXml implements Comparable { + public final String groupId; + public final String artifactId; + public final String version; + public final List hits; + + ArtifactXml(String groupId, String artifactId, String version, List hits) { + this.groupId = groupId; + this.artifactId = artifactId; + this.version = version; + this.hits = hits; + } + + public static Predicate repositoryFilter(final String repositoryId) { + return new Predicate() { + public boolean apply(ArtifactXml artifact) { + return Iterables.any(artifact.hits, new Predicate() { + public boolean apply(ArtifactHits hits) { + return hits.repositoryId.equals(repositoryId); + } + }); + } + }; + } + + public FlatArtifact flatten(String repositoryId) { + for (ArtifactHits hit : hits) { + if (hit.repositoryId.equals(repositoryId)) { + return new FlatArtifact(groupId, artifactId, version, hit.files); + } + } + + throw new RuntimeException("No hits in repository " + repositoryId); + } + + public int compareTo(ArtifactXml o) { + int c = groupId.compareTo(o.groupId); + + if(c != 0) { + return c; + } + + c = artifactId.compareTo(o.artifactId); + + if(c != 0) { + return c; + } + + return version.compareTo(o.version); + } + + public String getId() { + return groupId + ":" + artifactId + ":" + version; + } + + public Set repositories() { + Set repositories = new HashSet<>(10); + + for (ArtifactHits hit : hits) { + repositories.add(hit.repositoryId); + } + + return repositories; + } +} + +class FlatArtifact { + public final String groupId; + public final String artifactId; + public final String version; + public final List files; + + FlatArtifact(String groupId, String artifactId, String version, List files) { + this.groupId = groupId; + this.artifactId = artifactId; + this.version = version; + this.files = files; + } +} + +class ArtifactHits { + public final String repositoryId; + public final List files; + + ArtifactHits(String repositoryId, List files) { + this.repositoryId = repositoryId; + this.files = files; + } +} + +class ArtifactFile { + public final Option classifier; + public final String extension; + + ArtifactFile(Option classifier, String extension) { + this.classifier = classifier; + this.extension = extension; + } +} -- cgit v1.2.3