diff options
22 files changed, 690 insertions, 329 deletions
@@ -38,7 +38,7 @@ <dependency> <groupId>org.jdom</groupId> <artifactId>jdom</artifactId> - <version>2.0.2</version> + <version>2.0.4</version> </dependency> <dependency> <groupId>com.jolbox</groupId> diff --git a/src/main/java/io/trygvis/esper/testing/Http.java b/src/main/java/io/trygvis/esper/testing/Http.java deleted file mode 100755 index 38a66a4..0000000 --- a/src/main/java/io/trygvis/esper/testing/Http.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.trygvis.esper.testing; - -import org.apache.http.conn.scheme.*; -import org.apache.http.conn.ssl.*; -import org.apache.http.impl.client.*; -import org.apache.http.impl.conn.tsccm.*; -import org.apache.http.params.*; -import org.codehaus.httpcache4j.cache.*; -import org.codehaus.httpcache4j.resolver.*; - -public class Http { - - public static final HTTPCache http; - - static { - SchemeRegistry schemeRegistry = new SchemeRegistry(); - schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); - schemeRegistry.register(new Scheme("https", new EasySSLSocketFactory(), 443)); - - BasicHttpParams params = new BasicHttpParams(); - ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(params, schemeRegistry); - DefaultHttpClient httpClient = new DefaultHttpClient(cm, params); - HTTPClientResponseResolver resolver = new HTTPClientResponseResolver(httpClient); - http = new HTTPCache(new MemoryCacheStorage(), resolver); - } -} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java index a236e45..9bafc17 100755 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java @@ -10,6 +10,7 @@ import io.trygvis.esper.testing.object.ObjectManager; import io.trygvis.esper.testing.object.ObjectUtil; import io.trygvis.esper.testing.object.TransactionalActor; +import io.trygvis.esper.testing.util.*; import static java.lang.System.*; import org.apache.abdera.parser.*; @@ -31,7 +32,7 @@ public class GitoriousProjectDiscovery { public GitoriousProjectDiscovery(final Config config) throws Exception { boneCp = config.createBoneCp(); - gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitorious.url); + gitoriousClient = new GitoriousClient(HttpClient.createHttpCache(config), config.gitorious.url); final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); @@ -44,11 +45,11 @@ public class GitoriousProjectDiscovery { final ObjectManager<GitoriousRepositoryDto, ActorRef<GitoriousRepository>> repositoryManager = new ObjectManager<>("", repositories, new ObjectFactory<GitoriousRepositoryDto, ActorRef<GitoriousRepository>>() { public ActorRef<GitoriousRepository> create(GitoriousRepositoryDto repository) { - return ObjectUtil.scheduledActorWithFixedDelay(service, 0, 60, TimeUnit.SECONDS, boneCp, new GitoriousRepository(gitoriousClient, repository)); + return ObjectUtil.scheduledActorWithFixedDelay(service, 0, 60, TimeUnit.SECONDS, boneCp, "Gitorious", new GitoriousRepository(gitoriousClient, repository)); } }); - ObjectUtil.scheduledActorWithFixedDelay(service, config.gitorious.projectListUpdateDelay, config.gitorious.projectListUpdateInterval, TimeUnit.MILLISECONDS, boneCp, new TransactionalActor() { + ObjectUtil.scheduledActorWithFixedDelay(service, config.gitorious.projectListUpdateDelay, config.gitorious.projectListUpdateInterval, TimeUnit.MILLISECONDS, boneCp, "Gitorious", new TransactionalActor() { @Override public void act(Connection c) throws Exception { Daos daos = new Daos(c); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java index ed4cb59..e31fcfe 100644..100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsClient.java @@ -5,38 +5,32 @@ import fj.data.*; import io.trygvis.esper.testing.*; import static io.trygvis.esper.testing.Util.*; import io.trygvis.esper.testing.jenkins.JenkinsJobXml.*; -import static java.lang.Integer.parseInt; +import io.trygvis.esper.testing.util.*; import static org.apache.commons.lang.StringUtils.*; -import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; -import org.h2.util.*; import org.jdom2.*; -import org.jdom2.input.*; -import javax.xml.stream.*; import java.io.*; import java.net.*; import java.util.*; import java.util.List; +import javax.xml.stream.*; public class JenkinsClient { - private static final XMLInputFactory xmlReader = XMLInputFactory.newFactory(); - private static final StAXStreamBuilder streamBuilder = new StAXStreamBuilder(); - private final HTTPCache http; - - private boolean debugXml; + private final XmlHttpClient xmlHttpClient; public JenkinsClient(HTTPCache http) { - this.http = http; - this.debugXml = false; - } - - public void setDebugXml(boolean debugXml) { - this.debugXml = debugXml; + this.xmlHttpClient = new XmlHttpClient(http); } public JenkinsXml fetchJobs(URI uri) throws XMLStreamException, JDOMException, IOException { - Element root = fetchXml(uri).getRootElement(); + Option<Document> d = xmlHttpClient.fetch(uri); + + if(d.isNone()) { + return new JenkinsXml(Option.<String>none(), Option.<String>none(), Option.<String>none(), Collections.<JenkinsJobEntryXml>emptyList()); + } + + Element root = d.some().getRootElement(); List<JenkinsJobEntryXml> jobs = new ArrayList<>(); for (Element job : root.getChildren("job")) { @@ -57,60 +51,25 @@ public class JenkinsClient { Option.fromNull(root.getChildText("description")), jobs); } - public JenkinsJobXml fetchJob(URI uri) throws IOException, JDOMException, XMLStreamException { - Element root = fetchXml(uri).getRootElement(); + public Option<JenkinsJobXml> fetchJob(URI uri) throws IOException, JDOMException, XMLStreamException { + Option<Document> d = xmlHttpClient.fetch(uri); + + if(d.isNone()) { + return Option.none(); + } + + Element root = d.some().getRootElement(); String name = root.getName(); switch (name) { case "freeStyleProject": - return JenkinsJobXml.parse(uri, JenkinsJobType.FREE_STYLE, root); + return Option.some(JenkinsJobXml.parse(uri, JenkinsJobType.FREE_STYLE, root)); case "mavenModuleSet": - return JenkinsJobXml.parse(uri, JenkinsJobType.MAVEN, root); + return Option.some(JenkinsJobXml.parse(uri, JenkinsJobType.MAVEN, root)); default: - throw new IOException("Unknown project type: " + name); - } - } - - private Document fetchXml(URI uri) throws IOException, XMLStreamException, JDOMException { - HTTPResponse response = null; - - try { - response = http.execute(new HTTPRequest(uri)); - - if (response.getStatus().getCode() != 200) { - throw new IOException("Did not get 200 back, got " + response.getStatus().getCode()); - } - - InputStream stream = response.getPayload().getInputStream(); - - if (debugXml) { - int size; - try { - size = parseInt(response.getHeaders().getFirstHeader("Content-Length").getValue()); - } catch (Throwable e) { - size = 10 * 1024; - } - - // TODO: Pretty print - - ByteArrayOutputStream buffer = new ByteArrayOutputStream(size); - IOUtils.copy(stream, buffer); - byte[] bytes = buffer.toByteArray(); - System.out.println("------------------------------------------------"); - System.out.write(bytes); - System.out.println(); - System.out.println("------------------------------------------------"); - stream = new ByteArrayInputStream(bytes); - } - - return streamBuilder.build(xmlReader.createXMLStreamReader(stream)); - } catch (HTTPException e) { - throw new IOException(e); - } finally { - if (response != null) { - response.consume(); - } + System.out.println("Unknown project type: " + name); + return Option.none(); } } } diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java index b929576..6be720f 100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsImporter.java @@ -4,7 +4,9 @@ import fj.*; import fj.data.*; import io.trygvis.esper.testing.*; import io.trygvis.esper.testing.object.*; +import io.trygvis.esper.testing.util.*; import static java.lang.Thread.currentThread; +import org.codehaus.httpcache4j.cache.*; import org.joda.time.*; import java.net.URI; @@ -16,9 +18,9 @@ public class JenkinsImporter { public static void main(String[] args) throws Exception { Config config = Config.loadFromDisk(); - final JenkinsClient jenkinsClient = new JenkinsClient(HttpClient.createHttpClient(config)); + HTTPCache httpCache = HttpClient.createHttpCache(config); - jenkinsClient.setDebugXml(false); + final JenkinsClient jenkinsClient = new JenkinsClient(httpCache); HashSet<URI> servers = new HashSet<>(); servers.add(URI.create("https://builds.apache.org")); diff --git a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java index 9aad891..7a7f6af 100644..100755 --- a/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java +++ b/src/main/java/io/trygvis/esper/testing/jenkins/JenkinsJob.java @@ -8,7 +8,6 @@ import org.slf4j.*; import java.io.*; import java.net.*; -import java.util.*; import java.util.concurrent.*; public class JenkinsJob implements Closeable { @@ -49,7 +48,7 @@ public class JenkinsJob implements Closeable { try { logger.info("Updating " + name); long start = currentTimeMillis(); - latestStatus = some(client.fetchJob(uri)); + latestStatus = client.fetchJob(uri); long end = currentTimeMillis(); logger.info("Updated " + name + " in " + (end - start) + "ms"); } catch (Throwable e) { diff --git a/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java b/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java index 144913e..b732bfa 100644..100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java @@ -2,32 +2,20 @@ package io.trygvis.esper.testing.nexus; import fj.data.*; -import java.net.*; +import java.util.*; public class ArtifactDto implements Comparable<ArtifactDto> { - public final URI serverUrl; + public final UUID uuid; public final String repositoryId; public final ArtifactId id; - public ArtifactDto(URI serverUrl, String repositoryId, ArtifactId id) { - this.serverUrl = serverUrl; + public ArtifactDto(UUID uuid, String repositoryId, ArtifactId id) { + this.uuid = uuid; this.repositoryId = repositoryId; this.id = id; } public int compareTo(ArtifactDto o) { - int i = serverUrl.compareTo(o.serverUrl); - - if (i != 0) { - return i; - } - - i = repositoryId.compareTo(o.repositoryId); - - if (i != 0) { - return i; - } - return id.compareTo(o.id); } @@ -37,17 +25,11 @@ public class ArtifactDto implements Comparable<ArtifactDto> { ArtifactDto that = (ArtifactDto) o; - if (!serverUrl.equals(that.serverUrl)) return false; - if (!repositoryId.equals(that.repositoryId)) return false; - - return id.equals(that.id); + return uuid.equals(that.uuid); } public int hashCode() { - int result = serverUrl.hashCode(); - result = 31 * result + repositoryId.hashCode(); - result = 31 * result + id.hashCode(); - return result; + return uuid.hashCode(); } } diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java index 9b2b899..02dc28c 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java @@ -3,31 +3,52 @@ package io.trygvis.esper.testing.nexus; import fj.data.*; import static fj.data.Option.*; import static io.trygvis.esper.testing.nexus.SearchNGResponseParser.*; +import io.trygvis.esper.testing.util.*; import org.apache.commons.io.*; import org.apache.commons.lang.*; import static org.codehaus.httpcache4j.HTTPMethod.*; import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.codehaus.httpcache4j.util.*; +import org.jdom2.*; import java.io.*; import java.net.*; +import java.util.*; import javax.xml.stream.*; public class NexusClient { private final HTTPCache http; + private final URI nexusUrl; + private final XmlHttpClient xmlHttpClient; + public NexusClient(HTTPCache http, URI nexusUrl) { this.http = http; this.nexusUrl = nexusUrl; + this.xmlHttpClient = new XmlHttpClient(http); + } + + public NexusFeed fetchTimeline(String timeline) throws IOException { + URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline); + + Option<Document> d = xmlHttpClient.fetch(uri); + + if (d.isNone()) { + return new NexusFeed(Collections.<NexusEvent>emptyList()); + } + + Document document = d.some(); + + return NexusFeedParser.parseDocument(document); } public ArtifactSearchResult fetchIndex(String groupId, Option<String> repositoryId) throws IOException { ArtifactSearchResult aggregate = fetchIndexPage(groupId, repositoryId, Option.<Integer>none()); ArtifactSearchResult result = aggregate; - while(result.artifacts.size() > 0) { + while (result.artifacts.size() > 0) { result = fetchIndexPage(groupId, repositoryId, some(aggregate.artifacts.size())); aggregate = aggregate.append(result); } diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java index c5e8742..cb93a63 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java @@ -3,8 +3,7 @@ package io.trygvis.esper.testing.nexus; import fj.data.*; import static fj.data.Option.*; import io.trygvis.esper.testing.*; -import static io.trygvis.esper.testing.DaoUtil.timestampToLocalDateTime; -import org.joda.time.*; +import static java.lang.System.currentTimeMillis; import java.net.*; import java.sql.*; @@ -13,6 +12,14 @@ import java.util.Date; import java.util.List; public class NexusDao { + public static final String NEXUS_SERVER = "uuid, url, name"; + + public static final String NEXUS_REPOSITORY = "uuid, server, id, group_ids"; + + private final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version"; + + private final String NEXUS_ARTIFACT = "uuid, repository, " + NEXUS_ARTIFACT_ID; + private final Connection c; public NexusDao(Connection c) { @@ -28,59 +35,65 @@ public class NexusDao { } } + private static ArtifactDto artifactDto(ResultSet rs) throws SQLException { + int i = 1; + + return new ArtifactDto( + UUID.fromString(rs.getString(i++)), + rs.getString(i++), + new ArtifactId(rs.getString(i++), + rs.getString(i++), + rs.getString(i))); + } + private NexusRepositoryDto nexusRepositoryDto(ResultSet rs) throws SQLException { int i = 1; return new NexusRepositoryDto( - rs.getString(i++), - uri(rs.getString(i++)), - (String[]) rs.getArray(i++).getArray(), - fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime), - fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime), - fromNull(rs.getTimestamp(i)).map(timestampToLocalDateTime) + UUID.fromString(rs.getString(i++)), + UUID.fromString(rs.getString(i++)), + rs.getString(i++), + (String[]) rs.getArray(i).getArray() ); } - /* - public void insertRepository(String repositoryId, URI nexusUri, LocalDateTime discoveryDate) throws SQLException { - int i = 1; - try (PreparedStatement s = prepareStatement("INSERT INTO nexus_repository(id, uri, discovered_date) VALUES(?, ?, ?)")) { - s.setString(i++, repositoryId); - s.setString(i++, nexusUri.toASCIIString()); - s.setTimestamp(i, new Timestamp(discoveryDate.toDateTime().getMillis())); - s.executeUpdate(); - } - } - */ + // ----------------------------------------------------------------------- + // Nexus Artifact + // ----------------------------------------------------------------------- public List<NexusServerDto> selectServer() throws SQLException { - try (PreparedStatement s = c.prepareStatement("SELECT url, name FROM nexus_server")) { + try (PreparedStatement s = c.prepareStatement("SELECT " + NEXUS_SERVER + " FROM nexus_server")) { ResultSet rs = s.executeQuery(); List<NexusServerDto> servers = new ArrayList<>(); while (rs.next()) { - servers.add(new NexusServerDto(uri(rs.getString(1)), rs.getString(2))); + int i = 1; + servers.add(new NexusServerDto( + UUID.fromString(rs.getString(i++)), + uri(rs.getString(i++)), + rs.getString(i))); } return servers; } } - public Option<NexusRepositoryDto> findRepository(String repositoryId) throws SQLException { - try (PreparedStatement s = c.prepareStatement("SELECT id, server_url, group_ids, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?")) { - s.setString(1, repositoryId); + public Option<NexusRepositoryDto> findRepository(UUID server, String repositoryId) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT " + NEXUS_REPOSITORY + " FROM nexus_repository WHERE server=? AND id=?")) { + s.setString(1, server.toString()); + s.setString(2, repositoryId); - try (ResultSet rs = s.executeQuery()) { - if (!rs.next()) { - return Option.none(); - } + ResultSet rs = s.executeQuery(); - return some(nexusRepositoryDto(rs)); + if (!rs.next()) { + return Option.none(); } + + return some(nexusRepositoryDto(rs)); } } public List<NexusRepositoryDto> findRepositories(URI nexusUrl) throws SQLException { - try (PreparedStatement s = c.prepareStatement("SELECT id, server_url, group_ids, created_date, last_update, last_successful_update FROM nexus_repository WHERE server_url=?")) { + try (PreparedStatement s = c.prepareStatement("SELECT " + NEXUS_REPOSITORY + " FROM nexus_repository WHERE server=?")) { s.setString(1, nexusUrl.toASCIIString()); List<NexusRepositoryDto> list = new ArrayList<>(); @@ -94,14 +107,6 @@ public class NexusDao { } } - // ----------------------------------------------------------------------- - // Nexus Artifact - // ----------------------------------------------------------------------- - - private final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version"; - - private final String NEXUS_ARTIFACT = NEXUS_ARTIFACT_ID + ", snapshot_version, classifiers, packagings, created_date"; - private int setArtifactId(int i, PreparedStatement s, ArtifactId id) throws SQLException { s.setString(i++, id.groupId); s.setString(i++, id.artifactId); @@ -109,45 +114,24 @@ public class NexusDao { return i; } - private static ArtifactDto artifactDto(URI serverUrl, String repositoryId, ResultSet rs) throws SQLException { - int i = 1; + public UUID insertArtifact(UUID repository, ArtifactId id) throws SQLException { + try (PreparedStatement s = c.prepareStatement("INSERT INTO nexus_artifact(" + NEXUS_ARTIFACT + ") VALUES(?, ?, ?, ?, ?)")) { + UUID uuid = UUID.randomUUID(); - return new ArtifactDto( - serverUrl, - repositoryId, - new ArtifactId(rs.getString(i++), - rs.getString(i++), - rs.getString(i))); - } - - public void insertArtifact(URI nexusUrl, String repositoryId, ArtifactId id, Option<String> snapshotVersion, List<ArtifactFile> files, Date createdDate) throws SQLException { - String[] classifiers = new String[files.size()]; - String[] packagings = new String[files.size()]; - - for (int i = 0; i < files.size(); i++) { - classifiers[i] = files.get(i).classifier.toNull(); - packagings[i] = files.get(i).extension; - } - - int i = 1; - try (PreparedStatement s = c.prepareStatement("INSERT INTO nexus_artifact(server_url, repository_id, " + NEXUS_ARTIFACT + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")) { - s.setString(i++, nexusUrl.toASCIIString()); - s.setString(i++, repositoryId); - i = setArtifactId(i, s, id); - s.setString(i++, snapshotVersion.toNull()); - s.setArray(i++, c.createArrayOf("varchar", classifiers)); - s.setArray(i++, c.createArrayOf("varchar", packagings)); - s.setTimestamp(i, DaoUtil.dateToTimestamp.f(createdDate)); + int i = 1; + s.setString(i++, uuid.toString()); + s.setString(i++, repository.toString()); + setArtifactId(i, s, id); s.executeUpdate(); + + return uuid; } } - public void deleteArtifact(URI nexusUrl, String repositoryId, ArtifactId id) throws SQLException { + public void deleteArtifact(UUID uuid) throws SQLException { int i = 1; - try (PreparedStatement s = c.prepareStatement("DELETE FROM nexus_artifact WHERE server_url=? AND repository_id=? AND group_id=? AND artifact_id=? AND version=?")) { - s.setString(i++, nexusUrl.toASCIIString()); - s.setString(i++, repositoryId); - i += setArtifactId(i, s, id); + try (PreparedStatement s = c.prepareStatement("DELETE FROM nexus_artifact WHERE uuid=?")) { + s.setString(i, uuid.toString()); s.executeUpdate(); } } @@ -160,53 +144,101 @@ public class NexusDao { List<ArtifactDto> list = new ArrayList<>(); while (rs.next()) { - list.add(artifactDto(url, repositoryId, rs)); + list.add(artifactDto(rs)); } return list; } } + + public Option<ArtifactDto> findArtifact(UUID repository, ArtifactId id) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT " + NEXUS_ARTIFACT + " FROM nexus_artifact WHERE repository=? AND group_id=? AND artifact_id=? AND version =?")) { + int i = 1; + s.setString(i++, repository.toString()); + setArtifactId(i, s, id); + + ResultSet rs = s.executeQuery(); + if (!rs.next()) { + return none(); + } + + return some(artifactDto(rs)); + } + } + +// public void updateSnapshotTimestamp(UUID uuid, String snapshotTimestamp) throws SQLException { +// try(PreparedStatement s = c.prepareStatement("UPDATE nexus_artifact SET snapshot_timestamp=? WHERE uuid=?")) { +// s.setString(1, snapshotTimestamp); +// s.setString(2, uuid.toString()); +// s.executeUpdate(); +// } +// } + + public UUID insertNewSnapshotEvent(UUID artifact, String guid, String file, String snapshotTimestamp) throws SQLException { + try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, timestamp, guid, type, snapshot_timestamp, file) VALUES(?, ?, ?, ?, ?, ?, ?)")) { + UUID uuid = UUID.randomUUID(); + int i = 1; + s.setString(i++, uuid.toString()); + s.setString(i++, artifact.toString()); + s.setTimestamp(i++, new Timestamp(currentTimeMillis())); + s.setString(i++, guid); + s.setString(i++, "new_snapshot"); + s.setString(i++, snapshotTimestamp); + s.setString(i, file); + s.executeUpdate(); + return uuid; + } + } + + public int countEventByGuid(String guid) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT count(guid) FROM nexus_event WHERE guid=?")) { + s.setString(1, guid); + ResultSet rs = s.executeQuery(); + rs.next(); + return rs.getInt(1); + } + } } class NexusServerDto { + public final UUID uuid; + public final URI url; + public final String name; - NexusServerDto(URI url, String name) { + NexusServerDto(UUID uuid, URI url, String name) { + this.uuid = uuid; this.url = url; this.name = name; } public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof NexusServerDto)) return false; + if (o == null || getClass() != o.getClass()) return false; NexusServerDto that = (NexusServerDto) o; - if (!url.equals(that.url)) return false; - - return true; + return uuid.equals(that.uuid); } public int hashCode() { - return url.hashCode(); + return uuid.hashCode(); } } class NexusRepositoryDto { + public final UUID uuid; + + public final UUID server; + public final String repositoryId; - public final URI nexusUrl; + public final String[] groupIds; - public final Option<LocalDateTime> discoveryDate; - public final Option<LocalDateTime> lastUpdate; - public final Option<LocalDateTime> lastSuccessfulUpdate; - NexusRepositoryDto(String repositoryId, URI nexusUrl, String[] groupIds, Option<LocalDateTime> discoveryDate, Option<LocalDateTime> lastUpdate, Option<LocalDateTime> lastSuccessfulUpdate) { + NexusRepositoryDto(UUID uuid, UUID server, String repositoryId, String[] groupIds) { + this.uuid = uuid; + this.server = server; this.repositoryId = repositoryId; - this.nexusUrl = nexusUrl; this.groupIds = groupIds; - this.discoveryDate = discoveryDate; - this.lastUpdate = lastUpdate; - this.lastSuccessfulUpdate = lastSuccessfulUpdate; } } - diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java new file mode 100755 index 0000000..d85803f --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java @@ -0,0 +1,132 @@ +package io.trygvis.esper.testing.nexus; + +import fj.*; +import fj.data.*; +import static fj.data.Option.*; +import org.jdom2.*; +import static org.jdom2.filter.Filters.*; +import org.joda.time.*; +import org.joda.time.format.*; + +import java.net.*; +import java.util.*; +import java.util.List; +import java.util.regex.*; + +public class NexusFeedParser { + private static Namespace dc = Namespace.getNamespace("http://purl.org/dc/elements/1.1/"); + + public static NexusFeed parseDocument(Document document) { + List<Element> channels = document.getRootElement().getContent(element("channel")); + + List<NexusEvent> events = new ArrayList<>(); + + if (channels.size() != 1) { + return new NexusFeed(events); + } + + Element channel = channels.get(0); + + for (Element item : channel.getContent(element("item"))) { + Option<NexusEvent> e = parseEvent(item); + + if (e.isNone()) { + continue; + } + + events.add(e.some()); + } + + return new NexusFeed(events); + } + + public static Option<NexusEvent> parseEvent(Element item) { + String title = item.getChildText("title"); + + Option<String> guid = Option.fromNull(item.getChildText("guid")); + Option<String> creator = Option.fromNull(item.getChildText("creator", dc)); + Option<DateTime> date = Option.fromNull(item.getChildText("date", dc)).bind(new F<String, Option<DateTime>>() { + @Override + public Option<DateTime> f(String s) { + try { + return some(ISODateTimeFormat.dateTimeNoMillis().parseDateTime(s)); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + return Option.none(); + } + } + }); + + String[] strings = title.split(":"); + + String groupId; + String artifactId; + String version; + Option<String> classifier; + + if (strings.length == 3) { + groupId = strings[0]; + artifactId = strings[1]; + version = strings[2]; + classifier = Option.none(); + } else if (strings.length == 4) { + groupId = strings[0]; + artifactId = strings[1]; + version = strings[2]; + classifier = some(strings[3]); + } else { + return null; + } + + if (guid.isNone() || creator.isNone() || date.isNone()) { + return null; + } + + Pattern regexp = Pattern.compile("(.*)-([0-9]{8}\\.[0-9]{6}-[0-9]*)$"); + + Matcher matcher = regexp.matcher(version); + + if (matcher.matches()) { + ArtifactId id = new ArtifactId(groupId, artifactId, matcher.group(1) + "-SNAPSHOT"); + return Option.<NexusEvent>some(new NewSnapshotEvent(guid.some(), id, classifier, creator.some(), + date.some(), matcher.group(2), URI.create(item.getChildText("link")))); + } + + return none(); + } +} + +class NexusFeed { + List<NexusEvent> events = new ArrayList<>(); + + NexusFeed(List<NexusEvent> events) { + this.events = events; + } +} + +abstract class NexusEvent { + public final String guid; + public final ArtifactId artifactId; + public final Option<String> classifier; + public final String creator; + public final DateTime date; + + NexusEvent(String guid, ArtifactId artifactId, Option<String> classifier, String creator, DateTime date) { + this.guid = guid; + this.artifactId = artifactId; + this.classifier = classifier; + this.creator = creator; + this.date = date; + } +} + +class NewSnapshotEvent extends NexusEvent { + public final String snapshotTimestamp; + public final URI url; + + NewSnapshotEvent(String guid, ArtifactId artifactId, Option<String> classifier, String creator, DateTime date, String snapshotTimestamp, URI url) { + super(guid, artifactId, classifier, creator, date); + this.snapshotTimestamp = snapshotTimestamp; + this.url = url; + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java index ea57c43..8e49074 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java @@ -2,34 +2,34 @@ package io.trygvis.esper.testing.nexus; import com.jolbox.bonecp.*; import fj.data.*; -import static fj.data.Option.*; import io.trygvis.esper.testing.*; import io.trygvis.esper.testing.object.*; +import io.trygvis.esper.testing.util.*; import static java.lang.Thread.*; +import static java.util.regex.Pattern.quote; import org.codehaus.httpcache4j.cache.*; import java.sql.*; import java.util.*; -import java.util.Date; -import java.util.HashMap; -import java.util.TreeMap; +import java.util.List; import java.util.concurrent.atomic.*; public class NexusImporter { public static void main(String[] args) throws Exception { final Config config = Config.loadFromDisk(); - - final HTTPCache http = HttpClient.createHttpClient(config); - + final HTTPCache http = HttpClient.createHttpCache(config); + final XmlParser xmlParser = new XmlParser(); final BoneCPDataSource boneCp = config.createBoneCp(); + XmlParser.debugXml = true; + ObjectManager<NexusServerDto, ActorRef<NexusServer>> serverManager = new ObjectManager<>("Nexus server", Collections.<NexusServerDto>emptySet(), new ObjectFactory<NexusServerDto, ActorRef<NexusServer>>() { public ActorRef<NexusServer> create(NexusServerDto server) { final NexusClient client = new NexusClient(http, server.url); String name = server.name; - return ObjectUtil.threadedActor(name, config.nexusUpdateInterval, boneCp, new NexusServer(client, server)); + return ObjectUtil.threadedActor(name, config.nexusUpdateInterval, boneCp, "Nexus Server: " + name, new NexusServer(client, server, xmlParser)); } }); @@ -38,9 +38,13 @@ public class NexusImporter { while (shouldRun.get()) { try { + List<NexusServerDto> newKeys; + try (Connection c = boneCp.getConnection()) { - serverManager.update(new NexusDao(c).selectServer()); + newKeys = new NexusDao(c).selectServer(); } + + serverManager.update(newKeys); } catch (SQLException e) { e.printStackTrace(System.out); } @@ -58,92 +62,156 @@ class NexusServer implements TransactionalActor { public final NexusClient client; public final NexusServerDto server; + public final XmlParser xmlParser; - NexusServer(NexusClient client, NexusServerDto server) { + NexusServer(NexusClient client, NexusServerDto server, XmlParser xmlParser) { this.client = client; this.server = server; + this.xmlParser = xmlParser; } + @Override public void act(Connection c) throws Exception { - Date timestamp = new Date(); + NexusDao dao = new NexusDao(c); - String p = server.name; + NexusFeed feed = client.fetchTimeline("recentlyDeployedArtifacts"); - for (NexusRepositoryDto repository : dao.findRepositories(server.url)) { - String p2 = p + "/" + repository.repositoryId; + int newEvents = 0, oldEvents = 0; - System.out.println(p2 + ": Updating repository: " + repository.repositoryId); + for (NexusEvent event : feed.events) { - TreeMap<ArtifactId, ArtifactXml> artifactsInNexus = new TreeMap<>(); + if(dao.countEventByGuid(event.guid) != 0) { + oldEvents++; + continue; + } - for (String groupId : repository.groupIds) { - String p3 = p2 + "/" + groupId; + newEvents++; - System.out.println(p3 + ": Updating group id"); - ArtifactSearchResult result = client.fetchIndex(groupId, some(repository.repositoryId)); - System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts"); + String repositoryId = event.guid.replaceAll("^" + quote(server.url.toASCIIString()) + "/content/repositories/([-a-zA-Z0-9]*)/.*", "$1"); - for (ArtifactXml xml : result.artifacts) { - artifactsInNexus.put(xml.id, xml); - } + if(repositoryId.length() == 0) { + continue; + } - System.out.println(p3 + ": Updating everything under group id"); - result = client.fetchIndex(groupId + ".*", some(repository.repositoryId)); - System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts"); + Option<NexusRepositoryDto> r = dao.findRepository(server.uuid, repositoryId); - for (ArtifactXml xml : result.artifacts) { - artifactsInNexus.put(xml.id, xml); - } + if(r.isNone()) { + continue; } - Map<ArtifactId, ArtifactDto> artifactsInDatabase = new HashMap<>(); - for (ArtifactDto dto : dao.findArtifactsInRepository(server.url, repository.repositoryId)) { - artifactsInDatabase.put(dto.id, dto); - } + NexusRepositoryDto repository = r.some(); - ArrayList<FlatArtifact> created = new ArrayList<>(); - ArrayList<FlatArtifact> kept = new ArrayList<>(); - ArrayList<ArtifactDto> removed = new ArrayList<>(); + Option<ArtifactDto> a = dao.findArtifact(repository.uuid, event.artifactId); - for (ArtifactXml xml : artifactsInNexus.values()) { - Option<FlatArtifact> o = xml.flatten(repository.repositoryId); + if(event instanceof NewSnapshotEvent) { + NewSnapshotEvent newSnapshotEvent = (NewSnapshotEvent) event; - if(o.isNone()) { - continue; - } + Option<String> snapshotTimestamp = Option.some(newSnapshotEvent.snapshotTimestamp); - FlatArtifact artifact = o.some(); + UUID uuid; - if(!artifactsInDatabase.containsKey(xml.id)) { - created.add(artifact); + if(a.isNone()) { + System.out.println("New artifact: " + event.artifactId); + List<ArtifactFile> files = Collections.emptyList(); + uuid = dao.insertArtifact(repository.uuid, event.artifactId); } else { - kept.add(artifact); - } - } - - for (ArtifactDto dto : artifactsInDatabase.values()) { - if(!artifactsInNexus.containsKey(dto.id)) { - removed.add(dto); - } - } + ArtifactDto artifact = a.some(); - System.out.println(p2 + ": found " + created.size() + " new artifacts, " + removed.size() + " removed artifacts and " + kept.size() + " existing artifacts."); + System.out.println("Updated artifact: " + event.artifactId); +// dao.updateSnapshotTimestamp(artifact.uuid, newSnapshotEvent.snapshotTimestamp); - System.out.println(p2 + ": inserting new artifacts"); - for (FlatArtifact artifact : created) { - dao.insertArtifact(repository.nexusUrl, repository.repositoryId, artifact.id, Option.<String>none(), artifact.files, timestamp); - } - System.out.println(p2 + ": inserted"); + uuid = artifact.uuid; + } - System.out.println(p2 + ": deleting removed artifacts"); - for (ArtifactDto artifact : removed) { - dao.deleteArtifact(repository.nexusUrl, repository.repositoryId, artifact.id); + dao.insertNewSnapshotEvent(uuid, event.guid, newSnapshotEvent.url.toASCIIString(), newSnapshotEvent.snapshotTimestamp); } - System.out.println(p2 + ": deleted"); } - c.commit(); + System.out.println("Timeline updated. New=" + newEvents + ", old=" + oldEvents); } + +// public void act(Connection c) throws Exception { +// Date timestamp = new Date(); +// NexusDao dao = new NexusDao(c); +// +// String p = server.name; +// +// for (NexusRepositoryDto repository : dao.findRepositories(server.url)) { +// String p2 = p + "/" + repository.repositoryId; +// +// System.out.println(p2 + ": Updating repository: " + repository.repositoryId); +// +// TreeMap<ArtifactId, ArtifactXml> artifactsInNexus = new TreeMap<>(); +// +// for (String groupId : repository.groupIds) { +// String p3 = p2 + "/" + groupId; +// +// System.out.println(p3 + ": Updating group id"); +// ArtifactSearchResult result = client.fetchIndex(groupId, some(repository.repositoryId)); +// System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts"); +// +// for (ArtifactXml xml : result.artifacts) { +// artifactsInNexus.put(xml.id, xml); +// } +// +// System.out.println(p3 + ": Updating everything under group id"); +// result = client.fetchIndex(groupId + ".*", some(repository.repositoryId)); +// System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts"); +// +// for (ArtifactXml xml : result.artifacts) { +// artifactsInNexus.put(xml.id, xml); +// } +// } +// +// Map<ArtifactId, ArtifactDto> artifactsInDatabase = new HashMap<>(); +// for (ArtifactDto dto : dao.findArtifactsInRepository(server.url, repository.repositoryId)) { +// artifactsInDatabase.put(dto.id, dto); +// } +// +// ArrayList<FlatArtifact> created = new ArrayList<>(); +// ArrayList<FlatArtifact> kept = new ArrayList<>(); +// ArrayList<ArtifactDto> removed = new ArrayList<>(); +// +// for (ArtifactXml xml : artifactsInNexus.values()) { +// Option<FlatArtifact> o = xml.flatten(repository.repositoryId); +// +// if(o.isNone()) { +// continue; +// } +// +// FlatArtifact artifact = o.some(); +// +// if(!artifactsInDatabase.containsKey(xml.id)) { +// created.add(artifact); +// } +// else { +// kept.add(artifact); +// } +// } +// +// for (ArtifactDto dto : artifactsInDatabase.values()) { +// if(!artifactsInNexus.containsKey(dto.id)) { +// removed.add(dto); +// } +// } +// +// System.out.println(p2 + ": found " + created.size() + " new artifacts, " + removed.size() + " removed artifacts and " + kept.size() + " existing artifacts."); +// +// System.out.println(p2 + ": inserting new artifacts"); +// for (FlatArtifact artifact : created) { +// dao.insertArtifact(repository.nexusUrl, repository.repositoryId, artifact.id, Option.<String>none(), artifact.files, timestamp); +// } +// System.out.println(p2 + ": inserted"); +// +// System.out.println(p2 + ": deleting removed artifacts"); +// for (ArtifactDto artifact : removed) { +// dao.deleteArtifact(repository.nexusUrl, repository.repositoryId, artifact.id); +// } +// System.out.println(p2 + ": deleted"); +// } +// +// c.commit(); +// } } diff --git a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java index ecfaa34..93cc5fe 100755 --- a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java +++ b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java @@ -7,32 +7,42 @@ import java.util.concurrent.*; public class ObjectUtil { - public static <A extends TransactionalActor> ActorRef<A> threadedActor(String threadName, long delay, DataSource dataSource, A actor) { - return new ThreadedActor<>(dataSource, threadName, actor, delay); + public static <A extends TransactionalActor> ActorRef<A> threadedActor(String threadName, long delay, DataSource dataSource, String name, A actor) { + return new ThreadedActor<>(dataSource, threadName, name, actor, delay); } - public static <A extends TransactionalActor> ActorRef<A> scheduledActorWithFixedDelay(ScheduledExecutorService scheduledExecutorService, long initialDelay, long delay, TimeUnit unit, DataSource dataSource, A actor) { - return new ScheduledActor<>(scheduledExecutorService, initialDelay, delay, unit, dataSource, actor); + public static <A extends TransactionalActor> ActorRef<A> scheduledActorWithFixedDelay(ScheduledExecutorService scheduledExecutorService, long initialDelay, long delay, TimeUnit unit, DataSource dataSource, String name, A actor) { + return new ScheduledActor<>(scheduledExecutorService, initialDelay, delay, unit, dataSource, name, actor); } private static class TransactionalActorWrapper<A extends TransactionalActor> implements Runnable { private final DataSource dataSource; + private final String name; private final A actor; - TransactionalActorWrapper(DataSource dataSource, A actor) { + TransactionalActorWrapper(DataSource dataSource, String name, A actor) { this.dataSource = dataSource; + this.name = name; this.actor = actor; } public void run() { try { Connection c = dataSource.getConnection(); + try { + try (PreparedStatement s = c.prepareStatement("set application_name = 'Actor: " + name + "';")) { +// s.setString(1, "Actor: " + name); + s.executeUpdate(); + s.close(); + } + actor.act(c); c.commit(); } catch(SQLException e) { c.rollback(); + throw e; } finally { c.close(); } @@ -48,9 +58,9 @@ public class ObjectUtil { private final TransactionalActorWrapper<A> actor; - ScheduledActor(ScheduledExecutorService executorService, long initialDelay, long delay, TimeUnit unit, DataSource dataSource, A actor) { + ScheduledActor(ScheduledExecutorService executorService, long initialDelay, long delay, TimeUnit unit, DataSource dataSource, String name, A actor) { future = executorService.scheduleWithFixedDelay(this, initialDelay, delay, unit); - this.actor = new TransactionalActorWrapper<>(dataSource, actor); + this.actor = new TransactionalActorWrapper<>(dataSource, name, actor); } public A underlying() { @@ -74,8 +84,8 @@ public class ObjectUtil { private final Thread thread; private boolean shouldRun = true; - ThreadedActor(DataSource dataSource, String threadName, A actor, long delay) { - this.actor = new TransactionalActorWrapper<A>(dataSource, actor); + ThreadedActor(DataSource dataSource, String threadName, String name, A actor, long delay) { + this.actor = new TransactionalActorWrapper<>(dataSource, name, actor); this.delay = delay; thread = new Thread(this, threadName); thread.setDaemon(true); diff --git a/src/main/java/io/trygvis/esper/testing/EasySSLSocketFactory.java b/src/main/java/io/trygvis/esper/testing/util/EasySSLSocketFactory.java index b8a2904..c1f1a7e 100755 --- a/src/main/java/io/trygvis/esper/testing/EasySSLSocketFactory.java +++ b/src/main/java/io/trygvis/esper/testing/util/EasySSLSocketFactory.java @@ -1,4 +1,4 @@ -package io.trygvis.esper.testing; +package io.trygvis.esper.testing.util; /* * ==================================================================== diff --git a/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java b/src/main/java/io/trygvis/esper/testing/util/EasyX509TrustManager.java index 1762c6f..5a4c4d3 100755 --- a/src/main/java/io/trygvis/esper/testing/EasyX509TrustManager.java +++ b/src/main/java/io/trygvis/esper/testing/util/EasyX509TrustManager.java @@ -1,4 +1,4 @@ -package io.trygvis.esper.testing; +package io.trygvis.esper.testing.util; /* * ==================================================================== diff --git a/src/main/java/io/trygvis/esper/testing/HttpClient.java b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java index ff2effd..ef6f2a9 100755 --- a/src/main/java/io/trygvis/esper/testing/HttpClient.java +++ b/src/main/java/io/trygvis/esper/testing/util/HttpClient.java @@ -1,5 +1,8 @@ -package io.trygvis.esper.testing; +package io.trygvis.esper.testing.util; +import fj.*; +import fj.data.*; +import io.trygvis.esper.testing.*; import static java.lang.System.*; import org.apache.http.conn.scheme.*; @@ -9,12 +12,45 @@ import org.apache.http.params.*; import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.codehaus.httpcache4j.resolver.*; +import org.jdom2.*; import java.io.*; +import java.net.*; +import javax.xml.stream.*; -public class HttpClient { +public class HttpClient<A> { - public static HTTPCache createHttpClient(Config config) { + private final HTTPCache http; + private final F<InputStream, Option<A>> f; + + public HttpClient(HTTPCache http, F<InputStream, Option<A>> f) { + this.http = http; + this.f = f; + } + + public Option<A> fetch(URI uri) throws IOException { + HTTPResponse response = null; + + try { + response = http.execute(new HTTPRequest(uri)); + + if (response.getStatus().getCode() != 200) { + throw new IOException("Did not get 200 back, got " + response.getStatus().getCode()); + } + +// return getDocument(response.getPayload().getInputStream()); + return f.f(response.getPayload().getInputStream()); + } catch (HTTPException e) { + throw new IOException(e); + } finally { + if (response != null) { + response.consume(); + } + } + } + + + public static HTTPCache createHttpCache(Config config) { SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80)); schemeRegistry.register(new Scheme("https", new EasySSLSocketFactory(), 443)); diff --git a/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java b/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java new file mode 100755 index 0000000..2c2f922 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/util/XmlHttpClient.java @@ -0,0 +1,27 @@ +package io.trygvis.esper.testing.util; + +import fj.*; +import fj.data.*; +import org.codehaus.httpcache4j.cache.*; +import org.jdom2.*; + +import java.io.*; +import java.net.*; + +public class XmlHttpClient { + + private final HttpClient<Document> httpClient; + + public XmlHttpClient(HTTPCache http) { + final XmlParser parser = new XmlParser(); + httpClient = new HttpClient<>(http, new F<InputStream, Option<Document>>() { + public Option<Document> f(InputStream inputStream) { + return parser.parseDocument(inputStream); + } + }); + } + + public Option<Document> fetch(URI uri) throws IOException { + return httpClient.fetch(uri); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/util/XmlParser.java b/src/main/java/io/trygvis/esper/testing/util/XmlParser.java new file mode 100755 index 0000000..e1b463f --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/util/XmlParser.java @@ -0,0 +1,66 @@ +package io.trygvis.esper.testing.util; + +import fj.data.*; +import static fj.data.Option.*; +import static javax.xml.stream.XMLStreamConstants.*; +import org.h2.util.*; +import org.jdom2.*; +import org.jdom2.input.*; + +import java.io.*; +import javax.xml.stream.*; + +public class XmlParser { + public static boolean debugXml; + + private final XMLInputFactory xmlInputFactory; + + private final StAXStreamBuilder streamBuilder = new StAXStreamBuilder(); + + public XmlParser() { + xmlInputFactory = XMLInputFactory.newFactory(); + } + + public Option<Document> parseDocument(InputStream stream) { + + try { + if (debugXml) { + // TODO: Pretty print + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + IOUtils.copy(stream, buffer); + byte[] bytes = buffer.toByteArray(); + System.out.println("------------------------------------------------"); + System.out.write(bytes); + System.out.println(); + System.out.println("------------------------------------------------"); + stream = new ByteArrayInputStream(bytes); + } + + // https://github.com/hunterhacker/jdom/issues/101 + XMLStreamReader readerX = xmlInputFactory.createXMLStreamReader(stream); + + XMLStreamReader reader = xmlInputFactory.createFilteredReader(readerX, new StreamFilter() { + + boolean seenStartDocument; + + @Override + public boolean accept(XMLStreamReader reader) { + if(reader.getEventType() == SPACE && !seenStartDocument) { + return false; + } + + if(reader.getEventType() == START_DOCUMENT) { + seenStartDocument = false; + } + + return true; + } + }); + + return some(streamBuilder.build(reader)); + } catch (Exception e) { + e.printStackTrace(); + return none(); + } + } +} diff --git a/src/main/resources/ddl-jenkins.sql b/src/main/resources/ddl-jenkins.sql new file mode 100755 index 0000000..37e82c1 --- /dev/null +++ b/src/main/resources/ddl-jenkins.sql @@ -0,0 +1,9 @@ +BEGIN; + +DROP TABLE IF EXISTS jenkins_server; + +CREATE TABLE jenkins_server( + url VARCHAR(1000) NOT NULL PRIMARY KEY +); + +COMMIT; diff --git a/src/main/resources/ddl-nexus.sql b/src/main/resources/ddl-nexus.sql index 15ed482..1f7514f 100755 --- a/src/main/resources/ddl-nexus.sql +++ b/src/main/resources/ddl-nexus.sql @@ -6,55 +6,56 @@ DROP TABLE IF EXISTS nexus_repository; DROP TABLE IF EXISTS nexus_server; CREATE TABLE nexus_server ( - url VARCHAR(1000) PRIMARY KEY, - name VARCHAR(100) NOT NULL + uuid CHAR(36) NOT NULL, + url VARCHAR(1000) NOT NULL, + name VARCHAR(1000) NOT NULL, + CONSTRAINT pk_nexus_server PRIMARY KEY (uuid), + CONSTRAINT uq_nexus_server_url UNIQUE (url) ); CREATE TABLE nexus_repository ( - id VARCHAR(1000), - server_url VARCHAR(1000) NOT NULL, - group_ids VARCHAR(100) [], - created_date TIMESTAMP, - last_update TIMESTAMP, - last_successful_update TIMESTAMP, - CONSTRAINT nexus_repository_pk PRIMARY KEY (id, server_url), - CONSTRAINT nexus_server_fk FOREIGN KEY (server_url) REFERENCES nexus_server (url) + uuid CHAR(36) NOT NULL, + server CHAR(36) NOT NULL, + id VARCHAR(100), + group_ids VARCHAR(100) [], + CONSTRAINT pk_nexus_repository PRIMARY KEY (uuid), + CONSTRAINT fk_nexus_server FOREIGN KEY (server) REFERENCES nexus_server (uuid), + CONSTRAINT uq_nexus_repository_id UNIQUE (server, id) ); CREATE TABLE nexus_artifact ( - server_url VARCHAR(1000) NOT NULL, - repository_id VARCHAR(1000) NOT NULL, - group_id VARCHAR(100) NOT NULL, - artifact_id VARCHAR(100) NOT NULL, - version VARCHAR(100) NOT NULL, - snapshot_version VARCHAR(100), - classifiers VARCHAR(100) [], - packagings VARCHAR(100) [], - created_date TIMESTAMP, - last_update TIMESTAMP, - last_successful_update TIMESTAMP, - CONSTRAINT nexus_artifact_pk PRIMARY KEY (server_url, repository_id, group_id, artifact_id, version), - CONSTRAINT nexus_repository_fk FOREIGN KEY (server_url, repository_id) REFERENCES nexus_repository (server_url, id) + uuid CHAR(36) NOT NULL, + repository CHAR(36) NOT NULL, + group_id VARCHAR(100) NOT NULL, + artifact_id VARCHAR(100) NOT NULL, + version VARCHAR(100) NOT NULL, + CONSTRAINT pk_nexus_artifact PRIMARY KEY (uuid), + CONSTRAINT uq_nexus_artifact_gid_aid_version UNIQUE (group_id, artifact_id, version), + CONSTRAINT fk_nexus_repository FOREIGN KEY (repository) REFERENCES nexus_repository (uuid) ); CREATE TABLE nexus_event ( - timestamp VARCHAR(100), - uuid VARCHAR(100), - server_url VARCHAR(1000) NOT NULL, - repository_id VARCHAR(1000) NOT NULL, - group_id VARCHAR(100) NOT NULL, - artifact_id VARCHAR(100) NOT NULL, - version VARCHAR(100) NOT NULL, - files VARCHAR(100) [], - CONSTRAINT nexus_event_pk PRIMARY KEY (timestamp, server_url, repository_id, group_id, artifact_id, version), - CONSTRAINT nexus_artifact_fk FOREIGN KEY (server_url, repository_id, group_id, artifact_id, version) REFERENCES nexus_artifact (server_url, repository_id, group_id, artifact_id, version) + uuid CHAR(36) NOT NULL, + artifact CHAR(36) NOT NULL, + timestamp VARCHAR(100), + guid VARCHAR(1000), + + type VARCHAR(100) NOT NULL, + +-- new snapshot event + snapshot_timestamp VARCHAR(1000), + file VARCHAR(1000), + CONSTRAINT pk_nexus_event PRIMARY KEY (uuid), + CONSTRAINT fk_nexus_artifact FOREIGN KEY (artifact) REFERENCES nexus_artifact (uuid), + CONSTRAINT uq_guid UNIQUE (guid), + CONSTRAINT check_event_type CHECK (type IN ('new_snapshot')) +-- CONSTRAINT pk_nexus_event PRIMARY KEY (timestamp, server_url, repository_id, group_id, artifact_id, version) ); -INSERT INTO nexus_server (url, name) VALUES ('https://oss.sonatype.org', 'Central Repository'); --- INSERT INTO nexus_repository (id, server_url, group_ids) VALUES ('releases', 'https://oss.sonatype.org', ARRAY ['io.trygvis', 'no.arktekk', 'org.codehaus']); -INSERT INTO nexus_repository (id, server_url, group_ids) VALUES ('releases', 'https://oss.sonatype.org', ARRAY ['io.trygvis', 'no.arktekk', 'org.codehaus']); +INSERT INTO nexus_server (uuid, url, name) VALUES ('4666dba4-3e2e-11e2-8a1b-0bd430e00b36', 'https://oss.sonatype.org', 'Central Repository'); +INSERT INTO nexus_repository (uuid, server, id, group_ids) VALUES ('4a2d7ab2-3e2f-11e2-af03-eb1ace2381bb', '4666dba4-3e2e-11e2-8a1b-0bd430e00b36', 'releases', ARRAY ['io.trygvis', 'no.arktekk', 'org.codehaus']); -INSERT INTO nexus_server (url, name) VALUES ('http://nexus.codehaus.org', 'Codehaus Snapshots'); -INSERT INTO nexus_repository (id, server_url, group_ids) VALUES ('snapshots', 'http://nexus.codehaus.org', ARRAY ['org.codehaus.mojo']); +INSERT INTO nexus_server (uuid, url, name) VALUES ('91d942d8-3e2f-11e2-aaa0-a70628365abd', 'http://nexus.codehaus.org', 'Codehaus Snapshots'); +INSERT INTO nexus_repository (uuid, server, id, group_ids) VALUES ('a2415b88-3e2f-11e2-a2b8-2f066b90cf13', '91d942d8-3e2f-11e2-aaa0-a70628365abd', 'snapshots', ARRAY ['org.codehaus.mojo']); COMMIT; diff --git a/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java b/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java index 6495d86..038de34 100755 --- a/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java +++ b/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java @@ -3,7 +3,9 @@ package io.trygvis.esper.testing.nexus; import static com.google.common.collect.Iterables.*; import static com.google.common.collect.Lists.*; import static io.trygvis.esper.testing.nexus.ArtifactXml.repositoryFilter; +import io.trygvis.esper.testing.util.*; import junit.framework.*; +import org.jdom2.*; import java.io.*; import java.util.*; @@ -44,4 +46,21 @@ public class TestXmlParsing extends TestCase { assertEquals(2, flatArtifact.files.size()); } } + + public void testTimelineParsing() throws Exception { + XmlParser parser = new XmlParser(); + + try (InputStream stream = getClass().getResourceAsStream("/nexus/recentlyDeployedArtifacts.xml")) { + Document document = parser.parseDocument(stream).some(); + + NexusEvent event = NexusFeedParser.parseEvent(document.getRootElement().getChild("channel").getChild("item")).some(); + + assertTrue(event instanceof NewSnapshotEvent); + NewSnapshotEvent nse = (NewSnapshotEvent) event; + assertEquals("org.example", nse.artifactId.groupId); + assertEquals("example", nse.artifactId.artifactId); + assertEquals("1.0-SNAPSHOT", nse.artifactId.version); + assertEquals("20121204.122640-536", nse.snapshotTimestamp); + } + } } diff --git a/src/test/resources/nexus/recentlyDeployedArtifacts.xml b/src/test/resources/nexus/recentlyDeployedArtifacts.xml new file mode 100755 index 0000000..1fe22d9 --- /dev/null +++ b/src/test/resources/nexus/recentlyDeployedArtifacts.xml @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="UTF-8"?> +<rss xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0"> + <channel> + <title>New deployed artifacts</title> + <link>http://localhost:8081/repository/service/local/feeds/recentlyDeployedArtifacts</link> + <description>New deployed artifacts in all Nexus repositories (deployed).</description> + <pubDate>Tue, 04 Dec 2012 12:26:41 GMT</pubDate> + <dc:creator>Nexus 1.9.2.3</dc:creator> + <dc:date>2012-12-04T12:26:41Z</dc:date> + <item> + <title>org.example:example:1.0-20121204.122640-536</title> + <link>http://hostname/repository/content/repositories/snapshots/org/example/example/1.0-SNAPSHOT/example-1.0-20121204.122640-536.pom</link> + <description> + The artifact 'org.example:example:1.0-20121204.122640-536' in repository 'Snapshots' was + deployed.Action was initiated by user "developer". + Request originated from IP address 1.2.3.4. + </description> + <pubDate>Tue, 04 Dec 2012 12:26:40 GMT</pubDate> + <guid>http://hostname/repository/content/repositories/snapshots/org/example/example/1.0-SNAPSHOT/example-1.0-20121204.122640-536.pom</guid> + <dc:creator>developer</dc:creator> + <dc:date>2012-12-04T12:26:40Z</dc:date> + </item> + </channel> +</rss> diff --git a/src/test/resources/nexus/search-1.xml b/src/test/resources/nexus/search-1.xml index b501491..75bfdc9 100755 --- a/src/test/resources/nexus/search-1.xml +++ b/src/test/resources/nexus/search-1.xml @@ -2674,4 +2674,3 @@ </artifact> </data> </searchNGResponse> -- |