diff options
Diffstat (limited to 'src/main')
6 files changed, 180 insertions, 108 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java index 02dc28c..9900bfc 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java @@ -30,13 +30,25 @@ public class NexusClient { this.xmlHttpClient = new XmlHttpClient(http); } - public NexusFeed fetchTimeline(String timeline) throws IOException { - URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline); + public NexusFeed fetchTimeline(String timeline, int count, int from) throws IOException { + + String args = ""; + + if (count != 0) { + args += (args.length() == 0 ? "?" : "&") + "count=" + count; + } + + if (from != 0) { + args += (args.length() == 0 ? "?" : "&") + "from=" + from; + } + + System.out.println("args = " + args); + URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline + args); Option<Document> d = xmlHttpClient.fetch(uri); if (d.isNone()) { - return new NexusFeed(Collections.<NexusEvent>emptyList()); + return new NexusFeed(Collections.<HasNexusEvent>emptyList()); } Document document = d.some(); diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java index d8d013c..2b2dad7 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java @@ -2,23 +2,24 @@ package io.trygvis.esper.testing.nexus; import fj.data.*; import static fj.data.Option.*; -import io.trygvis.esper.testing.*; import static java.lang.System.currentTimeMillis; +import org.joda.time.*; import java.net.*; import java.sql.*; import java.util.*; -import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class NexusDao { - public static final String NEXUS_SERVER = "uuid, url, name"; + private static final String NEXUS_SERVER = "uuid, url, name"; - public static final String NEXUS_REPOSITORY = "uuid, server, id, group_ids"; + private static final String NEXUS_REPOSITORY = "uuid, server, id, group_ids"; - private final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version"; + private static final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version"; - private final String NEXUS_ARTIFACT = "uuid, repository, " + NEXUS_ARTIFACT_ID; + private static final String NEXUS_ARTIFACT = "uuid, repository, " + NEXUS_ARTIFACT_ID; private final Connection c; @@ -165,24 +166,14 @@ public class NexusDao { } } -// public void updateSnapshotTimestamp(UUID uuid, String snapshotTimestamp) throws SQLException { -// try(PreparedStatement s = c.prepareStatement("UPDATE nexus_artifact SET snapshot_timestamp=? WHERE uuid=?")) { -// s.setString(1, snapshotTimestamp); -// s.setString(2, uuid.toString()); -// s.executeUpdate(); -// } -// } - - public UUID insertNewSnapshotEvent(UUID artifact, String guid, String snapshotTimestamp, int buildNumber, String file) throws SQLException { - try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, timestamp, guid, type, snapshot_timestamp, build_number, file) VALUES(?, ?, ?, ?, ?, ?, ?, ?)")) { + public UUID insertNewSnapshotEvent(UUID artifact, String guid, String who, DateTime date, DateTime snapshotTimestamp, int buildNumber, String file) throws SQLException { + try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, created, guid, date, who, type, snapshot_timestamp, build_number, file) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { UUID uuid = UUID.randomUUID(); int i = 1; - s.setString(i++, uuid.toString()); - s.setString(i++, artifact.toString()); - s.setTimestamp(i++, new Timestamp(currentTimeMillis())); - s.setString(i++, guid); + i = insertEvent(s, i, artifact, guid, who, date, uuid); + s.setString(i++, "new_snapshot"); - s.setString(i++, snapshotTimestamp); + s.setTimestamp(i++, new Timestamp(snapshotTimestamp.getMillis())); s.setInt(i++, buildNumber); s.setString(i, file); s.executeUpdate(); @@ -190,14 +181,12 @@ public class NexusDao { } } - public UUID insertNewReleaseEvent(UUID artifact, String guid, String file) throws SQLException { - try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, timestamp, guid, type, file) VALUES(?, ?, ?, ?, ?, ?)")) { + public UUID insertNewReleaseEvent(UUID artifact, String guid, String who, DateTime date, String file) throws SQLException { + try (PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, created, guid, date, who, type, file) VALUES(?, ?, ?, ?, ?, ?, ?, ?)")) { UUID uuid = UUID.randomUUID(); int i = 1; - s.setString(i++, uuid.toString()); - s.setString(i++, artifact.toString()); - s.setTimestamp(i++, new Timestamp(currentTimeMillis())); - s.setString(i++, guid); + i = insertEvent(s, i, artifact, guid, who, date, uuid); + s.setString(i++, "new_release"); s.setString(i, file); s.executeUpdate(); @@ -205,6 +194,16 @@ public class NexusDao { } } + private int insertEvent(PreparedStatement s, int i, UUID artifact, String guid, String who, DateTime date, UUID uuid) throws SQLException { + s.setString(i++, uuid.toString()); + s.setString(i++, artifact.toString()); + s.setTimestamp(i++, new Timestamp(currentTimeMillis())); + s.setString(i++, guid); + s.setTimestamp(i++, new Timestamp(date.getMillis())); + s.setString(i++, who); + return i; + } + public int countEventByGuid(String guid) throws SQLException { try (PreparedStatement s = c.prepareStatement("SELECT count(guid) FROM nexus_event WHERE guid=?")) { s.setString(1, guid); @@ -213,6 +212,18 @@ public class NexusDao { return rs.getInt(1); } } + + public Set<String> selectGuidsByGuids(Collection<String> guids) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT guid FROM nexus_event WHERE guid = ANY (?)")) { + s.setArray(1, c.createArrayOf("varchar", guids.toArray())); + ResultSet rs = s.executeQuery(); + Set<String> list = new HashSet<>(); + while (rs.next()) { + list.add(rs.getString(1)); + } + return list; + } + } } class NexusServerDto { diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java index 3d27436..eb89f30 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java @@ -3,6 +3,7 @@ package io.trygvis.esper.testing.nexus; import fj.*; import fj.data.*; import static fj.data.Option.*; +import static java.util.regex.Pattern.compile; import org.jdom2.*; import static org.jdom2.filter.Filters.*; import org.joda.time.*; @@ -14,12 +15,15 @@ import java.util.List; import java.util.regex.*; public class NexusFeedParser { - private static Namespace dc = Namespace.getNamespace("http://purl.org/dc/elements/1.1/"); + private static final Namespace dc = Namespace.getNamespace("http://purl.org/dc/elements/1.1/"); + + private static final Pattern snapshotPattern = + compile("(.*)-([0-9]{4})([0-9]{2})([0-9]{2})\\.([0-9]{2})([0-9]{2})([0-9]{2})-([0-9]*)$"); public static NexusFeed parseDocument(Document document) { List<Element> channels = document.getRootElement().getContent(element("channel")); - List<NexusEvent> events = new ArrayList<>(); + List<HasNexusEvent> events = new ArrayList<>(); if (channels.size() != 1) { return new NexusFeed(events); @@ -28,7 +32,7 @@ public class NexusFeedParser { Element channel = channels.get(0); for (Element item : channel.getContent(element("item"))) { - Option<NexusEvent> e = parseEvent(item); + Option<HasNexusEvent> e = parseEvent(item); if (e.isNone()) { continue; @@ -40,7 +44,7 @@ public class NexusFeedParser { return new NexusFeed(events); } - public static Option<NexusEvent> parseEvent(Element item) { + public static Option<HasNexusEvent> parseEvent(Element item) { String title = item.getChildText("title"); Option<String> guid = Option.fromNull(item.getChildText("guid")); @@ -91,43 +95,47 @@ public class NexusFeedParser { return null; } - Pattern regexp = Pattern.compile("(.*)-([0-9]{8}\\.[0-9]{6})-([0-9]*)$"); - - Matcher matcher = regexp.matcher(version); + Matcher matcher = snapshotPattern.matcher(version); if (matcher.matches()) { ArtifactId id = new ArtifactId(groupId, artifactId, matcher.group(1) + "-SNAPSHOT"); - Option<Integer> buildNumber = parseInt.f(matcher.group(3)); + Option<Integer> buildNumber = parseInt.f(matcher.group(8)); if(buildNumber.isNone()) { System.out.println("Could not parse build number: " + matcher.group(3)); return none(); } - return Option.<NexusEvent>some(new NewSnapshotEvent(guid.some(), id, classifier, creator.some(), - date.some(), matcher.group(2), buildNumber.some(), URI.create(item.getChildText("link")))); + NexusEvent event = new NexusEvent(guid.some(), id, classifier, creator.some(), date.some()); + + int year = Integer.parseInt(matcher.group(2)); + int month = Integer.parseInt(matcher.group(3)); + int day = Integer.parseInt(matcher.group(4)); + int hour = Integer.parseInt(matcher.group(5)); + int minute = Integer.parseInt(matcher.group(6)); + int second = Integer.parseInt(matcher.group(7)); + + return Option.<HasNexusEvent>some(new NewSnapshotEvent(event, new DateTime(year, month, day, hour, minute, second, 0), buildNumber.some(), + URI.create(item.getChildText("link")))); } else { ArtifactId id = new ArtifactId(groupId, artifactId, version); - return Option.<NexusEvent>some(new NewReleaseEvent(guid.some(), id, classifier, creator.some(), - date.some(), URI.create(item.getChildText("link")))); - } + NexusEvent event = new NexusEvent(guid.some(), id, classifier, creator.some(), date.some()); -// System.out.println("Unknown event type."); -// -// return none(); + return Option.<HasNexusEvent>some(new NewReleaseEvent(event, URI.create(item.getChildText("link")))); + } } } class NexusFeed { - List<NexusEvent> events = new ArrayList<>(); + List<HasNexusEvent> events = new ArrayList<>(); - NexusFeed(List<NexusEvent> events) { + NexusFeed(List<HasNexusEvent> events) { this.events = events; } } -abstract class NexusEvent { +final class NexusEvent { public final String guid; public final ArtifactId artifactId; public final Option<String> classifier; @@ -143,24 +151,32 @@ abstract class NexusEvent { } } -class NewSnapshotEvent extends NexusEvent { - public final String snapshotTimestamp; +abstract class HasNexusEvent { + public final NexusEvent event; + + protected HasNexusEvent(NexusEvent event) { + this.event = event; + } +} + +class NewSnapshotEvent extends HasNexusEvent { + public final DateTime snapshotTimestamp; public final int buildNumber; public final URI url; - NewSnapshotEvent(String guid, ArtifactId artifactId, Option<String> classifier, String creator, DateTime date, String snapshotTimestamp, int buildNumber, URI url) { - super(guid, artifactId, classifier, creator, date); + NewSnapshotEvent(NexusEvent event, DateTime snapshotTimestamp, int buildNumber, URI url) { + super(event); this.snapshotTimestamp = snapshotTimestamp; this.buildNumber = buildNumber; this.url = url; } } -class NewReleaseEvent extends NexusEvent { +class NewReleaseEvent extends HasNexusEvent { public final URI url; - NewReleaseEvent(String guid, ArtifactId artifactId, Option<String> classifier, String creator, DateTime date, URI url) { - super(guid, artifactId, classifier, creator, date); + NewReleaseEvent(NexusEvent event, URI url) { + super(event); this.url = url; } } diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java index 3d0aaf0..3bf93c4 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java @@ -12,6 +12,7 @@ import org.codehaus.httpcache4j.cache.*; import java.sql.*; import java.util.*; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.*; public class NexusImporter { @@ -75,65 +76,97 @@ class NexusServer implements TransactionalActor { NexusDao dao = new NexusDao(c); - NexusFeed feed = client.fetchTimeline("recentlyDeployedArtifacts"); + int newEvents = 0, oldEvents = 0, count = 0, pages = 0; - int newEvents = 0, oldEvents = 0; + int newInFeed; + do { + newInFeed = 0; + NexusFeed feed = client.fetchTimeline("recentlyDeployedArtifacts", 100, count); - for (NexusEvent event : feed.events) { + pages++; + count += feed.events.size(); - if(dao.countEventByGuid(event.guid) != 0) { - oldEvents++; - continue; + List<String> guids = new ArrayList<>(); + + for (HasNexusEvent event : feed.events) { + guids.add(event.event.guid); } - newEvents++; + Set<String> oldGuids = dao.selectGuidsByGuids(guids); - String repositoryId = event.guid.replaceAll("^" + quote(server.url.toASCIIString()) + "/content/repositories/([-a-zA-Z0-9]*)/.*", "$1"); + System.out.println("oldGuids.size() = " + oldGuids.size()); - if(repositoryId.length() == 0) { - continue; - } + for (HasNexusEvent event : feed.events) { + +// if (oldGuids.contains(event.event.guid)) { +// oldEvents++; +// continue; +// } - Option<NexusRepositoryDto> r = dao.findRepository(server.uuid, repositoryId); + if (dao.countEventByGuid(event.event.guid) != 0) { + oldEvents++; + continue; + } - if(r.isNone()) { - continue; + newEvents++; + newInFeed++; + + onEvent(dao, event); } - NexusRepositoryDto repository = r.some(); + System.out.println("newInFeed = " + newInFeed); + } while (newInFeed > 0); - Option<ArtifactDto> a = dao.findArtifact(repository.uuid, event.artifactId); + System.out.println("Timeline updated. New=" + newEvents + ", old=" + oldEvents + ", count=" + count + ", pages=" + pages); + } - UUID uuid; + private void onEvent(NexusDao dao, HasNexusEvent e) throws SQLException { - if(a.isNone()) { - System.out.println("New artifact: " + event.artifactId); - uuid = dao.insertArtifact(repository.uuid, event.artifactId); - } - else { - ArtifactDto artifact = a.some(); + NexusEvent event = e.event; - System.out.println("Updated artifact: " + event.artifactId); -// dao.updateSnapshotTimestamp(artifact.uuid, newSnapshotEvent.snapshotTimestamp); + String repositoryId = event.guid.replaceAll("^" + quote(server.url.toASCIIString()) + "/content/repositories/([-a-zA-Z0-9]*)/.*", "$1"); - uuid = artifact.uuid; - } + if(repositoryId.length() == 0) { + return; + } - if (event instanceof NewSnapshotEvent) { - NewSnapshotEvent newSnapshotEvent = (NewSnapshotEvent) event; + Option<NexusRepositoryDto> r = dao.findRepository(server.uuid, repositoryId); - dao.insertNewSnapshotEvent(uuid, event.guid, newSnapshotEvent.snapshotTimestamp, - newSnapshotEvent.buildNumber, newSnapshotEvent.url.toASCIIString()); - } else if (event instanceof NewReleaseEvent) { - NewReleaseEvent nre = (NewReleaseEvent) event; + if(r.isNone()) { + return; + } - dao.insertNewReleaseEvent(uuid, event.guid, nre.url.toASCIIString()); - } else { - System.out.println("Unknown event type: " + event.getClass().getName()); - } + NexusRepositoryDto repository = r.some(); + + Option<ArtifactDto> a = dao.findArtifact(repository.uuid, event.artifactId); + + UUID uuid; + + if(a.isNone()) { + System.out.println("New artifact: " + event.artifactId); + uuid = dao.insertArtifact(repository.uuid, event.artifactId); } + else { + ArtifactDto artifact = a.some(); + +// System.out.println("Updated artifact: " + event.artifactId); - System.out.println("Timeline updated. New=" + newEvents + ", old=" + oldEvents); + uuid = artifact.uuid; + } + + if (e instanceof NewSnapshotEvent) { + NewSnapshotEvent newSnapshotEvent = (NewSnapshotEvent) e; + + dao.insertNewSnapshotEvent(uuid, event.guid, event.creator, event.date, + newSnapshotEvent.snapshotTimestamp, newSnapshotEvent.buildNumber, + newSnapshotEvent.url.toASCIIString()); + } else if (e instanceof NewReleaseEvent) { + NewReleaseEvent nre = (NewReleaseEvent) e; + + dao.insertNewReleaseEvent(uuid, event.guid, event.creator, event.date, nre.url.toASCIIString()); + } else { + System.out.println("Unknown event type: " + event.getClass().getName()); + } } // public void act(Connection c) throws Exception { diff --git a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java index 93cc5fe..f421acf 100755 --- a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java +++ b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java @@ -115,8 +115,7 @@ public class ObjectUtil { while (thread.isAlive()) { try { thread.join(); - } catch (InterruptedException e) { - continue; + } catch (InterruptedException ignore) { } } } diff --git a/src/main/resources/ddl-nexus.sql b/src/main/resources/ddl-nexus.sql index 7f84dac..8acf741 100755 --- a/src/main/resources/ddl-nexus.sql +++ b/src/main/resources/ddl-nexus.sql @@ -19,8 +19,8 @@ CREATE TABLE nexus_repository ( id VARCHAR(100), group_ids VARCHAR(100) [], CONSTRAINT pk_nexus_repository PRIMARY KEY (uuid), - CONSTRAINT fk_nexus_server FOREIGN KEY (server) REFERENCES nexus_server (uuid), - CONSTRAINT uq_nexus_repository_id UNIQUE (server, id) + CONSTRAINT fk_nexus_repository__nexus_server FOREIGN KEY (server) REFERENCES nexus_server (uuid), + CONSTRAINT uq_nexus_repository__id UNIQUE (server, id) ); CREATE TABLE nexus_artifact ( @@ -30,29 +30,30 @@ CREATE TABLE nexus_artifact ( artifact_id VARCHAR(100) NOT NULL, version VARCHAR(100) NOT NULL, CONSTRAINT pk_nexus_artifact PRIMARY KEY (uuid), - CONSTRAINT uq_nexus_artifact_gid_aid_version UNIQUE (group_id, artifact_id, version), - CONSTRAINT fk_nexus_repository FOREIGN KEY (repository) REFERENCES nexus_repository (uuid) + CONSTRAINT uq_nexus_artifact__gid__aid__version UNIQUE (group_id, artifact_id, version), + CONSTRAINT fk_nexus_artifact__nexus_repository FOREIGN KEY (repository) REFERENCES nexus_repository (uuid) ); CREATE TABLE nexus_event ( - uuid CHAR(36) NOT NULL, - artifact CHAR(36) NOT NULL, - timestamp VARCHAR(100), + uuid CHAR(36) NOT NULL, + artifact CHAR(36) NOT NULL, + created TIMESTAMP NOT NULL, -- From the RSS - guid VARCHAR(1000), + guid VARCHAR(1000) NOT NULL, + date TIMESTAMP NOT NULL, -- Our type flag - type VARCHAR(100) NOT NULL, + type VARCHAR(100) NOT NULL, -- new snapshot event - snapshot_timestamp TIMESTAMP, + snapshot_timestamp VARCHAR(100), build_number INT, file VARCHAR(1000), who VARCHAR(1000), CONSTRAINT pk_nexus_event PRIMARY KEY (uuid), - CONSTRAINT fk_nexus_artifact FOREIGN KEY (artifact) REFERENCES nexus_artifact (uuid), - CONSTRAINT uq_guid UNIQUE (guid), + CONSTRAINT fk_nexus_event__artifact FOREIGN KEY (artifact) REFERENCES nexus_artifact (uuid), + CONSTRAINT uq_nexus_event__guid UNIQUE (guid), CONSTRAINT check_event_type CHECK (type IN ('new_snapshot', 'new_release')) -- CONSTRAINT pk_nexus_event PRIMARY KEY (timestamp, server_url, repository_id, group_id, artifact_id, version) ); |