aboutsummaryrefslogtreecommitdiff
path: root/src/main
diff options
context:
space:
mode:
Diffstat (limited to 'src/main')
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusClient.java18
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusDao.java65
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java68
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java109
-rwxr-xr-xsrc/main/java/io/trygvis/esper/testing/object/ObjectUtil.java3
-rwxr-xr-xsrc/main/resources/ddl-nexus.sql25
6 files changed, 180 insertions, 108 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
index 02dc28c..9900bfc 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
@@ -30,13 +30,25 @@ public class NexusClient {
this.xmlHttpClient = new XmlHttpClient(http);
}
- public NexusFeed fetchTimeline(String timeline) throws IOException {
- URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline);
+ public NexusFeed fetchTimeline(String timeline, int count, int from) throws IOException {
+
+ String args = "";
+
+ if (count != 0) {
+ args += (args.length() == 0 ? "?" : "&") + "count=" + count;
+ }
+
+ if (from != 0) {
+ args += (args.length() == 0 ? "?" : "&") + "from=" + from;
+ }
+
+ System.out.println("args = " + args);
+ URI uri = URI.create(nexusUrl.toASCIIString() + "/service/local/feeds/" + timeline + args);
Option<Document> d = xmlHttpClient.fetch(uri);
if (d.isNone()) {
- return new NexusFeed(Collections.<NexusEvent>emptyList());
+ return new NexusFeed(Collections.<HasNexusEvent>emptyList());
}
Document document = d.some();
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
index d8d013c..2b2dad7 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
@@ -2,23 +2,24 @@ package io.trygvis.esper.testing.nexus;
import fj.data.*;
import static fj.data.Option.*;
-import io.trygvis.esper.testing.*;
import static java.lang.System.currentTimeMillis;
+import org.joda.time.*;
import java.net.*;
import java.sql.*;
import java.util.*;
-import java.util.Date;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
public class NexusDao {
- public static final String NEXUS_SERVER = "uuid, url, name";
+ private static final String NEXUS_SERVER = "uuid, url, name";
- public static final String NEXUS_REPOSITORY = "uuid, server, id, group_ids";
+ private static final String NEXUS_REPOSITORY = "uuid, server, id, group_ids";
- private final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version";
+ private static final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version";
- private final String NEXUS_ARTIFACT = "uuid, repository, " + NEXUS_ARTIFACT_ID;
+ private static final String NEXUS_ARTIFACT = "uuid, repository, " + NEXUS_ARTIFACT_ID;
private final Connection c;
@@ -165,24 +166,14 @@ public class NexusDao {
}
}
-// public void updateSnapshotTimestamp(UUID uuid, String snapshotTimestamp) throws SQLException {
-// try(PreparedStatement s = c.prepareStatement("UPDATE nexus_artifact SET snapshot_timestamp=? WHERE uuid=?")) {
-// s.setString(1, snapshotTimestamp);
-// s.setString(2, uuid.toString());
-// s.executeUpdate();
-// }
-// }
-
- public UUID insertNewSnapshotEvent(UUID artifact, String guid, String snapshotTimestamp, int buildNumber, String file) throws SQLException {
- try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, timestamp, guid, type, snapshot_timestamp, build_number, file) VALUES(?, ?, ?, ?, ?, ?, ?, ?)")) {
+ public UUID insertNewSnapshotEvent(UUID artifact, String guid, String who, DateTime date, DateTime snapshotTimestamp, int buildNumber, String file) throws SQLException {
+ try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, created, guid, date, who, type, snapshot_timestamp, build_number, file) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) {
UUID uuid = UUID.randomUUID();
int i = 1;
- s.setString(i++, uuid.toString());
- s.setString(i++, artifact.toString());
- s.setTimestamp(i++, new Timestamp(currentTimeMillis()));
- s.setString(i++, guid);
+ i = insertEvent(s, i, artifact, guid, who, date, uuid);
+
s.setString(i++, "new_snapshot");
- s.setString(i++, snapshotTimestamp);
+ s.setTimestamp(i++, new Timestamp(snapshotTimestamp.getMillis()));
s.setInt(i++, buildNumber);
s.setString(i, file);
s.executeUpdate();
@@ -190,14 +181,12 @@ public class NexusDao {
}
}
- public UUID insertNewReleaseEvent(UUID artifact, String guid, String file) throws SQLException {
- try(PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, timestamp, guid, type, file) VALUES(?, ?, ?, ?, ?, ?)")) {
+ public UUID insertNewReleaseEvent(UUID artifact, String guid, String who, DateTime date, String file) throws SQLException {
+ try (PreparedStatement s = c.prepareStatement("INSERT INTO nexus_event(uuid, artifact, created, guid, date, who, type, file) VALUES(?, ?, ?, ?, ?, ?, ?, ?)")) {
UUID uuid = UUID.randomUUID();
int i = 1;
- s.setString(i++, uuid.toString());
- s.setString(i++, artifact.toString());
- s.setTimestamp(i++, new Timestamp(currentTimeMillis()));
- s.setString(i++, guid);
+ i = insertEvent(s, i, artifact, guid, who, date, uuid);
+
s.setString(i++, "new_release");
s.setString(i, file);
s.executeUpdate();
@@ -205,6 +194,16 @@ public class NexusDao {
}
}
+ private int insertEvent(PreparedStatement s, int i, UUID artifact, String guid, String who, DateTime date, UUID uuid) throws SQLException {
+ s.setString(i++, uuid.toString());
+ s.setString(i++, artifact.toString());
+ s.setTimestamp(i++, new Timestamp(currentTimeMillis()));
+ s.setString(i++, guid);
+ s.setTimestamp(i++, new Timestamp(date.getMillis()));
+ s.setString(i++, who);
+ return i;
+ }
+
public int countEventByGuid(String guid) throws SQLException {
try (PreparedStatement s = c.prepareStatement("SELECT count(guid) FROM nexus_event WHERE guid=?")) {
s.setString(1, guid);
@@ -213,6 +212,18 @@ public class NexusDao {
return rs.getInt(1);
}
}
+
+ public Set<String> selectGuidsByGuids(Collection<String> guids) throws SQLException {
+ try (PreparedStatement s = c.prepareStatement("SELECT guid FROM nexus_event WHERE guid = ANY (?)")) {
+ s.setArray(1, c.createArrayOf("varchar", guids.toArray()));
+ ResultSet rs = s.executeQuery();
+ Set<String> list = new HashSet<>();
+ while (rs.next()) {
+ list.add(rs.getString(1));
+ }
+ return list;
+ }
+ }
}
class NexusServerDto {
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java
index 3d27436..eb89f30 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusFeedParser.java
@@ -3,6 +3,7 @@ package io.trygvis.esper.testing.nexus;
import fj.*;
import fj.data.*;
import static fj.data.Option.*;
+import static java.util.regex.Pattern.compile;
import org.jdom2.*;
import static org.jdom2.filter.Filters.*;
import org.joda.time.*;
@@ -14,12 +15,15 @@ import java.util.List;
import java.util.regex.*;
public class NexusFeedParser {
- private static Namespace dc = Namespace.getNamespace("http://purl.org/dc/elements/1.1/");
+ private static final Namespace dc = Namespace.getNamespace("http://purl.org/dc/elements/1.1/");
+
+ private static final Pattern snapshotPattern =
+ compile("(.*)-([0-9]{4})([0-9]{2})([0-9]{2})\\.([0-9]{2})([0-9]{2})([0-9]{2})-([0-9]*)$");
public static NexusFeed parseDocument(Document document) {
List<Element> channels = document.getRootElement().getContent(element("channel"));
- List<NexusEvent> events = new ArrayList<>();
+ List<HasNexusEvent> events = new ArrayList<>();
if (channels.size() != 1) {
return new NexusFeed(events);
@@ -28,7 +32,7 @@ public class NexusFeedParser {
Element channel = channels.get(0);
for (Element item : channel.getContent(element("item"))) {
- Option<NexusEvent> e = parseEvent(item);
+ Option<HasNexusEvent> e = parseEvent(item);
if (e.isNone()) {
continue;
@@ -40,7 +44,7 @@ public class NexusFeedParser {
return new NexusFeed(events);
}
- public static Option<NexusEvent> parseEvent(Element item) {
+ public static Option<HasNexusEvent> parseEvent(Element item) {
String title = item.getChildText("title");
Option<String> guid = Option.fromNull(item.getChildText("guid"));
@@ -91,43 +95,47 @@ public class NexusFeedParser {
return null;
}
- Pattern regexp = Pattern.compile("(.*)-([0-9]{8}\\.[0-9]{6})-([0-9]*)$");
-
- Matcher matcher = regexp.matcher(version);
+ Matcher matcher = snapshotPattern.matcher(version);
if (matcher.matches()) {
ArtifactId id = new ArtifactId(groupId, artifactId, matcher.group(1) + "-SNAPSHOT");
- Option<Integer> buildNumber = parseInt.f(matcher.group(3));
+ Option<Integer> buildNumber = parseInt.f(matcher.group(8));
if(buildNumber.isNone()) {
System.out.println("Could not parse build number: " + matcher.group(3));
return none();
}
- return Option.<NexusEvent>some(new NewSnapshotEvent(guid.some(), id, classifier, creator.some(),
- date.some(), matcher.group(2), buildNumber.some(), URI.create(item.getChildText("link"))));
+ NexusEvent event = new NexusEvent(guid.some(), id, classifier, creator.some(), date.some());
+
+ int year = Integer.parseInt(matcher.group(2));
+ int month = Integer.parseInt(matcher.group(3));
+ int day = Integer.parseInt(matcher.group(4));
+ int hour = Integer.parseInt(matcher.group(5));
+ int minute = Integer.parseInt(matcher.group(6));
+ int second = Integer.parseInt(matcher.group(7));
+
+ return Option.<HasNexusEvent>some(new NewSnapshotEvent(event, new DateTime(year, month, day, hour, minute, second, 0), buildNumber.some(),
+ URI.create(item.getChildText("link"))));
} else {
ArtifactId id = new ArtifactId(groupId, artifactId, version);
- return Option.<NexusEvent>some(new NewReleaseEvent(guid.some(), id, classifier, creator.some(),
- date.some(), URI.create(item.getChildText("link"))));
- }
+ NexusEvent event = new NexusEvent(guid.some(), id, classifier, creator.some(), date.some());
-// System.out.println("Unknown event type.");
-//
-// return none();
+ return Option.<HasNexusEvent>some(new NewReleaseEvent(event, URI.create(item.getChildText("link"))));
+ }
}
}
class NexusFeed {
- List<NexusEvent> events = new ArrayList<>();
+ List<HasNexusEvent> events = new ArrayList<>();
- NexusFeed(List<NexusEvent> events) {
+ NexusFeed(List<HasNexusEvent> events) {
this.events = events;
}
}
-abstract class NexusEvent {
+final class NexusEvent {
public final String guid;
public final ArtifactId artifactId;
public final Option<String> classifier;
@@ -143,24 +151,32 @@ abstract class NexusEvent {
}
}
-class NewSnapshotEvent extends NexusEvent {
- public final String snapshotTimestamp;
+abstract class HasNexusEvent {
+ public final NexusEvent event;
+
+ protected HasNexusEvent(NexusEvent event) {
+ this.event = event;
+ }
+}
+
+class NewSnapshotEvent extends HasNexusEvent {
+ public final DateTime snapshotTimestamp;
public final int buildNumber;
public final URI url;
- NewSnapshotEvent(String guid, ArtifactId artifactId, Option<String> classifier, String creator, DateTime date, String snapshotTimestamp, int buildNumber, URI url) {
- super(guid, artifactId, classifier, creator, date);
+ NewSnapshotEvent(NexusEvent event, DateTime snapshotTimestamp, int buildNumber, URI url) {
+ super(event);
this.snapshotTimestamp = snapshotTimestamp;
this.buildNumber = buildNumber;
this.url = url;
}
}
-class NewReleaseEvent extends NexusEvent {
+class NewReleaseEvent extends HasNexusEvent {
public final URI url;
- NewReleaseEvent(String guid, ArtifactId artifactId, Option<String> classifier, String creator, DateTime date, URI url) {
- super(guid, artifactId, classifier, creator, date);
+ NewReleaseEvent(NexusEvent event, URI url) {
+ super(event);
this.url = url;
}
}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
index 3d0aaf0..3bf93c4 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
@@ -12,6 +12,7 @@ import org.codehaus.httpcache4j.cache.*;
import java.sql.*;
import java.util.*;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.atomic.*;
public class NexusImporter {
@@ -75,65 +76,97 @@ class NexusServer implements TransactionalActor {
NexusDao dao = new NexusDao(c);
- NexusFeed feed = client.fetchTimeline("recentlyDeployedArtifacts");
+ int newEvents = 0, oldEvents = 0, count = 0, pages = 0;
- int newEvents = 0, oldEvents = 0;
+ int newInFeed;
+ do {
+ newInFeed = 0;
+ NexusFeed feed = client.fetchTimeline("recentlyDeployedArtifacts", 100, count);
- for (NexusEvent event : feed.events) {
+ pages++;
+ count += feed.events.size();
- if(dao.countEventByGuid(event.guid) != 0) {
- oldEvents++;
- continue;
+ List<String> guids = new ArrayList<>();
+
+ for (HasNexusEvent event : feed.events) {
+ guids.add(event.event.guid);
}
- newEvents++;
+ Set<String> oldGuids = dao.selectGuidsByGuids(guids);
- String repositoryId = event.guid.replaceAll("^" + quote(server.url.toASCIIString()) + "/content/repositories/([-a-zA-Z0-9]*)/.*", "$1");
+ System.out.println("oldGuids.size() = " + oldGuids.size());
- if(repositoryId.length() == 0) {
- continue;
- }
+ for (HasNexusEvent event : feed.events) {
+
+// if (oldGuids.contains(event.event.guid)) {
+// oldEvents++;
+// continue;
+// }
- Option<NexusRepositoryDto> r = dao.findRepository(server.uuid, repositoryId);
+ if (dao.countEventByGuid(event.event.guid) != 0) {
+ oldEvents++;
+ continue;
+ }
- if(r.isNone()) {
- continue;
+ newEvents++;
+ newInFeed++;
+
+ onEvent(dao, event);
}
- NexusRepositoryDto repository = r.some();
+ System.out.println("newInFeed = " + newInFeed);
+ } while (newInFeed > 0);
- Option<ArtifactDto> a = dao.findArtifact(repository.uuid, event.artifactId);
+ System.out.println("Timeline updated. New=" + newEvents + ", old=" + oldEvents + ", count=" + count + ", pages=" + pages);
+ }
- UUID uuid;
+ private void onEvent(NexusDao dao, HasNexusEvent e) throws SQLException {
- if(a.isNone()) {
- System.out.println("New artifact: " + event.artifactId);
- uuid = dao.insertArtifact(repository.uuid, event.artifactId);
- }
- else {
- ArtifactDto artifact = a.some();
+ NexusEvent event = e.event;
- System.out.println("Updated artifact: " + event.artifactId);
-// dao.updateSnapshotTimestamp(artifact.uuid, newSnapshotEvent.snapshotTimestamp);
+ String repositoryId = event.guid.replaceAll("^" + quote(server.url.toASCIIString()) + "/content/repositories/([-a-zA-Z0-9]*)/.*", "$1");
- uuid = artifact.uuid;
- }
+ if(repositoryId.length() == 0) {
+ return;
+ }
- if (event instanceof NewSnapshotEvent) {
- NewSnapshotEvent newSnapshotEvent = (NewSnapshotEvent) event;
+ Option<NexusRepositoryDto> r = dao.findRepository(server.uuid, repositoryId);
- dao.insertNewSnapshotEvent(uuid, event.guid, newSnapshotEvent.snapshotTimestamp,
- newSnapshotEvent.buildNumber, newSnapshotEvent.url.toASCIIString());
- } else if (event instanceof NewReleaseEvent) {
- NewReleaseEvent nre = (NewReleaseEvent) event;
+ if(r.isNone()) {
+ return;
+ }
- dao.insertNewReleaseEvent(uuid, event.guid, nre.url.toASCIIString());
- } else {
- System.out.println("Unknown event type: " + event.getClass().getName());
- }
+ NexusRepositoryDto repository = r.some();
+
+ Option<ArtifactDto> a = dao.findArtifact(repository.uuid, event.artifactId);
+
+ UUID uuid;
+
+ if(a.isNone()) {
+ System.out.println("New artifact: " + event.artifactId);
+ uuid = dao.insertArtifact(repository.uuid, event.artifactId);
}
+ else {
+ ArtifactDto artifact = a.some();
+
+// System.out.println("Updated artifact: " + event.artifactId);
- System.out.println("Timeline updated. New=" + newEvents + ", old=" + oldEvents);
+ uuid = artifact.uuid;
+ }
+
+ if (e instanceof NewSnapshotEvent) {
+ NewSnapshotEvent newSnapshotEvent = (NewSnapshotEvent) e;
+
+ dao.insertNewSnapshotEvent(uuid, event.guid, event.creator, event.date,
+ newSnapshotEvent.snapshotTimestamp, newSnapshotEvent.buildNumber,
+ newSnapshotEvent.url.toASCIIString());
+ } else if (e instanceof NewReleaseEvent) {
+ NewReleaseEvent nre = (NewReleaseEvent) e;
+
+ dao.insertNewReleaseEvent(uuid, event.guid, event.creator, event.date, nre.url.toASCIIString());
+ } else {
+ System.out.println("Unknown event type: " + event.getClass().getName());
+ }
}
// public void act(Connection c) throws Exception {
diff --git a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java
index 93cc5fe..f421acf 100755
--- a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java
+++ b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java
@@ -115,8 +115,7 @@ public class ObjectUtil {
while (thread.isAlive()) {
try {
thread.join();
- } catch (InterruptedException e) {
- continue;
+ } catch (InterruptedException ignore) {
}
}
}
diff --git a/src/main/resources/ddl-nexus.sql b/src/main/resources/ddl-nexus.sql
index 7f84dac..8acf741 100755
--- a/src/main/resources/ddl-nexus.sql
+++ b/src/main/resources/ddl-nexus.sql
@@ -19,8 +19,8 @@ CREATE TABLE nexus_repository (
id VARCHAR(100),
group_ids VARCHAR(100) [],
CONSTRAINT pk_nexus_repository PRIMARY KEY (uuid),
- CONSTRAINT fk_nexus_server FOREIGN KEY (server) REFERENCES nexus_server (uuid),
- CONSTRAINT uq_nexus_repository_id UNIQUE (server, id)
+ CONSTRAINT fk_nexus_repository__nexus_server FOREIGN KEY (server) REFERENCES nexus_server (uuid),
+ CONSTRAINT uq_nexus_repository__id UNIQUE (server, id)
);
CREATE TABLE nexus_artifact (
@@ -30,29 +30,30 @@ CREATE TABLE nexus_artifact (
artifact_id VARCHAR(100) NOT NULL,
version VARCHAR(100) NOT NULL,
CONSTRAINT pk_nexus_artifact PRIMARY KEY (uuid),
- CONSTRAINT uq_nexus_artifact_gid_aid_version UNIQUE (group_id, artifact_id, version),
- CONSTRAINT fk_nexus_repository FOREIGN KEY (repository) REFERENCES nexus_repository (uuid)
+ CONSTRAINT uq_nexus_artifact__gid__aid__version UNIQUE (group_id, artifact_id, version),
+ CONSTRAINT fk_nexus_artifact__nexus_repository FOREIGN KEY (repository) REFERENCES nexus_repository (uuid)
);
CREATE TABLE nexus_event (
- uuid CHAR(36) NOT NULL,
- artifact CHAR(36) NOT NULL,
- timestamp VARCHAR(100),
+ uuid CHAR(36) NOT NULL,
+ artifact CHAR(36) NOT NULL,
+ created TIMESTAMP NOT NULL,
-- From the RSS
- guid VARCHAR(1000),
+ guid VARCHAR(1000) NOT NULL,
+ date TIMESTAMP NOT NULL,
-- Our type flag
- type VARCHAR(100) NOT NULL,
+ type VARCHAR(100) NOT NULL,
-- new snapshot event
- snapshot_timestamp TIMESTAMP,
+ snapshot_timestamp VARCHAR(100),
build_number INT,
file VARCHAR(1000),
who VARCHAR(1000),
CONSTRAINT pk_nexus_event PRIMARY KEY (uuid),
- CONSTRAINT fk_nexus_artifact FOREIGN KEY (artifact) REFERENCES nexus_artifact (uuid),
- CONSTRAINT uq_guid UNIQUE (guid),
+ CONSTRAINT fk_nexus_event__artifact FOREIGN KEY (artifact) REFERENCES nexus_artifact (uuid),
+ CONSTRAINT uq_nexus_event__guid UNIQUE (guid),
CONSTRAINT check_event_type CHECK (type IN ('new_snapshot', 'new_release'))
-- CONSTRAINT pk_nexus_event PRIMARY KEY (timestamp, server_url, repository_id, group_id, artifact_id, version)
);