o Actually storing and deleting new/gone artifacts from Nexus.

5 files changed, 334 insertions, 100 deletions

diff --git a/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java b/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java
new file mode 100644
index 0000000..144913e
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java
@@ -0,0 +1,121 @@
+package io.trygvis.esper.testing.nexus;
+
+import fj.data.*;
+
+import java.net.*;
+
+public class ArtifactDto implements Comparable<ArtifactDto> {
+    public final URI serverUrl;
+    public final String repositoryId;
+    public final ArtifactId id;
+
+    public ArtifactDto(URI serverUrl, String repositoryId, ArtifactId id) {
+        this.serverUrl = serverUrl;
+        this.repositoryId = repositoryId;
+        this.id = id;
+    }
+
+    public int compareTo(ArtifactDto o) {
+        int i = serverUrl.compareTo(o.serverUrl);
+
+        if (i != 0) {
+            return i;
+        }
+
+        i = repositoryId.compareTo(o.repositoryId);
+
+        if (i != 0) {
+            return i;
+        }
+
+        return id.compareTo(o.id);
+    }
+
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof ArtifactDto)) return false;
+
+        ArtifactDto that = (ArtifactDto) o;
+
+        if (!serverUrl.equals(that.serverUrl)) return false;
+        if (!repositoryId.equals(that.repositoryId)) return false;
+
+        return id.equals(that.id);
+    }
+
+    public int hashCode() {
+        int result = serverUrl.hashCode();
+        result = 31 * result + repositoryId.hashCode();
+        result = 31 * result + id.hashCode();
+        return result;
+    }
+}
+
+class ArtifactFile {
+    public final Option<String> classifier;
+    public final String extension;
+
+    ArtifactFile(Option<String> classifier, String extension) {
+        this.classifier = classifier;
+        this.extension = extension;
+    }
+}
+
+class ArtifactId implements Comparable<ArtifactId> {
+    public final String groupId;
+    public final String artifactId;
+    public final String version;
+
+    ArtifactId(String groupId, String artifactId, String version) {
+        if(groupId == null) {
+            throw new NullPointerException("groupId");
+        }
+        if(artifactId == null) {
+            throw new NullPointerException("artifactId");
+        }
+        if(version == null) {
+            throw new NullPointerException("version");
+        }
+        this.groupId = groupId;
+        this.artifactId = artifactId;
+        this.version = version;
+    }
+
+    public int compareTo(ArtifactId o) {
+        int i = groupId.compareTo(o.groupId);
+
+        if (i != 0) {
+            return i;
+        }
+
+        i = artifactId.compareTo(o.artifactId);
+
+        if (i != 0) {
+            return i;
+        }
+
+        return version.compareTo(o.version);
+    }
+
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof ArtifactId)) return false;
+
+        ArtifactId that = (ArtifactId) o;
+
+        return groupId.equals(that.groupId) &&
+            artifactId.equals(that.artifactId) &&
+            version.equals(that.version);
+    }
+
+    public int hashCode() {
+        int result = groupId != null ? groupId.hashCode() : 0;
+        result = 31 * result + (artifactId != null ? artifactId.hashCode() : 0);
+        result = 31 * result + (version != null ? version.hashCode() : 0);
+        return result;
+    }
+
+    public String toString() {
+        return groupId + ":" + artifactId + ":" + version;
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
index 02b5e3a..42ed245 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java
@@ -38,7 +38,7 @@ public class NexusClient {
     public ArtifactSearchResult fetchIndexPage(String groupId, Option<String> repositoryId, Option<Integer> from) throws IOException {
         URIBuilder uriBuilder = URIBuilder.fromURI(nexusUrl).
                 addRawPath("/service/local/lucene/search").
-                addParameter("g", groupId + ".*");
+                addParameter("g", groupId);
 
         if (repositoryId.isSome()) {
             uriBuilder = uriBuilder.addParameter("repositoryId", repositoryId.some());
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
index 00e15fe..c5e8742 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java
@@ -3,17 +3,42 @@ package io.trygvis.esper.testing.nexus;
 import fj.data.*;
 import static fj.data.Option.*;
 import io.trygvis.esper.testing.*;
+import static io.trygvis.esper.testing.DaoUtil.timestampToLocalDateTime;
 import org.joda.time.*;
 
 import java.net.*;
-import java.sql.Array;
 import java.sql.*;
 import java.util.*;
+import java.util.Date;
 import java.util.List;
 
-public class NexusDao extends Dao {
-    protected NexusDao(Connection c) {
-        super(c);
+public class NexusDao {
+    private final Connection c;
+
+    public NexusDao(Connection c) {
+        this.c = c;
+    }
+
+    public static URI uri(String s) {
+        try {
+            return URI.create(s);
+        } catch (Exception e) {
+            e.printStackTrace(System.out);
+            return null;
+        }
+    }
+
+    private NexusRepositoryDto nexusRepositoryDto(ResultSet rs) throws SQLException {
+        int i = 1;
+
+        return new NexusRepositoryDto(
+            rs.getString(i++),
+            uri(rs.getString(i++)),
+            (String[]) rs.getArray(i++).getArray(),
+            fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime),
+            fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime),
+            fromNull(rs.getTimestamp(i)).map(timestampToLocalDateTime)
+        );
     }
 
     /*
@@ -29,71 +54,126 @@ public class NexusDao extends Dao {
     */
 
     public List<NexusServerDto> selectServer() throws SQLException {
-        try (PreparedStatement s = c.prepareStatement("SELECT url FROM nexus_server")) {
+        try (PreparedStatement s = c.prepareStatement("SELECT url, name FROM nexus_server")) {
             ResultSet rs = s.executeQuery();
 
             List<NexusServerDto> servers = new ArrayList<>();
-            while(rs.next()) {
-                servers.add(new NexusServerDto(uri(rs.getString(1))));
+            while (rs.next()) {
+                servers.add(new NexusServerDto(uri(rs.getString(1)), rs.getString(2)));
             }
             return servers;
         }
     }
 
-    public Option<NexusRepositoryDto> selectRepository(String repositoryId) throws SQLException {
-        PreparedStatement s = prepareStatement("SELECT id, nexus_server_url, group_ids, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?");
-        s.setString(1, repositoryId);
+    public Option<NexusRepositoryDto> findRepository(String repositoryId) throws SQLException {
+        try (PreparedStatement s = c.prepareStatement("SELECT id, server_url, group_ids, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?")) {
+            s.setString(1, repositoryId);
 
-        try (ResultSet rs = s.executeQuery()) {
-            if (!rs.next()) {
-                return Option.none();
-            }
+            try (ResultSet rs = s.executeQuery()) {
+                if (!rs.next()) {
+                    return Option.none();
+                }
 
-            return some(toRepository(rs));
+                return some(nexusRepositoryDto(rs));
+            }
         }
     }
 
     public List<NexusRepositoryDto> findRepositories(URI nexusUrl) throws SQLException {
-        PreparedStatement s = prepareStatement("SELECT id, nexus_server_url, group_ids, created_date, last_update, last_successful_update FROM nexus_repository WHERE nexus_server_url=?");
-        s.setString(1, nexusUrl.toASCIIString());
-
-        List<NexusRepositoryDto> list = new ArrayList<>();
-        try (ResultSet rs = s.executeQuery()) {
-            while(rs.next()) {
-                list.add(toRepository(rs));
+        try (PreparedStatement s = c.prepareStatement("SELECT id, server_url, group_ids, created_date, last_update, last_successful_update FROM nexus_repository WHERE server_url=?")) {
+            s.setString(1, nexusUrl.toASCIIString());
+
+            List<NexusRepositoryDto> list = new ArrayList<>();
+            try (ResultSet rs = s.executeQuery()) {
+                while (rs.next()) {
+                    list.add(nexusRepositoryDto(rs));
+                }
             }
+
+            return list;
         }
-        return list;
     }
 
-    private NexusRepositoryDto toRepository(ResultSet rs) throws SQLException {
+    // -----------------------------------------------------------------------
+    // Nexus Artifact
+    // -----------------------------------------------------------------------
+
+    private final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version";
+
+    private final String NEXUS_ARTIFACT = NEXUS_ARTIFACT_ID + ", snapshot_version, classifiers, packagings, created_date";
+
+    private int setArtifactId(int i, PreparedStatement s, ArtifactId id) throws SQLException {
+        s.setString(i++, id.groupId);
+        s.setString(i++, id.artifactId);
+        s.setString(i++, id.version);
+        return i;
+    }
+
+    private static ArtifactDto artifactDto(URI serverUrl, String repositoryId, ResultSet rs) throws SQLException {
         int i = 1;
 
-        return new NexusRepositoryDto(
-            rs.getString(i++),
-            uri(rs.getString(i++)),
-            (String[]) rs.getArray(i++).getArray(),
-            fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime),
-            fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime),
-            fromNull(rs.getTimestamp(i)).map(timestampToLocalDateTime)
-        );
+        return new ArtifactDto(
+            serverUrl,
+            repositoryId,
+            new ArtifactId(rs.getString(i++),
+                rs.getString(i++),
+                rs.getString(i)));
     }
 
-    private URI uri(String s) {
-        try {
-            return URI.create(s);
-        } catch (Exception e) {
-            e.printStackTrace(System.out);
-            return null;
+    public void insertArtifact(URI nexusUrl, String repositoryId, ArtifactId id, Option<String> snapshotVersion, List<ArtifactFile> files, Date createdDate) throws SQLException {
+        String[] classifiers = new String[files.size()];
+        String[] packagings = new String[files.size()];
+
+        for (int i = 0; i < files.size(); i++) {
+            classifiers[i] = files.get(i).classifier.toNull();
+            packagings[i] = files.get(i).extension;
+        }
+
+        int i = 1;
+        try (PreparedStatement s = c.prepareStatement("INSERT INTO nexus_artifact(server_url, repository_id, " + NEXUS_ARTIFACT + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")) {
+            s.setString(i++, nexusUrl.toASCIIString());
+            s.setString(i++, repositoryId);
+            i = setArtifactId(i, s, id);
+            s.setString(i++, snapshotVersion.toNull());
+            s.setArray(i++, c.createArrayOf("varchar", classifiers));
+            s.setArray(i++, c.createArrayOf("varchar", packagings));
+            s.setTimestamp(i, DaoUtil.dateToTimestamp.f(createdDate));
+            s.executeUpdate();
+        }
+    }
+
+    public void deleteArtifact(URI nexusUrl, String repositoryId, ArtifactId id) throws SQLException {
+        int i = 1;
+        try (PreparedStatement s = c.prepareStatement("DELETE FROM nexus_artifact WHERE server_url=? AND repository_id=? AND group_id=? AND artifact_id=? AND version=?")) {
+            s.setString(i++, nexusUrl.toASCIIString());
+            s.setString(i++, repositoryId);
+            i += setArtifactId(i, s, id);
+            s.executeUpdate();
+        }
+    }
+
+    public List<ArtifactDto> findArtifactsInRepository(URI url, String repositoryId) throws SQLException {
+        try (PreparedStatement s = c.prepareStatement("SELECT " + NEXUS_ARTIFACT + " FROM nexus_artifact WHERE server_url=? AND repository_id=?")) {
+            s.setString(1, url.toASCIIString());
+            s.setString(2, repositoryId);
+            ResultSet rs = s.executeQuery();
+
+            List<ArtifactDto> list = new ArrayList<>();
+            while (rs.next()) {
+                list.add(artifactDto(url, repositoryId, rs));
+            }
+            return list;
         }
     }
 }
 
 class NexusServerDto {
     public final URI url;
+    public final String name;
 
-    NexusServerDto(URI url) {
+    NexusServerDto(URI url, String name) {
         this.url = url;
+        this.name = name;
     }
 
     public boolean equals(Object o) {
@@ -129,3 +209,4 @@ class NexusRepositoryDto {
         this.lastSuccessfulUpdate = lastSuccessfulUpdate;
     }
 }
+
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
index 12d8c9c..c6e2460 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java
@@ -1,16 +1,18 @@
 package io.trygvis.esper.testing.nexus;
 
-import com.google.common.collect.*;
 import com.jolbox.bonecp.*;
 import fj.data.*;
+import static fj.data.Option.*;
 import io.trygvis.esper.testing.*;
 import io.trygvis.esper.testing.object.*;
 import static java.lang.Thread.*;
-import org.apache.commons.lang.*;
 import org.codehaus.httpcache4j.cache.*;
 
 import java.sql.*;
 import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.TreeMap;
 import java.util.concurrent.atomic.*;
 
 public class NexusImporter {
@@ -25,7 +27,9 @@ public class NexusImporter {
             public ActorRef<NexusServer> create(NexusServerDto server) {
                 final NexusClient client = new NexusClient(http, server.url);
 
-                return ObjectUtil.threadedActor(boneCp, config.nexusUpdateInterval, new NexusServer(client, server));
+                String name = server.name;
+
+                return ObjectUtil.threadedActor(boneCp, "", config.nexusUpdateInterval, new NexusServer(client, server));
             }
         });
 
@@ -42,7 +46,7 @@ public class NexusImporter {
             }
 
             synchronized (shouldRun) {
-                shouldRun.wait(1000);
+                shouldRun.wait(60 * 1000);
             }
         }
 
@@ -61,20 +65,83 @@ class NexusServer implements TransactionalActor {
     }
 
     public void act(Connection c) throws Exception {
+        Date timestamp = new Date();
         NexusDao dao = new NexusDao(c);
 
+        String p = server.name;
+
         for (NexusRepositoryDto repository : dao.findRepositories(server.url)) {
-            System.out.println("Updating repository: " + repository.repositoryId);
+            String p2 = p + "/" + repository.repositoryId;
+
+            System.out.println(p2 + ": Updating repository: " + repository.repositoryId);
+
+            TreeMap<ArtifactId, ArtifactXml> artifactsInNexus = new TreeMap<>();
+
             for (String groupId : repository.groupIds) {
-                System.out.println("Updating groupId: " + groupId);
-                ArtifactSearchResult result = client.fetchIndex(groupId, Option.<String>none());
+                String p3 = p2 + "/" + groupId;
+
+                System.out.println(p3 + ": Updating group id");
+                ArtifactSearchResult result = client.fetchIndex(groupId, some(repository.repositoryId));
+                System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts");
+
+                for (ArtifactXml xml : result.artifacts) {
+                    artifactsInNexus.put(xml.id, xml);
+                }
+
+                System.out.println(p3 + ": Updating everything under group id");
+                result = client.fetchIndex(groupId + ".*", some(repository.repositoryId));
+                System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts");
+
+                for (ArtifactXml xml : result.artifacts) {
+                    artifactsInNexus.put(xml.id, xml);
+                }
+            }
+
+            Map<ArtifactId, ArtifactDto> artifactsInDatabase = new HashMap<>();
+            for (ArtifactDto dto : dao.findArtifactsInRepository(server.url, repository.repositoryId)) {
+                artifactsInDatabase.put(dto.id, dto);
+            }
 
-                ArrayList<ArtifactXml> artifacts = Lists.newArrayList(result.artifacts);
-                Collections.sort(artifacts);
-                for (ArtifactXml artifact : artifacts) {
-                    System.out.println("repo=" + StringUtils.join(artifact.repositories(), ", ") + ", artifact=" + artifact.getId());
+            ArrayList<FlatArtifact> created = new ArrayList<>();
+            ArrayList<FlatArtifact> kept = new ArrayList<>();
+            ArrayList<ArtifactDto> removed = new ArrayList<>();
+
+            for (ArtifactXml xml : artifactsInNexus.values()) {
+                Option<FlatArtifact> o = xml.flatten(repository.repositoryId);
+
+                if(o.isNone()) {
+                    continue;
+                }
+
+                FlatArtifact artifact = o.some();
+
+                if(!artifactsInDatabase.containsKey(xml.id)) {
+                    created.add(artifact);
+                }
+                else {
+                    kept.add(artifact);
+                }
+            }
+
+            for (ArtifactDto dto : artifactsInDatabase.values()) {
+                if(!artifactsInNexus.containsKey(dto.id)) {
+                    removed.add(dto);
                 }
             }
+
+            System.out.println(p2 + ": found " + created.size() + " new artifacts, " + removed.size() + " removed artifacts and " + kept.size() + " existing artifacts.");
+
+            System.out.println(p2 + ": inserting new artifacts");
+            for (FlatArtifact artifact : created) {
+                dao.insertArtifact(repository.nexusUrl, repository.repositoryId, artifact.id, Option.<String>none(), artifact.files, timestamp);
+            }
+            System.out.println(p2 + ": inserted");
+
+            System.out.println(p2 + ": deleting removed artifacts");
+            for (ArtifactDto artifact : removed) {
+                dao.deleteArtifact(repository.nexusUrl, repository.repositoryId, artifact.id);
+            }
+            System.out.println(p2 + ": deleted");
         }
 
         c.commit();
diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
index 05dfd5b..6bfca99 100755
--- a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
+++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java
@@ -2,19 +2,18 @@ package io.trygvis.esper.testing.nexus;
 
 import com.google.common.base.*;
 import com.google.common.collect.*;
-import fj.*;
 import fj.data.*;
-import static fj.data.Option.fromNull;
+import static fj.data.Option.*;
 import static org.apache.commons.lang.StringUtils.*;
 import org.dom4j.*;
 import org.dom4j.io.*;
 
+import javax.xml.stream.*;
 import java.io.*;
 import java.util.*;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-import javax.xml.stream.*;
 
 public class NexusParser {
     public static final STAXEventReader xmlReader = new STAXEventReader();
@@ -23,7 +22,7 @@ public class NexusParser {
         Document doc = xmlReader.readDocument(is);
 
         Option<Integer> totalCount = fromNull(trimToNull(doc.getRootElement().elementText("totalCount"))).
-                bind(Option.parseInt);
+            bind(Option.parseInt);
         if (totalCount.isNone()) {
             throw new RuntimeException("Could not find required element <totalCount>");
         }
@@ -71,7 +70,7 @@ public class NexusParser {
                 artifactHitsList.add(new ArtifactHits(repositoryId, files));
             }
 
-            list.add(new ArtifactXml(groupId, artifactId, version, artifactHitsList));
+            list.add(new ArtifactXml(new ArtifactId(groupId, artifactId, version), artifactHitsList));
         }
 
         return new ArtifactSearchResult(totalCount.some(), tooManyResults, list);
@@ -97,15 +96,11 @@ class ArtifactSearchResult {
 }
 
 class ArtifactXml implements Comparable<ArtifactXml> {
-    public final String groupId;
-    public final String artifactId;
-    public final String version;
+    public final ArtifactId id;
     public final List<ArtifactHits> hits;
 
-    ArtifactXml(String groupId, String artifactId, String version, List<ArtifactHits> hits) {
-        this.groupId = groupId;
-        this.artifactId = artifactId;
-        this.version = version;
+    ArtifactXml(ArtifactId id, List<ArtifactHits> hits) {
+        this.id = id;
         this.hits = hits;
     }
 
@@ -121,34 +116,18 @@ class ArtifactXml implements Comparable<ArtifactXml> {
         };
     }
 
-    public FlatArtifact flatten(String repositoryId) {
+    public Option<FlatArtifact> flatten(String repositoryId) {
         for (ArtifactHits hit : hits) {
             if (hit.repositoryId.equals(repositoryId)) {
-                return new FlatArtifact(groupId, artifactId, version, hit.files);
+                return some(new FlatArtifact(id, hit.files));
             }
         }
 
-        throw new RuntimeException("No hits in repository " + repositoryId);
+        return none();
     }
 
     public int compareTo(ArtifactXml o) {
-        int c = groupId.compareTo(o.groupId);
-
-        if(c != 0) {
-            return c;
-        }
-
-        c = artifactId.compareTo(o.artifactId);
-
-        if(c != 0) {
-            return c;
-        }
-
-        return version.compareTo(o.version);
-    }
-
-    public String getId() {
-        return groupId + ":" + artifactId + ":" + version;
+        return id.compareTo(o.id);
     }
 
     public Set<String> repositories() {
@@ -163,15 +142,11 @@ class ArtifactXml implements Comparable<ArtifactXml> {
 }
 
 class FlatArtifact {
-    public final String groupId;
-    public final String artifactId;
-    public final String version;
+    public final ArtifactId id;
     public final List<ArtifactFile> files;
 
-    FlatArtifact(String groupId, String artifactId, String version, List<ArtifactFile> files) {
-        this.groupId = groupId;
-        this.artifactId = artifactId;
-        this.version = version;
+    FlatArtifact(ArtifactId id, List<ArtifactFile> files) {
+        this.id = id;
         this.files = files;
     }
 }
@@ -185,13 +160,3 @@ class ArtifactHits {
         this.files = files;
     }
 }
-
-class ArtifactFile {
-    public final Option<String> classifier;
-    public final String extension;
-
-    ArtifactFile(Option<String> classifier, String extension) {
-        this.classifier = classifier;
-        this.extension = extension;
-    }
-}