diff options
author | Trygve Laugstøl <trygvis@inamo.no> | 2012-11-24 18:02:55 +0100 |
---|---|---|
committer | Trygve Laugstøl <trygvis@inamo.no> | 2012-11-24 18:02:55 +0100 |
commit | e243a6fd6c444b451398ceb659ea4963a19122d0 (patch) | |
tree | 532ca0710468deee4e6fa076546a2d53f1cff498 | |
parent | cac8228f38136cfc41673458c58c25f168b1e1ff (diff) | |
download | esper-testing-e243a6fd6c444b451398ceb659ea4963a19122d0.tar.gz esper-testing-e243a6fd6c444b451398ceb659ea4963a19122d0.tar.bz2 esper-testing-e243a6fd6c444b451398ceb659ea4963a19122d0.tar.xz esper-testing-e243a6fd6c444b451398ceb659ea4963a19122d0.zip |
o Actually storing and deleting new/gone artifacts from Nexus.
-rwxr-xr-x | pom.xml | 6 | ||||
-rwxr-xr-x | src/main/java/io/trygvis/esper/testing/DaoUtil.java (renamed from src/main/java/io/trygvis/esper/testing/Dao.java) | 24 | ||||
-rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java | 68 | ||||
-rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java | 52 | ||||
-rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java | 82 | ||||
-rw-r--r-- | src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java | 121 | ||||
-rwxr-xr-x | src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java | 2 | ||||
-rwxr-xr-x | src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java | 159 | ||||
-rwxr-xr-x | src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java | 89 | ||||
-rwxr-xr-x | src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java | 63 | ||||
-rw-r--r-- | src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java | 9 | ||||
-rwxr-xr-x | src/main/resources/ddl-nexus.sql | 50 | ||||
-rwxr-xr-x | src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java | 20 |
13 files changed, 500 insertions, 245 deletions
@@ -8,6 +8,12 @@ <groupId>com.espertech</groupId> <artifactId>esper</artifactId> <version>4.7.0</version> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.abdera</groupId> diff --git a/src/main/java/io/trygvis/esper/testing/Dao.java b/src/main/java/io/trygvis/esper/testing/DaoUtil.java index e1a65fe..84b1afa 100755 --- a/src/main/java/io/trygvis/esper/testing/Dao.java +++ b/src/main/java/io/trygvis/esper/testing/DaoUtil.java @@ -4,30 +4,8 @@ import fj.*; import org.joda.time.*; import java.sql.*; -import java.util.*; - -public class Dao { - protected final Connection c; - private final Map<String, PreparedStatement> statements = new HashMap<>(); - - protected Dao(Connection c) { - this.c = c; - } - - protected PreparedStatement prepareStatement(String sql) throws SQLException { - PreparedStatement s = statements.get(sql); - - if (s != null) { - return s; - } - - s = c.prepareStatement(sql); - - statements.put(sql, s); - - return s; - } +public class DaoUtil { public static final F<Timestamp, java.util.Date> timestampToDate = new F<Timestamp, java.util.Date>() { public java.util.Date f(Timestamp timestamp) { return new java.util.Date(timestamp.getTime()); diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java index 79f2ef2..737358e 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java @@ -1,47 +1,53 @@ package io.trygvis.esper.testing.gitorious; -import io.trygvis.esper.testing.*; +import static io.trygvis.esper.testing.DaoUtil.*; import java.sql.*; -public class GitoriousEventDao extends Dao { +public class GitoriousEventDao { - public GitoriousEventDao(Connection c) throws SQLException { - super(c); - } + protected final Connection c; - private final PreparedStatement countEntryId = prepareStatement("SELECT count(entry_id) FROM gitorious_event WHERE entry_id=?"); + public GitoriousEventDao(Connection c) { + this.c = c; + } public int countEntryId(String entryId) throws SQLException { - countEntryId.setString(1, entryId); - try (ResultSet rs = countEntryId.executeQuery()) { - rs.next(); - return rs.getInt(1); + try (PreparedStatement s = c.prepareStatement("SELECT count(entry_id) FROM gitorious_event WHERE entry_id=?")) { + s.setString(1, entryId); + try (ResultSet rs = s.executeQuery()) { + rs.next(); + return rs.getInt(1); + } } } - private final PreparedStatement insertPush = prepareStatement("INSERT INTO gitorious_event(project_slug, name, entry_id, published, title, content, event_type, who, \"from\", \"to\", branch, commit_count) VALUES(?, ?, ?, ?, ?, ?, 'PUSH', ?, ?, ?, ?, ?)"); - public void insertEvent(GitoriousEvent event) throws SQLException { - PreparedStatement s; - if (event instanceof GitoriousPush) { - GitoriousPush push = (GitoriousPush) event; - s = insertPush; - s.setString(7, push.who); - s.setString(8, push.from); - s.setString(9, push.to); - s.setString(10, push.branch); - s.setInt(11, push.commitCount); - } else { - throw new SQLException("Unknown event type: " + event.getClass().getName()); + PreparedStatement s = null; + try { + if (event instanceof GitoriousPush) { + GitoriousPush push = (GitoriousPush) event; + s = c.prepareStatement("INSERT INTO gitorious_event(project_slug, name, entry_id, published, title, content, event_type, who, \"from\", \"to\", branch, commit_count) VALUES(?, ?, ?, ?, ?, ?, 'PUSH', ?, ?, ?, ?, ?)"); + s.setString(7, push.who); + s.setString(8, push.from); + s.setString(9, push.to); + s.setString(10, push.branch); + s.setInt(11, push.commitCount); + } else { + throw new SQLException("Unknown event type: " + event.getClass().getName()); + } + + s.setString(1, event.projectSlug); + s.setString(2, event.name); + s.setString(3, event.entryId); + s.setTimestamp(4, dateToTimestamp.f(event.published)); + s.setString(5, event.title); + s.setString(6, event.content); + s.executeUpdate(); + } finally { + if(s != null) { + s.close(); + } } - - s.setString(1, event.projectSlug); - s.setString(2, event.name); - s.setString(3, event.entryId); - s.setTimestamp(4, dateToTimestamp.f(event.published)); - s.setString(5, event.title); - s.setString(6, event.content); - s.executeUpdate(); } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java index 3caf8a1..18aad96 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java @@ -1,48 +1,48 @@ package io.trygvis.esper.testing.gitorious; -import io.trygvis.esper.testing.*; - import java.sql.*; import java.util.*; -public class GitoriousProjectDao extends Dao { - public GitoriousProjectDao(Connection c) throws SQLException { - super(c); - } +public class GitoriousProjectDao { + private final Connection c; - private final PreparedStatement countProjects = prepareStatement("SELECT count(*) FROM gitorious_project WHERE slug=?"); + public GitoriousProjectDao(Connection c) { + this.c = c; + } public int countProjects(String slug) throws SQLException { - countProjects.setString(1, slug); - try (ResultSet rs = countProjects.executeQuery()) { - rs.next(); - return rs.getInt(1); + try (PreparedStatement s = c.prepareStatement("SELECT count(*) FROM gitorious_project WHERE slug=?")) { + s.setString(1, slug); + try (ResultSet rs = s.executeQuery()) { + rs.next(); + return rs.getInt(1); + } } } - private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)"); - public void insertProject(String slug) throws SQLException { - insertProject.setString(1, slug); - insertProject.executeUpdate(); + try (PreparedStatement s = c.prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)")) { + s.setString(1, slug); + s.executeUpdate(); + } } - private final PreparedStatement selectSlugs = prepareStatement("SELECT slug FROM gitorious_project"); - public List<String> selectSlugs() throws SQLException { - try (ResultSet rs = selectSlugs.executeQuery()) { - List<String> list = new ArrayList<>(); - while (rs.next()) { - list.add(rs.getString(1)); + try (PreparedStatement s = c.prepareStatement("SELECT slug FROM gitorious_project")) { + try (ResultSet rs = s.executeQuery()) { + List<String> list = new ArrayList<>(); + while (rs.next()) { + list.add(rs.getString(1)); + } + return list; } - return list; } } - private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?"); - public void delete(String slug) throws SQLException { - delete.setString(1, slug); - delete.executeUpdate(); + try (PreparedStatement s = c.prepareStatement("DELETE FROM gitorious_project WHERE slug=?")) { + s.setString(1, slug); + s.executeUpdate(); + } } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java index 0dcd57a..2cf9fce 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java @@ -1,7 +1,8 @@ package io.trygvis.esper.testing.gitorious; import fj.data.*; -import io.trygvis.esper.testing.*; +import static io.trygvis.esper.testing.DaoUtil.dateToTimestamp; +import static io.trygvis.esper.testing.DaoUtil.timestampToDate; import java.net.*; import java.sql.*; @@ -9,9 +10,11 @@ import java.util.*; import java.util.Date; import java.util.List; -public class GitoriousRepositoryDao extends Dao { - public GitoriousRepositoryDao(Connection c) throws SQLException { - super(c); +public class GitoriousRepositoryDao { + private final Connection c; + + public GitoriousRepositoryDao(Connection c) { + this.c = c; } private static final String ALL_FIELDS = "project_slug, name, atom_feed, last_update, last_successful_update"; @@ -33,61 +36,62 @@ public class GitoriousRepositoryDao extends Dao { } } - private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?"); public int countRepositories(String projectSlug, String name) throws SQLException { - countRepositories.setString(1, projectSlug); - countRepositories.setString(2, name); - try (ResultSet rs = countRepositories.executeQuery()) { - rs.next(); - return rs.getInt(1); + try (PreparedStatement s = c.prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?")) { + s.setString(1, projectSlug); + s.setString(2, name); + try (ResultSet rs = s.executeQuery()) { + rs.next(); + return rs.getInt(1); + } } } - private final PreparedStatement selectForProject = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository WHERE project_slug=?"); - public List<GitoriousRepository> selectForProject(String projectSlug) throws SQLException { - selectForProject.setString(1, projectSlug); - return executeQuery(selectForProject); + try (PreparedStatement s = c.prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository WHERE project_slug=?")) { + s.setString(1, projectSlug); + return executeQuery(s); + } } - private final PreparedStatement select = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository"); - public List<GitoriousRepository> select() throws SQLException { - return executeQuery(select); + try (PreparedStatement s = c.prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository")) { + return executeQuery(s); + } } - private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)"); - public void insertRepository(String projectSlug, String name, URI atomFeed) throws SQLException { - insertRepository.setString(1, projectSlug); - insertRepository.setString(2, name); - insertRepository.setString(3, atomFeed.toASCIIString()); - insertRepository.executeUpdate(); + try (PreparedStatement s = c.prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)")) { + s.setString(1, projectSlug); + s.setString(2, name); + s.setString(3, atomFeed.toASCIIString()); + s.executeUpdate(); + } } - private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?"); - public void delete(GitoriousRepository repository) throws SQLException { - delete.setString(1, repository.projectSlug); - delete.setString(2, repository.name); - delete.executeUpdate(); + try (PreparedStatement s = c.prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?")) { + s.setString(1, repository.projectSlug); + s.setString(2, repository.name); + s.executeUpdate(); + } } - private final PreparedStatement deleteForProject = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=?"); - public void deleteForProject(String project) throws SQLException { - deleteForProject.setString(1, project); - deleteForProject.executeUpdate(); + try (PreparedStatement s = c.prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=?")) { + s.setString(1, project); + s.executeUpdate(); + } } - private final PreparedStatement updateTimestamp = prepareStatement("UPDATE gitorious_repository SET last_update=?, last_successful_update=? WHERE project_slug=? AND name=?"); - public void updateTimestamp(String projectName, String slug, Timestamp lastUpdate, Option<Date> lastSuccessfulUpdate) throws SQLException { - updateTimestamp.setTimestamp(1, lastUpdate); - updateTimestamp.setTimestamp(2, lastSuccessfulUpdate.map(dateToTimestamp).toNull()); - updateTimestamp.setString(3, slug); - updateTimestamp.setString(4, projectName); - updateTimestamp.executeUpdate(); + try (PreparedStatement s = c.prepareStatement("UPDATE gitorious_repository SET last_update=?, last_successful_update=? WHERE project_slug=? AND name=?")) { + s.setTimestamp(1, lastUpdate); + s.setTimestamp(2, lastSuccessfulUpdate.map(dateToTimestamp).toNull()); + s.setString(3, slug); + s.setString(4, projectName); + s.executeUpdate(); + } } } diff --git a/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java b/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java new file mode 100644 index 0000000..144913e --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/nexus/ArtifactDto.java @@ -0,0 +1,121 @@ +package io.trygvis.esper.testing.nexus; + +import fj.data.*; + +import java.net.*; + +public class ArtifactDto implements Comparable<ArtifactDto> { + public final URI serverUrl; + public final String repositoryId; + public final ArtifactId id; + + public ArtifactDto(URI serverUrl, String repositoryId, ArtifactId id) { + this.serverUrl = serverUrl; + this.repositoryId = repositoryId; + this.id = id; + } + + public int compareTo(ArtifactDto o) { + int i = serverUrl.compareTo(o.serverUrl); + + if (i != 0) { + return i; + } + + i = repositoryId.compareTo(o.repositoryId); + + if (i != 0) { + return i; + } + + return id.compareTo(o.id); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ArtifactDto)) return false; + + ArtifactDto that = (ArtifactDto) o; + + if (!serverUrl.equals(that.serverUrl)) return false; + if (!repositoryId.equals(that.repositoryId)) return false; + + return id.equals(that.id); + } + + public int hashCode() { + int result = serverUrl.hashCode(); + result = 31 * result + repositoryId.hashCode(); + result = 31 * result + id.hashCode(); + return result; + } +} + +class ArtifactFile { + public final Option<String> classifier; + public final String extension; + + ArtifactFile(Option<String> classifier, String extension) { + this.classifier = classifier; + this.extension = extension; + } +} + +class ArtifactId implements Comparable<ArtifactId> { + public final String groupId; + public final String artifactId; + public final String version; + + ArtifactId(String groupId, String artifactId, String version) { + if(groupId == null) { + throw new NullPointerException("groupId"); + } + if(artifactId == null) { + throw new NullPointerException("artifactId"); + } + if(version == null) { + throw new NullPointerException("version"); + } + this.groupId = groupId; + this.artifactId = artifactId; + this.version = version; + } + + public int compareTo(ArtifactId o) { + int i = groupId.compareTo(o.groupId); + + if (i != 0) { + return i; + } + + i = artifactId.compareTo(o.artifactId); + + if (i != 0) { + return i; + } + + return version.compareTo(o.version); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ArtifactId)) return false; + + ArtifactId that = (ArtifactId) o; + + return groupId.equals(that.groupId) && + artifactId.equals(that.artifactId) && + version.equals(that.version); + } + + public int hashCode() { + int result = groupId != null ? groupId.hashCode() : 0; + result = 31 * result + (artifactId != null ? artifactId.hashCode() : 0); + result = 31 * result + (version != null ? version.hashCode() : 0); + return result; + } + + public String toString() { + return groupId + ":" + artifactId + ":" + version; + } +} diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java index 02b5e3a..42ed245 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusClient.java @@ -38,7 +38,7 @@ public class NexusClient { public ArtifactSearchResult fetchIndexPage(String groupId, Option<String> repositoryId, Option<Integer> from) throws IOException { URIBuilder uriBuilder = URIBuilder.fromURI(nexusUrl). addRawPath("/service/local/lucene/search"). - addParameter("g", groupId + ".*"); + addParameter("g", groupId); if (repositoryId.isSome()) { uriBuilder = uriBuilder.addParameter("repositoryId", repositoryId.some()); diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java index 00e15fe..c5e8742 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusDao.java @@ -3,17 +3,42 @@ package io.trygvis.esper.testing.nexus; import fj.data.*; import static fj.data.Option.*; import io.trygvis.esper.testing.*; +import static io.trygvis.esper.testing.DaoUtil.timestampToLocalDateTime; import org.joda.time.*; import java.net.*; -import java.sql.Array; import java.sql.*; import java.util.*; +import java.util.Date; import java.util.List; -public class NexusDao extends Dao { - protected NexusDao(Connection c) { - super(c); +public class NexusDao { + private final Connection c; + + public NexusDao(Connection c) { + this.c = c; + } + + public static URI uri(String s) { + try { + return URI.create(s); + } catch (Exception e) { + e.printStackTrace(System.out); + return null; + } + } + + private NexusRepositoryDto nexusRepositoryDto(ResultSet rs) throws SQLException { + int i = 1; + + return new NexusRepositoryDto( + rs.getString(i++), + uri(rs.getString(i++)), + (String[]) rs.getArray(i++).getArray(), + fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime), + fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime), + fromNull(rs.getTimestamp(i)).map(timestampToLocalDateTime) + ); } /* @@ -29,71 +54,126 @@ public class NexusDao extends Dao { */ public List<NexusServerDto> selectServer() throws SQLException { - try (PreparedStatement s = c.prepareStatement("SELECT url FROM nexus_server")) { + try (PreparedStatement s = c.prepareStatement("SELECT url, name FROM nexus_server")) { ResultSet rs = s.executeQuery(); List<NexusServerDto> servers = new ArrayList<>(); - while(rs.next()) { - servers.add(new NexusServerDto(uri(rs.getString(1)))); + while (rs.next()) { + servers.add(new NexusServerDto(uri(rs.getString(1)), rs.getString(2))); } return servers; } } - public Option<NexusRepositoryDto> selectRepository(String repositoryId) throws SQLException { - PreparedStatement s = prepareStatement("SELECT id, nexus_server_url, group_ids, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?"); - s.setString(1, repositoryId); + public Option<NexusRepositoryDto> findRepository(String repositoryId) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT id, server_url, group_ids, discovery_date, last_update, last_successful_update FROM nexus_repository WHERE id=?")) { + s.setString(1, repositoryId); - try (ResultSet rs = s.executeQuery()) { - if (!rs.next()) { - return Option.none(); - } + try (ResultSet rs = s.executeQuery()) { + if (!rs.next()) { + return Option.none(); + } - return some(toRepository(rs)); + return some(nexusRepositoryDto(rs)); + } } } public List<NexusRepositoryDto> findRepositories(URI nexusUrl) throws SQLException { - PreparedStatement s = prepareStatement("SELECT id, nexus_server_url, group_ids, created_date, last_update, last_successful_update FROM nexus_repository WHERE nexus_server_url=?"); - s.setString(1, nexusUrl.toASCIIString()); - - List<NexusRepositoryDto> list = new ArrayList<>(); - try (ResultSet rs = s.executeQuery()) { - while(rs.next()) { - list.add(toRepository(rs)); + try (PreparedStatement s = c.prepareStatement("SELECT id, server_url, group_ids, created_date, last_update, last_successful_update FROM nexus_repository WHERE server_url=?")) { + s.setString(1, nexusUrl.toASCIIString()); + + List<NexusRepositoryDto> list = new ArrayList<>(); + try (ResultSet rs = s.executeQuery()) { + while (rs.next()) { + list.add(nexusRepositoryDto(rs)); + } } + + return list; } - return list; } - private NexusRepositoryDto toRepository(ResultSet rs) throws SQLException { + // ----------------------------------------------------------------------- + // Nexus Artifact + // ----------------------------------------------------------------------- + + private final String NEXUS_ARTIFACT_ID = "group_id, artifact_id, version"; + + private final String NEXUS_ARTIFACT = NEXUS_ARTIFACT_ID + ", snapshot_version, classifiers, packagings, created_date"; + + private int setArtifactId(int i, PreparedStatement s, ArtifactId id) throws SQLException { + s.setString(i++, id.groupId); + s.setString(i++, id.artifactId); + s.setString(i++, id.version); + return i; + } + + private static ArtifactDto artifactDto(URI serverUrl, String repositoryId, ResultSet rs) throws SQLException { int i = 1; - return new NexusRepositoryDto( - rs.getString(i++), - uri(rs.getString(i++)), - (String[]) rs.getArray(i++).getArray(), - fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime), - fromNull(rs.getTimestamp(i++)).map(timestampToLocalDateTime), - fromNull(rs.getTimestamp(i)).map(timestampToLocalDateTime) - ); + return new ArtifactDto( + serverUrl, + repositoryId, + new ArtifactId(rs.getString(i++), + rs.getString(i++), + rs.getString(i))); } - private URI uri(String s) { - try { - return URI.create(s); - } catch (Exception e) { - e.printStackTrace(System.out); - return null; + public void insertArtifact(URI nexusUrl, String repositoryId, ArtifactId id, Option<String> snapshotVersion, List<ArtifactFile> files, Date createdDate) throws SQLException { + String[] classifiers = new String[files.size()]; + String[] packagings = new String[files.size()]; + + for (int i = 0; i < files.size(); i++) { + classifiers[i] = files.get(i).classifier.toNull(); + packagings[i] = files.get(i).extension; + } + + int i = 1; + try (PreparedStatement s = c.prepareStatement("INSERT INTO nexus_artifact(server_url, repository_id, " + NEXUS_ARTIFACT + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")) { + s.setString(i++, nexusUrl.toASCIIString()); + s.setString(i++, repositoryId); + i = setArtifactId(i, s, id); + s.setString(i++, snapshotVersion.toNull()); + s.setArray(i++, c.createArrayOf("varchar", classifiers)); + s.setArray(i++, c.createArrayOf("varchar", packagings)); + s.setTimestamp(i, DaoUtil.dateToTimestamp.f(createdDate)); + s.executeUpdate(); + } + } + + public void deleteArtifact(URI nexusUrl, String repositoryId, ArtifactId id) throws SQLException { + int i = 1; + try (PreparedStatement s = c.prepareStatement("DELETE FROM nexus_artifact WHERE server_url=? AND repository_id=? AND group_id=? AND artifact_id=? AND version=?")) { + s.setString(i++, nexusUrl.toASCIIString()); + s.setString(i++, repositoryId); + i += setArtifactId(i, s, id); + s.executeUpdate(); + } + } + + public List<ArtifactDto> findArtifactsInRepository(URI url, String repositoryId) throws SQLException { + try (PreparedStatement s = c.prepareStatement("SELECT " + NEXUS_ARTIFACT + " FROM nexus_artifact WHERE server_url=? AND repository_id=?")) { + s.setString(1, url.toASCIIString()); + s.setString(2, repositoryId); + ResultSet rs = s.executeQuery(); + + List<ArtifactDto> list = new ArrayList<>(); + while (rs.next()) { + list.add(artifactDto(url, repositoryId, rs)); + } + return list; } } } class NexusServerDto { public final URI url; + public final String name; - NexusServerDto(URI url) { + NexusServerDto(URI url, String name) { this.url = url; + this.name = name; } public boolean equals(Object o) { @@ -129,3 +209,4 @@ class NexusRepositoryDto { this.lastSuccessfulUpdate = lastSuccessfulUpdate; } } + diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java index 12d8c9c..c6e2460 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusImporter.java @@ -1,16 +1,18 @@ package io.trygvis.esper.testing.nexus; -import com.google.common.collect.*; import com.jolbox.bonecp.*; import fj.data.*; +import static fj.data.Option.*; import io.trygvis.esper.testing.*; import io.trygvis.esper.testing.object.*; import static java.lang.Thread.*; -import org.apache.commons.lang.*; import org.codehaus.httpcache4j.cache.*; import java.sql.*; import java.util.*; +import java.util.Date; +import java.util.HashMap; +import java.util.TreeMap; import java.util.concurrent.atomic.*; public class NexusImporter { @@ -25,7 +27,9 @@ public class NexusImporter { public ActorRef<NexusServer> create(NexusServerDto server) { final NexusClient client = new NexusClient(http, server.url); - return ObjectUtil.threadedActor(boneCp, config.nexusUpdateInterval, new NexusServer(client, server)); + String name = server.name; + + return ObjectUtil.threadedActor(boneCp, "", config.nexusUpdateInterval, new NexusServer(client, server)); } }); @@ -42,7 +46,7 @@ public class NexusImporter { } synchronized (shouldRun) { - shouldRun.wait(1000); + shouldRun.wait(60 * 1000); } } @@ -61,20 +65,83 @@ class NexusServer implements TransactionalActor { } public void act(Connection c) throws Exception { + Date timestamp = new Date(); NexusDao dao = new NexusDao(c); + String p = server.name; + for (NexusRepositoryDto repository : dao.findRepositories(server.url)) { - System.out.println("Updating repository: " + repository.repositoryId); + String p2 = p + "/" + repository.repositoryId; + + System.out.println(p2 + ": Updating repository: " + repository.repositoryId); + + TreeMap<ArtifactId, ArtifactXml> artifactsInNexus = new TreeMap<>(); + for (String groupId : repository.groupIds) { - System.out.println("Updating groupId: " + groupId); - ArtifactSearchResult result = client.fetchIndex(groupId, Option.<String>none()); + String p3 = p2 + "/" + groupId; + + System.out.println(p3 + ": Updating group id"); + ArtifactSearchResult result = client.fetchIndex(groupId, some(repository.repositoryId)); + System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts"); + + for (ArtifactXml xml : result.artifacts) { + artifactsInNexus.put(xml.id, xml); + } + + System.out.println(p3 + ": Updating everything under group id"); + result = client.fetchIndex(groupId + ".*", some(repository.repositoryId)); + System.out.println(p3 + ": Found " + result.artifacts.size() + " artifacts"); + + for (ArtifactXml xml : result.artifacts) { + artifactsInNexus.put(xml.id, xml); + } + } + + Map<ArtifactId, ArtifactDto> artifactsInDatabase = new HashMap<>(); + for (ArtifactDto dto : dao.findArtifactsInRepository(server.url, repository.repositoryId)) { + artifactsInDatabase.put(dto.id, dto); + } - ArrayList<ArtifactXml> artifacts = Lists.newArrayList(result.artifacts); - Collections.sort(artifacts); - for (ArtifactXml artifact : artifacts) { - System.out.println("repo=" + StringUtils.join(artifact.repositories(), ", ") + ", artifact=" + artifact.getId()); + ArrayList<FlatArtifact> created = new ArrayList<>(); + ArrayList<FlatArtifact> kept = new ArrayList<>(); + ArrayList<ArtifactDto> removed = new ArrayList<>(); + + for (ArtifactXml xml : artifactsInNexus.values()) { + Option<FlatArtifact> o = xml.flatten(repository.repositoryId); + + if(o.isNone()) { + continue; + } + + FlatArtifact artifact = o.some(); + + if(!artifactsInDatabase.containsKey(xml.id)) { + created.add(artifact); + } + else { + kept.add(artifact); + } + } + + for (ArtifactDto dto : artifactsInDatabase.values()) { + if(!artifactsInNexus.containsKey(dto.id)) { + removed.add(dto); } } + + System.out.println(p2 + ": found " + created.size() + " new artifacts, " + removed.size() + " removed artifacts and " + kept.size() + " existing artifacts."); + + System.out.println(p2 + ": inserting new artifacts"); + for (FlatArtifact artifact : created) { + dao.insertArtifact(repository.nexusUrl, repository.repositoryId, artifact.id, Option.<String>none(), artifact.files, timestamp); + } + System.out.println(p2 + ": inserted"); + + System.out.println(p2 + ": deleting removed artifacts"); + for (ArtifactDto artifact : removed) { + dao.deleteArtifact(repository.nexusUrl, repository.repositoryId, artifact.id); + } + System.out.println(p2 + ": deleted"); } c.commit(); diff --git a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java index 05dfd5b..6bfca99 100755 --- a/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java +++ b/src/main/java/io/trygvis/esper/testing/nexus/NexusParser.java @@ -2,19 +2,18 @@ package io.trygvis.esper.testing.nexus; import com.google.common.base.*; import com.google.common.collect.*; -import fj.*; import fj.data.*; -import static fj.data.Option.fromNull; +import static fj.data.Option.*; import static org.apache.commons.lang.StringUtils.*; import org.dom4j.*; import org.dom4j.io.*; +import javax.xml.stream.*; import java.io.*; import java.util.*; import java.util.HashSet; import java.util.List; import java.util.Set; -import javax.xml.stream.*; public class NexusParser { public static final STAXEventReader xmlReader = new STAXEventReader(); @@ -23,7 +22,7 @@ public class NexusParser { Document doc = xmlReader.readDocument(is); Option<Integer> totalCount = fromNull(trimToNull(doc.getRootElement().elementText("totalCount"))). - bind(Option.parseInt); + bind(Option.parseInt); if (totalCount.isNone()) { throw new RuntimeException("Could not find required element <totalCount>"); } @@ -71,7 +70,7 @@ public class NexusParser { artifactHitsList.add(new ArtifactHits(repositoryId, files)); } - list.add(new ArtifactXml(groupId, artifactId, version, artifactHitsList)); + list.add(new ArtifactXml(new ArtifactId(groupId, artifactId, version), artifactHitsList)); } return new ArtifactSearchResult(totalCount.some(), tooManyResults, list); @@ -97,15 +96,11 @@ class ArtifactSearchResult { } class ArtifactXml implements Comparable<ArtifactXml> { - public final String groupId; - public final String artifactId; - public final String version; + public final ArtifactId id; public final List<ArtifactHits> hits; - ArtifactXml(String groupId, String artifactId, String version, List<ArtifactHits> hits) { - this.groupId = groupId; - this.artifactId = artifactId; - this.version = version; + ArtifactXml(ArtifactId id, List<ArtifactHits> hits) { + this.id = id; this.hits = hits; } @@ -121,34 +116,18 @@ class ArtifactXml implements Comparable<ArtifactXml> { }; } - public FlatArtifact flatten(String repositoryId) { + public Option<FlatArtifact> flatten(String repositoryId) { for (ArtifactHits hit : hits) { if (hit.repositoryId.equals(repositoryId)) { - return new FlatArtifact(groupId, artifactId, version, hit.files); + return some(new FlatArtifact(id, hit.files)); } } - throw new RuntimeException("No hits in repository " + repositoryId); + return none(); } public int compareTo(ArtifactXml o) { - int c = groupId.compareTo(o.groupId); - - if(c != 0) { - return c; - } - - c = artifactId.compareTo(o.artifactId); - - if(c != 0) { - return c; - } - - return version.compareTo(o.version); - } - - public String getId() { - return groupId + ":" + artifactId + ":" + version; + return id.compareTo(o.id); } public Set<String> repositories() { @@ -163,15 +142,11 @@ class ArtifactXml implements Comparable<ArtifactXml> { } class FlatArtifact { - public final String groupId; - public final String artifactId; - public final String version; + public final ArtifactId id; public final List<ArtifactFile> files; - FlatArtifact(String groupId, String artifactId, String version, List<ArtifactFile> files) { - this.groupId = groupId; - this.artifactId = artifactId; - this.version = version; + FlatArtifact(ArtifactId id, List<ArtifactFile> files) { + this.id = id; this.files = files; } } @@ -185,13 +160,3 @@ class ArtifactHits { this.files = files; } } - -class ArtifactFile { - public final Option<String> classifier; - public final String extension; - - ArtifactFile(Option<String> classifier, String extension) { - this.classifier = classifier; - this.extension = extension; - } -} diff --git a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java index 6b6fe75..2d23822 100644 --- a/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java +++ b/src/main/java/io/trygvis/esper/testing/object/ObjectUtil.java @@ -6,8 +6,8 @@ import java.sql.*; public class ObjectUtil { - public static <A extends TransactionalActor> ActorRef<A> threadedActor(DataSource dataSource, long delay, A actor) { - return new ThreadedActor<>(dataSource, actor, delay); + public static <A extends TransactionalActor> ActorRef<A> threadedActor(DataSource dataSource, String threadName, long delay, A actor) { + return new ThreadedActor<>(dataSource, threadName, actor, delay); } static class ThreadedActor<A extends TransactionalActor> implements ActorRef<A>, Runnable, Closeable { @@ -18,11 +18,11 @@ public class ObjectUtil { private final Thread thread; private boolean shouldRun = true; - ThreadedActor(DataSource dataSource, A actor, long delay) { + ThreadedActor(DataSource dataSource, String threadName, A actor, long delay) { this.dataSource = dataSource; this.actor = actor; this.delay = delay; - thread = new Thread(this); + thread = new Thread(this, threadName); thread.setDaemon(true); thread.start(); } @@ -43,6 +43,7 @@ public class ObjectUtil { } } } catch (Exception e) { + System.out.println("Exception in thread " + Thread.currentThread().getName()); e.printStackTrace(System.out); } diff --git a/src/main/resources/ddl-nexus.sql b/src/main/resources/ddl-nexus.sql index 9ee35e0..15ed482 100755 --- a/src/main/resources/ddl-nexus.sql +++ b/src/main/resources/ddl-nexus.sql @@ -1,34 +1,60 @@ BEGIN; DROP TABLE IF EXISTS nexus_event; +DROP TABLE IF EXISTS nexus_artifact; DROP TABLE IF EXISTS nexus_repository; DROP TABLE IF EXISTS nexus_server; CREATE TABLE nexus_server ( - url VARCHAR(1000) PRIMARY KEY + url VARCHAR(1000) PRIMARY KEY, + name VARCHAR(100) NOT NULL ); CREATE TABLE nexus_repository ( - id VARCHAR(1000) PRIMARY KEY, - nexus_server_url VARCHAR(1000), + id VARCHAR(1000), + server_url VARCHAR(1000) NOT NULL, group_ids VARCHAR(100) [], created_date TIMESTAMP, last_update TIMESTAMP, last_successful_update TIMESTAMP, - CONSTRAINT nexus_server_fk FOREIGN KEY (nexus_server_url) REFERENCES nexus_server (url) + CONSTRAINT nexus_repository_pk PRIMARY KEY (id, server_url), + CONSTRAINT nexus_server_fk FOREIGN KEY (server_url) REFERENCES nexus_server (url) +); + +CREATE TABLE nexus_artifact ( + server_url VARCHAR(1000) NOT NULL, + repository_id VARCHAR(1000) NOT NULL, + group_id VARCHAR(100) NOT NULL, + artifact_id VARCHAR(100) NOT NULL, + version VARCHAR(100) NOT NULL, + snapshot_version VARCHAR(100), + classifiers VARCHAR(100) [], + packagings VARCHAR(100) [], + created_date TIMESTAMP, + last_update TIMESTAMP, + last_successful_update TIMESTAMP, + CONSTRAINT nexus_artifact_pk PRIMARY KEY (server_url, repository_id, group_id, artifact_id, version), + CONSTRAINT nexus_repository_fk FOREIGN KEY (server_url, repository_id) REFERENCES nexus_repository (server_url, id) ); CREATE TABLE nexus_event ( + timestamp VARCHAR(100), + uuid VARCHAR(100), + server_url VARCHAR(1000) NOT NULL, repository_id VARCHAR(1000) NOT NULL, - - groupId VARCHAR(100), - artifactId VARCHAR(100), - version VARCHAR(100), - files VARCHAR(100), - CONSTRAINT nexus_repository_fk FOREIGN KEY (repository_id) REFERENCES nexus_repository (id) + group_id VARCHAR(100) NOT NULL, + artifact_id VARCHAR(100) NOT NULL, + version VARCHAR(100) NOT NULL, + files VARCHAR(100) [], + CONSTRAINT nexus_event_pk PRIMARY KEY (timestamp, server_url, repository_id, group_id, artifact_id, version), + CONSTRAINT nexus_artifact_fk FOREIGN KEY (server_url, repository_id, group_id, artifact_id, version) REFERENCES nexus_artifact (server_url, repository_id, group_id, artifact_id, version) ); -INSERT INTO nexus_server(url) VALUES('https://oss.sonatype.org'); -INSERT INTO nexus_repository(id, nexus_server_url, group_ids) VALUES('codehaus', 'https://oss.sonatype.org', ARRAY['io.trygvis', 'org.codehaus.mojo']); +INSERT INTO nexus_server (url, name) VALUES ('https://oss.sonatype.org', 'Central Repository'); +-- INSERT INTO nexus_repository (id, server_url, group_ids) VALUES ('releases', 'https://oss.sonatype.org', ARRAY ['io.trygvis', 'no.arktekk', 'org.codehaus']); +INSERT INTO nexus_repository (id, server_url, group_ids) VALUES ('releases', 'https://oss.sonatype.org', ARRAY ['io.trygvis', 'no.arktekk', 'org.codehaus']); + +INSERT INTO nexus_server (url, name) VALUES ('http://nexus.codehaus.org', 'Codehaus Snapshots'); +INSERT INTO nexus_repository (id, server_url, group_ids) VALUES ('snapshots', 'http://nexus.codehaus.org', ARRAY ['org.codehaus.mojo']); COMMIT; diff --git a/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java b/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java index c9aba8d..b52d894 100755 --- a/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java +++ b/src/test/java/io/trygvis/esper/testing/nexus/TestXmlParsing.java @@ -19,14 +19,14 @@ public class TestXmlParsing extends TestCase { assertEquals(132, list.size()); ArtifactXml artifact = list.get(0); - assertEquals("org.codehaus.mojo.hibernate3", artifact.groupId); - assertEquals("maven-hibernate3-jdk15", artifact.artifactId); - assertEquals("2.0-alpha-1", artifact.version); + assertEquals("org.codehaus.mojo.hibernate3", artifact.id.groupId); + assertEquals("maven-hibernate3-jdk15", artifact.id.artifactId); + assertEquals("2.0-alpha-1", artifact.id.version); artifact = list.get(4); - assertEquals("org.codehaus.mojo.hibernate3", artifact.groupId); - assertEquals("maven-hibernate3", artifact.artifactId); - assertEquals("2.0-alpha-1", artifact.version); + assertEquals("org.codehaus.mojo.hibernate3", artifact.id.groupId); + assertEquals("maven-hibernate3", artifact.id.artifactId); + assertEquals("2.0-alpha-1", artifact.id.version); assertEquals(2, artifact.hits.size()); assertEquals("appfuse-releases", artifact.hits.get(0).repositoryId); assertEquals(1, artifact.hits.get(0).files.size()); @@ -37,10 +37,10 @@ public class TestXmlParsing extends TestCase { ArrayList<ArtifactXml> filtered = newArrayList(filter(list, repositoryFilter("appfuse-releases"))); assertEquals(5, filtered.size()); - FlatArtifact flatArtifact = filtered.get(0).flatten("appfuse-releases"); - assertEquals("org.codehaus.mojo.hibernate3", flatArtifact.groupId); - assertEquals("maven-hibernate3-jdk15", flatArtifact.artifactId); - assertEquals("2.0-alpha-1", flatArtifact.version); + FlatArtifact flatArtifact = filtered.get(0).flatten("appfuse-releases").some(); + assertEquals("org.codehaus.mojo.hibernate3", flatArtifact.id.groupId); + assertEquals("maven-hibernate3-jdk15", flatArtifact.id.artifactId); + assertEquals("2.0-alpha-1", flatArtifact.id.version); assertEquals(2, flatArtifact.files.size()); } } |