diff options
Diffstat (limited to 'src/main/java/io/trygvis/esper')
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/Daos.java | 25 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/Main.java | 2 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/Dao.java | 15 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java (renamed from src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java) | 8 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java | 136 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java | 46 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java | 42 | ||||
| -rw-r--r-- | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java | 69 |
8 files changed, 286 insertions, 57 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/Daos.java b/src/main/java/io/trygvis/esper/testing/Daos.java new file mode 100644 index 0000000..db97f9d --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/Daos.java @@ -0,0 +1,25 @@ +package io.trygvis.esper.testing; + +import io.trygvis.esper.testing.gitorious.*; + +import java.sql.*; + +public class Daos { + public final AtomDao atomDao; + public final GitoriousEventDao gitoriousEventDao; + public final GitoriousProjectDao gitoriousProjectDao; + public final GitoriousRepositoryDao gitoriousRepositoryDao; + public final PreparedStatement begin; + + public Daos(Connection c) throws SQLException { + atomDao = new AtomDao(c); + gitoriousEventDao = new GitoriousEventDao(c); + gitoriousProjectDao = new GitoriousProjectDao(c); + gitoriousRepositoryDao = new GitoriousRepositoryDao(c); + begin = c.prepareStatement("BEGIN"); + } + + public void begin() throws SQLException { + begin.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java index 69e0bce..ad7c71a 100644 --- a/src/main/java/io/trygvis/esper/testing/Main.java +++ b/src/main/java/io/trygvis/esper/testing/Main.java @@ -19,6 +19,8 @@ public class Main { Properties properties = new Properties(); properties.setProperty("log4j.rootLogger", "DEBUG, A1"); properties.setProperty("log4j.logger.httpclient.wire.content", "INFO"); + properties.setProperty("log4j.logger.httpclient.wire.header", "INFO"); + properties.setProperty("log4j.logger.org.apache.commons.httpclient", "INFO"); properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender"); properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout"); properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java new file mode 100644 index 0000000..4714a87 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java @@ -0,0 +1,15 @@ +package io.trygvis.esper.testing.gitorious; + +import java.sql.*; + +public class Dao { + private final Connection c; + + protected Dao(Connection c) { + this.c = c; + } + + protected PreparedStatement prepareStatement(String sql) throws SQLException { + return c.prepareStatement(sql); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java index 766a4a9..603609e 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java @@ -2,13 +2,13 @@ package io.trygvis.esper.testing.gitorious; import java.sql.*; -public class GitoriousDao { +public class GitoriousEventDao { private final PreparedStatement countEntryId; private final PreparedStatement insertChange; - public GitoriousDao(Connection c) throws SQLException { - countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?"); - insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)"); + public GitoriousEventDao(Connection c) throws SQLException { + countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_event WHERE entry_id=?"); + insertChange = c.prepareStatement("INSERT INTO gitorious_event(entry_id, text) VALUES(?, ?)"); } public int countEntryId(String entryId) throws SQLException { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java index c77d7db..ee89527 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java @@ -1,24 +1,22 @@ package io.trygvis.esper.testing.gitorious; +import com.jolbox.bonecp.*; +import com.jolbox.bonecp.hooks.*; import io.trygvis.esper.testing.*; -import io.trygvis.esper.testing.ResourceManager.*; import org.apache.abdera.*; -import org.apache.abdera.model.*; import org.apache.abdera.protocol.client.*; import org.apache.abdera.protocol.client.cache.*; import org.codehaus.httpcache4j.cache.*; import org.codehaus.httpcache4j.client.*; import java.sql.*; -import java.util.Date; import java.util.*; import java.util.concurrent.*; public class GitoriousImporter { - private final AbderaClient abderaClient; - private final Connection c; - private final AtomDao atomDao; - private final GitoriousDao gitoriousDao; +// private final AbderaClient abderaClient; + private final BoneCP boneCp; + private final GitoriousClient gitoriousClient; public static void main(String[] args) throws Exception { Main.configureLog4j(); @@ -27,70 +25,109 @@ public class GitoriousImporter { public GitoriousImporter() throws Exception { Abdera abdera = new Abdera(); - abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); - - c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); - c.setAutoCommit(false); +// abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); + + BoneCPConfig config = new BoneCPConfig(); + config.setJdbcUrl(DbMain.JDBC_URL); + config.setUsername("esper"); + config.setPassword(""); + config.setDefaultAutoCommit(false); + config.setMaxConnectionsPerPartition(1); + + config.setConnectionHook(new AbstractConnectionHook() { + public void onAcquire(ConnectionHandle c) { + try { + c.setDebugHandle(new Daos(c)); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + }); - atomDao = new AtomDao(c); - gitoriousDao = new GitoriousDao(c); + boneCp = new BoneCP(config); HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance()); - final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); - -// Set<GitoriousProject> projects = gitoriousClient.findProjects(); -// -// System.out.println("projects.size() = " + projects.size()); -// for (GitoriousProject project : projects) { -// System.out.println("project.repositories = " + project.repositories); -// } - -// new GitoriousImporter(abderaClient, c).work(); + gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); int projectsUpdateInterval = 1000; final int projectUpdateInterval = 1000; - ResourceManager<GitoriousProject, GitoriousProjectResourceManager> gitoriousProjects = new ResourceManager<>(service, 1000, +// service.scheduleAtFixedRate(new Runnable() { +// public void run() { +// try { +// discoverProjects(); +// } catch (Exception e) { +// e.printStackTrace(System.out); +// } +// } +// }, projectsUpdateInterval, projectsUpdateInterval, TimeUnit.MILLISECONDS); + + discoverProjects(); + } - new ResourceManagerCallbacks<GitoriousProject, GitoriousProjectResourceManager>() { - public Set<GitoriousProject> discover() throws Exception { - return gitoriousClient.findProjects(); + private void discoverProjects() throws Exception { + Set<GitoriousProject> projects = gitoriousClient.findProjects(); + + try (ConnectionHandle connection = (ConnectionHandle) boneCp.getConnection()) { + Daos daos = (Daos) connection.getDebugHandle(); + GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao; + GitoriousProjectDao projectDao = daos.gitoriousProjectDao; + + daos.begin(); + System.out.println("Processing " + projects.size() + " projects."); + for (GitoriousProject project : projects) { + if(projectDao.countProjects(project.slug) == 0) { + System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories."); + projectDao.insertProject(project); + for (GitoriousRepository repository : project.repositories) { + repoDao.insertRepository(repository); + } } - - public GitoriousProjectResourceManager onNew(GitoriousProject key) { - return new GitoriousProjectResourceManager(service, projectUpdateInterval, key); + else { + for (GitoriousRepository repository : project.repositories) { + if(repoDao.countRepositories(repository) == 0) { + System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name); + repoDao.insertRepository(repository); + } + } + + for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) { + if(project.repositories.contains(repository)) { + continue; + } + System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name); + repoDao.delete(repository); + } } + } - public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) { - System.out.println("Project gone."); - manager.close(); + for (String project : projectDao.selectAll()) { + boolean found = false; + for (Iterator<GitoriousProject> it = projects.iterator(); it.hasNext(); ) { + GitoriousProject p = it.next(); + if(p.slug.equals(project)) { + found = true; + break; + } } - }); - ; - } - - class GitoriousProjectResourceManager extends ResourceManager<GitoriousRepository, GitoriousRepository> { - public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) { - super(executorService, delay, new ResourceManagerCallbacks<GitoriousRepository, GitoriousRepository>() { - public Set<GitoriousRepository> discover() throws Exception { - key + if (found) { + continue; } - public GitoriousRepository onNew(GitoriousRepository key) { - throw new RuntimeException("Not implemented"); - } + System.out.println("Gone project: " + project); + repoDao.deleteForProject(project); + projectDao.delete(project); + } - public void onGone(GitoriousRepository key, GitoriousRepository value) { - throw new RuntimeException("Not implemented"); - } - }); + connection.commit(); } } + /* private void work() throws SQLException, InterruptedException { String url = "http://qt.gitorious.org/projects/show/qt.atom"; @@ -153,4 +190,5 @@ public class GitoriousImporter { Thread.sleep(10 * 1000); } } + */ } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java index e5b3fdd..6947512 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java @@ -68,15 +68,33 @@ public class GitoriousProject implements Comparable<GitoriousProject> { public int compareTo(GitoriousProject other) { return slug.compareTo(other.slug); } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousProject)) return false; + + GitoriousProject that = (GitoriousProject) o; + + if (!repositories.equals(that.repositories)) return false; + if (!slug.equals(that.slug)) return false; + + return true; + } + + public int hashCode() { + int result = slug.hashCode(); + result = 31 * result + repositories.hashCode(); + return result; + } } class GitoriousRepository implements Comparable<GitoriousRepository> { - public final String project; + public final String projectSlug; public final String name; public final URI atom; - GitoriousRepository(String project, String name, URI atom) { - this.project = project; + GitoriousRepository(String projectSlug, String name, URI atom) { + this.projectSlug = projectSlug; this.name = name; this.atom = atom; } @@ -92,7 +110,7 @@ class GitoriousRepository implements Comparable<GitoriousRepository> { } public int compareTo(GitoriousRepository o) { - int a = project.compareTo(o.project); + int a = projectSlug.compareTo(o.projectSlug); if (a != 0) { return a; @@ -100,4 +118,24 @@ class GitoriousRepository implements Comparable<GitoriousRepository> { return name.compareTo(o.name); } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousRepository)) return false; + + GitoriousRepository that = (GitoriousRepository) o; + + if (!atom.equals(that.atom)) return false; + if (!name.equals(that.name)) return false; + if (!projectSlug.equals(that.projectSlug)) return false; + + return true; + } + + public int hashCode() { + int result = projectSlug.hashCode(); + result = 31 * result + name.hashCode(); + result = 31 * result + atom.hashCode(); + return result; + } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java new file mode 100644 index 0000000..f47b126 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java @@ -0,0 +1,42 @@ +package io.trygvis.esper.testing.gitorious; + +import java.sql.*; +import java.util.*; + +public class GitoriousProjectDao extends Dao { + public GitoriousProjectDao(Connection c) throws SQLException { + super(c); + } + + private final PreparedStatement countProjects = prepareStatement("SELECT count(*) FROM gitorious_project WHERE slug=?"); + public int countProjects(String slug) throws SQLException { + countProjects.setString(1, slug); + try(ResultSet rs = countProjects.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + + private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)"); + public void insertProject(GitoriousProject project) throws SQLException { + insertProject.setString(1, project.slug); + insertProject.executeUpdate(); + } + + private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project"); + public List<String> selectAll() throws SQLException { + try (ResultSet rs = selectAll.executeQuery()) { + List<String> list = new ArrayList<>(); + while(rs.next()) { + list.add(rs.getString(1)); + } + return list; + } + } + + private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?"); + public void delete(String slug) throws SQLException { + delete.setString(1, slug); + delete.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java new file mode 100644 index 0000000..a88f122 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java @@ -0,0 +1,69 @@ +package io.trygvis.esper.testing.gitorious; + +import java.net.*; +import java.sql.*; +import java.util.*; + +public class GitoriousRepositoryDao extends Dao { + public GitoriousRepositoryDao(Connection c) throws SQLException { + super(c); + } + + private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?"); + public int countRepositories(GitoriousRepository repository) throws SQLException { + countRepositories.setString(1, repository.projectSlug); + countRepositories.setString(2, repository.name); + try (ResultSet rs = countRepositories.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + + private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository WHERE project_slug=?"); + public List<GitoriousRepository> selectForProject(String projectSlug) throws Exception { + selectForProject.setString(1, projectSlug); + return executeQuery(selectForProject); + } + + private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository"); + public List<GitoriousRepository> selectAll() throws Exception { + return executeQuery(selectAll); + } + + private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)"); + public void insertRepository(GitoriousRepository repository) throws SQLException { + insertRepository.setString(1, repository.projectSlug); + insertRepository.setString(2, repository.name); + insertRepository.setString(3, repository.atom.toASCIIString()); + insertRepository.executeUpdate(); + } + + private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?"); + public void delete(GitoriousRepository repository) throws SQLException { + delete.setString(1, repository.projectSlug); + delete.setString(2, repository.name); + delete.executeUpdate(); + } + + private final PreparedStatement deleteForProject = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=?"); + public void deleteForProject(String project) throws SQLException { + deleteForProject.setString(1, project); + deleteForProject.executeUpdate(); + } + + private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException, URISyntaxException { + try (ResultSet rs = statement.executeQuery()) { + List<GitoriousRepository> list = new ArrayList<>(); + + while(rs.next()) { + list.add(new GitoriousRepository( + rs.getString(1), + rs.getString(2), + new URI(rs.getString(3)) + )); + } + + return list; + } + } +} |