From 2eae4836279646050e7e342752cde6e8f7c5b6cb Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Thu, 8 Nov 2012 22:24:01 +0100 Subject: wip --- src/main/java/io/trygvis/esper/testing/Daos.java | 25 ++++ src/main/java/io/trygvis/esper/testing/Main.java | 2 + .../io/trygvis/esper/testing/gitorious/Dao.java | 15 +++ .../esper/testing/gitorious/GitoriousDao.java | 27 ---- .../esper/testing/gitorious/GitoriousEventDao.java | 27 ++++ .../esper/testing/gitorious/GitoriousImporter.java | 136 +++++++++++++-------- .../esper/testing/gitorious/GitoriousProject.java | 46 ++++++- .../testing/gitorious/GitoriousProjectDao.java | 42 +++++++ .../testing/gitorious/GitoriousRepositoryDao.java | 69 +++++++++++ src/main/resources/ddl.sql | 13 +- 10 files changed, 320 insertions(+), 82 deletions(-) create mode 100644 src/main/java/io/trygvis/esper/testing/Daos.java create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/Dao.java delete mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java (limited to 'src') diff --git a/src/main/java/io/trygvis/esper/testing/Daos.java b/src/main/java/io/trygvis/esper/testing/Daos.java new file mode 100644 index 0000000..db97f9d --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/Daos.java @@ -0,0 +1,25 @@ +package io.trygvis.esper.testing; + +import io.trygvis.esper.testing.gitorious.*; + +import java.sql.*; + +public class Daos { + public final AtomDao atomDao; + public final GitoriousEventDao gitoriousEventDao; + public final GitoriousProjectDao gitoriousProjectDao; + public final GitoriousRepositoryDao gitoriousRepositoryDao; + public final PreparedStatement begin; + + public Daos(Connection c) throws SQLException { + atomDao = new AtomDao(c); + gitoriousEventDao = new GitoriousEventDao(c); + gitoriousProjectDao = new GitoriousProjectDao(c); + gitoriousRepositoryDao = new GitoriousRepositoryDao(c); + begin = c.prepareStatement("BEGIN"); + } + + public void begin() throws SQLException { + begin.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java index 69e0bce..ad7c71a 100644 --- a/src/main/java/io/trygvis/esper/testing/Main.java +++ b/src/main/java/io/trygvis/esper/testing/Main.java @@ -19,6 +19,8 @@ public class Main { Properties properties = new Properties(); properties.setProperty("log4j.rootLogger", "DEBUG, A1"); properties.setProperty("log4j.logger.httpclient.wire.content", "INFO"); + properties.setProperty("log4j.logger.httpclient.wire.header", "INFO"); + properties.setProperty("log4j.logger.org.apache.commons.httpclient", "INFO"); properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender"); properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout"); properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java new file mode 100644 index 0000000..4714a87 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java @@ -0,0 +1,15 @@ +package io.trygvis.esper.testing.gitorious; + +import java.sql.*; + +public class Dao { + private final Connection c; + + protected Dao(Connection c) { + this.c = c; + } + + protected PreparedStatement prepareStatement(String sql) throws SQLException { + return c.prepareStatement(sql); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java deleted file mode 100644 index 766a4a9..0000000 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java +++ /dev/null @@ -1,27 +0,0 @@ -package io.trygvis.esper.testing.gitorious; - -import java.sql.*; - -public class GitoriousDao { - private final PreparedStatement countEntryId; - private final PreparedStatement insertChange; - - public GitoriousDao(Connection c) throws SQLException { - countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?"); - insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)"); - } - - public int countEntryId(String entryId) throws SQLException { - countEntryId.setString(1, entryId); - try(ResultSet rs = countEntryId.executeQuery()) { - rs.next(); - return rs.getInt(1); - } - } - - public void insertChange(String entryId, String text) throws SQLException { - insertChange.setString(1, entryId); - insertChange.setString(2, text); - insertChange.executeUpdate(); - } -} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java new file mode 100644 index 0000000..603609e --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java @@ -0,0 +1,27 @@ +package io.trygvis.esper.testing.gitorious; + +import java.sql.*; + +public class GitoriousEventDao { + private final PreparedStatement countEntryId; + private final PreparedStatement insertChange; + + public GitoriousEventDao(Connection c) throws SQLException { + countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_event WHERE entry_id=?"); + insertChange = c.prepareStatement("INSERT INTO gitorious_event(entry_id, text) VALUES(?, ?)"); + } + + public int countEntryId(String entryId) throws SQLException { + countEntryId.setString(1, entryId); + try(ResultSet rs = countEntryId.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + + public void insertChange(String entryId, String text) throws SQLException { + insertChange.setString(1, entryId); + insertChange.setString(2, text); + insertChange.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java index c77d7db..ee89527 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java @@ -1,24 +1,22 @@ package io.trygvis.esper.testing.gitorious; +import com.jolbox.bonecp.*; +import com.jolbox.bonecp.hooks.*; import io.trygvis.esper.testing.*; -import io.trygvis.esper.testing.ResourceManager.*; import org.apache.abdera.*; -import org.apache.abdera.model.*; import org.apache.abdera.protocol.client.*; import org.apache.abdera.protocol.client.cache.*; import org.codehaus.httpcache4j.cache.*; import org.codehaus.httpcache4j.client.*; import java.sql.*; -import java.util.Date; import java.util.*; import java.util.concurrent.*; public class GitoriousImporter { - private final AbderaClient abderaClient; - private final Connection c; - private final AtomDao atomDao; - private final GitoriousDao gitoriousDao; +// private final AbderaClient abderaClient; + private final BoneCP boneCp; + private final GitoriousClient gitoriousClient; public static void main(String[] args) throws Exception { Main.configureLog4j(); @@ -27,70 +25,109 @@ public class GitoriousImporter { public GitoriousImporter() throws Exception { Abdera abdera = new Abdera(); - abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); - - c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); - c.setAutoCommit(false); +// abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); + + BoneCPConfig config = new BoneCPConfig(); + config.setJdbcUrl(DbMain.JDBC_URL); + config.setUsername("esper"); + config.setPassword(""); + config.setDefaultAutoCommit(false); + config.setMaxConnectionsPerPartition(1); + + config.setConnectionHook(new AbstractConnectionHook() { + public void onAcquire(ConnectionHandle c) { + try { + c.setDebugHandle(new Daos(c)); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + }); - atomDao = new AtomDao(c); - gitoriousDao = new GitoriousDao(c); + boneCp = new BoneCP(config); HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance()); - final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); - -// Set projects = gitoriousClient.findProjects(); -// -// System.out.println("projects.size() = " + projects.size()); -// for (GitoriousProject project : projects) { -// System.out.println("project.repositories = " + project.repositories); -// } - -// new GitoriousImporter(abderaClient, c).work(); + gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); int projectsUpdateInterval = 1000; final int projectUpdateInterval = 1000; - ResourceManager gitoriousProjects = new ResourceManager<>(service, 1000, +// service.scheduleAtFixedRate(new Runnable() { +// public void run() { +// try { +// discoverProjects(); +// } catch (Exception e) { +// e.printStackTrace(System.out); +// } +// } +// }, projectsUpdateInterval, projectsUpdateInterval, TimeUnit.MILLISECONDS); + + discoverProjects(); + } - new ResourceManagerCallbacks() { - public Set discover() throws Exception { - return gitoriousClient.findProjects(); + private void discoverProjects() throws Exception { + Set projects = gitoriousClient.findProjects(); + + try (ConnectionHandle connection = (ConnectionHandle) boneCp.getConnection()) { + Daos daos = (Daos) connection.getDebugHandle(); + GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao; + GitoriousProjectDao projectDao = daos.gitoriousProjectDao; + + daos.begin(); + System.out.println("Processing " + projects.size() + " projects."); + for (GitoriousProject project : projects) { + if(projectDao.countProjects(project.slug) == 0) { + System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories."); + projectDao.insertProject(project); + for (GitoriousRepository repository : project.repositories) { + repoDao.insertRepository(repository); + } } - - public GitoriousProjectResourceManager onNew(GitoriousProject key) { - return new GitoriousProjectResourceManager(service, projectUpdateInterval, key); + else { + for (GitoriousRepository repository : project.repositories) { + if(repoDao.countRepositories(repository) == 0) { + System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name); + repoDao.insertRepository(repository); + } + } + + for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) { + if(project.repositories.contains(repository)) { + continue; + } + System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name); + repoDao.delete(repository); + } } + } - public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) { - System.out.println("Project gone."); - manager.close(); + for (String project : projectDao.selectAll()) { + boolean found = false; + for (Iterator it = projects.iterator(); it.hasNext(); ) { + GitoriousProject p = it.next(); + if(p.slug.equals(project)) { + found = true; + break; + } } - }); - ; - } - - class GitoriousProjectResourceManager extends ResourceManager { - public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) { - super(executorService, delay, new ResourceManagerCallbacks() { - public Set discover() throws Exception { - key + if (found) { + continue; } - public GitoriousRepository onNew(GitoriousRepository key) { - throw new RuntimeException("Not implemented"); - } + System.out.println("Gone project: " + project); + repoDao.deleteForProject(project); + projectDao.delete(project); + } - public void onGone(GitoriousRepository key, GitoriousRepository value) { - throw new RuntimeException("Not implemented"); - } - }); + connection.commit(); } } + /* private void work() throws SQLException, InterruptedException { String url = "http://qt.gitorious.org/projects/show/qt.atom"; @@ -153,4 +190,5 @@ public class GitoriousImporter { Thread.sleep(10 * 1000); } } + */ } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java index e5b3fdd..6947512 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java @@ -68,15 +68,33 @@ public class GitoriousProject implements Comparable { public int compareTo(GitoriousProject other) { return slug.compareTo(other.slug); } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousProject)) return false; + + GitoriousProject that = (GitoriousProject) o; + + if (!repositories.equals(that.repositories)) return false; + if (!slug.equals(that.slug)) return false; + + return true; + } + + public int hashCode() { + int result = slug.hashCode(); + result = 31 * result + repositories.hashCode(); + return result; + } } class GitoriousRepository implements Comparable { - public final String project; + public final String projectSlug; public final String name; public final URI atom; - GitoriousRepository(String project, String name, URI atom) { - this.project = project; + GitoriousRepository(String projectSlug, String name, URI atom) { + this.projectSlug = projectSlug; this.name = name; this.atom = atom; } @@ -92,7 +110,7 @@ class GitoriousRepository implements Comparable { } public int compareTo(GitoriousRepository o) { - int a = project.compareTo(o.project); + int a = projectSlug.compareTo(o.projectSlug); if (a != 0) { return a; @@ -100,4 +118,24 @@ class GitoriousRepository implements Comparable { return name.compareTo(o.name); } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousRepository)) return false; + + GitoriousRepository that = (GitoriousRepository) o; + + if (!atom.equals(that.atom)) return false; + if (!name.equals(that.name)) return false; + if (!projectSlug.equals(that.projectSlug)) return false; + + return true; + } + + public int hashCode() { + int result = projectSlug.hashCode(); + result = 31 * result + name.hashCode(); + result = 31 * result + atom.hashCode(); + return result; + } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java new file mode 100644 index 0000000..f47b126 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java @@ -0,0 +1,42 @@ +package io.trygvis.esper.testing.gitorious; + +import java.sql.*; +import java.util.*; + +public class GitoriousProjectDao extends Dao { + public GitoriousProjectDao(Connection c) throws SQLException { + super(c); + } + + private final PreparedStatement countProjects = prepareStatement("SELECT count(*) FROM gitorious_project WHERE slug=?"); + public int countProjects(String slug) throws SQLException { + countProjects.setString(1, slug); + try(ResultSet rs = countProjects.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + + private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)"); + public void insertProject(GitoriousProject project) throws SQLException { + insertProject.setString(1, project.slug); + insertProject.executeUpdate(); + } + + private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project"); + public List selectAll() throws SQLException { + try (ResultSet rs = selectAll.executeQuery()) { + List list = new ArrayList<>(); + while(rs.next()) { + list.add(rs.getString(1)); + } + return list; + } + } + + private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?"); + public void delete(String slug) throws SQLException { + delete.setString(1, slug); + delete.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java new file mode 100644 index 0000000..a88f122 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java @@ -0,0 +1,69 @@ +package io.trygvis.esper.testing.gitorious; + +import java.net.*; +import java.sql.*; +import java.util.*; + +public class GitoriousRepositoryDao extends Dao { + public GitoriousRepositoryDao(Connection c) throws SQLException { + super(c); + } + + private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?"); + public int countRepositories(GitoriousRepository repository) throws SQLException { + countRepositories.setString(1, repository.projectSlug); + countRepositories.setString(2, repository.name); + try (ResultSet rs = countRepositories.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + + private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository WHERE project_slug=?"); + public List selectForProject(String projectSlug) throws Exception { + selectForProject.setString(1, projectSlug); + return executeQuery(selectForProject); + } + + private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository"); + public List selectAll() throws Exception { + return executeQuery(selectAll); + } + + private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)"); + public void insertRepository(GitoriousRepository repository) throws SQLException { + insertRepository.setString(1, repository.projectSlug); + insertRepository.setString(2, repository.name); + insertRepository.setString(3, repository.atom.toASCIIString()); + insertRepository.executeUpdate(); + } + + private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?"); + public void delete(GitoriousRepository repository) throws SQLException { + delete.setString(1, repository.projectSlug); + delete.setString(2, repository.name); + delete.executeUpdate(); + } + + private final PreparedStatement deleteForProject = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=?"); + public void deleteForProject(String project) throws SQLException { + deleteForProject.setString(1, project); + deleteForProject.executeUpdate(); + } + + private List executeQuery(PreparedStatement statement) throws SQLException, URISyntaxException { + try (ResultSet rs = statement.executeQuery()) { + List list = new ArrayList<>(); + + while(rs.next()) { + list.add(new GitoriousRepository( + rs.getString(1), + rs.getString(2), + new URI(rs.getString(3)) + )); + } + + return list; + } + } +} diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql index a8bec8f..6f213dc 100644 --- a/src/main/resources/ddl.sql +++ b/src/main/resources/ddl.sql @@ -1,6 +1,8 @@ BEGIN; DROP TABLE IF EXISTS gitorious_change; +DROP TABLE IF EXISTS gitorious_repository; +DROP TABLE IF EXISTS gitorious_project; DROP TABLE IF EXISTS atom_feed; CREATE TABLE atom_feed ( @@ -8,9 +10,16 @@ CREATE TABLE atom_feed ( last_update TIMESTAMP NOT NULL ); +CREATE TABLE gitorious_project ( + slug VARCHAR(1000) PRIMARY KEY +); + CREATE TABLE gitorious_repository ( - entry_id VARCHAR(1000) PRIMARY KEY, - text VARCHAR(1000) + project_slug VARCHAR(1000) NOT NULL, + name VARCHAR(1000) NOT NULL, + atom_feed VARCHAR(1000) NOT NULL, + CONSTRAINT gitorious_repository_pk PRIMARY KEY(project_slug, name), + CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY(project_slug) REFERENCES gitorious_project(slug) ); CREATE TABLE gitorious_change ( -- cgit v1.2.3