From 2eae4836279646050e7e342752cde6e8f7c5b6cb Mon Sep 17 00:00:00 2001
From: Trygve Laugstøl <trygvis@inamo.no>
Date: Thu, 8 Nov 2012 22:24:01 +0100
Subject: wip
---
pom.xml | 18 ++-
src/main/java/io/trygvis/esper/testing/Daos.java | 25 ++++
src/main/java/io/trygvis/esper/testing/Main.java | 2 +
.../io/trygvis/esper/testing/gitorious/Dao.java | 15 +++
.../esper/testing/gitorious/GitoriousDao.java | 27 ----
.../esper/testing/gitorious/GitoriousEventDao.java | 27 ++++
.../esper/testing/gitorious/GitoriousImporter.java | 136 +++++++++++++--------
.../esper/testing/gitorious/GitoriousProject.java | 46 ++++++-
.../testing/gitorious/GitoriousProjectDao.java | 42 +++++++
.../testing/gitorious/GitoriousRepositoryDao.java | 69 +++++++++++
src/main/resources/ddl.sql | 13 +-
11 files changed, 333 insertions(+), 87 deletions(-)
create mode 100644 src/main/java/io/trygvis/esper/testing/Daos.java
create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
delete mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java
create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java
create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
create mode 100644 src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
diff --git a/pom.xml b/pom.xml
index 7021cf6..64fabf8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,13 +29,21 @@
<artifactId>dom4j</artifactId>
<version>1.6.1</version>
</dependency>
- <!--
<dependency>
- <groupId>org.codehaus.woodstox</groupId>
- <artifactId>stax2-api</artifactId>
- <version>3.0.2</version>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
+ <version>0.7.1.RELEASE</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.6.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.6.4</version>
</dependency>
- -->
<dependency>
<groupId>org.functionaljava</groupId>
<artifactId>functionaljava</artifactId>
diff --git a/src/main/java/io/trygvis/esper/testing/Daos.java b/src/main/java/io/trygvis/esper/testing/Daos.java
new file mode 100644
index 0000000..db97f9d
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/Daos.java
@@ -0,0 +1,25 @@
+package io.trygvis.esper.testing;
+
+import io.trygvis.esper.testing.gitorious.*;
+
+import java.sql.*;
+
+public class Daos {
+ public final AtomDao atomDao;
+ public final GitoriousEventDao gitoriousEventDao;
+ public final GitoriousProjectDao gitoriousProjectDao;
+ public final GitoriousRepositoryDao gitoriousRepositoryDao;
+ public final PreparedStatement begin;
+
+ public Daos(Connection c) throws SQLException {
+ atomDao = new AtomDao(c);
+ gitoriousEventDao = new GitoriousEventDao(c);
+ gitoriousProjectDao = new GitoriousProjectDao(c);
+ gitoriousRepositoryDao = new GitoriousRepositoryDao(c);
+ begin = c.prepareStatement("BEGIN");
+ }
+
+ public void begin() throws SQLException {
+ begin.executeUpdate();
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java
index 69e0bce..ad7c71a 100644
--- a/src/main/java/io/trygvis/esper/testing/Main.java
+++ b/src/main/java/io/trygvis/esper/testing/Main.java
@@ -19,6 +19,8 @@ public class Main {
Properties properties = new Properties();
properties.setProperty("log4j.rootLogger", "DEBUG, A1");
properties.setProperty("log4j.logger.httpclient.wire.content", "INFO");
+ properties.setProperty("log4j.logger.httpclient.wire.header", "INFO");
+ properties.setProperty("log4j.logger.org.apache.commons.httpclient", "INFO");
properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender");
properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
new file mode 100644
index 0000000..4714a87
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
@@ -0,0 +1,15 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.sql.*;
+
+public class Dao {
+ private final Connection c;
+
+ protected Dao(Connection c) {
+ this.c = c;
+ }
+
+ protected PreparedStatement prepareStatement(String sql) throws SQLException {
+ return c.prepareStatement(sql);
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java
deleted file mode 100644
index 766a4a9..0000000
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package io.trygvis.esper.testing.gitorious;
-
-import java.sql.*;
-
-public class GitoriousDao {
- private final PreparedStatement countEntryId;
- private final PreparedStatement insertChange;
-
- public GitoriousDao(Connection c) throws SQLException {
- countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?");
- insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)");
- }
-
- public int countEntryId(String entryId) throws SQLException {
- countEntryId.setString(1, entryId);
- try(ResultSet rs = countEntryId.executeQuery()) {
- rs.next();
- return rs.getInt(1);
- }
- }
-
- public void insertChange(String entryId, String text) throws SQLException {
- insertChange.setString(1, entryId);
- insertChange.setString(2, text);
- insertChange.executeUpdate();
- }
-}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java
new file mode 100644
index 0000000..603609e
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java
@@ -0,0 +1,27 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.sql.*;
+
+public class GitoriousEventDao {
+ private final PreparedStatement countEntryId;
+ private final PreparedStatement insertChange;
+
+ public GitoriousEventDao(Connection c) throws SQLException {
+ countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_event WHERE entry_id=?");
+ insertChange = c.prepareStatement("INSERT INTO gitorious_event(entry_id, text) VALUES(?, ?)");
+ }
+
+ public int countEntryId(String entryId) throws SQLException {
+ countEntryId.setString(1, entryId);
+ try(ResultSet rs = countEntryId.executeQuery()) {
+ rs.next();
+ return rs.getInt(1);
+ }
+ }
+
+ public void insertChange(String entryId, String text) throws SQLException {
+ insertChange.setString(1, entryId);
+ insertChange.setString(2, text);
+ insertChange.executeUpdate();
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index c77d7db..ee89527 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -1,24 +1,22 @@
package io.trygvis.esper.testing.gitorious;
+import com.jolbox.bonecp.*;
+import com.jolbox.bonecp.hooks.*;
import io.trygvis.esper.testing.*;
-import io.trygvis.esper.testing.ResourceManager.*;
import org.apache.abdera.*;
-import org.apache.abdera.model.*;
import org.apache.abdera.protocol.client.*;
import org.apache.abdera.protocol.client.cache.*;
import org.codehaus.httpcache4j.cache.*;
import org.codehaus.httpcache4j.client.*;
import java.sql.*;
-import java.util.Date;
import java.util.*;
import java.util.concurrent.*;
public class GitoriousImporter {
- private final AbderaClient abderaClient;
- private final Connection c;
- private final AtomDao atomDao;
- private final GitoriousDao gitoriousDao;
+// private final AbderaClient abderaClient;
+ private final BoneCP boneCp;
+ private final GitoriousClient gitoriousClient;
public static void main(String[] args) throws Exception {
Main.configureLog4j();
@@ -27,70 +25,109 @@ public class GitoriousImporter {
public GitoriousImporter() throws Exception {
Abdera abdera = new Abdera();
- abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
-
- c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
- c.setAutoCommit(false);
+// abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
+
+ BoneCPConfig config = new BoneCPConfig();
+ config.setJdbcUrl(DbMain.JDBC_URL);
+ config.setUsername("esper");
+ config.setPassword("");
+ config.setDefaultAutoCommit(false);
+ config.setMaxConnectionsPerPartition(1);
+
+ config.setConnectionHook(new AbstractConnectionHook() {
+ public void onAcquire(ConnectionHandle c) {
+ try {
+ c.setDebugHandle(new Daos(c));
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
- atomDao = new AtomDao(c);
- gitoriousDao = new GitoriousDao(c);
+ boneCp = new BoneCP(config);
HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
- final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
-
-// Set<GitoriousProject> projects = gitoriousClient.findProjects();
-//
-// System.out.println("projects.size() = " + projects.size());
-// for (GitoriousProject project : projects) {
-// System.out.println("project.repositories = " + project.repositories);
-// }
-
-// new GitoriousImporter(abderaClient, c).work();
+ gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1);
int projectsUpdateInterval = 1000;
final int projectUpdateInterval = 1000;
- ResourceManager<GitoriousProject, GitoriousProjectResourceManager> gitoriousProjects = new ResourceManager<>(service, 1000,
+// service.scheduleAtFixedRate(new Runnable() {
+// public void run() {
+// try {
+// discoverProjects();
+// } catch (Exception e) {
+// e.printStackTrace(System.out);
+// }
+// }
+// }, projectsUpdateInterval, projectsUpdateInterval, TimeUnit.MILLISECONDS);
+
+ discoverProjects();
+ }
- new ResourceManagerCallbacks<GitoriousProject, GitoriousProjectResourceManager>() {
- public Set<GitoriousProject> discover() throws Exception {
- return gitoriousClient.findProjects();
+ private void discoverProjects() throws Exception {
+ Set<GitoriousProject> projects = gitoriousClient.findProjects();
+
+ try (ConnectionHandle connection = (ConnectionHandle) boneCp.getConnection()) {
+ Daos daos = (Daos) connection.getDebugHandle();
+ GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao;
+ GitoriousProjectDao projectDao = daos.gitoriousProjectDao;
+
+ daos.begin();
+ System.out.println("Processing " + projects.size() + " projects.");
+ for (GitoriousProject project : projects) {
+ if(projectDao.countProjects(project.slug) == 0) {
+ System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories.");
+ projectDao.insertProject(project);
+ for (GitoriousRepository repository : project.repositories) {
+ repoDao.insertRepository(repository);
+ }
}
-
- public GitoriousProjectResourceManager onNew(GitoriousProject key) {
- return new GitoriousProjectResourceManager(service, projectUpdateInterval, key);
+ else {
+ for (GitoriousRepository repository : project.repositories) {
+ if(repoDao.countRepositories(repository) == 0) {
+ System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name);
+ repoDao.insertRepository(repository);
+ }
+ }
+
+ for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) {
+ if(project.repositories.contains(repository)) {
+ continue;
+ }
+ System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name);
+ repoDao.delete(repository);
+ }
}
+ }
- public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) {
- System.out.println("Project gone.");
- manager.close();
+ for (String project : projectDao.selectAll()) {
+ boolean found = false;
+ for (Iterator<GitoriousProject> it = projects.iterator(); it.hasNext(); ) {
+ GitoriousProject p = it.next();
+ if(p.slug.equals(project)) {
+ found = true;
+ break;
+ }
}
- });
- ;
- }
-
- class GitoriousProjectResourceManager extends ResourceManager<GitoriousRepository, GitoriousRepository> {
- public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) {
- super(executorService, delay, new ResourceManagerCallbacks<GitoriousRepository, GitoriousRepository>() {
- public Set<GitoriousRepository> discover() throws Exception {
- key
+ if (found) {
+ continue;
}
- public GitoriousRepository onNew(GitoriousRepository key) {
- throw new RuntimeException("Not implemented");
- }
+ System.out.println("Gone project: " + project);
+ repoDao.deleteForProject(project);
+ projectDao.delete(project);
+ }
- public void onGone(GitoriousRepository key, GitoriousRepository value) {
- throw new RuntimeException("Not implemented");
- }
- });
+ connection.commit();
}
}
+ /*
private void work() throws SQLException, InterruptedException {
String url = "http://qt.gitorious.org/projects/show/qt.atom";
@@ -153,4 +190,5 @@ public class GitoriousImporter {
Thread.sleep(10 * 1000);
}
}
+ */
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
index e5b3fdd..6947512 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
@@ -68,15 +68,33 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
public int compareTo(GitoriousProject other) {
return slug.compareTo(other.slug);
}
+
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof GitoriousProject)) return false;
+
+ GitoriousProject that = (GitoriousProject) o;
+
+ if (!repositories.equals(that.repositories)) return false;
+ if (!slug.equals(that.slug)) return false;
+
+ return true;
+ }
+
+ public int hashCode() {
+ int result = slug.hashCode();
+ result = 31 * result + repositories.hashCode();
+ return result;
+ }
}
class GitoriousRepository implements Comparable<GitoriousRepository> {
- public final String project;
+ public final String projectSlug;
public final String name;
public final URI atom;
- GitoriousRepository(String project, String name, URI atom) {
- this.project = project;
+ GitoriousRepository(String projectSlug, String name, URI atom) {
+ this.projectSlug = projectSlug;
this.name = name;
this.atom = atom;
}
@@ -92,7 +110,7 @@ class GitoriousRepository implements Comparable<GitoriousRepository> {
}
public int compareTo(GitoriousRepository o) {
- int a = project.compareTo(o.project);
+ int a = projectSlug.compareTo(o.projectSlug);
if (a != 0) {
return a;
@@ -100,4 +118,24 @@ class GitoriousRepository implements Comparable<GitoriousRepository> {
return name.compareTo(o.name);
}
+
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof GitoriousRepository)) return false;
+
+ GitoriousRepository that = (GitoriousRepository) o;
+
+ if (!atom.equals(that.atom)) return false;
+ if (!name.equals(that.name)) return false;
+ if (!projectSlug.equals(that.projectSlug)) return false;
+
+ return true;
+ }
+
+ public int hashCode() {
+ int result = projectSlug.hashCode();
+ result = 31 * result + name.hashCode();
+ result = 31 * result + atom.hashCode();
+ return result;
+ }
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
new file mode 100644
index 0000000..f47b126
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
@@ -0,0 +1,42 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.sql.*;
+import java.util.*;
+
+public class GitoriousProjectDao extends Dao {
+ public GitoriousProjectDao(Connection c) throws SQLException {
+ super(c);
+ }
+
+ private final PreparedStatement countProjects = prepareStatement("SELECT count(*) FROM gitorious_project WHERE slug=?");
+ public int countProjects(String slug) throws SQLException {
+ countProjects.setString(1, slug);
+ try(ResultSet rs = countProjects.executeQuery()) {
+ rs.next();
+ return rs.getInt(1);
+ }
+ }
+
+ private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)");
+ public void insertProject(GitoriousProject project) throws SQLException {
+ insertProject.setString(1, project.slug);
+ insertProject.executeUpdate();
+ }
+
+ private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project");
+ public List<String> selectAll() throws SQLException {
+ try (ResultSet rs = selectAll.executeQuery()) {
+ List<String> list = new ArrayList<>();
+ while(rs.next()) {
+ list.add(rs.getString(1));
+ }
+ return list;
+ }
+ }
+
+ private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?");
+ public void delete(String slug) throws SQLException {
+ delete.setString(1, slug);
+ delete.executeUpdate();
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
new file mode 100644
index 0000000..a88f122
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
@@ -0,0 +1,69 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.net.*;
+import java.sql.*;
+import java.util.*;
+
+public class GitoriousRepositoryDao extends Dao {
+ public GitoriousRepositoryDao(Connection c) throws SQLException {
+ super(c);
+ }
+
+ private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?");
+ public int countRepositories(GitoriousRepository repository) throws SQLException {
+ countRepositories.setString(1, repository.projectSlug);
+ countRepositories.setString(2, repository.name);
+ try (ResultSet rs = countRepositories.executeQuery()) {
+ rs.next();
+ return rs.getInt(1);
+ }
+ }
+
+ private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository WHERE project_slug=?");
+ public List<GitoriousRepository> selectForProject(String projectSlug) throws Exception {
+ selectForProject.setString(1, projectSlug);
+ return executeQuery(selectForProject);
+ }
+
+ private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository");
+ public List<GitoriousRepository> selectAll() throws Exception {
+ return executeQuery(selectAll);
+ }
+
+ private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)");
+ public void insertRepository(GitoriousRepository repository) throws SQLException {
+ insertRepository.setString(1, repository.projectSlug);
+ insertRepository.setString(2, repository.name);
+ insertRepository.setString(3, repository.atom.toASCIIString());
+ insertRepository.executeUpdate();
+ }
+
+ private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?");
+ public void delete(GitoriousRepository repository) throws SQLException {
+ delete.setString(1, repository.projectSlug);
+ delete.setString(2, repository.name);
+ delete.executeUpdate();
+ }
+
+ private final PreparedStatement deleteForProject = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=?");
+ public void deleteForProject(String project) throws SQLException {
+ deleteForProject.setString(1, project);
+ deleteForProject.executeUpdate();
+ }
+
+ private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException, URISyntaxException {
+ try (ResultSet rs = statement.executeQuery()) {
+ List<GitoriousRepository> list = new ArrayList<>();
+
+ while(rs.next()) {
+ list.add(new GitoriousRepository(
+ rs.getString(1),
+ rs.getString(2),
+ new URI(rs.getString(3))
+ ));
+ }
+
+ return list;
+ }
+ }
+}
diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql
index a8bec8f..6f213dc 100644
--- a/src/main/resources/ddl.sql
+++ b/src/main/resources/ddl.sql
@@ -1,6 +1,8 @@
BEGIN;
DROP TABLE IF EXISTS gitorious_change;
+DROP TABLE IF EXISTS gitorious_repository;
+DROP TABLE IF EXISTS gitorious_project;
DROP TABLE IF EXISTS atom_feed;
CREATE TABLE atom_feed (
@@ -8,9 +10,16 @@ CREATE TABLE atom_feed (
last_update TIMESTAMP NOT NULL
);
+CREATE TABLE gitorious_project (
+ slug VARCHAR(1000) PRIMARY KEY
+);
+
CREATE TABLE gitorious_repository (
- entry_id VARCHAR(1000) PRIMARY KEY,
- text VARCHAR(1000)
+ project_slug VARCHAR(1000) NOT NULL,
+ name VARCHAR(1000) NOT NULL,
+ atom_feed VARCHAR(1000) NOT NULL,
+ CONSTRAINT gitorious_repository_pk PRIMARY KEY(project_slug, name),
+ CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY(project_slug) REFERENCES gitorious_project(slug)
);
CREATE TABLE gitorious_change (
--
cgit v1.2.3