wip

9 files changed, 297 insertions, 59 deletions

diff --git a/src/main/java/io/trygvis/esper/testing/Daos.java b/src/main/java/io/trygvis/esper/testing/Daos.java
new file mode 100644
index 0000000..db97f9d
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/Daos.java
@@ -0,0 +1,25 @@
+package io.trygvis.esper.testing;
+
+import io.trygvis.esper.testing.gitorious.*;
+
+import java.sql.*;
+
+public class Daos {
+    public final AtomDao atomDao;
+    public final GitoriousEventDao gitoriousEventDao;
+    public final GitoriousProjectDao gitoriousProjectDao;
+    public final GitoriousRepositoryDao gitoriousRepositoryDao;
+    public final PreparedStatement begin;
+
+    public Daos(Connection c) throws SQLException {
+        atomDao = new AtomDao(c);
+        gitoriousEventDao = new GitoriousEventDao(c);
+        gitoriousProjectDao = new GitoriousProjectDao(c);
+        gitoriousRepositoryDao = new GitoriousRepositoryDao(c);
+        begin = c.prepareStatement("BEGIN");
+    }
+
+    public void begin() throws SQLException {
+        begin.executeUpdate();
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java
index 69e0bce..ad7c71a 100644
--- a/src/main/java/io/trygvis/esper/testing/Main.java
+++ b/src/main/java/io/trygvis/esper/testing/Main.java
@@ -19,6 +19,8 @@ public class Main {
         Properties properties = new Properties();
         properties.setProperty("log4j.rootLogger", "DEBUG, A1");
         properties.setProperty("log4j.logger.httpclient.wire.content", "INFO");
+        properties.setProperty("log4j.logger.httpclient.wire.header", "INFO");
+        properties.setProperty("log4j.logger.org.apache.commons.httpclient", "INFO");
         properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender");
         properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
         properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
new file mode 100644
index 0000000..4714a87
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
@@ -0,0 +1,15 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.sql.*;
+
+public class Dao {
+    private final Connection c;
+
+    protected Dao(Connection c) {
+        this.c = c;
+    }
+
+    protected PreparedStatement prepareStatement(String sql) throws SQLException {
+        return c.prepareStatement(sql);
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java
index 766a4a9..603609e 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousEventDao.java
@@ -2,13 +2,13 @@ package io.trygvis.esper.testing.gitorious;
 
 import java.sql.*;
 
-public class GitoriousDao {
+public class GitoriousEventDao {
     private final PreparedStatement countEntryId;
     private final PreparedStatement insertChange;
 
-    public GitoriousDao(Connection c) throws SQLException {
-        countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?");
-        insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)");
+    public GitoriousEventDao(Connection c) throws SQLException {
+        countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_event WHERE entry_id=?");
+        insertChange = c.prepareStatement("INSERT INTO gitorious_event(entry_id, text) VALUES(?, ?)");
     }
 
     public int countEntryId(String entryId) throws SQLException {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index c77d7db..ee89527 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -1,24 +1,22 @@
 package io.trygvis.esper.testing.gitorious;
 
+import com.jolbox.bonecp.*;
+import com.jolbox.bonecp.hooks.*;
 import io.trygvis.esper.testing.*;
-import io.trygvis.esper.testing.ResourceManager.*;
 import org.apache.abdera.*;
-import org.apache.abdera.model.*;
 import org.apache.abdera.protocol.client.*;
 import org.apache.abdera.protocol.client.cache.*;
 import org.codehaus.httpcache4j.cache.*;
 import org.codehaus.httpcache4j.client.*;
 
 import java.sql.*;
-import java.util.Date;
 import java.util.*;
 import java.util.concurrent.*;
 
 public class GitoriousImporter {
-    private final AbderaClient abderaClient;
-    private final Connection c;
-    private final AtomDao atomDao;
-    private final GitoriousDao gitoriousDao;
+//    private final AbderaClient abderaClient;
+    private final BoneCP boneCp;
+    private final GitoriousClient gitoriousClient;
 
     public static void main(String[] args) throws Exception {
         Main.configureLog4j();
@@ -27,70 +25,109 @@ public class GitoriousImporter {
 
     public GitoriousImporter() throws Exception {
         Abdera abdera = new Abdera();
-        abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
-
-        c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
-        c.setAutoCommit(false);
+//        abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
+
+        BoneCPConfig config = new BoneCPConfig();
+        config.setJdbcUrl(DbMain.JDBC_URL);
+        config.setUsername("esper");
+        config.setPassword("");
+        config.setDefaultAutoCommit(false);
+        config.setMaxConnectionsPerPartition(1);
+
+        config.setConnectionHook(new AbstractConnectionHook() {
+            public void onAcquire(ConnectionHandle c) {
+                try {
+                    c.setDebugHandle(new Daos(c));
+                } catch (SQLException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
 
-        atomDao = new AtomDao(c);
-        gitoriousDao = new GitoriousDao(c);
+        boneCp = new BoneCP(config);
 
         HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
 
-        final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
-
-//        Set<GitoriousProject> projects = gitoriousClient.findProjects();
-//
-//        System.out.println("projects.size() = " + projects.size());
-//        for (GitoriousProject project : projects) {
-//            System.out.println("project.repositories = " + project.repositories);
-//        }
-
-//        new GitoriousImporter(abderaClient, c).work();
+        gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
 
         final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1);
 
         int projectsUpdateInterval = 1000;
         final int projectUpdateInterval = 1000;
 
-        ResourceManager<GitoriousProject, GitoriousProjectResourceManager> gitoriousProjects = new ResourceManager<>(service, 1000,
+//        service.scheduleAtFixedRate(new Runnable() {
+//            public void run() {
+//                try {
+//                    discoverProjects();
+//                } catch (Exception e) {
+//                    e.printStackTrace(System.out);
+//                }
+//            }
+//        }, projectsUpdateInterval, projectsUpdateInterval, TimeUnit.MILLISECONDS);
+
+        discoverProjects();
+    }
 
-            new ResourceManagerCallbacks<GitoriousProject, GitoriousProjectResourceManager>() {
-                public Set<GitoriousProject> discover() throws Exception {
-                    return gitoriousClient.findProjects();
+    private void discoverProjects() throws Exception {
+        Set<GitoriousProject> projects = gitoriousClient.findProjects();
+
+        try (ConnectionHandle connection = (ConnectionHandle) boneCp.getConnection()) {
+            Daos daos = (Daos) connection.getDebugHandle();
+            GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao;
+            GitoriousProjectDao projectDao = daos.gitoriousProjectDao;
+
+            daos.begin();
+            System.out.println("Processing " + projects.size() + " projects.");
+            for (GitoriousProject project : projects) {
+                if(projectDao.countProjects(project.slug) == 0) {
+                    System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories.");
+                    projectDao.insertProject(project);
+                    for (GitoriousRepository repository : project.repositories) {
+                        repoDao.insertRepository(repository);
+                    }
                 }
-
-                public GitoriousProjectResourceManager onNew(GitoriousProject key) {
-                    return new GitoriousProjectResourceManager(service, projectUpdateInterval, key);
+                else {
+                    for (GitoriousRepository repository : project.repositories) {
+                        if(repoDao.countRepositories(repository) == 0) {
+                            System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name);
+                            repoDao.insertRepository(repository);
+                        }
+                    }
+
+                    for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) {
+                        if(project.repositories.contains(repository)) {
+                            continue;
+                        }
+                        System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name);
+                        repoDao.delete(repository);
+                    }
                 }
+            }
 
-                public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) {
-                    System.out.println("Project gone.");
-                    manager.close();
+            for (String project : projectDao.selectAll()) {
+                boolean found = false;
+                for (Iterator<GitoriousProject> it = projects.iterator(); it.hasNext(); ) {
+                    GitoriousProject p = it.next();
+                    if(p.slug.equals(project)) {
+                        found = true;
+                        break;
+                    }
                 }
-            });
-        ;
-    }
-
-    class GitoriousProjectResourceManager extends ResourceManager<GitoriousRepository, GitoriousRepository> {
 
-        public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) {
-            super(executorService, delay, new ResourceManagerCallbacks<GitoriousRepository, GitoriousRepository>() {
-                public Set<GitoriousRepository> discover() throws Exception {
-                    key
+                if (found) {
+                    continue;
                 }
 
-                public GitoriousRepository onNew(GitoriousRepository key) {
-                    throw new RuntimeException("Not implemented");
-                }
+                System.out.println("Gone project: " + project);
+                repoDao.deleteForProject(project);
+                projectDao.delete(project);
+            }
 
-                public void onGone(GitoriousRepository key, GitoriousRepository value) {
-                    throw new RuntimeException("Not implemented");
-                }
-            });
+            connection.commit();
         }
     }
 
+    /*
     private void work() throws SQLException, InterruptedException {
         String url = "http://qt.gitorious.org/projects/show/qt.atom";
 
@@ -153,4 +190,5 @@ public class GitoriousImporter {
             Thread.sleep(10 * 1000);
         }
     }
+    */
 }
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
index e5b3fdd..6947512 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
@@ -68,15 +68,33 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
     public int compareTo(GitoriousProject other) {
         return slug.compareTo(other.slug);
     }
+
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof GitoriousProject)) return false;
+
+        GitoriousProject that = (GitoriousProject) o;
+
+        if (!repositories.equals(that.repositories)) return false;
+        if (!slug.equals(that.slug)) return false;
+
+        return true;
+    }
+
+    public int hashCode() {
+        int result = slug.hashCode();
+        result = 31 * result + repositories.hashCode();
+        return result;
+    }
 }
 
 class GitoriousRepository implements Comparable<GitoriousRepository> {
-    public final String project;
+    public final String projectSlug;
     public final String name;
     public final URI atom;
 
-    GitoriousRepository(String project, String name, URI atom) {
-        this.project = project;
+    GitoriousRepository(String projectSlug, String name, URI atom) {
+        this.projectSlug = projectSlug;
         this.name = name;
         this.atom = atom;
     }
@@ -92,7 +110,7 @@ class GitoriousRepository implements Comparable<GitoriousRepository> {
     }
 
     public int compareTo(GitoriousRepository o) {
-        int a = project.compareTo(o.project);
+        int a = projectSlug.compareTo(o.projectSlug);
 
         if (a != 0) {
             return a;
@@ -100,4 +118,24 @@ class GitoriousRepository implements Comparable<GitoriousRepository> {
 
         return name.compareTo(o.name);
     }
+
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof GitoriousRepository)) return false;
+
+        GitoriousRepository that = (GitoriousRepository) o;
+
+        if (!atom.equals(that.atom)) return false;
+        if (!name.equals(that.name)) return false;
+        if (!projectSlug.equals(that.projectSlug)) return false;
+
+        return true;
+    }
+
+    public int hashCode() {
+        int result = projectSlug.hashCode();
+        result = 31 * result + name.hashCode();
+        result = 31 * result + atom.hashCode();
+        return result;
+    }
 }
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
new file mode 100644
index 0000000..f47b126
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
@@ -0,0 +1,42 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.sql.*;
+import java.util.*;
+
+public class GitoriousProjectDao extends Dao {
+    public GitoriousProjectDao(Connection c) throws SQLException {
+        super(c);
+    }
+
+    private final PreparedStatement countProjects = prepareStatement("SELECT count(*) FROM gitorious_project WHERE slug=?");
+    public int countProjects(String slug) throws SQLException {
+        countProjects.setString(1, slug);
+        try(ResultSet rs = countProjects.executeQuery()) {
+            rs.next();
+            return rs.getInt(1);
+        }
+    }
+
+    private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)");
+    public void insertProject(GitoriousProject project) throws SQLException {
+        insertProject.setString(1, project.slug);
+        insertProject.executeUpdate();
+    }
+
+    private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project");
+    public List<String> selectAll() throws SQLException {
+        try (ResultSet rs = selectAll.executeQuery()) {
+            List<String> list = new ArrayList<>();
+            while(rs.next()) {
+                list.add(rs.getString(1));
+            }
+            return list;
+        }
+    }
+
+    private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?");
+    public void delete(String slug) throws SQLException {
+        delete.setString(1, slug);
+        delete.executeUpdate();
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
new file mode 100644
index 0000000..a88f122
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
@@ -0,0 +1,69 @@
+package io.trygvis.esper.testing.gitorious;
+
+import java.net.*;
+import java.sql.*;
+import java.util.*;
+
+public class GitoriousRepositoryDao extends Dao {
+    public GitoriousRepositoryDao(Connection c) throws SQLException {
+        super(c);
+    }
+
+    private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?");
+    public int countRepositories(GitoriousRepository repository) throws SQLException {
+        countRepositories.setString(1, repository.projectSlug);
+        countRepositories.setString(2, repository.name);
+        try (ResultSet rs = countRepositories.executeQuery()) {
+            rs.next();
+            return rs.getInt(1);
+        }
+    }
+
+    private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository WHERE project_slug=?");
+    public List<GitoriousRepository> selectForProject(String projectSlug) throws Exception {
+        selectForProject.setString(1, projectSlug);
+        return executeQuery(selectForProject);
+    }
+
+    private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name, atom_feed FROM gitorious_repository");
+    public List<GitoriousRepository> selectAll() throws Exception {
+        return executeQuery(selectAll);
+    }
+
+    private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)");
+    public void insertRepository(GitoriousRepository repository) throws SQLException {
+        insertRepository.setString(1, repository.projectSlug);
+        insertRepository.setString(2, repository.name);
+        insertRepository.setString(3, repository.atom.toASCIIString());
+        insertRepository.executeUpdate();
+    }
+
+    private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?");
+    public void delete(GitoriousRepository repository) throws SQLException {
+        delete.setString(1, repository.projectSlug);
+        delete.setString(2, repository.name);
+        delete.executeUpdate();
+    }
+
+    private final PreparedStatement deleteForProject = prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=?");
+    public void deleteForProject(String project) throws SQLException {
+        deleteForProject.setString(1, project);
+        deleteForProject.executeUpdate();
+    }
+
+    private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException, URISyntaxException {
+        try (ResultSet rs = statement.executeQuery()) {
+            List<GitoriousRepository> list = new ArrayList<>();
+
+            while(rs.next()) {
+                list.add(new GitoriousRepository(
+                    rs.getString(1),
+                    rs.getString(2),
+                    new URI(rs.getString(3))
+                ));
+            }
+
+            return list;
+        }
+    }
+}
diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql
index a8bec8f..6f213dc 100644
--- a/src/main/resources/ddl.sql
+++ b/src/main/resources/ddl.sql
@@ -1,6 +1,8 @@
 BEGIN;
 
 DROP TABLE IF EXISTS gitorious_change;
+DROP TABLE IF EXISTS gitorious_repository;
+DROP TABLE IF EXISTS gitorious_project;
 DROP TABLE IF EXISTS atom_feed;
 
 CREATE TABLE atom_feed (
@@ -8,9 +10,16 @@ CREATE TABLE atom_feed (
   last_update TIMESTAMP NOT NULL
 );
 
+CREATE TABLE gitorious_project (
+  slug VARCHAR(1000) PRIMARY KEY
+);
+
 CREATE TABLE gitorious_repository (
-  entry_id VARCHAR(1000) PRIMARY KEY,
-  text VARCHAR(1000)
+  project_slug VARCHAR(1000) NOT NULL,
+  name VARCHAR(1000) NOT NULL,
+  atom_feed VARCHAR(1000) NOT NULL,
+  CONSTRAINT gitorious_repository_pk PRIMARY KEY(project_slug, name),
+  CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY(project_slug) REFERENCES gitorious_project(slug)
 );
 
 CREATE TABLE gitorious_change (