wip

8 files changed, 269 insertions, 191 deletions

diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
index 4714a87..05faa9a 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
@@ -1,5 +1,7 @@
 package io.trygvis.esper.testing.gitorious;
 
+import fj.*;
+
 import java.sql.*;
 
 public class Dao {
@@ -12,4 +14,16 @@ public class Dao {
     protected PreparedStatement prepareStatement(String sql) throws SQLException {
         return c.prepareStatement(sql);
     }
+
+    public static final F<Timestamp, java.util.Date> timestampToDate = new F<Timestamp, java.util.Date>() {
+        public java.util.Date f(Timestamp timestamp) {
+            return new java.util.Date(timestamp.getTime());
+        }
+    };
+
+    public static final F<java.util.Date, Timestamp> dateToTimestamp = new F<java.util.Date, Timestamp>() {
+        public Timestamp f(java.util.Date date) {
+            return new Timestamp(date.getTime());
+        }
+    };
 }
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
index a4f300f..a58f1de 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -2,6 +2,7 @@ package io.trygvis.esper.testing.gitorious;
 
 import static java.lang.System.*;
 import org.apache.commons.io.*;
+import static org.apache.commons.lang.StringUtils.*;
 import static org.codehaus.httpcache4j.HTTPMethod.*;
 import org.codehaus.httpcache4j.*;
 import org.codehaus.httpcache4j.cache.*;
@@ -16,20 +17,20 @@ import java.util.*;
 public class GitoriousClient {
     public static final STAXEventReader xmlReader = new STAXEventReader();
     private final HTTPCache httpCache;
-    private final String gitoriousUrl;
+    public final String baseUrl;
     private final String projectsUri;
 
-    public GitoriousClient(HTTPCache httpCache, String gitoriousUrl) throws URISyntaxException {
+    public GitoriousClient(HTTPCache httpCache, String baseUrl) throws URISyntaxException {
         this.httpCache = httpCache;
-        this.gitoriousUrl = new URI(gitoriousUrl).toASCIIString();
-        this.projectsUri = gitoriousUrl + "/projects.xml";
+        this.baseUrl = new URI(baseUrl).toASCIIString();
+        this.projectsUri = baseUrl + "/projects.xml";
     }
 
-    public Set<GitoriousProject> findProjects() throws Exception {
+    public Set<GitoriousProjectXml> findProjects() throws Exception {
         System.out.println("Fetching all projects");
         int page = 1;
 
-        Set<GitoriousProject> all = new HashSet<>();
+        Set<GitoriousProjectXml> all = new HashSet<>();
         while (page <= 10) {
             System.out.println("Fetching projects XML, page=" + page);
             long start = currentTimeMillis();
@@ -42,7 +43,7 @@ public class GitoriousClient {
             try {
                 Document doc = xmlReader.readDocument(new ByteArrayInputStream(bytes));
 
-                List<GitoriousProject> list = GitoriousProject.projectsFromXml(gitoriousUrl, doc.getRootElement());
+                List<GitoriousProjectXml> list = GitoriousProjectXml.projectsFromXml(doc.getRootElement());
 
                 // This indicates the last page.
                 if (list.size() == 0) {
@@ -61,4 +62,138 @@ public class GitoriousClient {
 
         return all;
     }
+
+    public URI atomFeed(String slug) {
+        return URI.create(baseUrl + "/" + slug + ".atom");
+    }
+}
+
+class GitoriousProjectXml implements Comparable<GitoriousProjectXml> {
+    public final String slug;
+    public final List<GitoriousRepositoryXml> repositories;
+
+    public GitoriousProjectXml(String slug, List<GitoriousRepositoryXml> repositories) {
+        this.slug = slug;
+        this.repositories = repositories;
+    }
+
+    public static GitoriousProjectXml fromXml(Element project) throws URISyntaxException {
+        String slug = trimToNull(project.elementText("slug"));
+
+        if (slug == null) {
+            System.out.println("Missing slug");
+            return null;
+        }
+
+        Element repositories = project.element("repositories");
+        if (repositories == null) {
+            System.out.println("Missing <repositories>");
+            return null;
+        }
+
+        Element mainlines = repositories.element("mainlines");
+        if (mainlines == null) {
+            System.out.println("Missing <mainlines>");
+            return null;
+        }
+
+        List<Element> list = (List<Element>) mainlines.elements("repository");
+        List<GitoriousRepositoryXml> repositoryList = new ArrayList<>(list.size());
+        for (Element repository : list) {
+            GitoriousRepositoryXml r = GitoriousRepositoryXml.fromXml(slug, repository);
+
+            if (r == null) {
+                continue;
+            }
+
+            repositoryList.add(r);
+        }
+
+        return new GitoriousProjectXml(slug, repositoryList);
+    }
+
+    public static List<GitoriousProjectXml> projectsFromXml(Element root) throws URISyntaxException {
+        List<GitoriousProjectXml> projects = new ArrayList<>();
+        for (Element project : (List<Element>) root.elements("project")) {
+
+            GitoriousProjectXml p = GitoriousProjectXml.fromXml(project);
+            if (p == null) {
+                System.out.println(project.toString());
+                continue;
+            }
+            projects.add(p);
+        }
+
+        return projects;
+    }
+
+    public int compareTo(GitoriousProjectXml other) {
+        return slug.compareTo(other.slug);
+    }
+
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof GitoriousProjectXml)) return false;
+
+        GitoriousProjectXml that = (GitoriousProjectXml) o;
+
+        if (!repositories.equals(that.repositories)) return false;
+        if (!slug.equals(that.slug)) return false;
+
+        return true;
+    }
+
+    public int hashCode() {
+        int result = slug.hashCode();
+        result = 31 * result + repositories.hashCode();
+        return result;
+    }
+}
+
+class GitoriousRepositoryXml implements Comparable<GitoriousRepositoryXml> {
+    public final String projectSlug;
+    public final String name;
+
+    GitoriousRepositoryXml(String projectSlug, String name) {
+        this.projectSlug = projectSlug;
+        this.name = name;
+    }
+
+    public static GitoriousRepositoryXml fromXml(String project, Element element) throws URISyntaxException {
+        String name = trimToNull(element.elementText("name"));
+
+        if (name == null) {
+            return null;
+        }
+
+        return new GitoriousRepositoryXml(project, name);
+    }
+
+    public int compareTo(GitoriousRepositoryXml o) {
+        int a = projectSlug.compareTo(o.projectSlug);
+
+        if (a != 0) {
+            return a;
+        }
+
+        return name.compareTo(o.name);
+    }
+
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof GitoriousRepositoryXml)) return false;
+
+        GitoriousRepositoryXml that = (GitoriousRepositoryXml) o;
+
+        if (!name.equals(that.name)) return false;
+        if (!projectSlug.equals(that.projectSlug)) return false;
+
+        return true;
+    }
+
+    public int hashCode() {
+        int result = projectSlug.hashCode();
+        result = 31 * result + name.hashCode();
+        return result;
+    }
 }
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index 6264bc7..4cd0916 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -1,22 +1,23 @@
 package io.trygvis.esper.testing.gitorious;
 
 import com.jolbox.bonecp.*;
-import fj.*;
+import fj.data.*;
+import static fj.data.Option.*;
 import io.trygvis.esper.testing.*;
 import static java.lang.System.*;
 import org.apache.abdera.*;
 import org.apache.abdera.model.*;
 import org.apache.abdera.parser.*;
-import org.apache.abdera.protocol.client.*;
 import org.codehaus.httpcache4j.*;
 import org.codehaus.httpcache4j.cache.*;
 import org.codehaus.httpcache4j.client.*;
 
 import java.io.*;
-import java.net.*;
 import java.sql.*;
 import java.util.Date;
 import java.util.*;
+import java.util.List;
+import java.util.Set;
 import java.util.concurrent.*;
 
 public class GitoriousImporter {
@@ -27,31 +28,20 @@ public class GitoriousImporter {
 
     public static void main(String[] args) throws Exception {
         Main.configureLog4j();
-        new GitoriousImporter();
+        new GitoriousImporter(DbMain.JDBC_URL, "esper", "");
     }
 
-    public GitoriousImporter() throws Exception {
+    public GitoriousImporter(String jdbcUrl, String jdbcUsername, String jdbcPassword) throws Exception {
         Abdera abdera = new Abdera();
         parser = abdera.getParser();
 
         BoneCPConfig config = new BoneCPConfig();
-        config.setJdbcUrl(DbMain.JDBC_URL);
-        config.setUsername("esper");
-        config.setPassword("");
+        config.setJdbcUrl(jdbcUrl);
+        config.setUsername(jdbcUsername);
+        config.setPassword(jdbcPassword);
         config.setDefaultAutoCommit(false);
         config.setMaxConnectionsPerPartition(10);
 
-//        config.setConnectionHook(new AbstractConnectionHook() {
-//            public void onAcquire(ConnectionHandle c) {
-//                try {
-//                    System.out.println("New SQL connection.");
-//                    c.setDebugHandle(new Daos(c));
-//                } catch (SQLException e) {connections
-//                    throw new RuntimeException(e);
-//                }
-//            }
-//        });
-
         boneCp = new BoneCP(config);
 
         httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
@@ -87,7 +77,7 @@ public class GitoriousImporter {
     }
 
     private void discoverProjects() throws Exception {
-        Set<GitoriousProject> projects = gitoriousClient.findProjects();
+        Set<GitoriousProjectXml> projects = gitoriousClient.findProjects();
 
         long start = currentTimeMillis();
         try (Daos daos = Daos.lookup(boneCp)) {
@@ -95,25 +85,35 @@ public class GitoriousImporter {
             GitoriousProjectDao projectDao = daos.gitoriousProjectDao;
 
             System.out.println("Processing " + projects.size() + " projects.");
-            for (GitoriousProject project : projects) {
+            for (GitoriousProjectXml project : projects) {
                 if (projectDao.countProjects(project.slug) == 0) {
                     System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories.");
-                    projectDao.insertProject(project);
-                    for (GitoriousRepository repository : project.repositories) {
-                        repoDao.insertRepository(repository);
+                    projectDao.insertProject(project.slug);
+                    for (GitoriousRepositoryXml repository : project.repositories) {
+                        repoDao.insertRepository(repository.projectSlug, repository.name, gitoriousClient.atomFeed(project.slug));
                     }
                 } else {
-                    for (GitoriousRepository repository : project.repositories) {
-                        if (repoDao.countRepositories(repository) == 0) {
+                    for (GitoriousRepositoryXml repository : project.repositories) {
+                        if (repoDao.countRepositories(repository.projectSlug, repository.name) == 0) {
                             System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name);
-                            repoDao.insertRepository(repository);
+                            repoDao.insertRepository(repository.projectSlug, repository.name, gitoriousClient.atomFeed(project.slug));
                         }
                     }
 
                     for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) {
-                        if (project.repositories.contains(repository)) {
+                        boolean found = false;
+                        for (Iterator<GitoriousRepositoryXml> it = project.repositories.iterator(); it.hasNext(); ) {
+                            GitoriousRepositoryXml repo = it.next();
+                            if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) {
+                                found = true;
+                                break;
+                            }
+                        }
+
+                        if (found) {
                             continue;
                         }
+
                         System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name);
                         repoDao.delete(repository);
                     }
@@ -122,8 +122,8 @@ public class GitoriousImporter {
 
             for (String project : projectDao.selectSlugs()) {
                 boolean found = false;
-                for (Iterator<GitoriousProject> it = projects.iterator(); it.hasNext(); ) {
-                    GitoriousProject p = it.next();
+                for (Iterator<GitoriousProjectXml> it = projects.iterator(); it.hasNext(); ) {
+                    GitoriousProjectXml p = it.next();
                     if (p.slug.equals(project)) {
                         found = true;
                         break;
@@ -147,29 +147,25 @@ public class GitoriousImporter {
 
     private void updateRepositories() throws SQLException, IOException {
         try (Daos daos = Daos.lookup(boneCp)) {
-            List<P2<String, URI>> list = daos.gitoriousProjectDao.selectFeeds();
+            List<GitoriousRepository> list = daos.gitoriousRepositoryDao.select();
             System.out.println("Updating " + list.size() + " feeds.");
-            for (P2<String, URI> pair : list) {
-                updateFeed(daos, pair._1(), pair._2());
+            for (GitoriousRepository repository : list) {
+                updateFeed(daos, repository);
                 daos.commit();
             }
         }
     }
 
-    private void updateFeed(Daos daos, String slug, URI uri) throws SQLException {
-        AtomDao atomDao = daos.atomDao;
+    private void updateFeed(Daos daos, GitoriousRepository repository) throws SQLException {
+        GitoriousRepositoryDao repositoryDao = daos.gitoriousRepositoryDao;
         GitoriousEventDao eventDao = daos.gitoriousEventDao;
 
-        Timestamp lastUpdate = atomDao.getAtomFeed(uri);
+        Option<Date> lastUpdate = repository.lastUpdate;
 
-        System.out.println("Fetching " + uri);
-        RequestOptions options = new RequestOptions();
-        if (lastUpdate != null) {
-            options.setIfModifiedSince(lastUpdate);
-        }
+        System.out.println("Fetching " + repository.atomFeed);
 
         long start = currentTimeMillis();
-        HTTPResponse response = httpCache.execute(new HTTPRequest(uri, HTTPMethod.GET));
+        HTTPResponse response = httpCache.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET));
         long end = currentTimeMillis();
         System.out.println("Fetched in " + (end - start) + "ms");
 
@@ -178,11 +174,12 @@ public class GitoriousImporter {
 
         System.out.println("responseDate = " + responseDate);
 
-        Document<Element> document = null;
+        Document<Element> document;
         try {
             document = parser.parse(response.getPayload().getInputStream());
         } catch (ParseException e) {
-            System.out.println("Error parsing " + uri);
+            repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none());
+            System.out.println("Error parsing " + repository.atomFeed);
             e.printStackTrace(System.out);
             return;
         }
@@ -199,25 +196,19 @@ public class GitoriousImporter {
                 continue;
             }
 
-            if (lastUpdate != null && lastUpdate.after(published)) {
-                System.out.println("Old entry: " + uri + ":" + entryId);
+            if (lastUpdate.isSome() && lastUpdate.some().after(published)) {
+                System.out.println("Old entry: " + repository.atomFeed + ":" + entryId);
                 continue;
             }
 
-            System.out.println("New entry: " + uri + ":" + entryId);
             if (eventDao.countEntryId(entryId) == 0) {
+                System.out.println("New entry: " + repository.atomFeed + ":" + entryId);
                 eventDao.insertChange(entryId, title);
             } else {
                 System.out.println("Already imported entry: " + entryId);
             }
         }
 
-        if (lastUpdate == null) {
-            System.out.println("New atom feed");
-            atomDao.insertAtomFeed(uri, new Timestamp(responseDate.getTime()));
-        } else {
-            System.out.println("Updating atom feed");
-            atomDao.updateAtomFeed(uri, lastUpdate);
-        }
+        repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), some(new Date()));
     }
 }
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
index 1ccfac3..068866e 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
@@ -1,70 +1,15 @@
 package io.trygvis.esper.testing.gitorious;
 
-import static org.apache.commons.lang.StringUtils.*;
-import org.dom4j.*;
+import fj.data.*;
 
 import java.net.*;
 import java.util.*;
 
 public class GitoriousProject implements Comparable<GitoriousProject> {
     public final String slug;
-    public final URI atomFeed;
-    public final List<GitoriousRepository> repositories;
 
-    public GitoriousProject(String slug, URI atomFeed, List<GitoriousRepository> repositories) {
+    public GitoriousProject(String slug) {
         this.slug = slug;
-        this.atomFeed = atomFeed;
-        this.repositories = repositories;
-    }
-
-    public static GitoriousProject fromXml(String gitoriousUrl, Element project) throws URISyntaxException {
-        String slug = trimToNull(project.elementText("slug"));
-
-        if (slug == null) {
-            System.out.println("Missing slug");
-            return null;
-        }
-
-        Element repositories = project.element("repositories");
-        if (repositories == null) {
-            System.out.println("Missing <repositories>");
-            return null;
-        }
-
-        Element mainlines = repositories.element("mainlines");
-        if (mainlines == null) {
-            System.out.println("Missing <mainlines>");
-            return null;
-        }
-
-        List<Element> list = (List<Element>) mainlines.elements("repository");
-        List<GitoriousRepository> repositoryList = new ArrayList<>(list.size());
-        for (Element repository : list) {
-            GitoriousRepository r = GitoriousRepository.fromXml(gitoriousUrl, slug, repository);
-
-            if (r == null) {
-                continue;
-            }
-
-            repositoryList.add(r);
-        }
-
-        return new GitoriousProject(slug, URI.create(gitoriousUrl + "/" + slug + ".atom"), repositoryList);
-    }
-
-    public static List<GitoriousProject> projectsFromXml(String gitoriousUrl, Element root) throws URISyntaxException {
-        List<GitoriousProject> projects = new ArrayList<>();
-        for (Element project : (List<Element>) root.elements("project")) {
-
-            GitoriousProject p = GitoriousProject.fromXml(gitoriousUrl, project);
-            if (p == null) {
-                System.out.println(project.toString());
-                continue;
-            }
-            projects.add(p);
-        }
-
-        return projects;
     }
 
     public int compareTo(GitoriousProject other) {
@@ -77,7 +22,6 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
 
         GitoriousProject that = (GitoriousProject) o;
 
-        if (!repositories.equals(that.repositories)) return false;
         if (!slug.equals(that.slug)) return false;
 
         return true;
@@ -85,7 +29,6 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
 
     public int hashCode() {
         int result = slug.hashCode();
-        result = 31 * result + repositories.hashCode();
         return result;
     }
 }
@@ -93,20 +36,16 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
 class GitoriousRepository implements Comparable<GitoriousRepository> {
     public final String projectSlug;
     public final String name;
+    public final URI atomFeed;
+    public final Option<Date> lastUpdate;
+    public final Option<Date> lastSuccessfulUpdate;
 
-    GitoriousRepository(String projectSlug, String name) {
+    GitoriousRepository(String projectSlug, String name, URI atomFeed, Option<Date> lastUpdate, Option<Date> lastSuccessfulUpdate) {
         this.projectSlug = projectSlug;
         this.name = name;
-    }
-
-    public static GitoriousRepository fromXml(String gitoriousUrl, String project, Element element) throws URISyntaxException {
-        String name = trimToNull(element.elementText("name"));
-
-        if (name == null) {
-            return null;
-        }
-
-        return new GitoriousRepository(project, name);
+        this.atomFeed = atomFeed;
+        this.lastUpdate = lastUpdate;
+        this.lastSuccessfulUpdate = lastSuccessfulUpdate;
     }
 
     public int compareTo(GitoriousRepository o) {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
index 432d154..262d23c 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
@@ -1,8 +1,5 @@
 package io.trygvis.esper.testing.gitorious;
 
-import fj.*;
-
-import java.net.*;
 import java.sql.*;
 import java.util.*;
 
@@ -21,18 +18,17 @@ public class GitoriousProjectDao extends Dao {
         }
     }
 
-    private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug, atom_feed) VALUES(?, ?)");
+    private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)");
 
-    public void insertProject(GitoriousProject project) throws SQLException {
-        insertProject.setString(1, project.slug);
-        insertProject.setString(2, project.atomFeed.toASCIIString());
+    public void insertProject(String slug) throws SQLException {
+        insertProject.setString(1, slug);
         insertProject.executeUpdate();
     }
 
-    private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project");
+    private final PreparedStatement selectSlugs = prepareStatement("SELECT slug FROM gitorious_project");
 
     public List<String> selectSlugs() throws SQLException {
-        try (ResultSet rs = selectAll.executeQuery()) {
+        try (ResultSet rs = selectSlugs.executeQuery()) {
             List<String> list = new ArrayList<>();
             while (rs.next()) {
                 list.add(rs.getString(1));
@@ -41,18 +37,6 @@ public class GitoriousProjectDao extends Dao {
         }
     }
 
-    private final PreparedStatement selectFeeds = prepareStatement("SELECT slug, atom_feed FROM gitorious_project");
-
-    public List<P2<String, URI>> selectFeeds() throws SQLException {
-        try (ResultSet rs = selectFeeds.executeQuery()) {
-            List<P2<String, URI>> list = new ArrayList<>();
-            while (rs.next()) {
-                list.add(P.p(rs.getString(1), URI.create(rs.getString(2))));
-            }
-            return list;
-        }
-    }
-
     private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?");
 
     public void delete(String slug) throws SQLException {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
index 5f57441..bcb27de 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
@@ -1,43 +1,67 @@
 package io.trygvis.esper.testing.gitorious;
 
+import fj.data.*;
+
 import java.net.*;
 import java.sql.*;
 import java.util.*;
+import java.util.Date;
+import java.util.List;
 
 public class GitoriousRepositoryDao extends Dao {
     public GitoriousRepositoryDao(Connection c) throws SQLException {
         super(c);
     }
 
+    private static final String ALL_FIELDS = "project_slug, name, atom_feed, last_update, last_successful_update";
+
+    private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException {
+        try (ResultSet rs = statement.executeQuery()) {
+            List<GitoriousRepository> list = new ArrayList<>();
+
+            while (rs.next()) {
+                list.add(new GitoriousRepository(
+                    rs.getString(1),
+                    rs.getString(2),
+                    URI.create(rs.getString(3)),
+                    Option.fromNull(rs.getTimestamp(4)).map(timestampToDate),
+                    Option.fromNull(rs.getTimestamp(5)).map(timestampToDate)));
+            }
+
+            return list;
+        }
+    }
+
     private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?");
 
-    public int countRepositories(GitoriousRepository repository) throws SQLException {
-        countRepositories.setString(1, repository.projectSlug);
-        countRepositories.setString(2, repository.name);
+    public int countRepositories(String projectSlug, String name) throws SQLException {
+        countRepositories.setString(1, projectSlug);
+        countRepositories.setString(2, name);
         try (ResultSet rs = countRepositories.executeQuery()) {
             rs.next();
             return rs.getInt(1);
         }
     }
 
-    private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name FROM gitorious_repository WHERE project_slug=?");
+    private final PreparedStatement selectForProject = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository WHERE project_slug=?");
 
     public List<GitoriousRepository> selectForProject(String projectSlug) throws SQLException {
         selectForProject.setString(1, projectSlug);
         return executeQuery(selectForProject);
     }
 
-    private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name FROM gitorious_repository");
+    private final PreparedStatement select = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository");
 
-    public List<GitoriousRepository> selectAll() throws SQLException {
-        return executeQuery(selectAll);
+    public List<GitoriousRepository> select() throws SQLException {
+        return executeQuery(select);
     }
 
-    private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name) VALUES(?, ?)");
+    private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)");
 
-    public void insertRepository(GitoriousRepository repository) throws SQLException {
-        insertRepository.setString(1, repository.projectSlug);
-        insertRepository.setString(2, repository.name);
+    public void insertRepository(String projectSlug, String name, URI atomFeed) throws SQLException {
+        insertRepository.setString(1, projectSlug);
+        insertRepository.setString(2, name);
+        insertRepository.setString(3, atomFeed.toASCIIString());
         insertRepository.executeUpdate();
     }
 
@@ -56,18 +80,13 @@ public class GitoriousRepositoryDao extends Dao {
         deleteForProject.executeUpdate();
     }
 
-    private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException {
-        try (ResultSet rs = statement.executeQuery()) {
-            List<GitoriousRepository> list = new ArrayList<>();
-
-            while (rs.next()) {
-                list.add(new GitoriousRepository(
-                    rs.getString(1),
-                    rs.getString(2)
-                ));
-            }
+    private final PreparedStatement updateTimestamp = prepareStatement("UPDATE gitorious_repository SET last_update=?, last_successful_update=? WHERE project_slug=? AND name=?");
 
-            return list;
-        }
+    public void updateTimestamp(String projectName, String slug, Timestamp lastUpdate, Option<Date> lastSuccessfulUpdate) throws SQLException {
+        updateTimestamp.setTimestamp(1, lastUpdate);
+        updateTimestamp.setTimestamp(2, lastSuccessfulUpdate.map(dateToTimestamp).toNull());
+        updateTimestamp.setString(3, slug);
+        updateTimestamp.setString(4, projectName);
+        updateTimestamp.executeUpdate();
     }
 }
diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql
index 279e544..4e641cc 100644
--- a/src/main/resources/ddl.sql
+++ b/src/main/resources/ddl.sql
@@ -3,28 +3,24 @@ BEGIN;
 DROP TABLE IF EXISTS gitorious_event;
 DROP TABLE IF EXISTS gitorious_repository;
 DROP TABLE IF EXISTS gitorious_project;
-DROP TABLE IF EXISTS atom_feed;
-
-CREATE TABLE atom_feed (
-  url VARCHAR(1000) PRIMARY KEY,
-  last_update TIMESTAMP NOT NULL
-);
 
 CREATE TABLE gitorious_project (
-  slug VARCHAR(1000) PRIMARY KEY,
-  atom_feed VARCHAR(1000) NOT NULL
+    slug VARCHAR(1000) PRIMARY KEY
 );
 
 CREATE TABLE gitorious_repository (
-  project_slug VARCHAR(1000) NOT NULL,
-  name VARCHAR(1000) NOT NULL,
-  CONSTRAINT gitorious_repository_pk PRIMARY KEY(project_slug, name),
-  CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY(project_slug) REFERENCES gitorious_project(slug)
+    project_slug           VARCHAR(1000) NOT NULL,
+    name                   VARCHAR(1000) NOT NULL,
+    atom_feed              VARCHAR(1000) NOT NULL,
+    last_update            TIMESTAMP,
+    last_successful_update TIMESTAMP,
+    CONSTRAINT gitorious_repository_pk PRIMARY KEY (project_slug, name),
+    CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY (project_slug) REFERENCES gitorious_project (slug)
 );
 
 CREATE TABLE gitorious_event (
-  entry_id VARCHAR(1000) PRIMARY KEY,
-  text VARCHAR(1000)
+    entry_id VARCHAR(1000) PRIMARY KEY,
+    text     VARCHAR(1000)
 );
 
 COMMIT;
diff --git a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java
index 63ae319..2d5945e 100644
--- a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java
+++ b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java
@@ -11,12 +11,12 @@ public class TestXmlParsing extends TestCase {
         try (InputStream stream = getClass().getResourceAsStream("/gitorious/projects-2.xml")) {
             Document document = GitoriousClient.xmlReader.readDocument(stream);
 
-            List<GitoriousProject> projects = GitoriousProject.projectsFromXml("http://gitorious.org", document.getRootElement());
+            List<GitoriousProjectXml> projects = GitoriousProjectXml.projectsFromXml(document.getRootElement());
 
             assertNotNull(projects);
             assertEquals(20, projects.size());
 
-            GitoriousProject project = projects.get(3);
+            GitoriousProjectXml project = projects.get(3);
             assertEquals("aed-ii", project.slug);
             assertEquals(2, project.repositories.size());
         }