o Basic gitorious atom importer.

6 files changed, 172 insertions, 25 deletions

diff --git a/src/main/java/io/trygvis/esper/testing/AtomDao.java b/src/main/java/io/trygvis/esper/testing/AtomDao.java
new file mode 100644
index 0000000..0215545
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/AtomDao.java
@@ -0,0 +1,37 @@
+package io.trygvis.esper.testing;
+
+import java.sql.*;
+
+public class AtomDao {
+    private final PreparedStatement selectLastUpdate;
+    private final PreparedStatement insertAtomFeed;
+    private final PreparedStatement updateAtomFeed;
+
+    public AtomDao(Connection c) throws SQLException {
+        selectLastUpdate = c.prepareStatement("SELECT last_update FROM atom_feed WHERE url=?");
+        insertAtomFeed = c.prepareStatement("INSERT INTO atom_feed(url, last_update) VALUES(?, ?)");
+        updateAtomFeed = c.prepareStatement("UPDATE atom_feed SET last_update=? WHERE url=?");
+    }
+
+    public Timestamp getAtomFeed(String url) throws SQLException {
+        selectLastUpdate.setString(1, url);
+        ResultSet rs = selectLastUpdate.executeQuery();
+        if (!rs.next()) {
+            return null;
+        }
+
+        return rs.getTimestamp(1);
+    }
+
+    public void insertAtomFeed(String url, Timestamp lastUpdate) throws SQLException {
+        insertAtomFeed.setString(1, url);
+        insertAtomFeed.setTimestamp(2, lastUpdate);
+        insertAtomFeed.executeUpdate();
+    }
+
+    public void updateAtomFeed(String url, Timestamp lastUpdate) throws SQLException {
+        updateAtomFeed.setTimestamp(1, lastUpdate);
+        updateAtomFeed.setString(2, url);
+        updateAtomFeed.executeUpdate();
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/AtomImporter.java b/src/main/java/io/trygvis/esper/testing/AtomImporter.java
deleted file mode 100644
index 7e70715..0000000
--- a/src/main/java/io/trygvis/esper/testing/AtomImporter.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package io.trygvis.esper.testing;
-
-import org.apache.abdera.*;
-import org.apache.abdera.protocol.client.*;
-import org.apache.abdera.protocol.client.cache.*;
-
-public class AtomImporter {
-    public static void main(String[] args) {
-        Abdera abdera = new Abdera();
-        AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
-
-        while(true) {
-            ClientResponse response = abderaClient.get("http://gitorious.org/qt.atom");
-
-//            response.
-        }
-    }
-}
diff --git a/src/main/java/io/trygvis/esper/testing/DbMain.java b/src/main/java/io/trygvis/esper/testing/DbMain.java
index 635e3cc..345d7a7 100644
--- a/src/main/java/io/trygvis/esper/testing/DbMain.java
+++ b/src/main/java/io/trygvis/esper/testing/DbMain.java
@@ -5,7 +5,8 @@ import org.h2.tools.*;
 import java.sql.*;
 
 public class DbMain {
-    private static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1";
+//    public static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1";
+    public static final String JDBC_URL = "jdbc:postgresql://localhost/esper";
 
     public static void main(String[] args) throws Exception {
         Server server = Server.createTcpServer(args).start();
diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/GitoriousDao.java
new file mode 100644
index 0000000..bf5d954
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/GitoriousDao.java
@@ -0,0 +1,27 @@
+package io.trygvis.esper.testing;
+
+import java.sql.*;
+
+public class GitoriousDao {
+    private final PreparedStatement countEntryId;
+    private final PreparedStatement insertChange;
+
+    public GitoriousDao(Connection c) throws SQLException {
+        countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?");
+        insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)");
+    }
+
+    public int countEntryId(String entryId) throws SQLException {
+        countEntryId.setString(1, entryId);
+        try(ResultSet rs = countEntryId.executeQuery()) {
+            rs.next();
+            return rs.getInt(1);
+        }
+    }
+
+    public void insertChange(String entryId, String text) throws SQLException {
+        insertChange.setString(1, entryId);
+        insertChange.setString(2, text);
+        insertChange.executeUpdate();
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java
new file mode 100644
index 0000000..c79d4f5
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java
@@ -0,0 +1,98 @@
+package io.trygvis.esper.testing;
+
+import org.apache.abdera.*;
+import org.apache.abdera.model.*;
+import org.apache.abdera.protocol.client.*;
+import org.apache.abdera.protocol.client.cache.*;
+
+import java.sql.*;
+import java.util.Date;
+
+public class GitoriousImporter {
+    private final AbderaClient abderaClient;
+    private final Connection connection;
+    private final AtomDao atomDao;
+    private final GitoriousDao gitoriousDao;
+
+    public GitoriousImporter(AbderaClient abderaClient, Connection c) throws SQLException {
+        this.abderaClient = abderaClient;
+        this.connection = c;
+        atomDao = new AtomDao(c);
+        gitoriousDao = new GitoriousDao(c);
+    }
+
+    public static void main(String[] args) throws InterruptedException, SQLException {
+        Main.configureLog4j();
+        Abdera abdera = new Abdera();
+        AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
+
+        Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
+        connection.setAutoCommit(false);
+
+        new GitoriousImporter(abderaClient, connection).work();
+    }
+
+    private void work() throws SQLException, InterruptedException {
+        String url = "http://qt.gitorious.org/projects/show/qt.atom";
+
+        while (true) {
+            Timestamp lastUpdate = atomDao.getAtomFeed(url);
+
+            System.out.println("Fetching " + url);
+            RequestOptions options = new RequestOptions();
+            if(lastUpdate != null) {
+                options.setIfModifiedSince(lastUpdate);
+            }
+
+            long start = System.currentTimeMillis();
+            ClientResponse response = abderaClient.get(url, options);
+            long end = System.currentTimeMillis();
+            System.out.println("Fetched in " + (end - start) + "ms");
+
+            // Use the server's timestamp
+            Date responseDate = response.getDateHeader("Date");
+
+            System.out.println("responseDate = " + responseDate);
+
+            Document<Element> document = response.getDocument();
+            Feed feed = (Feed) document.getRoot();
+
+            for (Entry entry : feed.getEntries()) {
+                String entryId = entry.getId().toASCIIString();
+                Date published = entry.getPublished();
+                String title = entry.getTitle();
+
+                // Validate element
+                if (entryId == null || published == null || title == null) {
+                    continue;
+                }
+
+                if (lastUpdate != null && lastUpdate.after(published)) {
+                    System.out.println("Old entry: " + url + ":" + entryId);
+                    continue;
+                }
+
+                System.out.println("New entry: " + url + ":" + entryId);
+                if(gitoriousDao.countEntryId(entryId) == 0) {
+                    gitoriousDao.insertChange(entryId, title);
+                }
+                else {
+                    System.out.println("Already imported entry: " + entryId);
+                }
+            }
+
+            if (lastUpdate == null) {
+                System.out.println("New atom feed");
+                atomDao.insertAtomFeed(url, new Timestamp(responseDate.getTime()));
+            } else {
+                System.out.println("Updating atom feed");
+                atomDao.updateAtomFeed(url, lastUpdate);
+            }
+
+            connection.commit();
+
+            System.out.println("Sleeping");
+            Thread.sleep(10 * 1000);
+        }
+    }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java
index 25b3fad..69e0bce 100644
--- a/src/main/java/io/trygvis/esper/testing/Main.java
+++ b/src/main/java/io/trygvis/esper/testing/Main.java
@@ -9,20 +9,22 @@ public class Main {
 //    private static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1";
     private static final String JDBC_URL = "jdbc:h2:tcp://127.0.0.1/esper;DB_CLOSE_DELAY=-1";
 
-    public Main() throws Exception {
+    public static void main(String[] args) throws Exception {
+        configureLog4j();
+        Main main = new Main();
+        main.work();
+    }
+
+    public static void configureLog4j() {
         Properties properties = new Properties();
         properties.setProperty("log4j.rootLogger", "DEBUG, A1");
+        properties.setProperty("log4j.logger.httpclient.wire.content", "INFO");
         properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender");
         properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
         properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
         PropertyConfigurator.configure(properties);
     }
 
-    public static void main(String[] args) throws Exception {
-        Main main = new Main();
-        main.work();
-    }
-
     private void work() throws Exception {
         Configuration config = new Configuration();