diff options
Diffstat (limited to 'src/main/java/io/trygvis/esper')
6 files changed, 172 insertions, 25 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/AtomDao.java b/src/main/java/io/trygvis/esper/testing/AtomDao.java new file mode 100644 index 0000000..0215545 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/AtomDao.java @@ -0,0 +1,37 @@ +package io.trygvis.esper.testing; + +import java.sql.*; + +public class AtomDao { + private final PreparedStatement selectLastUpdate; + private final PreparedStatement insertAtomFeed; + private final PreparedStatement updateAtomFeed; + + public AtomDao(Connection c) throws SQLException { + selectLastUpdate = c.prepareStatement("SELECT last_update FROM atom_feed WHERE url=?"); + insertAtomFeed = c.prepareStatement("INSERT INTO atom_feed(url, last_update) VALUES(?, ?)"); + updateAtomFeed = c.prepareStatement("UPDATE atom_feed SET last_update=? WHERE url=?"); + } + + public Timestamp getAtomFeed(String url) throws SQLException { + selectLastUpdate.setString(1, url); + ResultSet rs = selectLastUpdate.executeQuery(); + if (!rs.next()) { + return null; + } + + return rs.getTimestamp(1); + } + + public void insertAtomFeed(String url, Timestamp lastUpdate) throws SQLException { + insertAtomFeed.setString(1, url); + insertAtomFeed.setTimestamp(2, lastUpdate); + insertAtomFeed.executeUpdate(); + } + + public void updateAtomFeed(String url, Timestamp lastUpdate) throws SQLException { + updateAtomFeed.setTimestamp(1, lastUpdate); + updateAtomFeed.setString(2, url); + updateAtomFeed.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/AtomImporter.java b/src/main/java/io/trygvis/esper/testing/AtomImporter.java deleted file mode 100644 index 7e70715..0000000 --- a/src/main/java/io/trygvis/esper/testing/AtomImporter.java +++ /dev/null @@ -1,18 +0,0 @@ -package io.trygvis.esper.testing; - -import org.apache.abdera.*; -import org.apache.abdera.protocol.client.*; -import org.apache.abdera.protocol.client.cache.*; - -public class AtomImporter { - public static void main(String[] args) { - Abdera abdera = new Abdera(); - AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); - - while(true) { - ClientResponse response = abderaClient.get("http://gitorious.org/qt.atom"); - -// response. - } - } -} diff --git a/src/main/java/io/trygvis/esper/testing/DbMain.java b/src/main/java/io/trygvis/esper/testing/DbMain.java index 635e3cc..345d7a7 100644 --- a/src/main/java/io/trygvis/esper/testing/DbMain.java +++ b/src/main/java/io/trygvis/esper/testing/DbMain.java @@ -5,7 +5,8 @@ import org.h2.tools.*; import java.sql.*; public class DbMain { - private static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1"; +// public static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1"; + public static final String JDBC_URL = "jdbc:postgresql://localhost/esper"; public static void main(String[] args) throws Exception { Server server = Server.createTcpServer(args).start(); diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/GitoriousDao.java new file mode 100644 index 0000000..bf5d954 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/GitoriousDao.java @@ -0,0 +1,27 @@ +package io.trygvis.esper.testing; + +import java.sql.*; + +public class GitoriousDao { + private final PreparedStatement countEntryId; + private final PreparedStatement insertChange; + + public GitoriousDao(Connection c) throws SQLException { + countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?"); + insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)"); + } + + public int countEntryId(String entryId) throws SQLException { + countEntryId.setString(1, entryId); + try(ResultSet rs = countEntryId.executeQuery()) { + rs.next(); + return rs.getInt(1); + } + } + + public void insertChange(String entryId, String text) throws SQLException { + insertChange.setString(1, entryId); + insertChange.setString(2, text); + insertChange.executeUpdate(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java new file mode 100644 index 0000000..c79d4f5 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java @@ -0,0 +1,98 @@ +package io.trygvis.esper.testing; + +import org.apache.abdera.*; +import org.apache.abdera.model.*; +import org.apache.abdera.protocol.client.*; +import org.apache.abdera.protocol.client.cache.*; + +import java.sql.*; +import java.util.Date; + +public class GitoriousImporter { + private final AbderaClient abderaClient; + private final Connection connection; + private final AtomDao atomDao; + private final GitoriousDao gitoriousDao; + + public GitoriousImporter(AbderaClient abderaClient, Connection c) throws SQLException { + this.abderaClient = abderaClient; + this.connection = c; + atomDao = new AtomDao(c); + gitoriousDao = new GitoriousDao(c); + } + + public static void main(String[] args) throws InterruptedException, SQLException { + Main.configureLog4j(); + Abdera abdera = new Abdera(); + AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); + + Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); + connection.setAutoCommit(false); + + new GitoriousImporter(abderaClient, connection).work(); + } + + private void work() throws SQLException, InterruptedException { + String url = "http://qt.gitorious.org/projects/show/qt.atom"; + + while (true) { + Timestamp lastUpdate = atomDao.getAtomFeed(url); + + System.out.println("Fetching " + url); + RequestOptions options = new RequestOptions(); + if(lastUpdate != null) { + options.setIfModifiedSince(lastUpdate); + } + + long start = System.currentTimeMillis(); + ClientResponse response = abderaClient.get(url, options); + long end = System.currentTimeMillis(); + System.out.println("Fetched in " + (end - start) + "ms"); + + // Use the server's timestamp + Date responseDate = response.getDateHeader("Date"); + + System.out.println("responseDate = " + responseDate); + + Document<Element> document = response.getDocument(); + Feed feed = (Feed) document.getRoot(); + + for (Entry entry : feed.getEntries()) { + String entryId = entry.getId().toASCIIString(); + Date published = entry.getPublished(); + String title = entry.getTitle(); + + // Validate element + if (entryId == null || published == null || title == null) { + continue; + } + + if (lastUpdate != null && lastUpdate.after(published)) { + System.out.println("Old entry: " + url + ":" + entryId); + continue; + } + + System.out.println("New entry: " + url + ":" + entryId); + if(gitoriousDao.countEntryId(entryId) == 0) { + gitoriousDao.insertChange(entryId, title); + } + else { + System.out.println("Already imported entry: " + entryId); + } + } + + if (lastUpdate == null) { + System.out.println("New atom feed"); + atomDao.insertAtomFeed(url, new Timestamp(responseDate.getTime())); + } else { + System.out.println("Updating atom feed"); + atomDao.updateAtomFeed(url, lastUpdate); + } + + connection.commit(); + + System.out.println("Sleeping"); + Thread.sleep(10 * 1000); + } + } +} diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java index 25b3fad..69e0bce 100644 --- a/src/main/java/io/trygvis/esper/testing/Main.java +++ b/src/main/java/io/trygvis/esper/testing/Main.java @@ -9,20 +9,22 @@ public class Main { // private static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1"; private static final String JDBC_URL = "jdbc:h2:tcp://127.0.0.1/esper;DB_CLOSE_DELAY=-1"; - public Main() throws Exception { + public static void main(String[] args) throws Exception { + configureLog4j(); + Main main = new Main(); + main.work(); + } + + public static void configureLog4j() { Properties properties = new Properties(); properties.setProperty("log4j.rootLogger", "DEBUG, A1"); + properties.setProperty("log4j.logger.httpclient.wire.content", "INFO"); properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender"); properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout"); properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n"); PropertyConfigurator.configure(properties); } - public static void main(String[] args) throws Exception { - Main main = new Main(); - main.work(); - } - private void work() throws Exception { Configuration config = new Configuration(); |