aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--pom.xml15
-rw-r--r--src/main/java/io/trygvis/esper/testing/AtomDao.java37
-rw-r--r--src/main/java/io/trygvis/esper/testing/AtomImporter.java18
-rw-r--r--src/main/java/io/trygvis/esper/testing/DbMain.java3
-rw-r--r--src/main/java/io/trygvis/esper/testing/GitoriousDao.java27
-rw-r--r--src/main/java/io/trygvis/esper/testing/GitoriousImporter.java98
-rw-r--r--src/main/java/io/trygvis/esper/testing/Main.java14
-rw-r--r--src/main/resources/ddl.sql16
8 files changed, 200 insertions, 28 deletions
diff --git a/pom.xml b/pom.xml
index 6340300..cdead83 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,14 +10,23 @@
<version>4.7.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.abdera</groupId>
+ <artifactId>abdera-client</artifactId>
+ <version>1.1.2</version>
+ </dependency>
+ <dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>1.3.168</version>
</dependency>
<dependency>
- <groupId>org.apache.abdera</groupId>
- <artifactId>abdera-client</artifactId>
- <version>1.1.2</version>
+ <groupId>postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ <version>9.1-901-1.jdbc4</version>
</dependency>
</dependencies>
+ <properties>
+ <maven.compiler.source>1.7</maven.compiler.source>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ </properties>
</project>
diff --git a/src/main/java/io/trygvis/esper/testing/AtomDao.java b/src/main/java/io/trygvis/esper/testing/AtomDao.java
new file mode 100644
index 0000000..0215545
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/AtomDao.java
@@ -0,0 +1,37 @@
+package io.trygvis.esper.testing;
+
+import java.sql.*;
+
+public class AtomDao {
+ private final PreparedStatement selectLastUpdate;
+ private final PreparedStatement insertAtomFeed;
+ private final PreparedStatement updateAtomFeed;
+
+ public AtomDao(Connection c) throws SQLException {
+ selectLastUpdate = c.prepareStatement("SELECT last_update FROM atom_feed WHERE url=?");
+ insertAtomFeed = c.prepareStatement("INSERT INTO atom_feed(url, last_update) VALUES(?, ?)");
+ updateAtomFeed = c.prepareStatement("UPDATE atom_feed SET last_update=? WHERE url=?");
+ }
+
+ public Timestamp getAtomFeed(String url) throws SQLException {
+ selectLastUpdate.setString(1, url);
+ ResultSet rs = selectLastUpdate.executeQuery();
+ if (!rs.next()) {
+ return null;
+ }
+
+ return rs.getTimestamp(1);
+ }
+
+ public void insertAtomFeed(String url, Timestamp lastUpdate) throws SQLException {
+ insertAtomFeed.setString(1, url);
+ insertAtomFeed.setTimestamp(2, lastUpdate);
+ insertAtomFeed.executeUpdate();
+ }
+
+ public void updateAtomFeed(String url, Timestamp lastUpdate) throws SQLException {
+ updateAtomFeed.setTimestamp(1, lastUpdate);
+ updateAtomFeed.setString(2, url);
+ updateAtomFeed.executeUpdate();
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/AtomImporter.java b/src/main/java/io/trygvis/esper/testing/AtomImporter.java
deleted file mode 100644
index 7e70715..0000000
--- a/src/main/java/io/trygvis/esper/testing/AtomImporter.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package io.trygvis.esper.testing;
-
-import org.apache.abdera.*;
-import org.apache.abdera.protocol.client.*;
-import org.apache.abdera.protocol.client.cache.*;
-
-public class AtomImporter {
- public static void main(String[] args) {
- Abdera abdera = new Abdera();
- AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
-
- while(true) {
- ClientResponse response = abderaClient.get("http://gitorious.org/qt.atom");
-
-// response.
- }
- }
-}
diff --git a/src/main/java/io/trygvis/esper/testing/DbMain.java b/src/main/java/io/trygvis/esper/testing/DbMain.java
index 635e3cc..345d7a7 100644
--- a/src/main/java/io/trygvis/esper/testing/DbMain.java
+++ b/src/main/java/io/trygvis/esper/testing/DbMain.java
@@ -5,7 +5,8 @@ import org.h2.tools.*;
import java.sql.*;
public class DbMain {
- private static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1";
+// public static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1";
+ public static final String JDBC_URL = "jdbc:postgresql://localhost/esper";
public static void main(String[] args) throws Exception {
Server server = Server.createTcpServer(args).start();
diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/GitoriousDao.java
new file mode 100644
index 0000000..bf5d954
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/GitoriousDao.java
@@ -0,0 +1,27 @@
+package io.trygvis.esper.testing;
+
+import java.sql.*;
+
+public class GitoriousDao {
+ private final PreparedStatement countEntryId;
+ private final PreparedStatement insertChange;
+
+ public GitoriousDao(Connection c) throws SQLException {
+ countEntryId = c.prepareStatement("SELECT count(entry_id) FROM gitorious_change WHERE entry_id=?");
+ insertChange = c.prepareStatement("INSERT INTO gitorious_change(entry_id, text) VALUES(?, ?)");
+ }
+
+ public int countEntryId(String entryId) throws SQLException {
+ countEntryId.setString(1, entryId);
+ try(ResultSet rs = countEntryId.executeQuery()) {
+ rs.next();
+ return rs.getInt(1);
+ }
+ }
+
+ public void insertChange(String entryId, String text) throws SQLException {
+ insertChange.setString(1, entryId);
+ insertChange.setString(2, text);
+ insertChange.executeUpdate();
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java
new file mode 100644
index 0000000..c79d4f5
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java
@@ -0,0 +1,98 @@
+package io.trygvis.esper.testing;
+
+import org.apache.abdera.*;
+import org.apache.abdera.model.*;
+import org.apache.abdera.protocol.client.*;
+import org.apache.abdera.protocol.client.cache.*;
+
+import java.sql.*;
+import java.util.Date;
+
+public class GitoriousImporter {
+ private final AbderaClient abderaClient;
+ private final Connection connection;
+ private final AtomDao atomDao;
+ private final GitoriousDao gitoriousDao;
+
+ public GitoriousImporter(AbderaClient abderaClient, Connection c) throws SQLException {
+ this.abderaClient = abderaClient;
+ this.connection = c;
+ atomDao = new AtomDao(c);
+ gitoriousDao = new GitoriousDao(c);
+ }
+
+ public static void main(String[] args) throws InterruptedException, SQLException {
+ Main.configureLog4j();
+ Abdera abdera = new Abdera();
+ AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
+
+ Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
+ connection.setAutoCommit(false);
+
+ new GitoriousImporter(abderaClient, connection).work();
+ }
+
+ private void work() throws SQLException, InterruptedException {
+ String url = "http://qt.gitorious.org/projects/show/qt.atom";
+
+ while (true) {
+ Timestamp lastUpdate = atomDao.getAtomFeed(url);
+
+ System.out.println("Fetching " + url);
+ RequestOptions options = new RequestOptions();
+ if(lastUpdate != null) {
+ options.setIfModifiedSince(lastUpdate);
+ }
+
+ long start = System.currentTimeMillis();
+ ClientResponse response = abderaClient.get(url, options);
+ long end = System.currentTimeMillis();
+ System.out.println("Fetched in " + (end - start) + "ms");
+
+ // Use the server's timestamp
+ Date responseDate = response.getDateHeader("Date");
+
+ System.out.println("responseDate = " + responseDate);
+
+ Document<Element> document = response.getDocument();
+ Feed feed = (Feed) document.getRoot();
+
+ for (Entry entry : feed.getEntries()) {
+ String entryId = entry.getId().toASCIIString();
+ Date published = entry.getPublished();
+ String title = entry.getTitle();
+
+ // Validate element
+ if (entryId == null || published == null || title == null) {
+ continue;
+ }
+
+ if (lastUpdate != null && lastUpdate.after(published)) {
+ System.out.println("Old entry: " + url + ":" + entryId);
+ continue;
+ }
+
+ System.out.println("New entry: " + url + ":" + entryId);
+ if(gitoriousDao.countEntryId(entryId) == 0) {
+ gitoriousDao.insertChange(entryId, title);
+ }
+ else {
+ System.out.println("Already imported entry: " + entryId);
+ }
+ }
+
+ if (lastUpdate == null) {
+ System.out.println("New atom feed");
+ atomDao.insertAtomFeed(url, new Timestamp(responseDate.getTime()));
+ } else {
+ System.out.println("Updating atom feed");
+ atomDao.updateAtomFeed(url, lastUpdate);
+ }
+
+ connection.commit();
+
+ System.out.println("Sleeping");
+ Thread.sleep(10 * 1000);
+ }
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/Main.java b/src/main/java/io/trygvis/esper/testing/Main.java
index 25b3fad..69e0bce 100644
--- a/src/main/java/io/trygvis/esper/testing/Main.java
+++ b/src/main/java/io/trygvis/esper/testing/Main.java
@@ -9,20 +9,22 @@ public class Main {
// private static final String JDBC_URL = "jdbc:h2:mem:esper;DB_CLOSE_DELAY=-1";
private static final String JDBC_URL = "jdbc:h2:tcp://127.0.0.1/esper;DB_CLOSE_DELAY=-1";
- public Main() throws Exception {
+ public static void main(String[] args) throws Exception {
+ configureLog4j();
+ Main main = new Main();
+ main.work();
+ }
+
+ public static void configureLog4j() {
Properties properties = new Properties();
properties.setProperty("log4j.rootLogger", "DEBUG, A1");
+ properties.setProperty("log4j.logger.httpclient.wire.content", "INFO");
properties.setProperty("log4j.appender.A1", "org.apache.log4j.ConsoleAppender");
properties.setProperty("log4j.appender.A1.layout", "org.apache.log4j.PatternLayout");
properties.setProperty("log4j.appender.A1.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
PropertyConfigurator.configure(properties);
}
- public static void main(String[] args) throws Exception {
- Main main = new Main();
- main.work();
- }
-
private void work() throws Exception {
Configuration config = new Configuration();
diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql
new file mode 100644
index 0000000..f545efd
--- /dev/null
+++ b/src/main/resources/ddl.sql
@@ -0,0 +1,16 @@
+BEGIN;
+
+DROP TABLE IF EXISTS gitorious_change;
+DROP TABLE IF EXISTS atom_feed;
+
+CREATE TABLE atom_feed (
+ url VARCHAR(1000) PRIMARY KEY,
+ last_update TIMESTAMP NOT NULL
+);
+
+CREATE TABLE gitorious_change (
+ entry_id VARCHAR(1000) PRIMARY KEY,
+ text VARCHAR(1000)
+);
+
+COMMIT;
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-29 11:20:18 +0000