aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/Dao.java14
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java149
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java99
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java79
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java26
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java65
-rw-r--r--src/main/resources/ddl.sql24
-rw-r--r--src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java4
8 files changed, 269 insertions, 191 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
index 4714a87..05faa9a 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java
@@ -1,5 +1,7 @@
package io.trygvis.esper.testing.gitorious;
+import fj.*;
+
import java.sql.*;
public class Dao {
@@ -12,4 +14,16 @@ public class Dao {
protected PreparedStatement prepareStatement(String sql) throws SQLException {
return c.prepareStatement(sql);
}
+
+ public static final F<Timestamp, java.util.Date> timestampToDate = new F<Timestamp, java.util.Date>() {
+ public java.util.Date f(Timestamp timestamp) {
+ return new java.util.Date(timestamp.getTime());
+ }
+ };
+
+ public static final F<java.util.Date, Timestamp> dateToTimestamp = new F<java.util.Date, Timestamp>() {
+ public Timestamp f(java.util.Date date) {
+ return new Timestamp(date.getTime());
+ }
+ };
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
index a4f300f..a58f1de 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -2,6 +2,7 @@ package io.trygvis.esper.testing.gitorious;
import static java.lang.System.*;
import org.apache.commons.io.*;
+import static org.apache.commons.lang.StringUtils.*;
import static org.codehaus.httpcache4j.HTTPMethod.*;
import org.codehaus.httpcache4j.*;
import org.codehaus.httpcache4j.cache.*;
@@ -16,20 +17,20 @@ import java.util.*;
public class GitoriousClient {
public static final STAXEventReader xmlReader = new STAXEventReader();
private final HTTPCache httpCache;
- private final String gitoriousUrl;
+ public final String baseUrl;
private final String projectsUri;
- public GitoriousClient(HTTPCache httpCache, String gitoriousUrl) throws URISyntaxException {
+ public GitoriousClient(HTTPCache httpCache, String baseUrl) throws URISyntaxException {
this.httpCache = httpCache;
- this.gitoriousUrl = new URI(gitoriousUrl).toASCIIString();
- this.projectsUri = gitoriousUrl + "/projects.xml";
+ this.baseUrl = new URI(baseUrl).toASCIIString();
+ this.projectsUri = baseUrl + "/projects.xml";
}
- public Set<GitoriousProject> findProjects() throws Exception {
+ public Set<GitoriousProjectXml> findProjects() throws Exception {
System.out.println("Fetching all projects");
int page = 1;
- Set<GitoriousProject> all = new HashSet<>();
+ Set<GitoriousProjectXml> all = new HashSet<>();
while (page <= 10) {
System.out.println("Fetching projects XML, page=" + page);
long start = currentTimeMillis();
@@ -42,7 +43,7 @@ public class GitoriousClient {
try {
Document doc = xmlReader.readDocument(new ByteArrayInputStream(bytes));
- List<GitoriousProject> list = GitoriousProject.projectsFromXml(gitoriousUrl, doc.getRootElement());
+ List<GitoriousProjectXml> list = GitoriousProjectXml.projectsFromXml(doc.getRootElement());
// This indicates the last page.
if (list.size() == 0) {
@@ -61,4 +62,138 @@ public class GitoriousClient {
return all;
}
+
+ public URI atomFeed(String slug) {
+ return URI.create(baseUrl + "/" + slug + ".atom");
+ }
+}
+
+class GitoriousProjectXml implements Comparable<GitoriousProjectXml> {
+ public final String slug;
+ public final List<GitoriousRepositoryXml> repositories;
+
+ public GitoriousProjectXml(String slug, List<GitoriousRepositoryXml> repositories) {
+ this.slug = slug;
+ this.repositories = repositories;
+ }
+
+ public static GitoriousProjectXml fromXml(Element project) throws URISyntaxException {
+ String slug = trimToNull(project.elementText("slug"));
+
+ if (slug == null) {
+ System.out.println("Missing slug");
+ return null;
+ }
+
+ Element repositories = project.element("repositories");
+ if (repositories == null) {
+ System.out.println("Missing <repositories>");
+ return null;
+ }
+
+ Element mainlines = repositories.element("mainlines");
+ if (mainlines == null) {
+ System.out.println("Missing <mainlines>");
+ return null;
+ }
+
+ List<Element> list = (List<Element>) mainlines.elements("repository");
+ List<GitoriousRepositoryXml> repositoryList = new ArrayList<>(list.size());
+ for (Element repository : list) {
+ GitoriousRepositoryXml r = GitoriousRepositoryXml.fromXml(slug, repository);
+
+ if (r == null) {
+ continue;
+ }
+
+ repositoryList.add(r);
+ }
+
+ return new GitoriousProjectXml(slug, repositoryList);
+ }
+
+ public static List<GitoriousProjectXml> projectsFromXml(Element root) throws URISyntaxException {
+ List<GitoriousProjectXml> projects = new ArrayList<>();
+ for (Element project : (List<Element>) root.elements("project")) {
+
+ GitoriousProjectXml p = GitoriousProjectXml.fromXml(project);
+ if (p == null) {
+ System.out.println(project.toString());
+ continue;
+ }
+ projects.add(p);
+ }
+
+ return projects;
+ }
+
+ public int compareTo(GitoriousProjectXml other) {
+ return slug.compareTo(other.slug);
+ }
+
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof GitoriousProjectXml)) return false;
+
+ GitoriousProjectXml that = (GitoriousProjectXml) o;
+
+ if (!repositories.equals(that.repositories)) return false;
+ if (!slug.equals(that.slug)) return false;
+
+ return true;
+ }
+
+ public int hashCode() {
+ int result = slug.hashCode();
+ result = 31 * result + repositories.hashCode();
+ return result;
+ }
+}
+
+class GitoriousRepositoryXml implements Comparable<GitoriousRepositoryXml> {
+ public final String projectSlug;
+ public final String name;
+
+ GitoriousRepositoryXml(String projectSlug, String name) {
+ this.projectSlug = projectSlug;
+ this.name = name;
+ }
+
+ public static GitoriousRepositoryXml fromXml(String project, Element element) throws URISyntaxException {
+ String name = trimToNull(element.elementText("name"));
+
+ if (name == null) {
+ return null;
+ }
+
+ return new GitoriousRepositoryXml(project, name);
+ }
+
+ public int compareTo(GitoriousRepositoryXml o) {
+ int a = projectSlug.compareTo(o.projectSlug);
+
+ if (a != 0) {
+ return a;
+ }
+
+ return name.compareTo(o.name);
+ }
+
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof GitoriousRepositoryXml)) return false;
+
+ GitoriousRepositoryXml that = (GitoriousRepositoryXml) o;
+
+ if (!name.equals(that.name)) return false;
+ if (!projectSlug.equals(that.projectSlug)) return false;
+
+ return true;
+ }
+
+ public int hashCode() {
+ int result = projectSlug.hashCode();
+ result = 31 * result + name.hashCode();
+ return result;
+ }
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index 6264bc7..4cd0916 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -1,22 +1,23 @@
package io.trygvis.esper.testing.gitorious;
import com.jolbox.bonecp.*;
-import fj.*;
+import fj.data.*;
+import static fj.data.Option.*;
import io.trygvis.esper.testing.*;
import static java.lang.System.*;
import org.apache.abdera.*;
import org.apache.abdera.model.*;
import org.apache.abdera.parser.*;
-import org.apache.abdera.protocol.client.*;
import org.codehaus.httpcache4j.*;
import org.codehaus.httpcache4j.cache.*;
import org.codehaus.httpcache4j.client.*;
import java.io.*;
-import java.net.*;
import java.sql.*;
import java.util.Date;
import java.util.*;
+import java.util.List;
+import java.util.Set;
import java.util.concurrent.*;
public class GitoriousImporter {
@@ -27,31 +28,20 @@ public class GitoriousImporter {
public static void main(String[] args) throws Exception {
Main.configureLog4j();
- new GitoriousImporter();
+ new GitoriousImporter(DbMain.JDBC_URL, "esper", "");
}
- public GitoriousImporter() throws Exception {
+ public GitoriousImporter(String jdbcUrl, String jdbcUsername, String jdbcPassword) throws Exception {
Abdera abdera = new Abdera();
parser = abdera.getParser();
BoneCPConfig config = new BoneCPConfig();
- config.setJdbcUrl(DbMain.JDBC_URL);
- config.setUsername("esper");
- config.setPassword("");
+ config.setJdbcUrl(jdbcUrl);
+ config.setUsername(jdbcUsername);
+ config.setPassword(jdbcPassword);
config.setDefaultAutoCommit(false);
config.setMaxConnectionsPerPartition(10);
-// config.setConnectionHook(new AbstractConnectionHook() {
-// public void onAcquire(ConnectionHandle c) {
-// try {
-// System.out.println("New SQL connection.");
-// c.setDebugHandle(new Daos(c));
-// } catch (SQLException e) {connections
-// throw new RuntimeException(e);
-// }
-// }
-// });
-
boneCp = new BoneCP(config);
httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
@@ -87,7 +77,7 @@ public class GitoriousImporter {
}
private void discoverProjects() throws Exception {
- Set<GitoriousProject> projects = gitoriousClient.findProjects();
+ Set<GitoriousProjectXml> projects = gitoriousClient.findProjects();
long start = currentTimeMillis();
try (Daos daos = Daos.lookup(boneCp)) {
@@ -95,25 +85,35 @@ public class GitoriousImporter {
GitoriousProjectDao projectDao = daos.gitoriousProjectDao;
System.out.println("Processing " + projects.size() + " projects.");
- for (GitoriousProject project : projects) {
+ for (GitoriousProjectXml project : projects) {
if (projectDao.countProjects(project.slug) == 0) {
System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories.");
- projectDao.insertProject(project);
- for (GitoriousRepository repository : project.repositories) {
- repoDao.insertRepository(repository);
+ projectDao.insertProject(project.slug);
+ for (GitoriousRepositoryXml repository : project.repositories) {
+ repoDao.insertRepository(repository.projectSlug, repository.name, gitoriousClient.atomFeed(project.slug));
}
} else {
- for (GitoriousRepository repository : project.repositories) {
- if (repoDao.countRepositories(repository) == 0) {
+ for (GitoriousRepositoryXml repository : project.repositories) {
+ if (repoDao.countRepositories(repository.projectSlug, repository.name) == 0) {
System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name);
- repoDao.insertRepository(repository);
+ repoDao.insertRepository(repository.projectSlug, repository.name, gitoriousClient.atomFeed(project.slug));
}
}
for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) {
- if (project.repositories.contains(repository)) {
+ boolean found = false;
+ for (Iterator<GitoriousRepositoryXml> it = project.repositories.iterator(); it.hasNext(); ) {
+ GitoriousRepositoryXml repo = it.next();
+ if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) {
+ found = true;
+ break;
+ }
+ }
+
+ if (found) {
continue;
}
+
System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name);
repoDao.delete(repository);
}
@@ -122,8 +122,8 @@ public class GitoriousImporter {
for (String project : projectDao.selectSlugs()) {
boolean found = false;
- for (Iterator<GitoriousProject> it = projects.iterator(); it.hasNext(); ) {
- GitoriousProject p = it.next();
+ for (Iterator<GitoriousProjectXml> it = projects.iterator(); it.hasNext(); ) {
+ GitoriousProjectXml p = it.next();
if (p.slug.equals(project)) {
found = true;
break;
@@ -147,29 +147,25 @@ public class GitoriousImporter {
private void updateRepositories() throws SQLException, IOException {
try (Daos daos = Daos.lookup(boneCp)) {
- List<P2<String, URI>> list = daos.gitoriousProjectDao.selectFeeds();
+ List<GitoriousRepository> list = daos.gitoriousRepositoryDao.select();
System.out.println("Updating " + list.size() + " feeds.");
- for (P2<String, URI> pair : list) {
- updateFeed(daos, pair._1(), pair._2());
+ for (GitoriousRepository repository : list) {
+ updateFeed(daos, repository);
daos.commit();
}
}
}
- private void updateFeed(Daos daos, String slug, URI uri) throws SQLException {
- AtomDao atomDao = daos.atomDao;
+ private void updateFeed(Daos daos, GitoriousRepository repository) throws SQLException {
+ GitoriousRepositoryDao repositoryDao = daos.gitoriousRepositoryDao;
GitoriousEventDao eventDao = daos.gitoriousEventDao;
- Timestamp lastUpdate = atomDao.getAtomFeed(uri);
+ Option<Date> lastUpdate = repository.lastUpdate;
- System.out.println("Fetching " + uri);
- RequestOptions options = new RequestOptions();
- if (lastUpdate != null) {
- options.setIfModifiedSince(lastUpdate);
- }
+ System.out.println("Fetching " + repository.atomFeed);
long start = currentTimeMillis();
- HTTPResponse response = httpCache.execute(new HTTPRequest(uri, HTTPMethod.GET));
+ HTTPResponse response = httpCache.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET));
long end = currentTimeMillis();
System.out.println("Fetched in " + (end - start) + "ms");
@@ -178,11 +174,12 @@ public class GitoriousImporter {
System.out.println("responseDate = " + responseDate);
- Document<Element> document = null;
+ Document<Element> document;
try {
document = parser.parse(response.getPayload().getInputStream());
} catch (ParseException e) {
- System.out.println("Error parsing " + uri);
+ repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none());
+ System.out.println("Error parsing " + repository.atomFeed);
e.printStackTrace(System.out);
return;
}
@@ -199,25 +196,19 @@ public class GitoriousImporter {
continue;
}
- if (lastUpdate != null && lastUpdate.after(published)) {
- System.out.println("Old entry: " + uri + ":" + entryId);
+ if (lastUpdate.isSome() && lastUpdate.some().after(published)) {
+ System.out.println("Old entry: " + repository.atomFeed + ":" + entryId);
continue;
}
- System.out.println("New entry: " + uri + ":" + entryId);
if (eventDao.countEntryId(entryId) == 0) {
+ System.out.println("New entry: " + repository.atomFeed + ":" + entryId);
eventDao.insertChange(entryId, title);
} else {
System.out.println("Already imported entry: " + entryId);
}
}
- if (lastUpdate == null) {
- System.out.println("New atom feed");
- atomDao.insertAtomFeed(uri, new Timestamp(responseDate.getTime()));
- } else {
- System.out.println("Updating atom feed");
- atomDao.updateAtomFeed(uri, lastUpdate);
- }
+ repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), some(new Date()));
}
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
index 1ccfac3..068866e 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
@@ -1,70 +1,15 @@
package io.trygvis.esper.testing.gitorious;
-import static org.apache.commons.lang.StringUtils.*;
-import org.dom4j.*;
+import fj.data.*;
import java.net.*;
import java.util.*;
public class GitoriousProject implements Comparable<GitoriousProject> {
public final String slug;
- public final URI atomFeed;
- public final List<GitoriousRepository> repositories;
- public GitoriousProject(String slug, URI atomFeed, List<GitoriousRepository> repositories) {
+ public GitoriousProject(String slug) {
this.slug = slug;
- this.atomFeed = atomFeed;
- this.repositories = repositories;
- }
-
- public static GitoriousProject fromXml(String gitoriousUrl, Element project) throws URISyntaxException {
- String slug = trimToNull(project.elementText("slug"));
-
- if (slug == null) {
- System.out.println("Missing slug");
- return null;
- }
-
- Element repositories = project.element("repositories");
- if (repositories == null) {
- System.out.println("Missing <repositories>");
- return null;
- }
-
- Element mainlines = repositories.element("mainlines");
- if (mainlines == null) {
- System.out.println("Missing <mainlines>");
- return null;
- }
-
- List<Element> list = (List<Element>) mainlines.elements("repository");
- List<GitoriousRepository> repositoryList = new ArrayList<>(list.size());
- for (Element repository : list) {
- GitoriousRepository r = GitoriousRepository.fromXml(gitoriousUrl, slug, repository);
-
- if (r == null) {
- continue;
- }
-
- repositoryList.add(r);
- }
-
- return new GitoriousProject(slug, URI.create(gitoriousUrl + "/" + slug + ".atom"), repositoryList);
- }
-
- public static List<GitoriousProject> projectsFromXml(String gitoriousUrl, Element root) throws URISyntaxException {
- List<GitoriousProject> projects = new ArrayList<>();
- for (Element project : (List<Element>) root.elements("project")) {
-
- GitoriousProject p = GitoriousProject.fromXml(gitoriousUrl, project);
- if (p == null) {
- System.out.println(project.toString());
- continue;
- }
- projects.add(p);
- }
-
- return projects;
}
public int compareTo(GitoriousProject other) {
@@ -77,7 +22,6 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
GitoriousProject that = (GitoriousProject) o;
- if (!repositories.equals(that.repositories)) return false;
if (!slug.equals(that.slug)) return false;
return true;
@@ -85,7 +29,6 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
public int hashCode() {
int result = slug.hashCode();
- result = 31 * result + repositories.hashCode();
return result;
}
}
@@ -93,20 +36,16 @@ public class GitoriousProject implements Comparable<GitoriousProject> {
class GitoriousRepository implements Comparable<GitoriousRepository> {
public final String projectSlug;
public final String name;
+ public final URI atomFeed;
+ public final Option<Date> lastUpdate;
+ public final Option<Date> lastSuccessfulUpdate;
- GitoriousRepository(String projectSlug, String name) {
+ GitoriousRepository(String projectSlug, String name, URI atomFeed, Option<Date> lastUpdate, Option<Date> lastSuccessfulUpdate) {
this.projectSlug = projectSlug;
this.name = name;
- }
-
- public static GitoriousRepository fromXml(String gitoriousUrl, String project, Element element) throws URISyntaxException {
- String name = trimToNull(element.elementText("name"));
-
- if (name == null) {
- return null;
- }
-
- return new GitoriousRepository(project, name);
+ this.atomFeed = atomFeed;
+ this.lastUpdate = lastUpdate;
+ this.lastSuccessfulUpdate = lastSuccessfulUpdate;
}
public int compareTo(GitoriousRepository o) {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
index 432d154..262d23c 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java
@@ -1,8 +1,5 @@
package io.trygvis.esper.testing.gitorious;
-import fj.*;
-
-import java.net.*;
import java.sql.*;
import java.util.*;
@@ -21,18 +18,17 @@ public class GitoriousProjectDao extends Dao {
}
}
- private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug, atom_feed) VALUES(?, ?)");
+ private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)");
- public void insertProject(GitoriousProject project) throws SQLException {
- insertProject.setString(1, project.slug);
- insertProject.setString(2, project.atomFeed.toASCIIString());
+ public void insertProject(String slug) throws SQLException {
+ insertProject.setString(1, slug);
insertProject.executeUpdate();
}
- private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project");
+ private final PreparedStatement selectSlugs = prepareStatement("SELECT slug FROM gitorious_project");
public List<String> selectSlugs() throws SQLException {
- try (ResultSet rs = selectAll.executeQuery()) {
+ try (ResultSet rs = selectSlugs.executeQuery()) {
List<String> list = new ArrayList<>();
while (rs.next()) {
list.add(rs.getString(1));
@@ -41,18 +37,6 @@ public class GitoriousProjectDao extends Dao {
}
}
- private final PreparedStatement selectFeeds = prepareStatement("SELECT slug, atom_feed FROM gitorious_project");
-
- public List<P2<String, URI>> selectFeeds() throws SQLException {
- try (ResultSet rs = selectFeeds.executeQuery()) {
- List<P2<String, URI>> list = new ArrayList<>();
- while (rs.next()) {
- list.add(P.p(rs.getString(1), URI.create(rs.getString(2))));
- }
- return list;
- }
- }
-
private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?");
public void delete(String slug) throws SQLException {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
index 5f57441..bcb27de 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java
@@ -1,43 +1,67 @@
package io.trygvis.esper.testing.gitorious;
+import fj.data.*;
+
import java.net.*;
import java.sql.*;
import java.util.*;
+import java.util.Date;
+import java.util.List;
public class GitoriousRepositoryDao extends Dao {
public GitoriousRepositoryDao(Connection c) throws SQLException {
super(c);
}
+ private static final String ALL_FIELDS = "project_slug, name, atom_feed, last_update, last_successful_update";
+
+ private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException {
+ try (ResultSet rs = statement.executeQuery()) {
+ List<GitoriousRepository> list = new ArrayList<>();
+
+ while (rs.next()) {
+ list.add(new GitoriousRepository(
+ rs.getString(1),
+ rs.getString(2),
+ URI.create(rs.getString(3)),
+ Option.fromNull(rs.getTimestamp(4)).map(timestampToDate),
+ Option.fromNull(rs.getTimestamp(5)).map(timestampToDate)));
+ }
+
+ return list;
+ }
+ }
+
private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?");
- public int countRepositories(GitoriousRepository repository) throws SQLException {
- countRepositories.setString(1, repository.projectSlug);
- countRepositories.setString(2, repository.name);
+ public int countRepositories(String projectSlug, String name) throws SQLException {
+ countRepositories.setString(1, projectSlug);
+ countRepositories.setString(2, name);
try (ResultSet rs = countRepositories.executeQuery()) {
rs.next();
return rs.getInt(1);
}
}
- private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name FROM gitorious_repository WHERE project_slug=?");
+ private final PreparedStatement selectForProject = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository WHERE project_slug=?");
public List<GitoriousRepository> selectForProject(String projectSlug) throws SQLException {
selectForProject.setString(1, projectSlug);
return executeQuery(selectForProject);
}
- private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name FROM gitorious_repository");
+ private final PreparedStatement select = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository");
- public List<GitoriousRepository> selectAll() throws SQLException {
- return executeQuery(selectAll);
+ public List<GitoriousRepository> select() throws SQLException {
+ return executeQuery(select);
}
- private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name) VALUES(?, ?)");
+ private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)");
- public void insertRepository(GitoriousRepository repository) throws SQLException {
- insertRepository.setString(1, repository.projectSlug);
- insertRepository.setString(2, repository.name);
+ public void insertRepository(String projectSlug, String name, URI atomFeed) throws SQLException {
+ insertRepository.setString(1, projectSlug);
+ insertRepository.setString(2, name);
+ insertRepository.setString(3, atomFeed.toASCIIString());
insertRepository.executeUpdate();
}
@@ -56,18 +80,13 @@ public class GitoriousRepositoryDao extends Dao {
deleteForProject.executeUpdate();
}
- private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException {
- try (ResultSet rs = statement.executeQuery()) {
- List<GitoriousRepository> list = new ArrayList<>();
-
- while (rs.next()) {
- list.add(new GitoriousRepository(
- rs.getString(1),
- rs.getString(2)
- ));
- }
+ private final PreparedStatement updateTimestamp = prepareStatement("UPDATE gitorious_repository SET last_update=?, last_successful_update=? WHERE project_slug=? AND name=?");
- return list;
- }
+ public void updateTimestamp(String projectName, String slug, Timestamp lastUpdate, Option<Date> lastSuccessfulUpdate) throws SQLException {
+ updateTimestamp.setTimestamp(1, lastUpdate);
+ updateTimestamp.setTimestamp(2, lastSuccessfulUpdate.map(dateToTimestamp).toNull());
+ updateTimestamp.setString(3, slug);
+ updateTimestamp.setString(4, projectName);
+ updateTimestamp.executeUpdate();
}
}
diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql
index 279e544..4e641cc 100644
--- a/src/main/resources/ddl.sql
+++ b/src/main/resources/ddl.sql
@@ -3,28 +3,24 @@ BEGIN;
DROP TABLE IF EXISTS gitorious_event;
DROP TABLE IF EXISTS gitorious_repository;
DROP TABLE IF EXISTS gitorious_project;
-DROP TABLE IF EXISTS atom_feed;
-
-CREATE TABLE atom_feed (
- url VARCHAR(1000) PRIMARY KEY,
- last_update TIMESTAMP NOT NULL
-);
CREATE TABLE gitorious_project (
- slug VARCHAR(1000) PRIMARY KEY,
- atom_feed VARCHAR(1000) NOT NULL
+ slug VARCHAR(1000) PRIMARY KEY
);
CREATE TABLE gitorious_repository (
- project_slug VARCHAR(1000) NOT NULL,
- name VARCHAR(1000) NOT NULL,
- CONSTRAINT gitorious_repository_pk PRIMARY KEY(project_slug, name),
- CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY(project_slug) REFERENCES gitorious_project(slug)
+ project_slug VARCHAR(1000) NOT NULL,
+ name VARCHAR(1000) NOT NULL,
+ atom_feed VARCHAR(1000) NOT NULL,
+ last_update TIMESTAMP,
+ last_successful_update TIMESTAMP,
+ CONSTRAINT gitorious_repository_pk PRIMARY KEY (project_slug, name),
+ CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY (project_slug) REFERENCES gitorious_project (slug)
);
CREATE TABLE gitorious_event (
- entry_id VARCHAR(1000) PRIMARY KEY,
- text VARCHAR(1000)
+ entry_id VARCHAR(1000) PRIMARY KEY,
+ text VARCHAR(1000)
);
COMMIT;
diff --git a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java
index 63ae319..2d5945e 100644
--- a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java
+++ b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java
@@ -11,12 +11,12 @@ public class TestXmlParsing extends TestCase {
try (InputStream stream = getClass().getResourceAsStream("/gitorious/projects-2.xml")) {
Document document = GitoriousClient.xmlReader.readDocument(stream);
- List<GitoriousProject> projects = GitoriousProject.projectsFromXml("http://gitorious.org", document.getRootElement());
+ List<GitoriousProjectXml> projects = GitoriousProjectXml.projectsFromXml(document.getRootElement());
assertNotNull(projects);
assertEquals(20, projects.size());
- GitoriousProject project = projects.get(3);
+ GitoriousProjectXml project = projects.get(3);
assertEquals("aed-ii", project.slug);
assertEquals(2, project.repositories.size());
}