diff options
8 files changed, 269 insertions, 191 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java index 4714a87..05faa9a 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/Dao.java @@ -1,5 +1,7 @@ package io.trygvis.esper.testing.gitorious; +import fj.*; + import java.sql.*; public class Dao { @@ -12,4 +14,16 @@ public class Dao { protected PreparedStatement prepareStatement(String sql) throws SQLException { return c.prepareStatement(sql); } + + public static final F<Timestamp, java.util.Date> timestampToDate = new F<Timestamp, java.util.Date>() { + public java.util.Date f(Timestamp timestamp) { + return new java.util.Date(timestamp.getTime()); + } + }; + + public static final F<java.util.Date, Timestamp> dateToTimestamp = new F<java.util.Date, Timestamp>() { + public Timestamp f(java.util.Date date) { + return new Timestamp(date.getTime()); + } + }; } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java index a4f300f..a58f1de 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -2,6 +2,7 @@ package io.trygvis.esper.testing.gitorious; import static java.lang.System.*; import org.apache.commons.io.*; +import static org.apache.commons.lang.StringUtils.*; import static org.codehaus.httpcache4j.HTTPMethod.*; import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; @@ -16,20 +17,20 @@ import java.util.*; public class GitoriousClient { public static final STAXEventReader xmlReader = new STAXEventReader(); private final HTTPCache httpCache; - private final String gitoriousUrl; + public final String baseUrl; private final String projectsUri; - public GitoriousClient(HTTPCache httpCache, String gitoriousUrl) throws URISyntaxException { + public GitoriousClient(HTTPCache httpCache, String baseUrl) throws URISyntaxException { this.httpCache = httpCache; - this.gitoriousUrl = new URI(gitoriousUrl).toASCIIString(); - this.projectsUri = gitoriousUrl + "/projects.xml"; + this.baseUrl = new URI(baseUrl).toASCIIString(); + this.projectsUri = baseUrl + "/projects.xml"; } - public Set<GitoriousProject> findProjects() throws Exception { + public Set<GitoriousProjectXml> findProjects() throws Exception { System.out.println("Fetching all projects"); int page = 1; - Set<GitoriousProject> all = new HashSet<>(); + Set<GitoriousProjectXml> all = new HashSet<>(); while (page <= 10) { System.out.println("Fetching projects XML, page=" + page); long start = currentTimeMillis(); @@ -42,7 +43,7 @@ public class GitoriousClient { try { Document doc = xmlReader.readDocument(new ByteArrayInputStream(bytes)); - List<GitoriousProject> list = GitoriousProject.projectsFromXml(gitoriousUrl, doc.getRootElement()); + List<GitoriousProjectXml> list = GitoriousProjectXml.projectsFromXml(doc.getRootElement()); // This indicates the last page. if (list.size() == 0) { @@ -61,4 +62,138 @@ public class GitoriousClient { return all; } + + public URI atomFeed(String slug) { + return URI.create(baseUrl + "/" + slug + ".atom"); + } +} + +class GitoriousProjectXml implements Comparable<GitoriousProjectXml> { + public final String slug; + public final List<GitoriousRepositoryXml> repositories; + + public GitoriousProjectXml(String slug, List<GitoriousRepositoryXml> repositories) { + this.slug = slug; + this.repositories = repositories; + } + + public static GitoriousProjectXml fromXml(Element project) throws URISyntaxException { + String slug = trimToNull(project.elementText("slug")); + + if (slug == null) { + System.out.println("Missing slug"); + return null; + } + + Element repositories = project.element("repositories"); + if (repositories == null) { + System.out.println("Missing <repositories>"); + return null; + } + + Element mainlines = repositories.element("mainlines"); + if (mainlines == null) { + System.out.println("Missing <mainlines>"); + return null; + } + + List<Element> list = (List<Element>) mainlines.elements("repository"); + List<GitoriousRepositoryXml> repositoryList = new ArrayList<>(list.size()); + for (Element repository : list) { + GitoriousRepositoryXml r = GitoriousRepositoryXml.fromXml(slug, repository); + + if (r == null) { + continue; + } + + repositoryList.add(r); + } + + return new GitoriousProjectXml(slug, repositoryList); + } + + public static List<GitoriousProjectXml> projectsFromXml(Element root) throws URISyntaxException { + List<GitoriousProjectXml> projects = new ArrayList<>(); + for (Element project : (List<Element>) root.elements("project")) { + + GitoriousProjectXml p = GitoriousProjectXml.fromXml(project); + if (p == null) { + System.out.println(project.toString()); + continue; + } + projects.add(p); + } + + return projects; + } + + public int compareTo(GitoriousProjectXml other) { + return slug.compareTo(other.slug); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousProjectXml)) return false; + + GitoriousProjectXml that = (GitoriousProjectXml) o; + + if (!repositories.equals(that.repositories)) return false; + if (!slug.equals(that.slug)) return false; + + return true; + } + + public int hashCode() { + int result = slug.hashCode(); + result = 31 * result + repositories.hashCode(); + return result; + } +} + +class GitoriousRepositoryXml implements Comparable<GitoriousRepositoryXml> { + public final String projectSlug; + public final String name; + + GitoriousRepositoryXml(String projectSlug, String name) { + this.projectSlug = projectSlug; + this.name = name; + } + + public static GitoriousRepositoryXml fromXml(String project, Element element) throws URISyntaxException { + String name = trimToNull(element.elementText("name")); + + if (name == null) { + return null; + } + + return new GitoriousRepositoryXml(project, name); + } + + public int compareTo(GitoriousRepositoryXml o) { + int a = projectSlug.compareTo(o.projectSlug); + + if (a != 0) { + return a; + } + + return name.compareTo(o.name); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousRepositoryXml)) return false; + + GitoriousRepositoryXml that = (GitoriousRepositoryXml) o; + + if (!name.equals(that.name)) return false; + if (!projectSlug.equals(that.projectSlug)) return false; + + return true; + } + + public int hashCode() { + int result = projectSlug.hashCode(); + result = 31 * result + name.hashCode(); + return result; + } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java index 6264bc7..4cd0916 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java @@ -1,22 +1,23 @@ package io.trygvis.esper.testing.gitorious; import com.jolbox.bonecp.*; -import fj.*; +import fj.data.*; +import static fj.data.Option.*; import io.trygvis.esper.testing.*; import static java.lang.System.*; import org.apache.abdera.*; import org.apache.abdera.model.*; import org.apache.abdera.parser.*; -import org.apache.abdera.protocol.client.*; import org.codehaus.httpcache4j.*; import org.codehaus.httpcache4j.cache.*; import org.codehaus.httpcache4j.client.*; import java.io.*; -import java.net.*; import java.sql.*; import java.util.Date; import java.util.*; +import java.util.List; +import java.util.Set; import java.util.concurrent.*; public class GitoriousImporter { @@ -27,31 +28,20 @@ public class GitoriousImporter { public static void main(String[] args) throws Exception { Main.configureLog4j(); - new GitoriousImporter(); + new GitoriousImporter(DbMain.JDBC_URL, "esper", ""); } - public GitoriousImporter() throws Exception { + public GitoriousImporter(String jdbcUrl, String jdbcUsername, String jdbcPassword) throws Exception { Abdera abdera = new Abdera(); parser = abdera.getParser(); BoneCPConfig config = new BoneCPConfig(); - config.setJdbcUrl(DbMain.JDBC_URL); - config.setUsername("esper"); - config.setPassword(""); + config.setJdbcUrl(jdbcUrl); + config.setUsername(jdbcUsername); + config.setPassword(jdbcPassword); config.setDefaultAutoCommit(false); config.setMaxConnectionsPerPartition(10); -// config.setConnectionHook(new AbstractConnectionHook() { -// public void onAcquire(ConnectionHandle c) { -// try { -// System.out.println("New SQL connection."); -// c.setDebugHandle(new Daos(c)); -// } catch (SQLException e) {connections -// throw new RuntimeException(e); -// } -// } -// }); - boneCp = new BoneCP(config); httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance()); @@ -87,7 +77,7 @@ public class GitoriousImporter { } private void discoverProjects() throws Exception { - Set<GitoriousProject> projects = gitoriousClient.findProjects(); + Set<GitoriousProjectXml> projects = gitoriousClient.findProjects(); long start = currentTimeMillis(); try (Daos daos = Daos.lookup(boneCp)) { @@ -95,25 +85,35 @@ public class GitoriousImporter { GitoriousProjectDao projectDao = daos.gitoriousProjectDao; System.out.println("Processing " + projects.size() + " projects."); - for (GitoriousProject project : projects) { + for (GitoriousProjectXml project : projects) { if (projectDao.countProjects(project.slug) == 0) { System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories."); - projectDao.insertProject(project); - for (GitoriousRepository repository : project.repositories) { - repoDao.insertRepository(repository); + projectDao.insertProject(project.slug); + for (GitoriousRepositoryXml repository : project.repositories) { + repoDao.insertRepository(repository.projectSlug, repository.name, gitoriousClient.atomFeed(project.slug)); } } else { - for (GitoriousRepository repository : project.repositories) { - if (repoDao.countRepositories(repository) == 0) { + for (GitoriousRepositoryXml repository : project.repositories) { + if (repoDao.countRepositories(repository.projectSlug, repository.name) == 0) { System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name); - repoDao.insertRepository(repository); + repoDao.insertRepository(repository.projectSlug, repository.name, gitoriousClient.atomFeed(project.slug)); } } for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) { - if (project.repositories.contains(repository)) { + boolean found = false; + for (Iterator<GitoriousRepositoryXml> it = project.repositories.iterator(); it.hasNext(); ) { + GitoriousRepositoryXml repo = it.next(); + if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) { + found = true; + break; + } + } + + if (found) { continue; } + System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name); repoDao.delete(repository); } @@ -122,8 +122,8 @@ public class GitoriousImporter { for (String project : projectDao.selectSlugs()) { boolean found = false; - for (Iterator<GitoriousProject> it = projects.iterator(); it.hasNext(); ) { - GitoriousProject p = it.next(); + for (Iterator<GitoriousProjectXml> it = projects.iterator(); it.hasNext(); ) { + GitoriousProjectXml p = it.next(); if (p.slug.equals(project)) { found = true; break; @@ -147,29 +147,25 @@ public class GitoriousImporter { private void updateRepositories() throws SQLException, IOException { try (Daos daos = Daos.lookup(boneCp)) { - List<P2<String, URI>> list = daos.gitoriousProjectDao.selectFeeds(); + List<GitoriousRepository> list = daos.gitoriousRepositoryDao.select(); System.out.println("Updating " + list.size() + " feeds."); - for (P2<String, URI> pair : list) { - updateFeed(daos, pair._1(), pair._2()); + for (GitoriousRepository repository : list) { + updateFeed(daos, repository); daos.commit(); } } } - private void updateFeed(Daos daos, String slug, URI uri) throws SQLException { - AtomDao atomDao = daos.atomDao; + private void updateFeed(Daos daos, GitoriousRepository repository) throws SQLException { + GitoriousRepositoryDao repositoryDao = daos.gitoriousRepositoryDao; GitoriousEventDao eventDao = daos.gitoriousEventDao; - Timestamp lastUpdate = atomDao.getAtomFeed(uri); + Option<Date> lastUpdate = repository.lastUpdate; - System.out.println("Fetching " + uri); - RequestOptions options = new RequestOptions(); - if (lastUpdate != null) { - options.setIfModifiedSince(lastUpdate); - } + System.out.println("Fetching " + repository.atomFeed); long start = currentTimeMillis(); - HTTPResponse response = httpCache.execute(new HTTPRequest(uri, HTTPMethod.GET)); + HTTPResponse response = httpCache.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET)); long end = currentTimeMillis(); System.out.println("Fetched in " + (end - start) + "ms"); @@ -178,11 +174,12 @@ public class GitoriousImporter { System.out.println("responseDate = " + responseDate); - Document<Element> document = null; + Document<Element> document; try { document = parser.parse(response.getPayload().getInputStream()); } catch (ParseException e) { - System.out.println("Error parsing " + uri); + repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none()); + System.out.println("Error parsing " + repository.atomFeed); e.printStackTrace(System.out); return; } @@ -199,25 +196,19 @@ public class GitoriousImporter { continue; } - if (lastUpdate != null && lastUpdate.after(published)) { - System.out.println("Old entry: " + uri + ":" + entryId); + if (lastUpdate.isSome() && lastUpdate.some().after(published)) { + System.out.println("Old entry: " + repository.atomFeed + ":" + entryId); continue; } - System.out.println("New entry: " + uri + ":" + entryId); if (eventDao.countEntryId(entryId) == 0) { + System.out.println("New entry: " + repository.atomFeed + ":" + entryId); eventDao.insertChange(entryId, title); } else { System.out.println("Already imported entry: " + entryId); } } - if (lastUpdate == null) { - System.out.println("New atom feed"); - atomDao.insertAtomFeed(uri, new Timestamp(responseDate.getTime())); - } else { - System.out.println("Updating atom feed"); - atomDao.updateAtomFeed(uri, lastUpdate); - } + repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), some(new Date())); } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java index 1ccfac3..068866e 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java @@ -1,70 +1,15 @@ package io.trygvis.esper.testing.gitorious; -import static org.apache.commons.lang.StringUtils.*; -import org.dom4j.*; +import fj.data.*; import java.net.*; import java.util.*; public class GitoriousProject implements Comparable<GitoriousProject> { public final String slug; - public final URI atomFeed; - public final List<GitoriousRepository> repositories; - public GitoriousProject(String slug, URI atomFeed, List<GitoriousRepository> repositories) { + public GitoriousProject(String slug) { this.slug = slug; - this.atomFeed = atomFeed; - this.repositories = repositories; - } - - public static GitoriousProject fromXml(String gitoriousUrl, Element project) throws URISyntaxException { - String slug = trimToNull(project.elementText("slug")); - - if (slug == null) { - System.out.println("Missing slug"); - return null; - } - - Element repositories = project.element("repositories"); - if (repositories == null) { - System.out.println("Missing <repositories>"); - return null; - } - - Element mainlines = repositories.element("mainlines"); - if (mainlines == null) { - System.out.println("Missing <mainlines>"); - return null; - } - - List<Element> list = (List<Element>) mainlines.elements("repository"); - List<GitoriousRepository> repositoryList = new ArrayList<>(list.size()); - for (Element repository : list) { - GitoriousRepository r = GitoriousRepository.fromXml(gitoriousUrl, slug, repository); - - if (r == null) { - continue; - } - - repositoryList.add(r); - } - - return new GitoriousProject(slug, URI.create(gitoriousUrl + "/" + slug + ".atom"), repositoryList); - } - - public static List<GitoriousProject> projectsFromXml(String gitoriousUrl, Element root) throws URISyntaxException { - List<GitoriousProject> projects = new ArrayList<>(); - for (Element project : (List<Element>) root.elements("project")) { - - GitoriousProject p = GitoriousProject.fromXml(gitoriousUrl, project); - if (p == null) { - System.out.println(project.toString()); - continue; - } - projects.add(p); - } - - return projects; } public int compareTo(GitoriousProject other) { @@ -77,7 +22,6 @@ public class GitoriousProject implements Comparable<GitoriousProject> { GitoriousProject that = (GitoriousProject) o; - if (!repositories.equals(that.repositories)) return false; if (!slug.equals(that.slug)) return false; return true; @@ -85,7 +29,6 @@ public class GitoriousProject implements Comparable<GitoriousProject> { public int hashCode() { int result = slug.hashCode(); - result = 31 * result + repositories.hashCode(); return result; } } @@ -93,20 +36,16 @@ public class GitoriousProject implements Comparable<GitoriousProject> { class GitoriousRepository implements Comparable<GitoriousRepository> { public final String projectSlug; public final String name; + public final URI atomFeed; + public final Option<Date> lastUpdate; + public final Option<Date> lastSuccessfulUpdate; - GitoriousRepository(String projectSlug, String name) { + GitoriousRepository(String projectSlug, String name, URI atomFeed, Option<Date> lastUpdate, Option<Date> lastSuccessfulUpdate) { this.projectSlug = projectSlug; this.name = name; - } - - public static GitoriousRepository fromXml(String gitoriousUrl, String project, Element element) throws URISyntaxException { - String name = trimToNull(element.elementText("name")); - - if (name == null) { - return null; - } - - return new GitoriousRepository(project, name); + this.atomFeed = atomFeed; + this.lastUpdate = lastUpdate; + this.lastSuccessfulUpdate = lastSuccessfulUpdate; } public int compareTo(GitoriousRepository o) { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java index 432d154..262d23c 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDao.java @@ -1,8 +1,5 @@ package io.trygvis.esper.testing.gitorious; -import fj.*; - -import java.net.*; import java.sql.*; import java.util.*; @@ -21,18 +18,17 @@ public class GitoriousProjectDao extends Dao { } } - private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug, atom_feed) VALUES(?, ?)"); + private final PreparedStatement insertProject = prepareStatement("INSERT INTO gitorious_project(slug) VALUES(?)"); - public void insertProject(GitoriousProject project) throws SQLException { - insertProject.setString(1, project.slug); - insertProject.setString(2, project.atomFeed.toASCIIString()); + public void insertProject(String slug) throws SQLException { + insertProject.setString(1, slug); insertProject.executeUpdate(); } - private final PreparedStatement selectAll = prepareStatement("SELECT slug FROM gitorious_project"); + private final PreparedStatement selectSlugs = prepareStatement("SELECT slug FROM gitorious_project"); public List<String> selectSlugs() throws SQLException { - try (ResultSet rs = selectAll.executeQuery()) { + try (ResultSet rs = selectSlugs.executeQuery()) { List<String> list = new ArrayList<>(); while (rs.next()) { list.add(rs.getString(1)); @@ -41,18 +37,6 @@ public class GitoriousProjectDao extends Dao { } } - private final PreparedStatement selectFeeds = prepareStatement("SELECT slug, atom_feed FROM gitorious_project"); - - public List<P2<String, URI>> selectFeeds() throws SQLException { - try (ResultSet rs = selectFeeds.executeQuery()) { - List<P2<String, URI>> list = new ArrayList<>(); - while (rs.next()) { - list.add(P.p(rs.getString(1), URI.create(rs.getString(2)))); - } - return list; - } - } - private final PreparedStatement delete = prepareStatement("DELETE FROM gitorious_project WHERE slug=?"); public void delete(String slug) throws SQLException { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java index 5f57441..bcb27de 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java @@ -1,43 +1,67 @@ package io.trygvis.esper.testing.gitorious; +import fj.data.*; + import java.net.*; import java.sql.*; import java.util.*; +import java.util.Date; +import java.util.List; public class GitoriousRepositoryDao extends Dao { public GitoriousRepositoryDao(Connection c) throws SQLException { super(c); } + private static final String ALL_FIELDS = "project_slug, name, atom_feed, last_update, last_successful_update"; + + private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException { + try (ResultSet rs = statement.executeQuery()) { + List<GitoriousRepository> list = new ArrayList<>(); + + while (rs.next()) { + list.add(new GitoriousRepository( + rs.getString(1), + rs.getString(2), + URI.create(rs.getString(3)), + Option.fromNull(rs.getTimestamp(4)).map(timestampToDate), + Option.fromNull(rs.getTimestamp(5)).map(timestampToDate))); + } + + return list; + } + } + private final PreparedStatement countRepositories = prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?"); - public int countRepositories(GitoriousRepository repository) throws SQLException { - countRepositories.setString(1, repository.projectSlug); - countRepositories.setString(2, repository.name); + public int countRepositories(String projectSlug, String name) throws SQLException { + countRepositories.setString(1, projectSlug); + countRepositories.setString(2, name); try (ResultSet rs = countRepositories.executeQuery()) { rs.next(); return rs.getInt(1); } } - private final PreparedStatement selectForProject = prepareStatement("SELECT project_slug, name FROM gitorious_repository WHERE project_slug=?"); + private final PreparedStatement selectForProject = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository WHERE project_slug=?"); public List<GitoriousRepository> selectForProject(String projectSlug) throws SQLException { selectForProject.setString(1, projectSlug); return executeQuery(selectForProject); } - private final PreparedStatement selectAll = prepareStatement("SELECT project_slug, name FROM gitorious_repository"); + private final PreparedStatement select = prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository"); - public List<GitoriousRepository> selectAll() throws SQLException { - return executeQuery(selectAll); + public List<GitoriousRepository> select() throws SQLException { + return executeQuery(select); } - private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name) VALUES(?, ?)"); + private final PreparedStatement insertRepository = prepareStatement("INSERT INTO gitorious_repository(project_slug, name, atom_feed) VALUES(?, ?, ?)"); - public void insertRepository(GitoriousRepository repository) throws SQLException { - insertRepository.setString(1, repository.projectSlug); - insertRepository.setString(2, repository.name); + public void insertRepository(String projectSlug, String name, URI atomFeed) throws SQLException { + insertRepository.setString(1, projectSlug); + insertRepository.setString(2, name); + insertRepository.setString(3, atomFeed.toASCIIString()); insertRepository.executeUpdate(); } @@ -56,18 +80,13 @@ public class GitoriousRepositoryDao extends Dao { deleteForProject.executeUpdate(); } - private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException { - try (ResultSet rs = statement.executeQuery()) { - List<GitoriousRepository> list = new ArrayList<>(); - - while (rs.next()) { - list.add(new GitoriousRepository( - rs.getString(1), - rs.getString(2) - )); - } + private final PreparedStatement updateTimestamp = prepareStatement("UPDATE gitorious_repository SET last_update=?, last_successful_update=? WHERE project_slug=? AND name=?"); - return list; - } + public void updateTimestamp(String projectName, String slug, Timestamp lastUpdate, Option<Date> lastSuccessfulUpdate) throws SQLException { + updateTimestamp.setTimestamp(1, lastUpdate); + updateTimestamp.setTimestamp(2, lastSuccessfulUpdate.map(dateToTimestamp).toNull()); + updateTimestamp.setString(3, slug); + updateTimestamp.setString(4, projectName); + updateTimestamp.executeUpdate(); } } diff --git a/src/main/resources/ddl.sql b/src/main/resources/ddl.sql index 279e544..4e641cc 100644 --- a/src/main/resources/ddl.sql +++ b/src/main/resources/ddl.sql @@ -3,28 +3,24 @@ BEGIN; DROP TABLE IF EXISTS gitorious_event; DROP TABLE IF EXISTS gitorious_repository; DROP TABLE IF EXISTS gitorious_project; -DROP TABLE IF EXISTS atom_feed; - -CREATE TABLE atom_feed ( - url VARCHAR(1000) PRIMARY KEY, - last_update TIMESTAMP NOT NULL -); CREATE TABLE gitorious_project ( - slug VARCHAR(1000) PRIMARY KEY, - atom_feed VARCHAR(1000) NOT NULL + slug VARCHAR(1000) PRIMARY KEY ); CREATE TABLE gitorious_repository ( - project_slug VARCHAR(1000) NOT NULL, - name VARCHAR(1000) NOT NULL, - CONSTRAINT gitorious_repository_pk PRIMARY KEY(project_slug, name), - CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY(project_slug) REFERENCES gitorious_project(slug) + project_slug VARCHAR(1000) NOT NULL, + name VARCHAR(1000) NOT NULL, + atom_feed VARCHAR(1000) NOT NULL, + last_update TIMESTAMP, + last_successful_update TIMESTAMP, + CONSTRAINT gitorious_repository_pk PRIMARY KEY (project_slug, name), + CONSTRAINT gitorious_repository_2_gitorious_project FOREIGN KEY (project_slug) REFERENCES gitorious_project (slug) ); CREATE TABLE gitorious_event ( - entry_id VARCHAR(1000) PRIMARY KEY, - text VARCHAR(1000) + entry_id VARCHAR(1000) PRIMARY KEY, + text VARCHAR(1000) ); COMMIT; diff --git a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java index 63ae319..2d5945e 100644 --- a/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java +++ b/src/test/java/io/trygvis/esper/testing/gitorious/TestXmlParsing.java @@ -11,12 +11,12 @@ public class TestXmlParsing extends TestCase { try (InputStream stream = getClass().getResourceAsStream("/gitorious/projects-2.xml")) { Document document = GitoriousClient.xmlReader.readDocument(stream); - List<GitoriousProject> projects = GitoriousProject.projectsFromXml("http://gitorious.org", document.getRootElement()); + List<GitoriousProjectXml> projects = GitoriousProjectXml.projectsFromXml(document.getRootElement()); assertNotNull(projects); assertEquals(20, projects.size()); - GitoriousProject project = projects.get(3); + GitoriousProjectXml project = projects.get(3); assertEquals("aed-ii", project.slug); assertEquals(2, project.repositories.size()); } |