diff options
Diffstat (limited to 'src/main/java/io/trygvis/esper/testing/gitorious')
6 files changed, 264 insertions, 261 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java index f44635f..986d4c1 100755 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -83,7 +83,7 @@ public class GitoriousClient { return URI.create(baseUrl + "/" + projectSlug + "/" + repositoryName + ".atom"); } - public Iterable<GitoriousEvent> fetchGitoriousEvents(GitoriousRepository repository, Option<Date> lastUpdate) throws SQLException, ParseException { + public Iterable<GitoriousEvent> fetchGitoriousEvents(GitoriousRepositoryDto repository, Option<Date> lastUpdate) throws SQLException, ParseException { System.out.println("Fetching " + repository.atomFeed); long start = currentTimeMillis(); diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java deleted file mode 100755 index a0efda5..0000000 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ /dev/null @@ -1,174 +0,0 @@ -package io.trygvis.esper.testing.gitorious; - -import com.jolbox.bonecp.*; -import fj.data.*; -import static fj.data.Option.*; -import io.trygvis.esper.testing.*; -import static java.lang.System.*; -import org.apache.abdera.parser.*; - -import java.io.*; -import java.net.*; -import java.sql.*; -import java.util.Date; -import java.util.List; -import java.util.Set; -import java.util.concurrent.*; - -public class GitoriousImporter { - private final BoneCPDataSource boneCp; - private final GitoriousClient gitoriousClient; - - public static void main(String[] args) throws Exception { - Config config = Config.loadFromDisk(); - new GitoriousImporter(config); - } - - public GitoriousImporter(final Config config) throws Exception { - boneCp = config.createBoneCp(); - - gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitoriousUrl); - - final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(2); - - boolean projectsUpdateEnabled = true; - int projectsUpdateDelay = 0 * 1000; - int projectsUpdateInterval = 60 * 1000; - - boolean repositoriesUpdateEnabled = false; - int repositoriesUpdateDelay = 0; - int repositoriesUpdateInterval = 60 * 1000; - - if (projectsUpdateEnabled) { - service.scheduleAtFixedRate(new Runnable() { - public void run() { - try { - discoverProjects(); - } catch (Exception e) { - e.printStackTrace(System.out); - } - } - }, projectsUpdateDelay, projectsUpdateInterval, TimeUnit.MILLISECONDS); - } - - if (repositoriesUpdateEnabled) { - service.scheduleAtFixedRate(new Runnable() { - public void run() { - try { - updateRepositories(); - } catch (Exception e) { - e.printStackTrace(System.out); - } - } - }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS); - } - } - - private void discoverProjects() throws Exception { - Set<GitoriousProjectXml> projects = gitoriousClient.findProjects(); - - long start = currentTimeMillis(); - try (Daos daos = Daos.lookup(boneCp)) { - GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao; - GitoriousProjectDao projectDao = daos.gitoriousProjectDao; - - System.out.println("Processing " + projects.size() + " projects."); - for (GitoriousProjectXml project : projects) { - if (projectDao.countProjects(project.slug) == 0) { - System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories."); - projectDao.insertProject(project.slug); - for (GitoriousRepositoryXml repository : project.repositories) { - URI atomFeed = gitoriousClient.atomFeed(repository.projectSlug, repository.name); - repoDao.insertRepository(repository.projectSlug, repository.name, atomFeed); - } - } else { - for (GitoriousRepositoryXml repository : project.repositories) { - if (repoDao.countRepositories(repository.projectSlug, repository.name) == 0) { - System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name); - URI atomFeed = gitoriousClient.atomFeed(repository.projectSlug, repository.name); - repoDao.insertRepository(repository.projectSlug, repository.name, atomFeed); - } - } - - for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) { - boolean found = false; - for (GitoriousRepositoryXml repo : project.repositories) { - if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) { - found = true; - break; - } - } - - if (found) { - continue; - } - - System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name); - repoDao.delete(repository); - } - } - } - - for (String project : projectDao.selectSlugs()) { - boolean found = false; - for (GitoriousProjectXml p : projects) { - if (p.slug.equals(project)) { - found = true; - break; - } - } - - if (found) { - continue; - } - - System.out.println("Gone project: " + project); - repoDao.deleteForProject(project); - projectDao.delete(project); - } - - daos.commit(); - } - long end = currentTimeMillis(); - System.out.println("Processed in " + (end - start) + " ms"); - } - - private void updateRepositories() throws SQLException, IOException { - try (Daos daos = Daos.lookup(boneCp)) { - List<GitoriousRepository> list = daos.gitoriousRepositoryDao.select(); - System.out.println("Updating " + list.size() + " feeds."); - for (GitoriousRepository repository : list) { - updateFeed(daos, repository); - daos.commit(); - } - } - } - - private void updateFeed(Daos daos, GitoriousRepository repository) throws SQLException { - GitoriousRepositoryDao repositoryDao = daos.gitoriousRepositoryDao; - GitoriousEventDao eventDao = daos.gitoriousEventDao; - - Option<Date> lastUpdate = repository.lastSuccessfulUpdate; - - Iterable<GitoriousEvent> events; - try { - events = gitoriousClient.fetchGitoriousEvents(repository, lastUpdate); - } catch (ParseException e) { - repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none()); - System.out.println("Error parsing " + repository.atomFeed); - e.printStackTrace(System.out); - return; - } - - for (GitoriousEvent event : events) { - if (eventDao.countEntryId(event.entryId) == 0) { - System.out.println("New entry in " + repository.atomFeed + ": " + event.entryId); - eventDao.insertEvent(event); - } else { - System.out.println("Already imported entry: " + event.entryId); - } - } - - repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), some(new Date())); - } -} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java deleted file mode 100644 index 068866e..0000000 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java +++ /dev/null @@ -1,78 +0,0 @@ -package io.trygvis.esper.testing.gitorious; - -import fj.data.*; - -import java.net.*; -import java.util.*; - -public class GitoriousProject implements Comparable<GitoriousProject> { - public final String slug; - - public GitoriousProject(String slug) { - this.slug = slug; - } - - public int compareTo(GitoriousProject other) { - return slug.compareTo(other.slug); - } - - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof GitoriousProject)) return false; - - GitoriousProject that = (GitoriousProject) o; - - if (!slug.equals(that.slug)) return false; - - return true; - } - - public int hashCode() { - int result = slug.hashCode(); - return result; - } -} - -class GitoriousRepository implements Comparable<GitoriousRepository> { - public final String projectSlug; - public final String name; - public final URI atomFeed; - public final Option<Date> lastUpdate; - public final Option<Date> lastSuccessfulUpdate; - - GitoriousRepository(String projectSlug, String name, URI atomFeed, Option<Date> lastUpdate, Option<Date> lastSuccessfulUpdate) { - this.projectSlug = projectSlug; - this.name = name; - this.atomFeed = atomFeed; - this.lastUpdate = lastUpdate; - this.lastSuccessfulUpdate = lastSuccessfulUpdate; - } - - public int compareTo(GitoriousRepository o) { - int a = projectSlug.compareTo(o.projectSlug); - - if (a != 0) { - return a; - } - - return name.compareTo(o.name); - } - - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof GitoriousRepository)) return false; - - GitoriousRepository that = (GitoriousRepository) o; - - if (!name.equals(that.name)) return false; - if (!projectSlug.equals(that.projectSlug)) return false; - - return true; - } - - public int hashCode() { - int result = projectSlug.hashCode(); - result = 31 * result + name.hashCode(); - return result; - } -} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java new file mode 100755 index 0000000..a236e45 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java @@ -0,0 +1,172 @@ +package io.trygvis.esper.testing.gitorious; + +import com.jolbox.bonecp.*; +import fj.data.Option; +import static fj.data.Option.*; +import io.trygvis.esper.testing.*; +import io.trygvis.esper.testing.object.ActorRef; +import io.trygvis.esper.testing.object.ObjectFactory; +import io.trygvis.esper.testing.object.ObjectManager; +import io.trygvis.esper.testing.object.ObjectUtil; +import io.trygvis.esper.testing.object.TransactionalActor; + +import static java.lang.System.*; +import org.apache.abdera.parser.*; + +import java.net.*; +import java.sql.*; +import java.util.*; +import java.util.Date; +import java.util.concurrent.*; + +public class GitoriousProjectDiscovery { + private final BoneCPDataSource boneCp; + private final GitoriousClient gitoriousClient; + + public static void main(String[] args) throws Exception { + Config config = Config.loadFromDisk(); + new GitoriousProjectDiscovery(config); + } + + public GitoriousProjectDiscovery(final Config config) throws Exception { + boneCp = config.createBoneCp(); + + gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitorious.url); + + final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); + + Set<GitoriousRepositoryDto> repositories = Collections.emptySet(); + try (Connection c = boneCp.getConnection()) { + repositories = new HashSet<>(new Daos(c).gitoriousRepositoryDao.select(Daos.OrderDirection.ASC)); + } catch (SQLException e) { + // ignore + } + + final ObjectManager<GitoriousRepositoryDto, ActorRef<GitoriousRepository>> repositoryManager = new ObjectManager<>("", repositories, new ObjectFactory<GitoriousRepositoryDto, ActorRef<GitoriousRepository>>() { + public ActorRef<GitoriousRepository> create(GitoriousRepositoryDto repository) { + return ObjectUtil.scheduledActorWithFixedDelay(service, 0, 60, TimeUnit.SECONDS, boneCp, new GitoriousRepository(gitoriousClient, repository)); + } + }); + + ObjectUtil.scheduledActorWithFixedDelay(service, config.gitorious.projectListUpdateDelay, config.gitorious.projectListUpdateInterval, TimeUnit.MILLISECONDS, boneCp, new TransactionalActor() { + @Override + public void act(Connection c) throws Exception { + Daos daos = new Daos(c); + discoverProjects(daos); + repositoryManager.update(daos.gitoriousRepositoryDao.select(Daos.OrderDirection.NONE)); + } + }); + } + + private void discoverProjects(Daos daos) throws Exception { + Set<GitoriousProjectXml> projects = gitoriousClient.findProjects(); + + long start = currentTimeMillis(); + GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao; + GitoriousProjectDao projectDao = daos.gitoriousProjectDao; + + System.out.println("Processing " + projects.size() + " projects."); + for (GitoriousProjectXml project : projects) { + if (projectDao.countProjects(project.slug) == 0) { + System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories."); + projectDao.insertProject(project.slug); + for (GitoriousRepositoryXml repository : project.repositories) { + URI atomFeed = gitoriousClient.atomFeed(repository.projectSlug, repository.name); + repoDao.insertRepository(repository.projectSlug, repository.name, atomFeed); + } + } else { + for (GitoriousRepositoryXml repository : project.repositories) { + if (repoDao.countRepositories(repository.projectSlug, repository.name) == 0) { + System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name); + URI atomFeed = gitoriousClient.atomFeed(repository.projectSlug, repository.name); + repoDao.insertRepository(repository.projectSlug, repository.name, atomFeed); + } + } + + for (GitoriousRepositoryDto repository : repoDao.selectForProject(project.slug)) { + boolean found = false; + for (GitoriousRepositoryXml repo : project.repositories) { + if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) { + found = true; + break; + } + } + + if (found) { + continue; + } + + System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name); + repoDao.delete(repository); + } + } + } + + for (String project : projectDao.selectSlugs()) { + boolean found = false; + for (GitoriousProjectXml p : projects) { + if (p.slug.equals(project)) { + found = true; + break; + } + } + + if (found) { + continue; + } + + System.out.println("Gone project: " + project); + repoDao.deleteForProject(project); + projectDao.delete(project); + } + + daos.commit(); + + long end = currentTimeMillis(); + System.out.println("Processed in " + (end - start) + " ms"); + } +} + +class GitoriousRepository implements TransactionalActor { + private final GitoriousClient gitoriousClient; + private final GitoriousRepositoryDto repository; + + public GitoriousRepository(GitoriousClient gitoriousClient, GitoriousRepositoryDto repository) { + this.gitoriousClient = gitoriousClient; + this.repository = repository; + } + + public void act(Connection c) throws Exception { + Daos daos = new Daos(c); + updateFeed(daos, repository); + c.commit(); + } + + private void updateFeed(Daos daos, GitoriousRepositoryDto repository) throws SQLException { + GitoriousRepositoryDao repositoryDao = daos.gitoriousRepositoryDao; + GitoriousEventDao eventDao = daos.gitoriousEventDao; + + Option<Date> lastUpdate = repository.lastSuccessfulUpdate; + + Iterable<GitoriousEvent> events; + try { + events = gitoriousClient.fetchGitoriousEvents(repository, lastUpdate); + } catch (ParseException e) { + repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none()); + System.out.println("Error parsing " + repository.atomFeed); + e.printStackTrace(System.out); + return; + } + + for (GitoriousEvent event : events) { + if (eventDao.countEntryId(event.entryId) == 0) { + System.out.println("New entry in " + repository.atomFeed + ": " + event.entryId); + eventDao.insertEvent(event); + } else { + System.out.println("Already imported entry: " + event.entryId); + } + } + + repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), some(new Date())); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDto.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDto.java new file mode 100755 index 0000000..f147ec6 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDto.java @@ -0,0 +1,70 @@ +package io.trygvis.esper.testing.gitorious; + +import fj.data.*; + +import java.net.*; +import java.util.*; + +public class GitoriousProjectDto implements Comparable<GitoriousProjectDto> { + public final String slug; + + public GitoriousProjectDto(String slug) { + this.slug = slug; + } + + public int compareTo(GitoriousProjectDto other) { + return slug.compareTo(other.slug); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousProjectDto)) return false; + + GitoriousProjectDto that = (GitoriousProjectDto) o; + + return slug.equals(that.slug); + } + + public int hashCode() { + return slug.hashCode(); + } +} + +class GitoriousRepositoryDto implements Comparable<GitoriousRepositoryDto> { + public final String projectSlug; + public final String name; + public final URI atomFeed; + public final Option<Date> lastUpdate; + public final Option<Date> lastSuccessfulUpdate; + + GitoriousRepositoryDto(String projectSlug, String name, URI atomFeed, Option<Date> lastUpdate, Option<Date> lastSuccessfulUpdate) { + this.projectSlug = projectSlug; + this.name = name; + this.atomFeed = atomFeed; + this.lastUpdate = lastUpdate; + this.lastSuccessfulUpdate = lastSuccessfulUpdate; + } + + public int compareTo(GitoriousRepositoryDto o) { + int a = projectSlug.compareTo(o.projectSlug); + + if (a != 0) { + return a; + } + + return name.compareTo(o.name); + } + + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof GitoriousRepositoryDto)) return false; + + GitoriousRepositoryDto that = (GitoriousRepositoryDto) o; + + return name.equals(that.name) && projectSlug.equals(that.projectSlug); + } + + public int hashCode() { + return 31 * projectSlug.hashCode() + name.hashCode(); + } +} diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java index 2cf9fce..bdc0e4e 100644..100755 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousRepositoryDao.java @@ -1,6 +1,7 @@ package io.trygvis.esper.testing.gitorious; import fj.data.*; +import io.trygvis.esper.testing.*; import static io.trygvis.esper.testing.DaoUtil.dateToTimestamp; import static io.trygvis.esper.testing.DaoUtil.timestampToDate; @@ -19,12 +20,12 @@ public class GitoriousRepositoryDao { private static final String ALL_FIELDS = "project_slug, name, atom_feed, last_update, last_successful_update"; - private List<GitoriousRepository> executeQuery(PreparedStatement statement) throws SQLException { + private List<GitoriousRepositoryDto> executeQuery(PreparedStatement statement) throws SQLException { try (ResultSet rs = statement.executeQuery()) { - List<GitoriousRepository> list = new ArrayList<>(); + List<GitoriousRepositoryDto> list = new ArrayList<>(); while (rs.next()) { - list.add(new GitoriousRepository( + list.add(new GitoriousRepositoryDto( rs.getString(1), rs.getString(2), URI.create(rs.getString(3)), @@ -36,7 +37,6 @@ public class GitoriousRepositoryDao { } } - public int countRepositories(String projectSlug, String name) throws SQLException { try (PreparedStatement s = c.prepareStatement("SELECT count(*) FROM gitorious_repository WHERE project_slug=? and name=?")) { s.setString(1, projectSlug); @@ -48,15 +48,28 @@ public class GitoriousRepositoryDao { } } - public List<GitoriousRepository> selectForProject(String projectSlug) throws SQLException { + public List<GitoriousRepositoryDto> selectForProject(String projectSlug) throws SQLException { try (PreparedStatement s = c.prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository WHERE project_slug=?")) { s.setString(1, projectSlug); return executeQuery(s); } } - public List<GitoriousRepository> select() throws SQLException { - try (PreparedStatement s = c.prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository")) { + public List<GitoriousRepositoryDto> select(Daos.OrderDirection order) throws SQLException { + String orderBy; + + switch (order) { + case ASC: + orderBy = "ORDER BY project_slug, name"; + break; + case DESC: + orderBy = "ORDER BY project_slug DESC, name DESC"; + break; + default: + orderBy = ""; + } + + try (PreparedStatement s = c.prepareStatement("SELECT " + ALL_FIELDS + " FROM gitorious_repository " + orderBy)) { return executeQuery(s); } } @@ -70,7 +83,7 @@ public class GitoriousRepositoryDao { } } - public void delete(GitoriousRepository repository) throws SQLException { + public void delete(GitoriousRepositoryDto repository) throws SQLException { try (PreparedStatement s = c.prepareStatement("DELETE FROM gitorious_repository WHERE project_slug=? and name=?")) { s.setString(1, repository.projectSlug); s.setString(2, repository.name); |