diff options
author | Trygve Laugstøl <trygvis@inamo.no> | 2012-11-27 16:24:01 +0100 |
---|---|---|
committer | Trygve Laugstøl <trygvis@inamo.no> | 2012-12-07 20:06:42 +0100 |
commit | 041bab815c5c554169835993735b8e5c35d436ed (patch) | |
tree | 473774956540013316dc10bd8428350c15f20933 /src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java | |
parent | e243a6fd6c444b451398ceb659ea4963a19122d0 (diff) | |
download | esper-testing-041bab815c5c554169835993735b8e5c35d436ed.tar.gz esper-testing-041bab815c5c554169835993735b8e5c35d436ed.tar.bz2 esper-testing-041bab815c5c554169835993735b8e5c35d436ed.tar.xz esper-testing-041bab815c5c554169835993735b8e5c35d436ed.zip |
o Switching the gitorious code to the actor structure.
Diffstat (limited to 'src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java')
-rwxr-xr-x | src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java | 172 |
1 files changed, 172 insertions, 0 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java new file mode 100755 index 0000000..a236e45 --- /dev/null +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProjectDiscovery.java @@ -0,0 +1,172 @@ +package io.trygvis.esper.testing.gitorious; + +import com.jolbox.bonecp.*; +import fj.data.Option; +import static fj.data.Option.*; +import io.trygvis.esper.testing.*; +import io.trygvis.esper.testing.object.ActorRef; +import io.trygvis.esper.testing.object.ObjectFactory; +import io.trygvis.esper.testing.object.ObjectManager; +import io.trygvis.esper.testing.object.ObjectUtil; +import io.trygvis.esper.testing.object.TransactionalActor; + +import static java.lang.System.*; +import org.apache.abdera.parser.*; + +import java.net.*; +import java.sql.*; +import java.util.*; +import java.util.Date; +import java.util.concurrent.*; + +public class GitoriousProjectDiscovery { + private final BoneCPDataSource boneCp; + private final GitoriousClient gitoriousClient; + + public static void main(String[] args) throws Exception { + Config config = Config.loadFromDisk(); + new GitoriousProjectDiscovery(config); + } + + public GitoriousProjectDiscovery(final Config config) throws Exception { + boneCp = config.createBoneCp(); + + gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitorious.url); + + final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); + + Set<GitoriousRepositoryDto> repositories = Collections.emptySet(); + try (Connection c = boneCp.getConnection()) { + repositories = new HashSet<>(new Daos(c).gitoriousRepositoryDao.select(Daos.OrderDirection.ASC)); + } catch (SQLException e) { + // ignore + } + + final ObjectManager<GitoriousRepositoryDto, ActorRef<GitoriousRepository>> repositoryManager = new ObjectManager<>("", repositories, new ObjectFactory<GitoriousRepositoryDto, ActorRef<GitoriousRepository>>() { + public ActorRef<GitoriousRepository> create(GitoriousRepositoryDto repository) { + return ObjectUtil.scheduledActorWithFixedDelay(service, 0, 60, TimeUnit.SECONDS, boneCp, new GitoriousRepository(gitoriousClient, repository)); + } + }); + + ObjectUtil.scheduledActorWithFixedDelay(service, config.gitorious.projectListUpdateDelay, config.gitorious.projectListUpdateInterval, TimeUnit.MILLISECONDS, boneCp, new TransactionalActor() { + @Override + public void act(Connection c) throws Exception { + Daos daos = new Daos(c); + discoverProjects(daos); + repositoryManager.update(daos.gitoriousRepositoryDao.select(Daos.OrderDirection.NONE)); + } + }); + } + + private void discoverProjects(Daos daos) throws Exception { + Set<GitoriousProjectXml> projects = gitoriousClient.findProjects(); + + long start = currentTimeMillis(); + GitoriousRepositoryDao repoDao = daos.gitoriousRepositoryDao; + GitoriousProjectDao projectDao = daos.gitoriousProjectDao; + + System.out.println("Processing " + projects.size() + " projects."); + for (GitoriousProjectXml project : projects) { + if (projectDao.countProjects(project.slug) == 0) { + System.out.println("New project: " + project.slug + ", has " + project.repositories.size() + " repositories."); + projectDao.insertProject(project.slug); + for (GitoriousRepositoryXml repository : project.repositories) { + URI atomFeed = gitoriousClient.atomFeed(repository.projectSlug, repository.name); + repoDao.insertRepository(repository.projectSlug, repository.name, atomFeed); + } + } else { + for (GitoriousRepositoryXml repository : project.repositories) { + if (repoDao.countRepositories(repository.projectSlug, repository.name) == 0) { + System.out.println("New repository for project " + repository.projectSlug + ": " + repository.name); + URI atomFeed = gitoriousClient.atomFeed(repository.projectSlug, repository.name); + repoDao.insertRepository(repository.projectSlug, repository.name, atomFeed); + } + } + + for (GitoriousRepositoryDto repository : repoDao.selectForProject(project.slug)) { + boolean found = false; + for (GitoriousRepositoryXml repo : project.repositories) { + if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) { + found = true; + break; + } + } + + if (found) { + continue; + } + + System.out.println("Gone repository for project " + repository.projectSlug + ": " + repository.name); + repoDao.delete(repository); + } + } + } + + for (String project : projectDao.selectSlugs()) { + boolean found = false; + for (GitoriousProjectXml p : projects) { + if (p.slug.equals(project)) { + found = true; + break; + } + } + + if (found) { + continue; + } + + System.out.println("Gone project: " + project); + repoDao.deleteForProject(project); + projectDao.delete(project); + } + + daos.commit(); + + long end = currentTimeMillis(); + System.out.println("Processed in " + (end - start) + " ms"); + } +} + +class GitoriousRepository implements TransactionalActor { + private final GitoriousClient gitoriousClient; + private final GitoriousRepositoryDto repository; + + public GitoriousRepository(GitoriousClient gitoriousClient, GitoriousRepositoryDto repository) { + this.gitoriousClient = gitoriousClient; + this.repository = repository; + } + + public void act(Connection c) throws Exception { + Daos daos = new Daos(c); + updateFeed(daos, repository); + c.commit(); + } + + private void updateFeed(Daos daos, GitoriousRepositoryDto repository) throws SQLException { + GitoriousRepositoryDao repositoryDao = daos.gitoriousRepositoryDao; + GitoriousEventDao eventDao = daos.gitoriousEventDao; + + Option<Date> lastUpdate = repository.lastSuccessfulUpdate; + + Iterable<GitoriousEvent> events; + try { + events = gitoriousClient.fetchGitoriousEvents(repository, lastUpdate); + } catch (ParseException e) { + repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none()); + System.out.println("Error parsing " + repository.atomFeed); + e.printStackTrace(System.out); + return; + } + + for (GitoriousEvent event : events) { + if (eventDao.countEntryId(event.entryId) == 0) { + System.out.println("New entry in " + repository.atomFeed + ": " + event.entryId); + eventDao.insertEvent(event); + } else { + System.out.println("Already imported entry: " + event.entryId); + } + } + + repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), some(new Date())); + } +} |