diff options
Diffstat (limited to 'src/main/java/io/trygvis/esper/testing/gitorious')
3 files changed, 81 insertions, 31 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java index 9479faa..a4f300f 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -25,12 +25,12 @@ public class GitoriousClient { this.projectsUri = gitoriousUrl + "/projects.xml"; } - public List<GitoriousProject> findProjects() throws Exception { + public Set<GitoriousProject> findProjects() throws Exception { System.out.println("Fetching all projects"); int page = 1; - List<GitoriousProject> all = new ArrayList<>(); - while (true) { + Set<GitoriousProject> all = new HashSet<>(); + while (page <= 10) { System.out.println("Fetching projects XML, page=" + page); long start = currentTimeMillis(); HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET); diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java index 05dfe43..c77d7db 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java @@ -1,6 +1,7 @@ package io.trygvis.esper.testing.gitorious; import io.trygvis.esper.testing.*; +import io.trygvis.esper.testing.ResourceManager.*; import org.apache.abdera.*; import org.apache.abdera.model.*; import org.apache.abdera.protocol.client.*; @@ -11,48 +12,83 @@ import org.codehaus.httpcache4j.client.*; import java.sql.*; import java.util.Date; import java.util.*; +import java.util.concurrent.*; public class GitoriousImporter { private final AbderaClient abderaClient; - private final Connection connection; + private final Connection c; private final AtomDao atomDao; private final GitoriousDao gitoriousDao; - public GitoriousImporter(AbderaClient abderaClient, Connection c) throws SQLException { - this.abderaClient = abderaClient; - this.connection = c; - atomDao = new AtomDao(c); - gitoriousDao = new GitoriousDao(c); - } - public static void main(String[] args) throws Exception { Main.configureLog4j(); + new GitoriousImporter(); + } + + public GitoriousImporter() throws Exception { Abdera abdera = new Abdera(); - AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); + abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); - Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); - connection.setAutoCommit(false); + c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); + c.setAutoCommit(false); + + atomDao = new AtomDao(c); + gitoriousDao = new GitoriousDao(c); HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance()); - GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); + final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); - List<GitoriousProject> projects = gitoriousClient.findProjects(); +// Set<GitoriousProject> projects = gitoriousClient.findProjects(); +// +// System.out.println("projects.size() = " + projects.size()); +// for (GitoriousProject project : projects) { +// System.out.println("project.repositories = " + project.repositories); +// } - System.out.println("projects.size() = " + projects.size()); - for (GitoriousProject project : projects) { - System.out.println("project.repositories = " + project.repositories); - } +// new GitoriousImporter(abderaClient, c).work(); -// new GitoriousImporter(abderaClient, connection).work(); -// -// ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); -// -// new ResourceManager<URL, URL>(Equal.<URL>anyEqual(), service, 1000, new Callable<List<URL>>() { -// public List<URL> call() throws Exception { -// -// } -// }); + final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); + + int projectsUpdateInterval = 1000; + final int projectUpdateInterval = 1000; + + ResourceManager<GitoriousProject, GitoriousProjectResourceManager> gitoriousProjects = new ResourceManager<>(service, 1000, + + new ResourceManagerCallbacks<GitoriousProject, GitoriousProjectResourceManager>() { + public Set<GitoriousProject> discover() throws Exception { + return gitoriousClient.findProjects(); + } + + public GitoriousProjectResourceManager onNew(GitoriousProject key) { + return new GitoriousProjectResourceManager(service, projectUpdateInterval, key); + } + + public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) { + System.out.println("Project gone."); + manager.close(); + } + }); + ; + } + + class GitoriousProjectResourceManager extends ResourceManager<GitoriousRepository, GitoriousRepository> { + + public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) { + super(executorService, delay, new ResourceManagerCallbacks<GitoriousRepository, GitoriousRepository>() { + public Set<GitoriousRepository> discover() throws Exception { + key + } + + public GitoriousRepository onNew(GitoriousRepository key) { + throw new RuntimeException("Not implemented"); + } + + public void onGone(GitoriousRepository key, GitoriousRepository value) { + throw new RuntimeException("Not implemented"); + } + }); + } } private void work() throws SQLException, InterruptedException { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java index 725b678..e5b3fdd 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java @@ -6,7 +6,7 @@ import org.dom4j.*; import java.net.*; import java.util.*; -public class GitoriousProject { +public class GitoriousProject implements Comparable<GitoriousProject> { public final String slug; public final List<GitoriousRepository> repositories; @@ -64,9 +64,13 @@ public class GitoriousProject { return projects; } + + public int compareTo(GitoriousProject other) { + return slug.compareTo(other.slug); + } } -class GitoriousRepository { +class GitoriousRepository implements Comparable<GitoriousRepository> { public final String project; public final String name; public final URI atom; @@ -86,4 +90,14 @@ class GitoriousRepository { return new GitoriousRepository(project, name, new URI(gitoriousUrl + "/" + project + "/" + name + ".atom")); } + + public int compareTo(GitoriousRepository o) { + int a = project.compareTo(o.project); + + if (a != 0) { + return a; + } + + return name.compareTo(o.name); + } } |