From ddbf5d1ac05d0f03f7cbe9275c303541cca945be Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Thu, 8 Nov 2012 17:01:10 +0100 Subject: wip --- .../io/trygvis/esper/testing/ResourceManager.java | 62 ++++++++++++--- .../esper/testing/gitorious/GitoriousClient.java | 6 +- .../esper/testing/gitorious/GitoriousImporter.java | 88 +++++++++++++++------- .../esper/testing/gitorious/GitoriousProject.java | 18 ++++- 4 files changed, 133 insertions(+), 41 deletions(-) (limited to 'src/main') diff --git a/src/main/java/io/trygvis/esper/testing/ResourceManager.java b/src/main/java/io/trygvis/esper/testing/ResourceManager.java index e9a0068..dff19bd 100644 --- a/src/main/java/io/trygvis/esper/testing/ResourceManager.java +++ b/src/main/java/io/trygvis/esper/testing/ResourceManager.java @@ -1,31 +1,73 @@ package io.trygvis.esper.testing; -import fj.*; - +import java.io.*; import java.util.*; import java.util.concurrent.*; -public class ResourceManager { - private final Equal equal; - private final Callable> discoverer; +public class ResourceManager, V> implements Closeable { + private final ResourceManagerCallbacks callbacks; + private final ScheduledFuture future; + private Map map = Collections.emptyMap(); - public ResourceManager(Equal equal, ScheduledExecutorService executorService, int delay, Callable> discoverer) { - this.equal = equal; - this.discoverer = discoverer; + public interface ResourceManagerCallbacks, V> { + Set discover() throws Exception; + + V onNew(K key); + + void onGone(K key, V value); + } + + public ResourceManager(ScheduledExecutorService executorService, int delay, + ResourceManagerCallbacks callbacks) { + this.callbacks = callbacks; - executorService.scheduleWithFixedDelay(new Runnable() { + future = executorService.scheduleWithFixedDelay(new Runnable() { public void run() { work(); } }, delay, delay, TimeUnit.MILLISECONDS); + } private void work() { try { - List keys = discoverer.call(); + System.out.println("Discovering..."); + Set keys = callbacks.discover(); + + Set found = new HashSet<>(); + + for (K key : keys) { + if (map.containsKey(key)) { + continue; + } + + found.add(key); + } + + Set lost = new HashSet<>(map.keySet()); + lost.retainAll(found); + + System.out.println("Discovered " + keys.size() + " keys, new: " + found.size() + ", gone=" + lost.size()); + + Map newMap = new HashMap<>(found.size()); + + for (K k : found) { + newMap.put(k, callbacks.onNew(k)); + } + + for (K k : lost) { + callbacks.onGone(k, map.get(k)); + } + + map = newMap; } catch (Exception e) { + e.printStackTrace(); return; } } + + public void close() { + future.cancel(true); + } } diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java index 9479faa..a4f300f 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java @@ -25,12 +25,12 @@ public class GitoriousClient { this.projectsUri = gitoriousUrl + "/projects.xml"; } - public List findProjects() throws Exception { + public Set findProjects() throws Exception { System.out.println("Fetching all projects"); int page = 1; - List all = new ArrayList<>(); - while (true) { + Set all = new HashSet<>(); + while (page <= 10) { System.out.println("Fetching projects XML, page=" + page); long start = currentTimeMillis(); HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET); diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java index 05dfe43..c77d7db 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java @@ -1,6 +1,7 @@ package io.trygvis.esper.testing.gitorious; import io.trygvis.esper.testing.*; +import io.trygvis.esper.testing.ResourceManager.*; import org.apache.abdera.*; import org.apache.abdera.model.*; import org.apache.abdera.protocol.client.*; @@ -11,48 +12,83 @@ import org.codehaus.httpcache4j.client.*; import java.sql.*; import java.util.Date; import java.util.*; +import java.util.concurrent.*; public class GitoriousImporter { private final AbderaClient abderaClient; - private final Connection connection; + private final Connection c; private final AtomDao atomDao; private final GitoriousDao gitoriousDao; - public GitoriousImporter(AbderaClient abderaClient, Connection c) throws SQLException { - this.abderaClient = abderaClient; - this.connection = c; - atomDao = new AtomDao(c); - gitoriousDao = new GitoriousDao(c); - } - public static void main(String[] args) throws Exception { Main.configureLog4j(); + new GitoriousImporter(); + } + + public GitoriousImporter() throws Exception { Abdera abdera = new Abdera(); - AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); + abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000)); - Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); - connection.setAutoCommit(false); + c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", ""); + c.setAutoCommit(false); + + atomDao = new AtomDao(c); + gitoriousDao = new GitoriousDao(c); HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance()); - GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); + final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org"); - List projects = gitoriousClient.findProjects(); +// Set projects = gitoriousClient.findProjects(); +// +// System.out.println("projects.size() = " + projects.size()); +// for (GitoriousProject project : projects) { +// System.out.println("project.repositories = " + project.repositories); +// } - System.out.println("projects.size() = " + projects.size()); - for (GitoriousProject project : projects) { - System.out.println("project.repositories = " + project.repositories); - } +// new GitoriousImporter(abderaClient, c).work(); -// new GitoriousImporter(abderaClient, connection).work(); -// -// ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); -// -// new ResourceManager(Equal.anyEqual(), service, 1000, new Callable>() { -// public List call() throws Exception { -// -// } -// }); + final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1); + + int projectsUpdateInterval = 1000; + final int projectUpdateInterval = 1000; + + ResourceManager gitoriousProjects = new ResourceManager<>(service, 1000, + + new ResourceManagerCallbacks() { + public Set discover() throws Exception { + return gitoriousClient.findProjects(); + } + + public GitoriousProjectResourceManager onNew(GitoriousProject key) { + return new GitoriousProjectResourceManager(service, projectUpdateInterval, key); + } + + public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) { + System.out.println("Project gone."); + manager.close(); + } + }); + ; + } + + class GitoriousProjectResourceManager extends ResourceManager { + + public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) { + super(executorService, delay, new ResourceManagerCallbacks() { + public Set discover() throws Exception { + key + } + + public GitoriousRepository onNew(GitoriousRepository key) { + throw new RuntimeException("Not implemented"); + } + + public void onGone(GitoriousRepository key, GitoriousRepository value) { + throw new RuntimeException("Not implemented"); + } + }); + } } private void work() throws SQLException, InterruptedException { diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java index 725b678..e5b3fdd 100644 --- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java +++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java @@ -6,7 +6,7 @@ import org.dom4j.*; import java.net.*; import java.util.*; -public class GitoriousProject { +public class GitoriousProject implements Comparable { public final String slug; public final List repositories; @@ -64,9 +64,13 @@ public class GitoriousProject { return projects; } + + public int compareTo(GitoriousProject other) { + return slug.compareTo(other.slug); + } } -class GitoriousRepository { +class GitoriousRepository implements Comparable { public final String project; public final String name; public final URI atom; @@ -86,4 +90,14 @@ class GitoriousRepository { return new GitoriousRepository(project, name, new URI(gitoriousUrl + "/" + project + "/" + name + ".atom")); } + + public int compareTo(GitoriousRepository o) { + int a = project.compareTo(o.project); + + if (a != 0) { + return a; + } + + return name.compareTo(o.name); + } } -- cgit v1.2.3