aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/main/java/io/trygvis/esper/testing/ResourceManager.java62
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java6
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java88
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java18
4 files changed, 133 insertions, 41 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/ResourceManager.java b/src/main/java/io/trygvis/esper/testing/ResourceManager.java
index e9a0068..dff19bd 100644
--- a/src/main/java/io/trygvis/esper/testing/ResourceManager.java
+++ b/src/main/java/io/trygvis/esper/testing/ResourceManager.java
@@ -1,31 +1,73 @@
package io.trygvis.esper.testing;
-import fj.*;
-
+import java.io.*;
import java.util.*;
import java.util.concurrent.*;
-public class ResourceManager<K, V> {
- private final Equal<K> equal;
- private final Callable<List<K>> discoverer;
+public class ResourceManager<K extends Comparable<K>, V> implements Closeable {
+ private final ResourceManagerCallbacks<K, V> callbacks;
+ private final ScheduledFuture<?> future;
+
private Map<K, V> map = Collections.emptyMap();
- public ResourceManager(Equal<K> equal, ScheduledExecutorService executorService, int delay, Callable<List<K>> discoverer) {
- this.equal = equal;
- this.discoverer = discoverer;
+ public interface ResourceManagerCallbacks<K extends Comparable<K>, V> {
+ Set<K> discover() throws Exception;
+
+ V onNew(K key);
+
+ void onGone(K key, V value);
+ }
+
+ public ResourceManager(ScheduledExecutorService executorService, int delay,
+ ResourceManagerCallbacks<K, V> callbacks) {
+ this.callbacks = callbacks;
- executorService.scheduleWithFixedDelay(new Runnable() {
+ future = executorService.scheduleWithFixedDelay(new Runnable() {
public void run() {
work();
}
}, delay, delay, TimeUnit.MILLISECONDS);
+
}
private void work() {
try {
- List<K> keys = discoverer.call();
+ System.out.println("Discovering...");
+ Set<K> keys = callbacks.discover();
+
+ Set<K> found = new HashSet<>();
+
+ for (K key : keys) {
+ if (map.containsKey(key)) {
+ continue;
+ }
+
+ found.add(key);
+ }
+
+ Set<K> lost = new HashSet<>(map.keySet());
+ lost.retainAll(found);
+
+ System.out.println("Discovered " + keys.size() + " keys, new: " + found.size() + ", gone=" + lost.size());
+
+ Map<K, V> newMap = new HashMap<>(found.size());
+
+ for (K k : found) {
+ newMap.put(k, callbacks.onNew(k));
+ }
+
+ for (K k : lost) {
+ callbacks.onGone(k, map.get(k));
+ }
+
+ map = newMap;
} catch (Exception e) {
+ e.printStackTrace();
return;
}
}
+
+ public void close() {
+ future.cancel(true);
+ }
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
index 9479faa..a4f300f 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -25,12 +25,12 @@ public class GitoriousClient {
this.projectsUri = gitoriousUrl + "/projects.xml";
}
- public List<GitoriousProject> findProjects() throws Exception {
+ public Set<GitoriousProject> findProjects() throws Exception {
System.out.println("Fetching all projects");
int page = 1;
- List<GitoriousProject> all = new ArrayList<>();
- while (true) {
+ Set<GitoriousProject> all = new HashSet<>();
+ while (page <= 10) {
System.out.println("Fetching projects XML, page=" + page);
long start = currentTimeMillis();
HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET);
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index 05dfe43..c77d7db 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -1,6 +1,7 @@
package io.trygvis.esper.testing.gitorious;
import io.trygvis.esper.testing.*;
+import io.trygvis.esper.testing.ResourceManager.*;
import org.apache.abdera.*;
import org.apache.abdera.model.*;
import org.apache.abdera.protocol.client.*;
@@ -11,48 +12,83 @@ import org.codehaus.httpcache4j.client.*;
import java.sql.*;
import java.util.Date;
import java.util.*;
+import java.util.concurrent.*;
public class GitoriousImporter {
private final AbderaClient abderaClient;
- private final Connection connection;
+ private final Connection c;
private final AtomDao atomDao;
private final GitoriousDao gitoriousDao;
- public GitoriousImporter(AbderaClient abderaClient, Connection c) throws SQLException {
- this.abderaClient = abderaClient;
- this.connection = c;
- atomDao = new AtomDao(c);
- gitoriousDao = new GitoriousDao(c);
- }
-
public static void main(String[] args) throws Exception {
Main.configureLog4j();
+ new GitoriousImporter();
+ }
+
+ public GitoriousImporter() throws Exception {
Abdera abdera = new Abdera();
- AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
+ abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
- Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
- connection.setAutoCommit(false);
+ c = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
+ c.setAutoCommit(false);
+
+ atomDao = new AtomDao(c);
+ gitoriousDao = new GitoriousDao(c);
HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
- GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
+ final GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
- List<GitoriousProject> projects = gitoriousClient.findProjects();
+// Set<GitoriousProject> projects = gitoriousClient.findProjects();
+//
+// System.out.println("projects.size() = " + projects.size());
+// for (GitoriousProject project : projects) {
+// System.out.println("project.repositories = " + project.repositories);
+// }
- System.out.println("projects.size() = " + projects.size());
- for (GitoriousProject project : projects) {
- System.out.println("project.repositories = " + project.repositories);
- }
+// new GitoriousImporter(abderaClient, c).work();
-// new GitoriousImporter(abderaClient, connection).work();
-//
-// ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1);
-//
-// new ResourceManager<URL, URL>(Equal.<URL>anyEqual(), service, 1000, new Callable<List<URL>>() {
-// public List<URL> call() throws Exception {
-//
-// }
-// });
+ final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1);
+
+ int projectsUpdateInterval = 1000;
+ final int projectUpdateInterval = 1000;
+
+ ResourceManager<GitoriousProject, GitoriousProjectResourceManager> gitoriousProjects = new ResourceManager<>(service, 1000,
+
+ new ResourceManagerCallbacks<GitoriousProject, GitoriousProjectResourceManager>() {
+ public Set<GitoriousProject> discover() throws Exception {
+ return gitoriousClient.findProjects();
+ }
+
+ public GitoriousProjectResourceManager onNew(GitoriousProject key) {
+ return new GitoriousProjectResourceManager(service, projectUpdateInterval, key);
+ }
+
+ public void onGone(GitoriousProject key, GitoriousProjectResourceManager manager) {
+ System.out.println("Project gone.");
+ manager.close();
+ }
+ });
+ ;
+ }
+
+ class GitoriousProjectResourceManager extends ResourceManager<GitoriousRepository, GitoriousRepository> {
+
+ public GitoriousProjectResourceManager(ScheduledExecutorService executorService, int delay, GitoriousProject key) {
+ super(executorService, delay, new ResourceManagerCallbacks<GitoriousRepository, GitoriousRepository>() {
+ public Set<GitoriousRepository> discover() throws Exception {
+ key
+ }
+
+ public GitoriousRepository onNew(GitoriousRepository key) {
+ throw new RuntimeException("Not implemented");
+ }
+
+ public void onGone(GitoriousRepository key, GitoriousRepository value) {
+ throw new RuntimeException("Not implemented");
+ }
+ });
+ }
}
private void work() throws SQLException, InterruptedException {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
index 725b678..e5b3fdd 100644
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
@@ -6,7 +6,7 @@ import org.dom4j.*;
import java.net.*;
import java.util.*;
-public class GitoriousProject {
+public class GitoriousProject implements Comparable<GitoriousProject> {
public final String slug;
public final List<GitoriousRepository> repositories;
@@ -64,9 +64,13 @@ public class GitoriousProject {
return projects;
}
+
+ public int compareTo(GitoriousProject other) {
+ return slug.compareTo(other.slug);
+ }
}
-class GitoriousRepository {
+class GitoriousRepository implements Comparable<GitoriousRepository> {
public final String project;
public final String name;
public final URI atom;
@@ -86,4 +90,14 @@ class GitoriousRepository {
return new GitoriousRepository(project, name, new URI(gitoriousUrl + "/" + project + "/" + name + ".atom"));
}
+
+ public int compareTo(GitoriousRepository o) {
+ int a = project.compareTo(o.project);
+
+ if (a != 0) {
+ return a;
+ }
+
+ return name.compareTo(o.name);
+ }
}