aboutsummaryrefslogtreecommitdiff
path: root/src/main/java/io/trygvis/esper
diff options
context:
space:
mode:
authorTrygve Laugstøl <trygvis@inamo.no>2012-11-08 13:20:47 +0100
committerTrygve Laugstøl <trygvis@inamo.no>2012-11-08 13:20:47 +0100
commitd6a532c420a93b211a9747c5fb807a3f2767fa22 (patch)
tree572bc13c87a726cbc7132043cc1d10af795f442b /src/main/java/io/trygvis/esper
parentff34cdc13bc8ba0943cded8009b9869a455894f6 (diff)
downloadesper-testing-d6a532c420a93b211a9747c5fb807a3f2767fa22.tar.gz
esper-testing-d6a532c420a93b211a9747c5fb807a3f2767fa22.tar.bz2
esper-testing-d6a532c420a93b211a9747c5fb807a3f2767fa22.tar.xz
esper-testing-d6a532c420a93b211a9747c5fb807a3f2767fa22.zip
o Slurping down the entire list of projects.
Diffstat (limited to 'src/main/java/io/trygvis/esper')
-rw-r--r--src/main/java/io/trygvis/esper/testing/ResourceManager.java31
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java64
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java (renamed from src/main/java/io/trygvis/esper/testing/GitoriousDao.java)2
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java (renamed from src/main/java/io/trygvis/esper/testing/GitoriousImporter.java)36
-rw-r--r--src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java89
5 files changed, 214 insertions, 8 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/ResourceManager.java b/src/main/java/io/trygvis/esper/testing/ResourceManager.java
new file mode 100644
index 0000000..e9a0068
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/ResourceManager.java
@@ -0,0 +1,31 @@
+package io.trygvis.esper.testing;
+
+import fj.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+public class ResourceManager<K, V> {
+ private final Equal<K> equal;
+ private final Callable<List<K>> discoverer;
+ private Map<K, V> map = Collections.emptyMap();
+
+ public ResourceManager(Equal<K> equal, ScheduledExecutorService executorService, int delay, Callable<List<K>> discoverer) {
+ this.equal = equal;
+ this.discoverer = discoverer;
+
+ executorService.scheduleWithFixedDelay(new Runnable() {
+ public void run() {
+ work();
+ }
+ }, delay, delay, TimeUnit.MILLISECONDS);
+ }
+
+ private void work() {
+ try {
+ List<K> keys = discoverer.call();
+ } catch (Exception e) {
+ return;
+ }
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
new file mode 100644
index 0000000..9479faa
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -0,0 +1,64 @@
+package io.trygvis.esper.testing.gitorious;
+
+import static java.lang.System.*;
+import org.apache.commons.io.*;
+import static org.codehaus.httpcache4j.HTTPMethod.*;
+import org.codehaus.httpcache4j.*;
+import org.codehaus.httpcache4j.cache.*;
+import org.dom4j.*;
+import org.dom4j.io.*;
+
+import javax.xml.stream.*;
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class GitoriousClient {
+ public static final STAXEventReader xmlReader = new STAXEventReader();
+ private final HTTPCache httpCache;
+ private final String gitoriousUrl;
+ private final String projectsUri;
+
+ public GitoriousClient(HTTPCache httpCache, String gitoriousUrl) throws URISyntaxException {
+ this.httpCache = httpCache;
+ this.gitoriousUrl = new URI(gitoriousUrl).toASCIIString();
+ this.projectsUri = gitoriousUrl + "/projects.xml";
+ }
+
+ public List<GitoriousProject> findProjects() throws Exception {
+ System.out.println("Fetching all projects");
+ int page = 1;
+
+ List<GitoriousProject> all = new ArrayList<>();
+ while (true) {
+ System.out.println("Fetching projects XML, page=" + page);
+ long start = currentTimeMillis();
+ HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET);
+ HTTPResponse response = httpCache.execute(request);
+ long end = currentTimeMillis();
+ System.out.println("Fetched XML in " + (end - start) + "ms.");
+
+ byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream());
+ try {
+ Document doc = xmlReader.readDocument(new ByteArrayInputStream(bytes));
+
+ List<GitoriousProject> list = GitoriousProject.projectsFromXml(gitoriousUrl, doc.getRootElement());
+
+ // This indicates the last page.
+ if (list.size() == 0) {
+ break;
+ }
+
+ System.out.println("Parsed out " + list.size() + " projects.");
+ all.addAll(list);
+ } catch (XMLStreamException e) {
+ System.out.println("Unable to parse XML.");
+ System.out.println(new String(bytes));
+ }
+
+ page++;
+ }
+
+ return all;
+ }
+}
diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousDao.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java
index bf5d954..766a4a9 100644
--- a/src/main/java/io/trygvis/esper/testing/GitoriousDao.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousDao.java
@@ -1,4 +1,4 @@
-package io.trygvis.esper.testing;
+package io.trygvis.esper.testing.gitorious;
import java.sql.*;
diff --git a/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index c79d4f5..05dfe43 100644
--- a/src/main/java/io/trygvis/esper/testing/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -1,12 +1,16 @@
-package io.trygvis.esper.testing;
+package io.trygvis.esper.testing.gitorious;
+import io.trygvis.esper.testing.*;
import org.apache.abdera.*;
import org.apache.abdera.model.*;
import org.apache.abdera.protocol.client.*;
import org.apache.abdera.protocol.client.cache.*;
+import org.codehaus.httpcache4j.cache.*;
+import org.codehaus.httpcache4j.client.*;
import java.sql.*;
import java.util.Date;
+import java.util.*;
public class GitoriousImporter {
private final AbderaClient abderaClient;
@@ -21,7 +25,7 @@ public class GitoriousImporter {
gitoriousDao = new GitoriousDao(c);
}
- public static void main(String[] args) throws InterruptedException, SQLException {
+ public static void main(String[] args) throws Exception {
Main.configureLog4j();
Abdera abdera = new Abdera();
AbderaClient abderaClient = new AbderaClient(abdera, new LRUCache(abdera, 1000));
@@ -29,7 +33,26 @@ public class GitoriousImporter {
Connection connection = DriverManager.getConnection(DbMain.JDBC_URL, "esper", "");
connection.setAutoCommit(false);
- new GitoriousImporter(abderaClient, connection).work();
+ HTTPCache httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
+
+ GitoriousClient gitoriousClient = new GitoriousClient(httpCache, "https://gitorious.org");
+
+ List<GitoriousProject> projects = gitoriousClient.findProjects();
+
+ System.out.println("projects.size() = " + projects.size());
+ for (GitoriousProject project : projects) {
+ System.out.println("project.repositories = " + project.repositories);
+ }
+
+// new GitoriousImporter(abderaClient, connection).work();
+//
+// ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(1);
+//
+// new ResourceManager<URL, URL>(Equal.<URL>anyEqual(), service, 1000, new Callable<List<URL>>() {
+// public List<URL> call() throws Exception {
+//
+// }
+// });
}
private void work() throws SQLException, InterruptedException {
@@ -40,7 +63,7 @@ public class GitoriousImporter {
System.out.println("Fetching " + url);
RequestOptions options = new RequestOptions();
- if(lastUpdate != null) {
+ if (lastUpdate != null) {
options.setIfModifiedSince(lastUpdate);
}
@@ -73,10 +96,9 @@ public class GitoriousImporter {
}
System.out.println("New entry: " + url + ":" + entryId);
- if(gitoriousDao.countEntryId(entryId) == 0) {
+ if (gitoriousDao.countEntryId(entryId) == 0) {
gitoriousDao.insertChange(entryId, title);
- }
- else {
+ } else {
System.out.println("Already imported entry: " + entryId);
}
}
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
new file mode 100644
index 0000000..725b678
--- /dev/null
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousProject.java
@@ -0,0 +1,89 @@
+package io.trygvis.esper.testing.gitorious;
+
+import static org.apache.commons.lang.StringUtils.*;
+import org.dom4j.*;
+
+import java.net.*;
+import java.util.*;
+
+public class GitoriousProject {
+ public final String slug;
+ public final List<GitoriousRepository> repositories;
+
+ public GitoriousProject(String slug, List<GitoriousRepository> repositories) {
+ this.slug = slug;
+ this.repositories = repositories;
+ }
+
+ public static GitoriousProject fromXml(String gitoriousUrl, Element project) throws URISyntaxException {
+ String slug = trimToNull(project.elementText("slug"));
+
+ if (slug == null) {
+ System.out.println("Missing slug");
+ return null;
+ }
+
+ Element repositories = project.element("repositories");
+ if (repositories == null) {
+ System.out.println("Missing <repositories>");
+ return null;
+ }
+
+ Element mainlines = repositories.element("mainlines");
+ if (mainlines == null) {
+ System.out.println("Missing <mainlines>");
+ return null;
+ }
+
+ List<Element> list = (List<Element>) mainlines.elements("repository");
+ List<GitoriousRepository> repositoryList = new ArrayList<>(list.size());
+ for (Element repository : list) {
+ GitoriousRepository r = GitoriousRepository.fromXml(gitoriousUrl, slug, repository);
+
+ if (r == null) {
+ continue;
+ }
+
+ repositoryList.add(r);
+ }
+
+ return new GitoriousProject(slug, repositoryList);
+ }
+
+ public static List<GitoriousProject> projectsFromXml(String gitoriousUrl, Element root) throws URISyntaxException {
+ List<GitoriousProject> projects = new ArrayList<>();
+ for (Element project : (List<Element>) root.elements("project")) {
+
+ GitoriousProject p = GitoriousProject.fromXml(gitoriousUrl, project);
+ if (p == null) {
+ System.out.println(project.toString());
+ continue;
+ }
+ projects.add(p);
+ }
+
+ return projects;
+ }
+}
+
+class GitoriousRepository {
+ public final String project;
+ public final String name;
+ public final URI atom;
+
+ GitoriousRepository(String project, String name, URI atom) {
+ this.project = project;
+ this.name = name;
+ this.atom = atom;
+ }
+
+ public static GitoriousRepository fromXml(String gitoriousUrl, String project, Element element) throws URISyntaxException {
+ String name = trimToNull(element.elementText("name"));
+
+ if (name == null) {
+ return null;
+ }
+
+ return new GitoriousRepository(project, name, new URI(gitoriousUrl + "/" + project + "/" + name + ".atom"));
+ }
+}