aboutsummaryrefslogtreecommitdiff
path: root/src/main/java/io/trygvis/esper/testing/gitorious
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/io/trygvis/esper/testing/gitorious')
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java4
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java58
-rwxr-xr-x[-rw-r--r--]src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java77
3 files changed, 74 insertions, 65 deletions
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java
index 7e0a1b7..b7b21d6 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousAtomFeedParser.java
@@ -2,9 +2,9 @@ package io.trygvis.esper.testing.gitorious;
import fj.data.*;
import org.apache.abdera.*;
-import org.apache.abdera.model.*;
import org.apache.abdera.model.Document;
import org.apache.abdera.model.Element;
+import org.apache.abdera.model.*;
import org.apache.abdera.parser.*;
import org.dom4j.*;
import org.dom4j.io.*;
@@ -82,7 +82,7 @@ public class GitoriousAtomFeedParser {
switch (event) {
case "created repository":
case "created branch":
- // This is similar "pushed", but doesn't contain any info on commit IDs or branches
+ // This is similar "pushed", but doesn't contain any info on commit IDs or branches
case "started development of":
return null;
case "pushed":
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
index 892d8d0..f44635f 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousClient.java
@@ -1,9 +1,13 @@
package io.trygvis.esper.testing.gitorious;
import static java.lang.System.*;
+
+import fj.data.Option;
+import org.apache.abdera.parser.ParseException;
import org.apache.commons.io.*;
import static org.apache.commons.lang.StringUtils.*;
import static org.codehaus.httpcache4j.HTTPMethod.*;
+
import org.codehaus.httpcache4j.*;
import org.codehaus.httpcache4j.cache.*;
import org.dom4j.*;
@@ -12,16 +16,18 @@ import org.dom4j.io.*;
import javax.xml.stream.*;
import java.io.*;
import java.net.*;
+import java.sql.SQLException;
import java.util.*;
public class GitoriousClient {
public static final STAXEventReader xmlReader = new STAXEventReader();
- private final HTTPCache httpCache;
public final String baseUrl;
+ private final HTTPCache http;
private final String projectsUri;
+ private final GitoriousAtomFeedParser parser = new GitoriousAtomFeedParser();
- public GitoriousClient(HTTPCache httpCache, String baseUrl) throws URISyntaxException {
- this.httpCache = httpCache;
+ public GitoriousClient(HTTPCache http, String baseUrl) throws URISyntaxException {
+ this.http = http;
this.baseUrl = new URI(baseUrl).toASCIIString();
this.projectsUri = baseUrl + "/projects.xml";
}
@@ -31,13 +37,23 @@ public class GitoriousClient {
int page = 1;
Set<GitoriousProjectXml> all = new HashSet<>();
- while (page <= 10) {
- System.out.println("Fetching projects XML, page=" + page);
+ while (true) {
+ System.out.println("Fetching projects, page=" + page);
long start = currentTimeMillis();
- HTTPRequest request = new HTTPRequest(new URI(projectsUri + "?page=" + page), GET);
- HTTPResponse response = httpCache.execute(request);
+ HTTPResponse response = http.execute(new HTTPRequest(new URI(projectsUri + "?page=" + page), GET));
long end = currentTimeMillis();
- System.out.println("Fetched XML in " + (end - start) + "ms.");
+ System.out.println("Fetched in " + (end - start) + "ms.");
+
+ if (!response.getStatus().equals(Status.OK)) {
+ System.out.println("Got non-200 status from server: " + response.getStatus());
+ break;
+ }
+
+ MIMEType mimeType = MIMEType.valueOf(trimToEmpty(response.getHeaders().getFirstHeaderValue("Content-Type")));
+ if (!mimeType.getPrimaryType().equals("application") || !mimeType.getSubType().equals("xml")) {
+ System.out.println("Unexpected mime type, probably at the end of the list: " + mimeType);
+ break;
+ }
byte[] bytes = IOUtils.toByteArray(response.getPayload().getInputStream());
try {
@@ -66,6 +82,22 @@ public class GitoriousClient {
public URI atomFeed(String projectSlug, String repositoryName) {
return URI.create(baseUrl + "/" + projectSlug + "/" + repositoryName + ".atom");
}
+
+ public Iterable<GitoriousEvent> fetchGitoriousEvents(GitoriousRepository repository, Option<Date> lastUpdate) throws SQLException, ParseException {
+ System.out.println("Fetching " + repository.atomFeed);
+
+ long start = currentTimeMillis();
+ HTTPResponse response = http.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET));
+ long end = currentTimeMillis();
+ System.out.println("Fetched in " + (end - start) + "ms");
+
+ // Use the server's timestamp
+ Date responseDate = response.getDate().toDate();
+
+ System.out.println("responseDate = " + responseDate);
+
+ return parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name);
+ }
}
class GitoriousProjectXml implements Comparable<GitoriousProjectXml> {
@@ -138,10 +170,7 @@ class GitoriousProjectXml implements Comparable<GitoriousProjectXml> {
GitoriousProjectXml that = (GitoriousProjectXml) o;
- if (!repositories.equals(that.repositories)) return false;
- if (!slug.equals(that.slug)) return false;
-
- return true;
+ return repositories.equals(that.repositories) && slug.equals(that.slug);
}
public int hashCode() {
@@ -186,10 +215,7 @@ class GitoriousRepositoryXml implements Comparable<GitoriousRepositoryXml> {
GitoriousRepositoryXml that = (GitoriousRepositoryXml) o;
- if (!name.equals(that.name)) return false;
- if (!projectSlug.equals(that.projectSlug)) return false;
-
- return true;
+ return name.equals(that.name) && projectSlug.equals(that.projectSlug);
}
public int hashCode() {
diff --git a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
index b4bc683..1e7a7fd 100644..100755
--- a/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
+++ b/src/main/java/io/trygvis/esper/testing/gitorious/GitoriousImporter.java
@@ -6,9 +6,7 @@ import static fj.data.Option.*;
import io.trygvis.esper.testing.*;
import static java.lang.System.*;
import org.apache.abdera.parser.*;
-import org.codehaus.httpcache4j.*;
-import org.codehaus.httpcache4j.cache.*;
-import org.codehaus.httpcache4j.resolver.*;
+import org.apache.commons.httpclient.protocol.*;
import java.io.*;
import java.net.*;
@@ -20,37 +18,34 @@ import java.util.Set;
import java.util.concurrent.*;
public class GitoriousImporter {
- private final GitoriousAtomFeedParser parser;
private final BoneCP boneCp;
private final GitoriousClient gitoriousClient;
- private final HTTPCache httpCache;
public static void main(String[] args) throws Exception {
- Main.configureLog4j();
- new GitoriousImporter(DbMain.JDBC_URL, "esper", "");
+ Config config = Config.loadFromDisk();
+ new GitoriousImporter(config, DbMain.JDBC_URL, "esper", "esper");
}
- public GitoriousImporter(String jdbcUrl, String jdbcUsername, String jdbcPassword) throws Exception {
- parser = new GitoriousAtomFeedParser();
+ public GitoriousImporter(Config config, final String jdbcUrl, final String jdbcUsername, final String jdbcPassword) throws Exception {
+ BoneCPConfig boneCPConfig = new BoneCPConfig(){{
+ setJdbcUrl(jdbcUrl);
+ setUsername(jdbcUsername);
+ setPassword(jdbcPassword);
+ setDefaultAutoCommit(false);
+ setMaxConnectionsPerPartition(10);
+ }};
- BoneCPConfig config = new BoneCPConfig();
- config.setJdbcUrl(jdbcUrl);
- config.setUsername(jdbcUsername);
- config.setPassword(jdbcPassword);
- config.setDefaultAutoCommit(false);
- config.setMaxConnectionsPerPartition(10);
+ boneCp = new BoneCP(boneCPConfig);
- boneCp = new BoneCP(config);
-
- httpCache = new HTTPCache(new MemoryCacheStorage(), HTTPClientResponseResolver.createMultithreadedInstance());
-
- gitoriousClient = new GitoriousClient(httpCache, "http://gitorious.org");
+ gitoriousClient = new GitoriousClient(HttpClient.createHttpClient(config), config.gitoriousUrl);
final ScheduledThreadPoolExecutor service = new ScheduledThreadPoolExecutor(2);
- boolean projectsUpdateEnabled = false;
+ boolean projectsUpdateEnabled = true;
int projectsUpdateDelay = 0 * 1000;
int projectsUpdateInterval = 60 * 1000;
+
+ boolean repositoriesUpdateEnabled = false;
int repositoriesUpdateDelay = 0;
int repositoriesUpdateInterval = 60 * 1000;
@@ -66,15 +61,17 @@ public class GitoriousImporter {
}, projectsUpdateDelay, projectsUpdateInterval, TimeUnit.MILLISECONDS);
}
- service.scheduleAtFixedRate(new Runnable() {
- public void run() {
- try {
- updateRepositories();
- } catch (Exception e) {
- e.printStackTrace(System.out);
+ if (repositoriesUpdateEnabled) {
+ service.scheduleAtFixedRate(new Runnable() {
+ public void run() {
+ try {
+ updateRepositories();
+ } catch (Exception e) {
+ e.printStackTrace(System.out);
+ }
}
- }
- }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS);
+ }, repositoriesUpdateDelay, repositoriesUpdateInterval, TimeUnit.MILLISECONDS);
+ }
}
private void discoverProjects() throws Exception {
@@ -105,8 +102,7 @@ public class GitoriousImporter {
for (GitoriousRepository repository : repoDao.selectForProject(project.slug)) {
boolean found = false;
- for (Iterator<GitoriousRepositoryXml> it = project.repositories.iterator(); it.hasNext(); ) {
- GitoriousRepositoryXml repo = it.next();
+ for (GitoriousRepositoryXml repo : project.repositories) {
if (repo.projectSlug.equals(repository.projectSlug) && repo.name.equals(repository.name)) {
found = true;
break;
@@ -125,8 +121,7 @@ public class GitoriousImporter {
for (String project : projectDao.selectSlugs()) {
boolean found = false;
- for (Iterator<GitoriousProjectXml> it = projects.iterator(); it.hasNext(); ) {
- GitoriousProjectXml p = it.next();
+ for (GitoriousProjectXml p : projects) {
if (p.slug.equals(project)) {
found = true;
break;
@@ -165,21 +160,9 @@ public class GitoriousImporter {
Option<Date> lastUpdate = repository.lastSuccessfulUpdate;
- System.out.println("Fetching " + repository.atomFeed);
-
- long start = currentTimeMillis();
- HTTPResponse response = httpCache.execute(new HTTPRequest(repository.atomFeed, HTTPMethod.GET));
- long end = currentTimeMillis();
- System.out.println("Fetched in " + (end - start) + "ms");
-
- // Use the server's timestamp
- Date responseDate = response.getDate().toDate();
-
- System.out.println("responseDate = " + responseDate);
-
- List<GitoriousEvent> events;
+ Iterable<GitoriousEvent> events;
try {
- events = parser.parseStream(response.getPayload().getInputStream(), lastUpdate, repository.projectSlug, repository.name);
+ events = gitoriousClient.fetchGitoriousEvents(repository, lastUpdate);
} catch (ParseException e) {
repositoryDao.updateTimestamp(repository.projectSlug, repository.name, new Timestamp(currentTimeMillis()), Option.<Date>none());
System.out.println("Error parsing " + repository.atomFeed);
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-02 17:16:25 +0000