diff options
Diffstat (limited to 'trygvis/eda/cli/add_to_db.py')
| -rw-r--r-- | trygvis/eda/cli/add_to_db.py | 33 |
1 files changed, 22 insertions, 11 deletions
diff --git a/trygvis/eda/cli/add_to_db.py b/trygvis/eda/cli/add_to_db.py index 84f8d74..cb035a4 100644 --- a/trygvis/eda/cli/add_to_db.py +++ b/trygvis/eda/cli/add_to_db.py @@ -3,16 +3,27 @@ import trygvis.eda.cli as cli def run(files, args): def load(g): - s = 0 - for f in files: - cli.info("Adding %s" % f) - pre = len(g) - g.load(f, format="turtle") - post = len(g) - diff = post - pre - s += diff - cli.info("Loaded %d tuples" % diff) - - cli.info("Done. Loaded %d tuples" % s) + agg = 0 + file_count = len(files) + cli.info("Loading %d files" % file_count) + for i, f in enumerate(files): + cli.info("Adding file #%d/%d: %s" % (i + 1, file_count, f)) + + tmp = cli.create_graph() + tmp.load(f, format="turtle") + size = len(tmp) + # g.add_graph(tmp) + + for idx, t in enumerate(tmp.triples((None, None, None))): + g.add(t) + + if idx % 10000 == 0: + g.commit() + g.commit() + + agg += size + cli.info("Loaded %d tuples" % size) + + cli.info("Done. Loaded %d tuples (including duplicates)" % agg) cli.with_database(load) |