o SPARQL doesn't like blank nodes to generate some URLs for blank nodes from KiCAD schematics.

3 files changed, 57 insertions, 26 deletions

diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py
index 8050d6d..04de836 100644
--- a/trygvis/eda/cli/__init__.py
+++ b/trygvis/eda/cli/__init__.py
@@ -44,19 +44,32 @@ def with_database(tx):
         elif rt != store.VALID_STORE:
             raise CliException("The database is corrupt: %s" % path)
     elif db_type == 'sparql':
-        g = sparqlstore.SPARQLStore()
-        g.open(config["db"]["url"])
+        query_endpoint = config["db"]["url"]
+        update_endpoint = config["db"]["update_url"]
+
+        if update_endpoint is None:
+            g = sparqlstore.SPARQLStore()
+            g.open(query_endpoint)
+        else:
+            # def my_bnode_ext(node):
+            #     if isinstance(node, BNode):
+            #         return '<bnode:b%s>' % node
+            #     else:
+            #         return sparqlstore._node_to_sparql(node)
+            g = sparqlstore.SPARQLUpdateStore(autocommit=False)
+            g.open((query_endpoint, update_endpoint))
     else:
         raise CliException("Unknown db.type: %s" % db_type)
 
     try:
         print("g=%s" % g)
         tx(g)
+        g.commit()
     finally:
         g.close()
 
 
-def create_graph(digikey=False, kicad=False):
+def create_graph(digikey=False, kicad=False) -> Graph:
     g = Graph()
 
     if digikey:
@@ -105,6 +118,9 @@ _initNs = {
 
 
 def sparql(g: Graph, query: str, init_bindings=None):
+    # for k, v in _initNs.items():
+    #     print("PREFIX %s: <%s>" % (k, v))
+
     if isinstance(g, sparqlstore.SPARQLStore):
         return g.query(query, initNs=_initNs, initBindings=init_bindings)
     else:
diff --git a/trygvis/eda/cli/add_to_db.py b/trygvis/eda/cli/add_to_db.py
index 84f8d74..cb035a4 100644
--- a/trygvis/eda/cli/add_to_db.py
+++ b/trygvis/eda/cli/add_to_db.py
@@ -3,16 +3,27 @@ import trygvis.eda.cli as cli
 
 def run(files, args):
     def load(g):
-        s = 0
-        for f in files:
-            cli.info("Adding %s" % f)
-            pre = len(g)
-            g.load(f, format="turtle")
-            post = len(g)
-            diff = post - pre
-            s += diff
-            cli.info("Loaded %d tuples" % diff)
-
-        cli.info("Done. Loaded %d tuples" % s)
+        agg = 0
+        file_count = len(files)
+        cli.info("Loading %d files" % file_count)
+        for i, f in enumerate(files):
+            cli.info("Adding file #%d/%d: %s" % (i + 1, file_count, f))
+
+            tmp = cli.create_graph()
+            tmp.load(f, format="turtle")
+            size = len(tmp)
+            # g.add_graph(tmp)
+
+            for idx, t in enumerate(tmp.triples((None, None, None))):
+                g.add(t)
+
+                if idx % 10000 == 0:
+                    g.commit()
+            g.commit()
+
+            agg += size
+            cli.info("Loaded %d tuples" % size)
+
+        cli.info("Done. Loaded %d tuples (including duplicates)" % agg)
 
     cli.with_database(load)
diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py
index b017767..651ad32 100755
--- a/trygvis/eda/cli/kicad_bom_to_ttl.py
+++ b/trygvis/eda/cli/kicad_bom_to_ttl.py
@@ -6,9 +6,15 @@ import re
 import sys
 import xml.etree.ElementTree
 from urllib.parse import quote_plus
+from operator import attrgetter
+import itertools
 
-from trygvis.eda import cli
-from trygvis.eda.kicad import rdf as kicad_rdf
+import rdflib
+
+from .. import cli
+from ..kicad import rdf as kicad_rdf
+from rdflib import Literal, URIRef
+from rdflib.namespace import RDF, RDFS
 
 
 def _clean(s):
@@ -280,9 +286,6 @@ def process(src, dst, schematic_url):
     # for name in export.component_fields():
     #     cli.info(name)
 
-    from operator import attrgetter
-    import itertools
-
     # cli.info("components:")
     # fmt = '%2s%3s'
     # cli.info(fmt % ('Ref', ''))
@@ -345,18 +348,19 @@ def process(src, dst, schematic_url):
     # cli.info('Assigned part:        %d / %2.f%%' % (has_part, (has_part / part_count) * 100))
     # cli.info('Assigned digikey:     %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100))
 
-    from ..kicad import rdf as kicad_rdf
-    from rdflib import Literal, BNode, URIRef
-    from rdflib.namespace import RDF, RDFS
-
     g = cli.create_graph(kicad=True)
 
     schematic = URIRef(schematic_url)
     g.add((schematic, RDF.type, kicad_rdf.KICAD_TYPE.schematic))
 
-    #  componentNodes = []
+    # The components and fields could/should have been a BNodes. Their generated names are not very nice.
+    # TODO: try using a hash of the current value and put that under a special generated namespace.
+    # "http://example.org/my-board" + ref="C10" => "http://example.org/my-boardC10"
+    # hash("http://example.org/my-boardC10") => 123456
+    # add_prefix(hash) => "https://trygvis.io/purl/kicad/generated#123456"
     for c in export.components:
-        node = BNode()
+        ns = rdflib.Namespace(schematic_url)
+        node = ns[c.ref]
         g.add((schematic, kicad_rdf.KICAD.component, node))
 
         g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component))
@@ -364,7 +368,7 @@ def process(src, dst, schematic_url):
         g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text)))
 
         for name, value in c.fields.items():
-            f = BNode()
+            f = ns["%s-%s" % (c.ref, name)]
             g.add((node, kicad_rdf.KICAD.field, f))
             g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field))
             g.add((f, kicad_rdf.KICAD.field_name, Literal(name)))