From db85189ea87b214d3649138fcd943a942bf8c530 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Wed, 28 Dec 2016 15:29:30 +0100 Subject: o SPARQL doesn't like blank nodes to generate some URLs for blank nodes from KiCAD schematics. --- trygvis/eda/cli/__init__.py | 22 +++++++++++++++++++--- trygvis/eda/cli/add_to_db.py | 33 ++++++++++++++++++++++----------- trygvis/eda/cli/kicad_bom_to_ttl.py | 28 ++++++++++++++++------------ 3 files changed, 57 insertions(+), 26 deletions(-) (limited to 'trygvis/eda/cli') diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py index 8050d6d..04de836 100644 --- a/trygvis/eda/cli/__init__.py +++ b/trygvis/eda/cli/__init__.py @@ -44,19 +44,32 @@ def with_database(tx): elif rt != store.VALID_STORE: raise CliException("The database is corrupt: %s" % path) elif db_type == 'sparql': - g = sparqlstore.SPARQLStore() - g.open(config["db"]["url"]) + query_endpoint = config["db"]["url"] + update_endpoint = config["db"]["update_url"] + + if update_endpoint is None: + g = sparqlstore.SPARQLStore() + g.open(query_endpoint) + else: + # def my_bnode_ext(node): + # if isinstance(node, BNode): + # return '' % node + # else: + # return sparqlstore._node_to_sparql(node) + g = sparqlstore.SPARQLUpdateStore(autocommit=False) + g.open((query_endpoint, update_endpoint)) else: raise CliException("Unknown db.type: %s" % db_type) try: print("g=%s" % g) tx(g) + g.commit() finally: g.close() -def create_graph(digikey=False, kicad=False): +def create_graph(digikey=False, kicad=False) -> Graph: g = Graph() if digikey: @@ -105,6 +118,9 @@ _initNs = { def sparql(g: Graph, query: str, init_bindings=None): + # for k, v in _initNs.items(): + # print("PREFIX %s: <%s>" % (k, v)) + if isinstance(g, sparqlstore.SPARQLStore): return g.query(query, initNs=_initNs, initBindings=init_bindings) else: diff --git a/trygvis/eda/cli/add_to_db.py b/trygvis/eda/cli/add_to_db.py index 84f8d74..cb035a4 100644 --- a/trygvis/eda/cli/add_to_db.py +++ b/trygvis/eda/cli/add_to_db.py @@ -3,16 +3,27 @@ import trygvis.eda.cli as cli def run(files, args): def load(g): - s = 0 - for f in files: - cli.info("Adding %s" % f) - pre = len(g) - g.load(f, format="turtle") - post = len(g) - diff = post - pre - s += diff - cli.info("Loaded %d tuples" % diff) - - cli.info("Done. Loaded %d tuples" % s) + agg = 0 + file_count = len(files) + cli.info("Loading %d files" % file_count) + for i, f in enumerate(files): + cli.info("Adding file #%d/%d: %s" % (i + 1, file_count, f)) + + tmp = cli.create_graph() + tmp.load(f, format="turtle") + size = len(tmp) + # g.add_graph(tmp) + + for idx, t in enumerate(tmp.triples((None, None, None))): + g.add(t) + + if idx % 10000 == 0: + g.commit() + g.commit() + + agg += size + cli.info("Loaded %d tuples" % size) + + cli.info("Done. Loaded %d tuples (including duplicates)" % agg) cli.with_database(load) diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py index b017767..651ad32 100755 --- a/trygvis/eda/cli/kicad_bom_to_ttl.py +++ b/trygvis/eda/cli/kicad_bom_to_ttl.py @@ -6,9 +6,15 @@ import re import sys import xml.etree.ElementTree from urllib.parse import quote_plus +from operator import attrgetter +import itertools -from trygvis.eda import cli -from trygvis.eda.kicad import rdf as kicad_rdf +import rdflib + +from .. import cli +from ..kicad import rdf as kicad_rdf +from rdflib import Literal, URIRef +from rdflib.namespace import RDF, RDFS def _clean(s): @@ -280,9 +286,6 @@ def process(src, dst, schematic_url): # for name in export.component_fields(): # cli.info(name) - from operator import attrgetter - import itertools - # cli.info("components:") # fmt = '%2s%3s' # cli.info(fmt % ('Ref', '')) @@ -345,18 +348,19 @@ def process(src, dst, schematic_url): # cli.info('Assigned part: %d / %2.f%%' % (has_part, (has_part / part_count) * 100)) # cli.info('Assigned digikey: %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100)) - from ..kicad import rdf as kicad_rdf - from rdflib import Literal, BNode, URIRef - from rdflib.namespace import RDF, RDFS - g = cli.create_graph(kicad=True) schematic = URIRef(schematic_url) g.add((schematic, RDF.type, kicad_rdf.KICAD_TYPE.schematic)) - # componentNodes = [] + # The components and fields could/should have been a BNodes. Their generated names are not very nice. + # TODO: try using a hash of the current value and put that under a special generated namespace. + # "http://example.org/my-board" + ref="C10" => "http://example.org/my-boardC10" + # hash("http://example.org/my-boardC10") => 123456 + # add_prefix(hash) => "https://trygvis.io/purl/kicad/generated#123456" for c in export.components: - node = BNode() + ns = rdflib.Namespace(schematic_url) + node = ns[c.ref] g.add((schematic, kicad_rdf.KICAD.component, node)) g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component)) @@ -364,7 +368,7 @@ def process(src, dst, schematic_url): g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text))) for name, value in c.fields.items(): - f = BNode() + f = ns["%s-%s" % (c.ref, name)] g.add((node, kicad_rdf.KICAD.field, f)) g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field)) g.add((f, kicad_rdf.KICAD.field_name, Literal(name))) -- cgit v1.2.3