From 9bf18900c05547a40bb3c3a7a819a137e81dae92 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Tue, 27 Dec 2016 22:24:37 +0100 Subject: o Adding a 'db-stats' application for summarizing the contents of the database. o Starting on make-bom. Extremely slow when using rdflib (~8 minutes), jena is very fast (~50 ms). --- trygvis/eda/cli/__init__.py | 25 ++++- trygvis/eda/cli/db_stats.py | 39 +++++++ ...igikey_download_attribute_types_for_category.py | 4 +- trygvis/eda/cli/eda_rdf.py | 18 +++- trygvis/eda/cli/make_bom.py | 120 +++++++++++++++++---- trygvis/eda/digikey/rdf.py | 12 +-- 6 files changed, 184 insertions(+), 34 deletions(-) create mode 100755 trygvis/eda/cli/db_stats.py (limited to 'trygvis') diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py index aa9e57e..9cfb710 100644 --- a/trygvis/eda/cli/__init__.py +++ b/trygvis/eda/cli/__init__.py @@ -1,9 +1,8 @@ import sys import logging -from rdflib import ConjunctiveGraph -from rdflib import Graph -from rdflib import store +from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS +from rdflib.plugins.sparql import prepareQuery from ..digikey import rdf as digikey_rdf from ..kicad import rdf as kicad_rdf @@ -24,7 +23,7 @@ def info(msg=None): sys.stderr.write("\n") -def exit(msg=None): +def do_exit(msg=None): sys.exit(msg) @@ -54,3 +53,21 @@ def create_graph(digikey=False, kicad=False): g.bind("kicad", kicad_rdf.KICAD) g.bind("kicad-type", kicad_rdf.KICAD_TYPE) return g + + +_initNs = { + "rdf": RDF, + "rdfs": RDFS, + "dk": digikey_rdf.DIGIKEY, + "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE, + "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE, + "dk-part": digikey_rdf.DIGIKEY_PART, + "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY, + "kicad": kicad_rdf.KICAD, + "kicad-type": kicad_rdf.KICAD_TYPE} + + +def sparql(g, query, init_bindings = None): + q = prepareQuery(query, initNs=_initNs) + + return g.query(q, initBindings=init_bindings) diff --git a/trygvis/eda/cli/db_stats.py b/trygvis/eda/cli/db_stats.py new file mode 100755 index 0000000..4780c20 --- /dev/null +++ b/trygvis/eda/cli/db_stats.py @@ -0,0 +1,39 @@ +from trygvis.eda import cli + + +def run(db_path): + g = cli.open_database(db_path) + + res = cli.sparql(g, """ +SELECT ?schematic ?label +WHERE { + ?schematic a kicad-type:schematic + OPTIONAL { + ?schematic rdfs:label ?label + } +} +""") + cli.info("Found %d schematics in database" % len(res)) + for row in res: + name = row.label if row.label is not None else "" + url = row.schematic + + cli.info("%s:" % name) + cli.info(" URL: %s" % url) + cli.info() + + res = cli.sparql(g, """ +SELECT + ?dk_part ?dk_part_number ?label +WHERE { + ?dk_part a dk:part ; + dk:partNumber ?dk_part_number . + OPTIONAL { + ?dk_part rdfs:label ?label + } +} +ORDER BY ?dk_part_number + """) + cli.info("Found %d Digi-Key parts:" % len(res)) + for row in res: + cli.info(" %-30s: %s" % (row.dk_part_number, row.label)) diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py index aef360c..1a33ab1 100644 --- a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py +++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py @@ -11,12 +11,12 @@ def run(category, sub_category, output_file, args): c = db.find_category(category) if c is None: - cli.exit("Could not find category \"%s\"" % category) + cli.do_exit("Could not find category \"%s\"" % category) sc = c.find_sub_category_by_label(sub_category) if c is None: - cli.exit("Could not find sub-category \"%s\" inside \"%s\"" % (sub_category, category)) + cli.do_exit("Could not find sub-category \"%s\" inside \"%s\"" % (sub_category, category)) attributes = download_attribute_types_from_category(sc, client) db.merge_attribute_types(attributes) diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py index 4e80e93..7bf0934 100644 --- a/trygvis/eda/cli/eda_rdf.py +++ b/trygvis/eda/cli/eda_rdf.py @@ -8,18 +8,20 @@ def main(): subparsers = parser.add_subparsers(dest="cmd") + # kicad-* p = subparsers.add_parser("kicad-bom-to-ttl") p.add_argument("-o", "--output", required=False) p.add_argument("-i", "--input", required=False) + # db-* p = subparsers.add_parser("add-to-db") p.add_argument("-d", "--db", required=True) p.add_argument("files", nargs='*') - p = subparsers.add_parser("make-bom") + p = subparsers.add_parser("db-stats") p.add_argument("-d", "--db", required=True) - p.add_argument("--schematic", required=True) + # digikey-* p = subparsers.add_parser("digikey-download-for-schematic") p.add_argument("-d", "--db", required=True) p.add_argument("--schematic", required=True) @@ -29,6 +31,11 @@ def main(): p.add_argument("-s", "--sub-category", required=True) p.add_argument("-o", "--output", required=False) + # Other + p = subparsers.add_parser("make-bom") + p.add_argument("-d", "--db", required=True) + p.add_argument("--schematic", required=True) + args = parser.parse_args() cli.init() @@ -54,10 +61,13 @@ def main(): add_to_db.run(args.files, args.db, args) + elif args.cmd == "db-stats": + from trygvis.eda.cli import db_stats + db_stats.run(args.db) + elif args.cmd == "make-bom": from trygvis.eda.cli import make_bom - - make_bom.run(args.schematic, args.db, args) + make_bom.run(args.schematic, args.db) elif args.cmd == "digikey-download-for-schematic": from trygvis.eda.cli import digikey_download_for_schematic diff --git a/trygvis/eda/cli/make_bom.py b/trygvis/eda/cli/make_bom.py index 28059cb..39d664b 100755 --- a/trygvis/eda/cli/make_bom.py +++ b/trygvis/eda/cli/make_bom.py @@ -1,21 +1,105 @@ from trygvis.eda import cli -from trygvis.eda.digikey import * -from trygvis.eda.digikey import rdf as digikey_rdf -from trygvis.eda.kicad import rdf as kicad_rdf - -initNs = { - "rdf": RDF, - "rdfs": RDFS, - "dk": digikey_rdf.DIGIKEY, - "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE, - "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE, - "dk-part": digikey_rdf.DIGIKEY_PART, - "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY, - "kicad": kicad_rdf.KICAD, - "kicad-type": kicad_rdf.KICAD_TYPE} - - -def run(schematic_url, db_path, args): +import rdflib + + +class DigiKeyPart(object): + def __init__(self, part_number): + self.part_number = part_number + self.attributes = {} + + def set_attribute(self, key, value): + self.attributes[key] = value + + +class Component(object): + def __init__(self, ref, value): + self.ref = ref + self.value = value + self.fields = {} + + def set_field(self, field_name, field_value): + self.fields[field_name] = field_value + + +def run(schematic_url, db_path): g = cli.open_database(db_path) - cli.info('implement..') + components = {} + dkParts = {} + + cli.info("Loading components") + + res = cli.sparql(g, """ +SELECT + ?ref ?value +WHERE { + ?schematic a kicad-type:schematic ; + kicad:component ?cmp . + ?cmp a kicad-type:schematic_component ; + kicad:value ?value ; + rdfs:label ?ref . +} +ORDER BY ?ref +""", init_bindings={"schematic": rdflib.URIRef(schematic_url)}) + + for row in res: + c = Component(row.ref, row.value) + components[row.ref] = c + cli.info('ref=%s, value=%s' % (c.ref, c.value)) + + cli.info("Loading custom component attributes") + res = cli.sparql(g, """ +SELECT + ?ref ?field ?field_name ?field_value +WHERE { + ?schematic a kicad-type:schematic ; + kicad:component ?cmp . + ?cmp a kicad-type:schematic_component ; + rdfs:label ?ref ; + kicad:field ?field . + ?field a kicad-type:field ; kicad:field_name ?field_name . + ?field a kicad-type:field ; kicad:field_value ?field_value . +} +ORDER BY ?ref ?field_name +""", init_bindings={"schematic": rdflib.URIRef(schematic_url)}) + + for row in res: + c = components[row.ref] + assert isinstance(c, Component) + c.set_field(row.field_name, row.field_value) + cli.info('%5s: %-20s %s' % (c.ref, row.field_name + ':', row.field_value)) + + cli.info("Loading Digi-Key parts for schematic") + res = cli.sparql(g, """ +SELECT +?ref ?part_number ?type ?value +WHERE { + ?schematic kicad:component ?cmp . + ?cmp a kicad-type:schematic_component ; + rdfs:label ?ref ; + kicad:field ?d . + ?d kicad:field_name "digikey" ; + kicad:field_value ?part_number . + ?part a dk:part ; + rdfs:label ?label ; + dk:partNumber ?part_number ; + dk:attribute-value ?attr_value . + ?attr_value rdfs:label ?value . + ?attr_type a dk:attributeType ; + rdfs:label ?type ; + dk:value ?attr_value . +} +ORDER BY ?ref ?attr_type ?attr_value +""", init_bindings={"schematic": rdflib.URIRef(schematic_url)}) + + for row in res: + pn = row.part_number + + if not hasattr(dkParts, pn): + part = DigiKeyPart(pn) + dkParts[pn] = part + else: + part = dkParts[pn] + + part.set_attribute(row.type, row.value) + cli.info('%5s: %-20s %s' % (pn, row.type + ':', row.value)) diff --git a/trygvis/eda/digikey/rdf.py b/trygvis/eda/digikey/rdf.py index 5f1dede..7a21275 100644 --- a/trygvis/eda/digikey/rdf.py +++ b/trygvis/eda/digikey/rdf.py @@ -1,7 +1,7 @@ -from rdflib import Namespace +import rdflib -DIGIKEY = Namespace("https://trygvis.io/purl/digikey#") -DIGIKEY_ATTRIBUTE_TYPE = Namespace("https://trygvis.io/purl/digikey-attribute-type#") -DIGIKEY_ATTRIBUTE_VALUE = Namespace("https://trygvis.io/purl/digikey-attribute-value#") -DIGIKEY_PART = Namespace("https://trygvis.io/purl/digikey-part#") -DIGIKEY_PRODUCT_CATEGORY = Namespace("https://trygvis.io/purl/digikey-product-category#") +DIGIKEY = rdflib.Namespace("https://trygvis.io/purl/digikey#") +DIGIKEY_ATTRIBUTE_TYPE = rdflib.Namespace("https://trygvis.io/purl/digikey-attribute-type#") +DIGIKEY_ATTRIBUTE_VALUE = rdflib.Namespace("https://trygvis.io/purl/digikey-attribute-value#") +DIGIKEY_PART = rdflib.Namespace("https://trygvis.io/purl/digikey-part#") +DIGIKEY_PRODUCT_CATEGORY = rdflib.Namespace("https://trygvis.io/purl/digikey-product-category#") -- cgit v1.2.3