o Initial import of some tools for working with KiCAD BOM files and Digi-key.

7 files changed, 681 insertions, 0 deletions

diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py
new file mode 100644
index 0000000..aa9e57e
--- /dev/null
+++ b/trygvis/eda/cli/__init__.py
@@ -0,0 +1,56 @@
+import sys
+import logging
+
+from rdflib import ConjunctiveGraph
+from rdflib import Graph
+from rdflib import store
+
+from ..digikey import rdf as digikey_rdf
+from ..kicad import rdf as kicad_rdf
+
+
+class CliException(Exception):
+    pass
+
+
+def init():
+    logging.basicConfig(level=logging.DEBUG)
+    pass
+
+
+def info(msg=None):
+    if msg is not None:
+        sys.stderr.write(msg)
+    sys.stderr.write("\n")
+
+
+def exit(msg=None):
+    sys.exit(msg)
+
+
+def open_database(path):
+    g = ConjunctiveGraph('Sleepycat')
+    rt = g.open(path, create=False)
+    if rt == store.NO_STORE:
+        info("Creating store in %s" % path)
+        g.open(path, create=True)
+    elif rt != store.VALID_STORE:
+        raise CliException("The database is corrupt: %s" % path)
+
+    return g
+
+
+def create_graph(digikey=False, kicad=False):
+    g = Graph()
+
+    if digikey:
+        g.bind("dk", digikey_rdf.DIGIKEY)
+        g.bind("dk-part", digikey_rdf.DIGIKEY_PART)
+        g.bind("dk-attr-type", digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE)
+        g.bind("dk-attr-value", digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE)
+        g.bind("dk-product-category", digikey_rdf.DIGIKEY_PRODUCT_CATEGORY)
+
+    if kicad:
+        g.bind("kicad", kicad_rdf.KICAD)
+        g.bind("kicad-type", kicad_rdf.KICAD_TYPE)
+    return g
diff --git a/trygvis/eda/cli/add_to_db.py b/trygvis/eda/cli/add_to_db.py
new file mode 100644
index 0000000..0511850
--- /dev/null
+++ b/trygvis/eda/cli/add_to_db.py
@@ -0,0 +1,17 @@
+import trygvis.eda.cli as cli
+
+
+def run(files, path, args):
+    g = cli.open_database(path)
+
+    s = 0
+    for f in files:
+        cli.info("Adding %s" % f)
+        pre = len(g)
+        g.load(f, format="turtle")
+        post = len(g)
+        diff = post - pre
+        s += diff
+        cli.info("Loaded %d tuples" % diff)
+
+    cli.info("Done. Loaded %d tuples" % s)
diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
new file mode 100644
index 0000000..aef360c
--- /dev/null
+++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
@@ -0,0 +1,29 @@
+from trygvis.eda import cli, write_graph
+
+from trygvis.eda.digikey import *
+
+
+def run(category, sub_category, output_file, args):
+    client = DigikeyClient()
+    db = DigikeyDatabase()
+
+    download_category_tree(db, client)
+    c = db.find_category(category)
+
+    if c is None:
+        cli.exit("Could not find category \"%s\"" % category)
+
+    sc = c.find_sub_category_by_label(sub_category)
+
+    if c is None:
+        cli.exit("Could not find sub-category \"%s\" inside \"%s\"" % (sub_category, category))
+
+    attributes = download_attribute_types_from_category(sc, client)
+    db.merge_attribute_types(attributes)
+
+    g = cli.create_graph()
+    for a in attributes:
+        [g.add(node) for node in a.to_nodes()]
+
+    filename = output_file if output_file is not "-" else None
+    write_graph(gen_g=lambda: g, filename=filename)
diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py
new file mode 100755
index 0000000..99f5266
--- /dev/null
+++ b/trygvis/eda/cli/digikey_download_for_schematic.py
@@ -0,0 +1,133 @@
+from os.path import isfile
+
+from rdflib.plugins.sparql import prepareQuery
+import rdflib.term
+
+from trygvis.eda import cli, write_graph
+from trygvis.eda.digikey import *
+from trygvis.eda.digikey import rdf as digikey_rdf
+from trygvis.eda.kicad import rdf as kicad_rdf
+
+initNs = {
+    "rdf": RDF,
+    "rdfs": RDFS,
+    "dk": digikey_rdf.DIGIKEY,
+    "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE,
+    "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE,
+    "dk-part": digikey_rdf.DIGIKEY_PART,
+    "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY,
+    "kicad": kicad_rdf.KICAD,
+    "kicad-type": kicad_rdf.KICAD_TYPE}
+
+
+def run(schematic_url, db_path, args):
+    cli.info("Schematic: %s" % schematic_url)
+    g = cli.open_database(db_path)
+
+    client = DigikeyClient()
+    db = DigikeyDatabase()
+    download_category_tree(db, client)
+
+    # Dump schematic:
+    # SELECT ?schematic
+    # WHERE {
+    #  ?schematic a kicad-type:schematic .
+    #  b:rateboard-2_a ? ?.
+    #  ?s ?predicate ?object .
+    # }
+
+    # allComponentsQ = prepareQuery('SELECT ?ref ?value WHERE { ?schematic_url a kicad-type:schematic .  ?schematic_url kicad:component ?o . ?o rdfs:label ?ref .  ?o kicad:value ?value . }', initNs = initNs)
+    #
+    # for row in g.query(allComponentsQ, initBindings={'schematic_url': schematic_url}):
+    #    print("ref=%s, value=%s" % (row.ref, row.value))
+
+    res = g.query(prepareQuery("SELECT ?schematic WHERE {?schematic a kicad-type:schematic}", initNs=initNs))
+    print("Found %d schematics in database" % len(res))
+    for row in res:
+        print("schematic: %s" % row.schematic)
+
+    res = g.query(prepareQuery("SELECT ?dk_part ?dk_part_number WHERE {?dk_part a dk:part ; dk:partNumber ?dk_part_number}", initNs=initNs))
+    print("Found %d Digikey parts in database" % len(res))
+    for row in res:
+        print("Part: url=%s, partNumber=%s" % (row.dk_part, row.dk_part_number))
+
+    q = prepareQuery("""
+SELECT
+    ?digikey_pn
+    (group_concat(distinct ?ref;separator=";") as ?refs)
+WHERE {
+    ?schematic_url kicad:component ?cmp .
+    ?cmp rdfs:label ?ref ;
+    kicad:value ?value .
+    OPTIONAL {
+        ?cmp kicad:field ?d .
+        ?d kicad:field_name "digikey" ;
+           kicad:field_value ?digikey_pn .
+#        OPTIONAL {
+            ?dk_part a             dk:part ;
+                     dk:partNumber ?digikey_pn .
+#        }
+    }
+}
+GROUP BY ?digikey_pn
+ORDER BY ?digikey_pn
+""", initNs=initNs)
+
+    res = g.query(q, initBindings={'schematic_url': rdflib.term.URIRef(schematic_url)})
+    for row in res:
+        pn = row.digikey_pn
+
+        if pn is None:
+            continue
+
+        refs = row.refs.split(';')
+
+        cli.info("Part \"%s\" is used by %s" % (pn, refs))
+
+        filename = 'ttl/digikey-part-' + normalize_filename(pn + ".ttl")
+
+        def download_graph():
+            cli.info("Downloading product: " + pn)
+            product = download_product(client, db, pn)
+
+            g = cli.create_graph(digikey=True)
+            [g.add(node) for node in product.to_nodes()]
+            return g
+
+        write_graph(download_graph, filename)
+
+            # q = prepareQuery("""
+    # SELECT
+    # DISTINCT ?category ?digikeyUrl
+    # WHERE {
+    #   ?schematic_url kicad:component ?cmp .
+    #   ?cmp rdfs:label ?ref .
+    #   ?cmp kicad:value ?value .
+    #   ?cmp kicad:field ?d .
+    #   ?d kicad:field_name "digikey" .
+    #   ?d kicad:field_value ?digikey .
+    #
+    #   ?part a dk:part .
+    #   ?part dk:partNumber ?digikey .
+    #   ?part dk:category ?category .
+    #   ?category dk:parent ?_ .
+    #   ?category dk:url ?digikeyUrl
+    # }""", initNs=initNs)
+    #
+    # res = g.query(q, initBindings={'schematic_url': schematic_url})
+    #
+    # cli.info("Found %d categories" % (len(res)))
+    # for row in res:
+    #     cli.info("Category: %s" % (row.category))
+    #     category = db.findSubCategoryByUrl(row.category)
+    #     if category is None:
+    #         raise Exception('could not find category: ' + row.category)
+    #     attributes = downloadAttributeTypesFromCategory(category, client)
+    #     db.mergeAttributeTypes(attributes)
+    #
+    #     filename = 'digikey-category-' + normalize_filename(str(category.label) + ".ttl")
+    #     if not isfile(filename):
+    #         tmpG = cli.create_graph()
+    #         for a in attributes:
+    #             [tmpG.add(node) for node in a.toNodes()]
+    #         writeGraph(tmpG, 'ttl/' + filename)
diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py
new file mode 100644
index 0000000..4e80e93
--- /dev/null
+++ b/trygvis/eda/cli/eda_rdf.py
@@ -0,0 +1,69 @@
+import argparse
+import sys
+import trygvis.eda.cli as cli
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(dest="cmd")
+
+    p = subparsers.add_parser("kicad-bom-to-ttl")
+    p.add_argument("-o", "--output", required=False)
+    p.add_argument("-i", "--input", required=False)
+
+    p = subparsers.add_parser("add-to-db")
+    p.add_argument("-d", "--db", required=True)
+    p.add_argument("files", nargs='*')
+
+    p = subparsers.add_parser("make-bom")
+    p.add_argument("-d", "--db", required=True)
+    p.add_argument("--schematic", required=True)
+
+    p = subparsers.add_parser("digikey-download-for-schematic")
+    p.add_argument("-d", "--db", required=True)
+    p.add_argument("--schematic", required=True)
+
+    p = subparsers.add_parser("digikey-download-attribute-types-for-category")
+    p.add_argument("-c", "--category", required=True)
+    p.add_argument("-s", "--sub-category", required=True)
+    p.add_argument("-o", "--output", required=False)
+
+    args = parser.parse_args()
+
+    cli.init()
+
+    if args.cmd == "kicad-bom-to-ttl":
+        from trygvis.eda.cli import kicad_bom_to_ttl
+
+        if args.input is not None:
+            src = open(args.input, "r")
+        else:
+            src = sys.stdin
+
+        if args.output is not None:
+            dst = open(args.output, "wb")
+        else:
+            dst = sys.stdout.buffer
+
+        with src, dst:
+            kicad_bom_to_ttl.run(src, dst, args)
+
+    elif args.cmd == "add-to-db":
+        from trygvis.eda.cli import add_to_db
+
+        add_to_db.run(args.files, args.db, args)
+
+    elif args.cmd == "make-bom":
+        from trygvis.eda.cli import make_bom
+
+        make_bom.run(args.schematic, args.db, args)
+
+    elif args.cmd == "digikey-download-for-schematic":
+        from trygvis.eda.cli import digikey_download_for_schematic
+        digikey_download_for_schematic.run(args.schematic, args.db, args)
+    elif args.cmd == "digikey-download-attribute-types-for-category":
+        from trygvis.eda.cli import digikey_download_attribute_types_for_category
+        digikey_download_attribute_types_for_category.run(args.category, args.sub_category, args.output, args)
+    else:
+        sys.exit("Unknown command: %s" % args.cmd)
diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py
new file mode 100755
index 0000000..c7d6899
--- /dev/null
+++ b/trygvis/eda/cli/kicad_bom_to_ttl.py
@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+
+import functools
+import os.path
+import re
+import xml.etree.ElementTree
+from urllib.parse import quote_plus
+
+from trygvis.eda import cli
+from trygvis.eda.kicad import rdf as kicad_rdf
+
+
+def _clean(s):
+    if s is None:
+        return None
+    s = s.strip()
+    return None if len(s) == 0 else s
+
+
+def _cleaned_text(e, name):
+    child = e.find(name)
+    if child is None:
+        return None
+    return child.text
+
+
+def _cleaned_attr(e, name):
+    value = e.get(name)
+    if value is None:
+        return None
+    return _clean(value)
+
+
+def _iso_size_to_factor(size):
+    if size == 'k':
+        return 3
+    if size == 'M':
+        return 6
+    if size == 'm':
+        return -3
+    if size == 'u' or size == '\N{GREEK SMALL LETTER MU}' or size == '\N{MICRO SIGN}':
+        return -6
+    if size == 'n':
+        return -9
+    if size == 'p':
+        return -12
+    return None
+
+
+@functools.total_ordering
+class Value:
+    valueRe = re.compile("([0-9]+\.?[0-9]*)(.*)")
+    typeRe = re.compile("(.*)(H|Hz)$")
+
+    def __init__(self, text, value, factor, type):
+        self.text = text
+        self.value = value
+        self.factor = factor
+        self.type = type
+
+    def __eq__(self, other):
+        return self.text == other.text
+
+    def __lt__(self, other):
+        if self.value is not None:
+            if other.value is None:
+                return 1
+            else:
+                return (self.factor, self.value) < (other.factor, other.value)
+        else:
+            if other.value is None:
+                return self.text < other.text
+            else:
+                return -1
+
+    def __str__(self):
+        if self.value is not None:
+            s = str(self.value) + 'e' + str(self.factor)
+            if self.type is not None:
+                s = s + ' ' + self.type
+            return s
+        if self.text is not None:
+            return self.text
+        return "unknown"
+
+    @staticmethod
+    def parse(text):
+        if text is None:
+            return Value(None, None, None, None)
+
+        m = Value.valueRe.match(text)
+        if m:
+            value_int = float(m.group(1))
+            suffix = m.group(2)
+            m = Value.typeRe.match(suffix)
+            if m:
+                suffix = m.group(1)
+                type = m.group(2)
+            else:
+                type = None
+
+            factor = _iso_size_to_factor(suffix)
+            # cli.info(text + ': suffix:' + suffix + ' => value_int: ' + str(value_int) + ', factor: ' + str(
+            #     factor) + ', type=' + str(type))
+
+            if value_int is not None and factor is not None:
+                return Value(text, value_int, factor, type)
+
+        return Value(text, None, None, None)
+
+
+class Component:
+    refRe = re.compile("([a-zA-Z]+)([0-9]+)")
+
+    def __init__(self, ref, value, footprint, fields):
+        self.ref = ref
+        self.value = Value.parse(value)
+        self.footprint = footprint
+        self.fields = fields
+        m = Component.refRe.match(ref)
+        if m:
+            self.ref_class = m.group(1)
+            self.ref_instance = int(m.group(2))
+        else:
+            self.ref_class = None
+            self.ref_instance = None
+
+    def __str__(self):
+        s = 'ref=' + self.ref
+        if self.value is not None:
+            s += ', value=' + str(self.value)
+        if self.footprint is not None:
+            s += ', footprint=' + self.footprint
+        for name, value in self.fields.items():
+            s += ', ' + name + '=' + value
+        return s
+
+    def find_field(self, key):
+        return self.fields[key] if key in self.fields else None
+
+    @staticmethod
+    def from_xml(e):
+        ref = e.get('ref')
+        value = _cleaned_text(e, 'value')
+        footprint = _cleaned_text(e, 'footprint')
+        fs = {}
+
+        fields = e.find('fields')
+        if fields is not None:
+            for field in fields.findall('field'):
+                fs[field.get('name')] = field.text
+
+        if ref is not None:
+            return Component(ref, value, footprint, fs)
+
+        return None
+
+
+class TitleBlock(object):
+    def __init__(self, title=None, rev=None, date=None, source=None):
+        self.title = title
+        self.rev = rev
+        self.date = date
+        self.source = source
+
+    @staticmethod
+    def from_xml(tb):
+        title = _cleaned_text(tb, 'title')
+        rev = _cleaned_text(tb, 'rev')
+        date = _cleaned_text(tb, 'date')
+        source = _cleaned_text(tb, 'source')
+        return TitleBlock(title, rev, date, source)
+
+
+class Sheet(object):
+    def __init__(self, number, name, title_block):
+        self.number = number
+        self.name = name
+        self.title_block = title_block
+
+    @staticmethod
+    def from_xml(s):
+        number = _cleaned_attr(s, "number")
+        name = _cleaned_attr(s, "name")
+        node = s.find('title_block')
+        title_block = TitleBlock.from_xml(node) if node is not None else TitleBlock()
+        return Sheet(int(number) if number is not None else None, name, title_block)
+
+
+class Design(object):
+    def __init__(self, source=None, date=None, tool=None, sheets=None):
+        self.source = source
+        self.date = date
+        self.tool = tool
+        self.sheets = sheets if sheets is not None else []
+
+    @staticmethod
+    def from_xml(d):
+        source = _cleaned_text(d, 'source')
+        date = _cleaned_text(d, 'date')
+        tool = _cleaned_text(d, 'tool')
+        sheets = []
+        for s in d.iterfind('sheet'):
+            sheets.append(Sheet.from_xml(s))
+        return Design(source, date, tool, sheets)
+
+
+class Export(object):
+    def __init__(self, design, components):
+        self.design = design if design is not None else Design()
+        self.components = components
+
+    @staticmethod
+    def from_xml(doc):
+        cs = []
+
+        node = doc.find('design')
+        design = Design.from_xml(node) if node is not None else None
+
+        components = doc.find('components')
+        for e in components.iter('comp'):
+            c = Component.from_xml(e)
+            if c is not None:
+                cs.append(c)
+
+        return Export(design, cs)
+
+    def component_fields(self):
+        fs = set()
+        for c in self.components:
+            for key, value in c.fields.items():
+                fs.add(key)
+        return fs
+
+    def generate_schematic_url(self):
+        s = next((s for s in self.design.sheets if s.number == 1), None)
+        title = s.title_block.title if s is not None else None
+        if title is None:
+            if self.design.source is not None:
+                title = _clean(os.path.basename(self.design.source))
+
+        if title is None:
+            raise cli.CliException("Could not generate a stable, identifying URL for the schematic")
+
+        title = quote_plus(title)
+        return kicad_rdf.KICAD_BOARD[title]
+
+
+def run(src, dst, args):
+    from trygvis.eda import cli
+
+    tree = xml.etree.ElementTree.parse(src)
+    root = tree.getroot()
+
+    export = Export.from_xml(root)
+
+    # print("components:")
+    # for c in export.components:
+    #  print(c)
+
+    # for name in export.component_fields():
+    #     cli.info(name)
+
+    from operator import attrgetter
+    import itertools
+
+    # cli.info("components:")
+    # fmt = '%2s%3s'
+    # cli.info(fmt % ('Ref', ''))
+
+    components = sorted(export.components, key=attrgetter('ref'))
+
+    component_count = 0
+    part_count = 0
+    has_part = 0
+    has_digikey = 0
+
+    for cls, cs in itertools.groupby(components, key=attrgetter('ref_class')):
+        # cli.info('--- Class: %s ---' % cls)
+        cs = sorted(cs, key=attrgetter('value'))
+        for c in cs:
+            component_count += 1
+
+            value = str(c.value)
+            part = c.find_field("part")
+            digikey = c.find_field("digikey")
+            footprint = c.footprint
+
+            if c.ref_class is not None:
+                ref_class = c.ref_class
+                ref_instance = c.ref_instance
+            else:
+                ref_class = c.ref
+                ref_instance = ""
+
+            # cli.info(fmt % (ref_class, ref_instance))
+            # cli.info('    Value: %s' % c.value.text)
+            # if c.footprint:
+            #     cli.info('    Footprint: %s' % c.footprint)
+
+            if part is not None:
+                if part != 'NA':
+                    has_part += 1 if part is not None else 0
+                    part_count += 1
+
+                    if digikey is not None:
+                        has_digikey += 1
+                else:
+                    part = None
+            else:
+                part_count += 1
+                part = "MISSING"
+
+                # if part is not None:
+                #     cli.info('    Part: %s' % part)
+                #     if digikey is not None:
+                #         cli.info('    Digikey: %s' % digikey)
+
+    schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url()
+
+    # cli.info()
+    # cli.info('=== Summary ===')
+    # cli.info('Schematic URL:        %s' % schematic_url)
+    # cli.info('Number of components: %d' % component_count)
+    # cli.info('Number of parts:      %d' % part_count)
+    # cli.info('Assigned part:        %d / %2.f%%' % (has_part, (has_part / part_count) * 100))
+    # cli.info('Assigned digikey:     %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100))
+
+    from ..kicad import rdf as kicad_rdf
+    from rdflib import Literal, BNode, URIRef
+    from rdflib.namespace import RDF, RDFS
+
+    g = cli.create_graph(kicad=True)
+
+    schematic = URIRef(schematic_url)
+    g.add((schematic, RDF.type, kicad_rdf.KICAD_TYPE.schematic))
+
+    #  componentNodes = []
+    for c in export.components:
+        node = BNode()
+        g.add((schematic, kicad_rdf.KICAD.component, node))
+
+        g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component))
+        g.add((node, RDFS.label, Literal(c.ref)))
+        g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text)))
+
+        for name, value in c.fields.items():
+            f = BNode()
+            g.add((node, kicad_rdf.KICAD.field, f))
+            g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field))
+            g.add((f, kicad_rdf.KICAD.field_name, Literal(name)))
+            g.add((f, kicad_rdf.KICAD.field_value, Literal(value)))
+
+    # TODO: serialize the data from <design> too
+
+    g.serialize(destination=dst, encoding='utf-8', format='turtle')
diff --git a/trygvis/eda/cli/make_bom.py b/trygvis/eda/cli/make_bom.py
new file mode 100755
index 0000000..28059cb
--- /dev/null
+++ b/trygvis/eda/cli/make_bom.py
@@ -0,0 +1,21 @@
+from trygvis.eda import cli
+from trygvis.eda.digikey import *
+from trygvis.eda.digikey import rdf as digikey_rdf
+from trygvis.eda.kicad import rdf as kicad_rdf
+
+initNs = {
+    "rdf": RDF,
+    "rdfs": RDFS,
+    "dk": digikey_rdf.DIGIKEY,
+    "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE,
+    "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE,
+    "dk-part": digikey_rdf.DIGIKEY_PART,
+    "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY,
+    "kicad": kicad_rdf.KICAD,
+    "kicad-type": kicad_rdf.KICAD_TYPE}
+
+
+def run(schematic_url, db_path, args):
+    g = cli.open_database(db_path)
+
+    cli.info('implement..')