diff options
Diffstat (limited to 'trygvis')
| -rw-r--r-- | trygvis/__init__.py | 0 | ||||
| -rw-r--r-- | trygvis/eda/__init__.py | 20 | ||||
| -rw-r--r-- | trygvis/eda/__main__.py | 3 | ||||
| -rw-r--r-- | trygvis/eda/cli/__init__.py | 56 | ||||
| -rw-r--r-- | trygvis/eda/cli/add_to_db.py | 17 | ||||
| -rw-r--r-- | trygvis/eda/cli/digikey_download_attribute_types_for_category.py | 29 | ||||
| -rwxr-xr-x | trygvis/eda/cli/digikey_download_for_schematic.py | 133 | ||||
| -rw-r--r-- | trygvis/eda/cli/eda_rdf.py | 69 | ||||
| -rwxr-xr-x | trygvis/eda/cli/kicad_bom_to_ttl.py | 356 | ||||
| -rwxr-xr-x | trygvis/eda/cli/make_bom.py | 21 | ||||
| -rw-r--r-- | trygvis/eda/digikey/__init__.py | 300 | ||||
| -rw-r--r-- | trygvis/eda/digikey/__main__.py | 42 | ||||
| -rw-r--r-- | trygvis/eda/digikey/rdf.py | 7 | ||||
| -rw-r--r-- | trygvis/eda/kicad/__init__.py | 0 | ||||
| -rw-r--r-- | trygvis/eda/kicad/rdf.py | 7 |
15 files changed, 1060 insertions, 0 deletions
diff --git a/trygvis/__init__.py b/trygvis/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/trygvis/__init__.py diff --git a/trygvis/eda/__init__.py b/trygvis/eda/__init__.py new file mode 100644 index 0000000..e6ccb44 --- /dev/null +++ b/trygvis/eda/__init__.py @@ -0,0 +1,20 @@ +import sys + +from os.path import isfile +from . import cli + + +def write_graph(gen_g, filename=None, force_write=False): + if filename is not None: + if force_write or not isfile(filename): + g = gen_g() + bs = g.serialize(encoding='utf-8', format='turtle') + with open(filename, "wb") as f: + f.write(bs) + cli.info("Wrote %s" % filename) + else: + cli.info("Skipped writing %s, already exists" % filename) + else: + g = gen_g() + bs = g.serialize(encoding='utf-8', format='turtle') + sys.stdout.buffer.write(bs) diff --git a/trygvis/eda/__main__.py b/trygvis/eda/__main__.py new file mode 100644 index 0000000..5f641d8 --- /dev/null +++ b/trygvis/eda/__main__.py @@ -0,0 +1,3 @@ +from .cli import eda_rdf + +eda_rdf.main() diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py new file mode 100644 index 0000000..aa9e57e --- /dev/null +++ b/trygvis/eda/cli/__init__.py @@ -0,0 +1,56 @@ +import sys +import logging + +from rdflib import ConjunctiveGraph +from rdflib import Graph +from rdflib import store + +from ..digikey import rdf as digikey_rdf +from ..kicad import rdf as kicad_rdf + + +class CliException(Exception): + pass + + +def init(): + logging.basicConfig(level=logging.DEBUG) + pass + + +def info(msg=None): + if msg is not None: + sys.stderr.write(msg) + sys.stderr.write("\n") + + +def exit(msg=None): + sys.exit(msg) + + +def open_database(path): + g = ConjunctiveGraph('Sleepycat') + rt = g.open(path, create=False) + if rt == store.NO_STORE: + info("Creating store in %s" % path) + g.open(path, create=True) + elif rt != store.VALID_STORE: + raise CliException("The database is corrupt: %s" % path) + + return g + + +def create_graph(digikey=False, kicad=False): + g = Graph() + + if digikey: + g.bind("dk", digikey_rdf.DIGIKEY) + g.bind("dk-part", digikey_rdf.DIGIKEY_PART) + g.bind("dk-attr-type", digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE) + g.bind("dk-attr-value", digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE) + g.bind("dk-product-category", digikey_rdf.DIGIKEY_PRODUCT_CATEGORY) + + if kicad: + g.bind("kicad", kicad_rdf.KICAD) + g.bind("kicad-type", kicad_rdf.KICAD_TYPE) + return g diff --git a/trygvis/eda/cli/add_to_db.py b/trygvis/eda/cli/add_to_db.py new file mode 100644 index 0000000..0511850 --- /dev/null +++ b/trygvis/eda/cli/add_to_db.py @@ -0,0 +1,17 @@ +import trygvis.eda.cli as cli + + +def run(files, path, args): + g = cli.open_database(path) + + s = 0 + for f in files: + cli.info("Adding %s" % f) + pre = len(g) + g.load(f, format="turtle") + post = len(g) + diff = post - pre + s += diff + cli.info("Loaded %d tuples" % diff) + + cli.info("Done. Loaded %d tuples" % s) diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py new file mode 100644 index 0000000..aef360c --- /dev/null +++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py @@ -0,0 +1,29 @@ +from trygvis.eda import cli, write_graph + +from trygvis.eda.digikey import * + + +def run(category, sub_category, output_file, args): + client = DigikeyClient() + db = DigikeyDatabase() + + download_category_tree(db, client) + c = db.find_category(category) + + if c is None: + cli.exit("Could not find category \"%s\"" % category) + + sc = c.find_sub_category_by_label(sub_category) + + if c is None: + cli.exit("Could not find sub-category \"%s\" inside \"%s\"" % (sub_category, category)) + + attributes = download_attribute_types_from_category(sc, client) + db.merge_attribute_types(attributes) + + g = cli.create_graph() + for a in attributes: + [g.add(node) for node in a.to_nodes()] + + filename = output_file if output_file is not "-" else None + write_graph(gen_g=lambda: g, filename=filename) diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py new file mode 100755 index 0000000..99f5266 --- /dev/null +++ b/trygvis/eda/cli/digikey_download_for_schematic.py @@ -0,0 +1,133 @@ +from os.path import isfile + +from rdflib.plugins.sparql import prepareQuery +import rdflib.term + +from trygvis.eda import cli, write_graph +from trygvis.eda.digikey import * +from trygvis.eda.digikey import rdf as digikey_rdf +from trygvis.eda.kicad import rdf as kicad_rdf + +initNs = { + "rdf": RDF, + "rdfs": RDFS, + "dk": digikey_rdf.DIGIKEY, + "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE, + "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE, + "dk-part": digikey_rdf.DIGIKEY_PART, + "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY, + "kicad": kicad_rdf.KICAD, + "kicad-type": kicad_rdf.KICAD_TYPE} + + +def run(schematic_url, db_path, args): + cli.info("Schematic: %s" % schematic_url) + g = cli.open_database(db_path) + + client = DigikeyClient() + db = DigikeyDatabase() + download_category_tree(db, client) + + # Dump schematic: + # SELECT ?schematic + # WHERE { + # ?schematic a kicad-type:schematic . + # b:rateboard-2_a ? ?. + # ?s ?predicate ?object . + # } + + # allComponentsQ = prepareQuery('SELECT ?ref ?value WHERE { ?schematic_url a kicad-type:schematic . ?schematic_url kicad:component ?o . ?o rdfs:label ?ref . ?o kicad:value ?value . }', initNs = initNs) + # + # for row in g.query(allComponentsQ, initBindings={'schematic_url': schematic_url}): + # print("ref=%s, value=%s" % (row.ref, row.value)) + + res = g.query(prepareQuery("SELECT ?schematic WHERE {?schematic a kicad-type:schematic}", initNs=initNs)) + print("Found %d schematics in database" % len(res)) + for row in res: + print("schematic: %s" % row.schematic) + + res = g.query(prepareQuery("SELECT ?dk_part ?dk_part_number WHERE {?dk_part a dk:part ; dk:partNumber ?dk_part_number}", initNs=initNs)) + print("Found %d Digikey parts in database" % len(res)) + for row in res: + print("Part: url=%s, partNumber=%s" % (row.dk_part, row.dk_part_number)) + + q = prepareQuery(""" +SELECT + ?digikey_pn + (group_concat(distinct ?ref;separator=";") as ?refs) +WHERE { + ?schematic_url kicad:component ?cmp . + ?cmp rdfs:label ?ref ; + kicad:value ?value . + OPTIONAL { + ?cmp kicad:field ?d . + ?d kicad:field_name "digikey" ; + kicad:field_value ?digikey_pn . +# OPTIONAL { + ?dk_part a dk:part ; + dk:partNumber ?digikey_pn . +# } + } +} +GROUP BY ?digikey_pn +ORDER BY ?digikey_pn +""", initNs=initNs) + + res = g.query(q, initBindings={'schematic_url': rdflib.term.URIRef(schematic_url)}) + for row in res: + pn = row.digikey_pn + + if pn is None: + continue + + refs = row.refs.split(';') + + cli.info("Part \"%s\" is used by %s" % (pn, refs)) + + filename = 'ttl/digikey-part-' + normalize_filename(pn + ".ttl") + + def download_graph(): + cli.info("Downloading product: " + pn) + product = download_product(client, db, pn) + + g = cli.create_graph(digikey=True) + [g.add(node) for node in product.to_nodes()] + return g + + write_graph(download_graph, filename) + + # q = prepareQuery(""" + # SELECT + # DISTINCT ?category ?digikeyUrl + # WHERE { + # ?schematic_url kicad:component ?cmp . + # ?cmp rdfs:label ?ref . + # ?cmp kicad:value ?value . + # ?cmp kicad:field ?d . + # ?d kicad:field_name "digikey" . + # ?d kicad:field_value ?digikey . + # + # ?part a dk:part . + # ?part dk:partNumber ?digikey . + # ?part dk:category ?category . + # ?category dk:parent ?_ . + # ?category dk:url ?digikeyUrl + # }""", initNs=initNs) + # + # res = g.query(q, initBindings={'schematic_url': schematic_url}) + # + # cli.info("Found %d categories" % (len(res))) + # for row in res: + # cli.info("Category: %s" % (row.category)) + # category = db.findSubCategoryByUrl(row.category) + # if category is None: + # raise Exception('could not find category: ' + row.category) + # attributes = downloadAttributeTypesFromCategory(category, client) + # db.mergeAttributeTypes(attributes) + # + # filename = 'digikey-category-' + normalize_filename(str(category.label) + ".ttl") + # if not isfile(filename): + # tmpG = cli.create_graph() + # for a in attributes: + # [tmpG.add(node) for node in a.toNodes()] + # writeGraph(tmpG, 'ttl/' + filename) diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py new file mode 100644 index 0000000..4e80e93 --- /dev/null +++ b/trygvis/eda/cli/eda_rdf.py @@ -0,0 +1,69 @@ +import argparse +import sys +import trygvis.eda.cli as cli + + +def main(): + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(dest="cmd") + + p = subparsers.add_parser("kicad-bom-to-ttl") + p.add_argument("-o", "--output", required=False) + p.add_argument("-i", "--input", required=False) + + p = subparsers.add_parser("add-to-db") + p.add_argument("-d", "--db", required=True) + p.add_argument("files", nargs='*') + + p = subparsers.add_parser("make-bom") + p.add_argument("-d", "--db", required=True) + p.add_argument("--schematic", required=True) + + p = subparsers.add_parser("digikey-download-for-schematic") + p.add_argument("-d", "--db", required=True) + p.add_argument("--schematic", required=True) + + p = subparsers.add_parser("digikey-download-attribute-types-for-category") + p.add_argument("-c", "--category", required=True) + p.add_argument("-s", "--sub-category", required=True) + p.add_argument("-o", "--output", required=False) + + args = parser.parse_args() + + cli.init() + + if args.cmd == "kicad-bom-to-ttl": + from trygvis.eda.cli import kicad_bom_to_ttl + + if args.input is not None: + src = open(args.input, "r") + else: + src = sys.stdin + + if args.output is not None: + dst = open(args.output, "wb") + else: + dst = sys.stdout.buffer + + with src, dst: + kicad_bom_to_ttl.run(src, dst, args) + + elif args.cmd == "add-to-db": + from trygvis.eda.cli import add_to_db + + add_to_db.run(args.files, args.db, args) + + elif args.cmd == "make-bom": + from trygvis.eda.cli import make_bom + + make_bom.run(args.schematic, args.db, args) + + elif args.cmd == "digikey-download-for-schematic": + from trygvis.eda.cli import digikey_download_for_schematic + digikey_download_for_schematic.run(args.schematic, args.db, args) + elif args.cmd == "digikey-download-attribute-types-for-category": + from trygvis.eda.cli import digikey_download_attribute_types_for_category + digikey_download_attribute_types_for_category.run(args.category, args.sub_category, args.output, args) + else: + sys.exit("Unknown command: %s" % args.cmd) diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py new file mode 100755 index 0000000..c7d6899 --- /dev/null +++ b/trygvis/eda/cli/kicad_bom_to_ttl.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 + +import functools +import os.path +import re +import xml.etree.ElementTree +from urllib.parse import quote_plus + +from trygvis.eda import cli +from trygvis.eda.kicad import rdf as kicad_rdf + + +def _clean(s): + if s is None: + return None + s = s.strip() + return None if len(s) == 0 else s + + +def _cleaned_text(e, name): + child = e.find(name) + if child is None: + return None + return child.text + + +def _cleaned_attr(e, name): + value = e.get(name) + if value is None: + return None + return _clean(value) + + +def _iso_size_to_factor(size): + if size == 'k': + return 3 + if size == 'M': + return 6 + if size == 'm': + return -3 + if size == 'u' or size == '\N{GREEK SMALL LETTER MU}' or size == '\N{MICRO SIGN}': + return -6 + if size == 'n': + return -9 + if size == 'p': + return -12 + return None + + +@functools.total_ordering +class Value: + valueRe = re.compile("([0-9]+\.?[0-9]*)(.*)") + typeRe = re.compile("(.*)(H|Hz)$") + + def __init__(self, text, value, factor, type): + self.text = text + self.value = value + self.factor = factor + self.type = type + + def __eq__(self, other): + return self.text == other.text + + def __lt__(self, other): + if self.value is not None: + if other.value is None: + return 1 + else: + return (self.factor, self.value) < (other.factor, other.value) + else: + if other.value is None: + return self.text < other.text + else: + return -1 + + def __str__(self): + if self.value is not None: + s = str(self.value) + 'e' + str(self.factor) + if self.type is not None: + s = s + ' ' + self.type + return s + if self.text is not None: + return self.text + return "unknown" + + @staticmethod + def parse(text): + if text is None: + return Value(None, None, None, None) + + m = Value.valueRe.match(text) + if m: + value_int = float(m.group(1)) + suffix = m.group(2) + m = Value.typeRe.match(suffix) + if m: + suffix = m.group(1) + type = m.group(2) + else: + type = None + + factor = _iso_size_to_factor(suffix) + # cli.info(text + ': suffix:' + suffix + ' => value_int: ' + str(value_int) + ', factor: ' + str( + # factor) + ', type=' + str(type)) + + if value_int is not None and factor is not None: + return Value(text, value_int, factor, type) + + return Value(text, None, None, None) + + +class Component: + refRe = re.compile("([a-zA-Z]+)([0-9]+)") + + def __init__(self, ref, value, footprint, fields): + self.ref = ref + self.value = Value.parse(value) + self.footprint = footprint + self.fields = fields + m = Component.refRe.match(ref) + if m: + self.ref_class = m.group(1) + self.ref_instance = int(m.group(2)) + else: + self.ref_class = None + self.ref_instance = None + + def __str__(self): + s = 'ref=' + self.ref + if self.value is not None: + s += ', value=' + str(self.value) + if self.footprint is not None: + s += ', footprint=' + self.footprint + for name, value in self.fields.items(): + s += ', ' + name + '=' + value + return s + + def find_field(self, key): + return self.fields[key] if key in self.fields else None + + @staticmethod + def from_xml(e): + ref = e.get('ref') + value = _cleaned_text(e, 'value') + footprint = _cleaned_text(e, 'footprint') + fs = {} + + fields = e.find('fields') + if fields is not None: + for field in fields.findall('field'): + fs[field.get('name')] = field.text + + if ref is not None: + return Component(ref, value, footprint, fs) + + return None + + +class TitleBlock(object): + def __init__(self, title=None, rev=None, date=None, source=None): + self.title = title + self.rev = rev + self.date = date + self.source = source + + @staticmethod + def from_xml(tb): + title = _cleaned_text(tb, 'title') + rev = _cleaned_text(tb, 'rev') + date = _cleaned_text(tb, 'date') + source = _cleaned_text(tb, 'source') + return TitleBlock(title, rev, date, source) + + +class Sheet(object): + def __init__(self, number, name, title_block): + self.number = number + self.name = name + self.title_block = title_block + + @staticmethod + def from_xml(s): + number = _cleaned_attr(s, "number") + name = _cleaned_attr(s, "name") + node = s.find('title_block') + title_block = TitleBlock.from_xml(node) if node is not None else TitleBlock() + return Sheet(int(number) if number is not None else None, name, title_block) + + +class Design(object): + def __init__(self, source=None, date=None, tool=None, sheets=None): + self.source = source + self.date = date + self.tool = tool + self.sheets = sheets if sheets is not None else [] + + @staticmethod + def from_xml(d): + source = _cleaned_text(d, 'source') + date = _cleaned_text(d, 'date') + tool = _cleaned_text(d, 'tool') + sheets = [] + for s in d.iterfind('sheet'): + sheets.append(Sheet.from_xml(s)) + return Design(source, date, tool, sheets) + + +class Export(object): + def __init__(self, design, components): + self.design = design if design is not None else Design() + self.components = components + + @staticmethod + def from_xml(doc): + cs = [] + + node = doc.find('design') + design = Design.from_xml(node) if node is not None else None + + components = doc.find('components') + for e in components.iter('comp'): + c = Component.from_xml(e) + if c is not None: + cs.append(c) + + return Export(design, cs) + + def component_fields(self): + fs = set() + for c in self.components: + for key, value in c.fields.items(): + fs.add(key) + return fs + + def generate_schematic_url(self): + s = next((s for s in self.design.sheets if s.number == 1), None) + title = s.title_block.title if s is not None else None + if title is None: + if self.design.source is not None: + title = _clean(os.path.basename(self.design.source)) + + if title is None: + raise cli.CliException("Could not generate a stable, identifying URL for the schematic") + + title = quote_plus(title) + return kicad_rdf.KICAD_BOARD[title] + + +def run(src, dst, args): + from trygvis.eda import cli + + tree = xml.etree.ElementTree.parse(src) + root = tree.getroot() + + export = Export.from_xml(root) + + # print("components:") + # for c in export.components: + # print(c) + + # for name in export.component_fields(): + # cli.info(name) + + from operator import attrgetter + import itertools + + # cli.info("components:") + # fmt = '%2s%3s' + # cli.info(fmt % ('Ref', '')) + + components = sorted(export.components, key=attrgetter('ref')) + + component_count = 0 + part_count = 0 + has_part = 0 + has_digikey = 0 + + for cls, cs in itertools.groupby(components, key=attrgetter('ref_class')): + # cli.info('--- Class: %s ---' % cls) + cs = sorted(cs, key=attrgetter('value')) + for c in cs: + component_count += 1 + + value = str(c.value) + part = c.find_field("part") + digikey = c.find_field("digikey") + footprint = c.footprint + + if c.ref_class is not None: + ref_class = c.ref_class + ref_instance = c.ref_instance + else: + ref_class = c.ref + ref_instance = "" + + # cli.info(fmt % (ref_class, ref_instance)) + # cli.info(' Value: %s' % c.value.text) + # if c.footprint: + # cli.info(' Footprint: %s' % c.footprint) + + if part is not None: + if part != 'NA': + has_part += 1 if part is not None else 0 + part_count += 1 + + if digikey is not None: + has_digikey += 1 + else: + part = None + else: + part_count += 1 + part = "MISSING" + + # if part is not None: + # cli.info(' Part: %s' % part) + # if digikey is not None: + # cli.info(' Digikey: %s' % digikey) + + schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url() + + # cli.info() + # cli.info('=== Summary ===') + # cli.info('Schematic URL: %s' % schematic_url) + # cli.info('Number of components: %d' % component_count) + # cli.info('Number of parts: %d' % part_count) + # cli.info('Assigned part: %d / %2.f%%' % (has_part, (has_part / part_count) * 100)) + # cli.info('Assigned digikey: %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100)) + + from ..kicad import rdf as kicad_rdf + from rdflib import Literal, BNode, URIRef + from rdflib.namespace import RDF, RDFS + + g = cli.create_graph(kicad=True) + + schematic = URIRef(schematic_url) + g.add((schematic, RDF.type, kicad_rdf.KICAD_TYPE.schematic)) + + # componentNodes = [] + for c in export.components: + node = BNode() + g.add((schematic, kicad_rdf.KICAD.component, node)) + + g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component)) + g.add((node, RDFS.label, Literal(c.ref))) + g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text))) + + for name, value in c.fields.items(): + f = BNode() + g.add((node, kicad_rdf.KICAD.field, f)) + g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field)) + g.add((f, kicad_rdf.KICAD.field_name, Literal(name))) + g.add((f, kicad_rdf.KICAD.field_value, Literal(value))) + + # TODO: serialize the data from <design> too + + g.serialize(destination=dst, encoding='utf-8', format='turtle') diff --git a/trygvis/eda/cli/make_bom.py b/trygvis/eda/cli/make_bom.py new file mode 100755 index 0000000..28059cb --- /dev/null +++ b/trygvis/eda/cli/make_bom.py @@ -0,0 +1,21 @@ +from trygvis.eda import cli +from trygvis.eda.digikey import * +from trygvis.eda.digikey import rdf as digikey_rdf +from trygvis.eda.kicad import rdf as kicad_rdf + +initNs = { + "rdf": RDF, + "rdfs": RDFS, + "dk": digikey_rdf.DIGIKEY, + "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE, + "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE, + "dk-part": digikey_rdf.DIGIKEY_PART, + "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY, + "kicad": kicad_rdf.KICAD, + "kicad-type": kicad_rdf.KICAD_TYPE} + + +def run(schematic_url, db_path, args): + g = cli.open_database(db_path) + + cli.info('implement..') diff --git a/trygvis/eda/digikey/__init__.py b/trygvis/eda/digikey/__init__.py new file mode 100644 index 0000000..5f4ad8a --- /dev/null +++ b/trygvis/eda/digikey/__init__.py @@ -0,0 +1,300 @@ +import re + +import requests +from cachecontrol import CacheControl +from cachecontrol.caches.file_cache import FileCache +from cachecontrol.heuristics import ExpiresAfter +from lxml import html +from rdflib import Literal +from rdflib.namespace import RDF, RDFS + +import trygvis.eda.digikey.rdf + +def normalize_filename(part): + return part.replace('/', '_').replace(' ', '_') + +def _clean(s): + if s is None: + return None + s = s.strip() + return None if len(s) == 0 else s + + +class DigikeyDatabase(object): + def __init__(self): + self.productCategories = [] + self.attributeTypes = {} + + def add_product_category(self, pc): + self.productCategories.append(pc) + + def find_category(self, label): + return next((c for c in self.productCategories if c.label == label), None) + + def find_sub_category_by_url(self, url): + for p in self.productCategories: + for sc in p.subCategories: + if sc.url() == url: + return sc + return None + + def merge_attribute_types(self, attributeTypes): + for a in attributeTypes: + if a.id in self.attributeTypes: + # TODO: implement merging + continue + self.attributeTypes[a.id] = a + + def find_type(self, id): + return self.attributeTypes.get(id, None) + + +class DigikeyProductCategory(object): + def __init__(self, id, label, digikey_url=None, parent=None): + self.id = _clean(id) + self.label = _clean(label) + self.digikey_url = digikey_url if digikey_url is None or digikey_url.startswith("http") else \ + "http://www.digikey.com" + digikey_url + self.parent = parent + self.subCategories = [] + + assert self.id is not None + assert self.label is not None + + def add_sub_category(self, id, label, digikey_url): + sc = DigikeyProductCategory(id, label, digikey_url=digikey_url, parent=self) + self.subCategories.append(sc) + + def find_sub_category_by_label(self, label): + return next((sc for sc in self.subCategories if sc.label == label), None) + + def url(self): + return rdf.DIGIKEY_PRODUCT_CATEGORY[self.id] + + def to_nodes(self): + node = self.url() + nodes = [ + (node, RDF.type, rdf.DIGIKEY.productCategory), + (node, RDFS.label, Literal(self.label)), + ] + if self.parent is not None: + parentUrl = rdf.DIGIKEY_PRODUCT_CATEGORY[self.parent.id] + nodes.append((node, rdf.DIGIKEY.parent, parentUrl)) + if self.digikey_url is not None: + nodes.append((node, rdf.DIGIKEY.url, Literal(self.digikey_url))) + return nodes + + +class DigikeyAttributeType(object): + def __init__(self, category, id, label, options): + self.category = category + self.id = _clean(id) + self.label = _clean(label) + self.options = options + + assert self.category is not None + assert self.id is not None + assert self.label is not None + assert self.options is not None + + def to_nodes(self): + nodes = [] + node = rdf.DIGIKEY_ATTRIBUTE_TYPE[self.id] + nodes.append((node, RDF.type, rdf.DIGIKEY.attributeType)) + nodes.append((node, RDFS.label, Literal(self.label))) + + for o in self.options: + optionNode = rdf.DIGIKEY_ATTRIBUTE_VALUE[self.id + '-' + o.id] + nodes.extend([ + (optionNode, rdf.DIGIKEY.id, Literal(o.id)), + (optionNode, RDFS.label, Literal(o.label)), + (node, rdf.DIGIKEY.value, optionNode)]) + return nodes + + +class DigikeyAttributeValue(object): + def __init__(self, id, label, type=None, type_id=None, type_label=None): + self.id = _clean(id) + self.label = _clean(label) + self.type = type + self.type_id = type_id + self.type_label = type_label + + assert self.id is not None + assert self.label is not None + + +class DigikeyProduct(object): + def __init__(self, part_id, part_number, values, categories): + self.part_id = _clean(part_id) + self.part_number = _clean(part_number) + self.values = values + self.categories = categories + self.quantity_available = None + self.description = None + + assert self.part_id is not None + assert self.part_number is not None + + def to_nodes(self): + nodes = [] + node = rdf.DIGIKEY_PART[self.part_id] + nodes.append((node, RDF.type, rdf.DIGIKEY.part)) + nodes.append((node, rdf.DIGIKEY.partNumber, Literal(self.part_number))) + nodes.append((node, RDFS.label, Literal(self.description))) + for v in self.values: + typeLabel = v.type.label if v.type is not None else v.typeLabel + typeId = v.type.id if v.type is not None else v.typeId + nodes.append((node, rdf.DIGIKEY['attribute-value'], rdf.DIGIKEY_ATTRIBUTE_VALUE[typeId + '-' + v.id])) + + for c in self.categories: + nodes.append((node, rdf.DIGIKEY.category, c.url())) + return nodes + + +class DigikeyClient(object): + def __init__(self): + cache = FileCache('digikey_cache', forever=True) + self.sess = CacheControl(requests.Session(), cache=cache, heuristic=ExpiresAfter(days=1)) + + def req(self, url, params=None): + if not url.startswith("http://"): + url = "http://www.digikey.com" + url + return self.sess.get(url, params=params) + + +def _to_string(e): + s = "" + for t in e.itertext(): + s += t + return s.strip() + + +def _id_from_url(url): + if url is None: + return None + m = re.search(r".*/([0-9]+)", url) + return m.group(1) if m else None + + +def download_category_tree(database, client, baseurl="http://www.digikey.com/products/en"): + page = client.req(baseurl) + dom = html.fromstring(page.content) + + items = dom.xpath("//h2[contains(@class, 'catfiltertopitem')]") + for h2 in items: + label = _to_string(h2) + # print(h2) + pcId = None + for a in h2.getchildren(): + url = a.get('href') + pcId = _id_from_url(url) + if pcId is None: + continue + + if pcId is None: + continue + + pc = DigikeyProductCategory(pcId, label) + n = h2.getnext() + if n.tag == 'span': + n = n.getnext() + if n.tag == 'ul': + for a in n.xpath('./li/a'): + label = _to_string(a) + url = a.get('href') + id = _id_from_url(url) + if id is None: + continue + # print(' ' + toString(a) + ', id=' + str(id) + ', url=' + url) + pc.add_sub_category(id, label, url) + + database.add_product_category(pc) + + +def download_attribute_types_from_category(category, client): + page = client.req(category.digikey_url) + tree = html.fromstring(page.content) + + attributes = [] + for form in tree.xpath("//form[contains(@class, 'search-form')]"): + print('form: ' + str(form)) + headers = form.xpath(".//tr[@id='appliedFilterHeaderRow']/th/text()") + print("headers: " + str(headers)) + for select in form.xpath(".//td/select[contains(@class, 'filter-selectors')]"): + td = select.getparent() + index = td.getparent().index(td) + try: + attributeLabel = headers[index] + except: + continue + attributeId = select.get('name') + print("label: " + attributeLabel + ", id: " + attributeId) + options = [] + type = DigikeyAttributeType(category, attributeId, attributeLabel, options) + for o in select.xpath("./option"): + id = o.get('value') + label = _to_string(o) + # print("o: %s" % str(o)) + options.append(DigikeyAttributeValue(id, label, type=type)) + attributes.append(type) + + return attributes + + +def download_product(client, db, query): + # http://www.digikey.com/products/en?x=0&y=0&lang=en&site=us&keywords=553-2320-1-ND + page = client.req("http://www.digikey.com/products/en", params={'lang': 'en', 'site': 'us', 'keywords': query}) + tree = html.fromstring(page.content) + + values = [] + categories = [] + for table in tree.xpath("//table[contains(@class, 'attributes-table-main')]"): + label = None + id = None + for tr in table.xpath(".//tr"): + if tr.get("id") is not None: + continue + tds = tr.xpath("./th | ./td") + if len(tds) != 3: + continue + type_label = _to_string(tds[0]) + label = _to_string(tds[1]) + for input in tds[2].xpath("./input[@name]"): + typeId = input.get("name") + id = input.get("value") + else: + typeId = None + + if id is None or typeId is None: + continue + if typeId == "t": # categories are handled later + continue + values.append(DigikeyAttributeValue(id, label, type_id=typeId, type_label=type_label)) + + for td in table.xpath(".//td[@class='attributes-td-categories-link']"): + tr = td.getparent() + id = None + url = None + for a in td.xpath(".//a[@href]"): + url = a.get("href") + id = _id_from_url(url) + + for input in tr.xpath(".//input[@name='t' and @value]"): + categoryId = input.get("value") + + if id is None: + continue + categories.append(DigikeyProductCategory(id, label, digikey_url=url)) + + part_id = part_number = None + for n in tree.xpath("//input[@name='partid' and @value]"): + part_id = n.get("value") + for n in tree.xpath("//*[@itemprop='productID' and @content]"): + part_number = n.get("content") + part_number = part_number.replace('sku:', '') + + p = DigikeyProduct(part_id, part_number, values, categories) + for n in tree.xpath("//*[@itemprop='description']"): + p.description = _to_string(n) + return p diff --git a/trygvis/eda/digikey/__main__.py b/trygvis/eda/digikey/__main__.py new file mode 100644 index 0000000..ceb341e --- /dev/null +++ b/trygvis/eda/digikey/__main__.py @@ -0,0 +1,42 @@ +import argparse + +from .. import write_graph +from ..cli import * +from ..digikey import * + +parser = argparse.ArgumentParser() +subparsers = parser.add_subparsers(dest='cmd') # help='sub-command help' + +dct_parser = subparsers.add_parser("download-category-tree") +dct_parser.add_argument("-o", "--output", required=False) + +dp_parser = subparsers.add_parser("download-product") +dp_parser.add_argument("-p", "--product") +dp_parser.add_argument("-o", "--output", required=False) + +args = parser.parse_args() + +client = DigikeyClient() +db = DigikeyDatabase() + +if args.cmd == "download-category-tree": + download_category_tree(db, client) + if args.output is not None: + def make_graph(): + g = create_graph(digikey=True) + for pc in db.productCategories: + [g.add(node) for node in pc.to_nodes()] + + for sc in pc.subCategories: + [g.add(node) for node in sc.to_nodes()] + write_graph(make_graph, args.output) + +elif args.cmd == "download-product": + download_category_tree(db, client) + product = download_product(client, db, args.product) + + if args.output is not None: + def make_graph(): + g = create_graph(digikey=True) + [g.add(node) for node in product.to_nodes()] + write_graph(make_graph, args.output) diff --git a/trygvis/eda/digikey/rdf.py b/trygvis/eda/digikey/rdf.py new file mode 100644 index 0000000..5f1dede --- /dev/null +++ b/trygvis/eda/digikey/rdf.py @@ -0,0 +1,7 @@ +from rdflib import Namespace + +DIGIKEY = Namespace("https://trygvis.io/purl/digikey#") +DIGIKEY_ATTRIBUTE_TYPE = Namespace("https://trygvis.io/purl/digikey-attribute-type#") +DIGIKEY_ATTRIBUTE_VALUE = Namespace("https://trygvis.io/purl/digikey-attribute-value#") +DIGIKEY_PART = Namespace("https://trygvis.io/purl/digikey-part#") +DIGIKEY_PRODUCT_CATEGORY = Namespace("https://trygvis.io/purl/digikey-product-category#") diff --git a/trygvis/eda/kicad/__init__.py b/trygvis/eda/kicad/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/trygvis/eda/kicad/__init__.py diff --git a/trygvis/eda/kicad/rdf.py b/trygvis/eda/kicad/rdf.py new file mode 100644 index 0000000..683eaad --- /dev/null +++ b/trygvis/eda/kicad/rdf.py @@ -0,0 +1,7 @@ +from rdflib import Namespace + +KICAD = Namespace("https://trygvis/purl/kicad#") +KICAD_TYPE = Namespace("https://trygvis/purl/kicad-type#") + +# Namespace for all unknown kicad boards +KICAD_BOARD = Namespace("https://trygvis/purl/kicad-board#") |