From 67013ae17af0436b930dce450a813239be969601 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Tue, 27 Dec 2016 01:14:37 +0100 Subject: o Initial import of some tools for working with KiCAD BOM files and Digi-key. --- .editorconfig | 2 + .gitignore | 13 + LICENSE.txt | 19 ++ README.md | 16 + requirement.txt | 14 + setup.cfg | 2 + setup.py | 106 ++++++ trygvis/__init__.py | 0 trygvis/eda/__init__.py | 20 ++ trygvis/eda/__main__.py | 3 + trygvis/eda/cli/__init__.py | 56 ++++ trygvis/eda/cli/add_to_db.py | 17 + ...igikey_download_attribute_types_for_category.py | 29 ++ trygvis/eda/cli/digikey_download_for_schematic.py | 133 ++++++++ trygvis/eda/cli/eda_rdf.py | 69 ++++ trygvis/eda/cli/kicad_bom_to_ttl.py | 356 +++++++++++++++++++++ trygvis/eda/cli/make_bom.py | 21 ++ trygvis/eda/digikey/__init__.py | 300 +++++++++++++++++ trygvis/eda/digikey/__main__.py | 42 +++ trygvis/eda/digikey/rdf.py | 7 + trygvis/eda/kicad/__init__.py | 0 trygvis/eda/kicad/rdf.py | 7 + 22 files changed, 1232 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 requirement.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 trygvis/__init__.py create mode 100644 trygvis/eda/__init__.py create mode 100644 trygvis/eda/__main__.py create mode 100644 trygvis/eda/cli/__init__.py create mode 100644 trygvis/eda/cli/add_to_db.py create mode 100644 trygvis/eda/cli/digikey_download_attribute_types_for_category.py create mode 100755 trygvis/eda/cli/digikey_download_for_schematic.py create mode 100644 trygvis/eda/cli/eda_rdf.py create mode 100755 trygvis/eda/cli/kicad_bom_to_ttl.py create mode 100755 trygvis/eda/cli/make_bom.py create mode 100644 trygvis/eda/digikey/__init__.py create mode 100644 trygvis/eda/digikey/__main__.py create mode 100644 trygvis/eda/digikey/rdf.py create mode 100644 trygvis/eda/kicad/__init__.py create mode 100644 trygvis/eda/kicad/rdf.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d0587c8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,2 @@ +[*] +ident = 4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..70e9a43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +env +*.db +*.pyc + +digikey_cache + +*.n3 +*.ttl + +.idea +*.iml + +*.tmp.* diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..7675ff5 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2016 Trygve Laugstøl + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1ff2319 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# Rules + +o Inconsistent unit usage (0.1uF vs 100nF) +o Different components for same values / constraints +o Tolerances are within requirements +o values match selected part +o correct footprint +o find skipped instance numbers (C1, C2, C4 => C3) + +o Availability + - status: ative/last buy/obsolete + - MOQ + +# Utils + +o Download datasheets, IBIS model, SPICE model diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..2567238 --- /dev/null +++ b/requirement.txt @@ -0,0 +1,14 @@ +bsddb3==6.2.1 +CacheControl==0.11.7 +html5lib==0.999999999 +isodate==0.5.4 +lockfile==0.12.2 +lxml==3.7.0 +mechanize==0.2.5 +pkg-resources==0.0.0 +pyparsing==2.1.10 +rdflib==4.2.1 +requests==2.12.4 +six==1.10.0 +SPARQLWrapper==1.8.0 +webencodings==0.5 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6fcf812 --- /dev/null +++ b/setup.py @@ -0,0 +1,106 @@ +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +# To use a consistent encoding +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +setup( + name='eda-rds', + + version='1.0.0', + + description='RDF-oriented for for EDA (electronic design automation)', + long_description=long_description, + + # The project's main homepage. + url='https://trygvis.io/git/2016/12/eda-rdf', + + author='Trygve Laugstøl', + author_email='trygvis@inamo.no', + + # Choose your license + license='MIT', + + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + 'Development Status :: 3 - Alpha', + + 'Intended Audience :: Developers', + 'Intended Audience :: Manufacturing', + 'Topic :: Software Development :: Build Tools', + 'Topic :: Database', + 'Topic :: Internet', + 'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)', + 'Topic :: Software Development :: Quality Assurance', + 'Environment :: Console', + + 'License :: OSI Approved :: MIT License', + + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Operating System :: OS Independent', + ], + + keywords='EDA KiCAD RDF Digi-Key', + + packages=find_packages(exclude=['docs', 'tests']), + + # List run-time dependencies here. These will be installed by pip when + # your project is installed. For an analysis of "install_requires" vs pip's + # requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + "bsddb3>=6.2.1", + "CacheControl>=0.11.7", + "html5lib>=0.999999999", + "isodate>=0.5.4", + "lockfile>=0.12.2", + "lxml>=3.7.0", + "mechanize>=0.2.5", + "pyparsing>=2.1.10", + "rdflib>=4.2.1", + "requests>=2.12.4", + "six>=1.10.0", + "SPARQLWrapper>=1.8.0", + "webencodings>=0.5", + ], + + # List additional groups of dependencies here (e.g. development + # dependencies). You can install these using the following syntax, + # for example: + # $ pip install -e .[dev,test] + extras_require={ + # 'dev': ['check-manifest'], + # 'test': ['coverage'], + }, + + # If there are data files included in your packages that need to be + # installed, specify them here. If using Python 2.6 or less, then these + # have to be included in MANIFEST.in as well. + package_data={ + # 'sample': ['package_data.dat'], + }, + + # Although 'package_data' is the preferred approach, in some case you may + # need to place data files outside of your packages. See: + # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa + # In this case, 'data_file' will be installed into '/my_data' + # data_files=[('my_data', ['data/data_file'])], + + # To provide executable scripts, use entry points in preference to the + # "scripts" keyword. Entry points provide cross-platform support and allow + # pip to create the appropriate form of executable for the target platform. + entry_points={ + 'console_scripts': [ + 'eda-rdf=trygvis.eda.cli.eda_rdf:main', + ], + }, +) diff --git a/trygvis/__init__.py b/trygvis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/trygvis/eda/__init__.py b/trygvis/eda/__init__.py new file mode 100644 index 0000000..e6ccb44 --- /dev/null +++ b/trygvis/eda/__init__.py @@ -0,0 +1,20 @@ +import sys + +from os.path import isfile +from . import cli + + +def write_graph(gen_g, filename=None, force_write=False): + if filename is not None: + if force_write or not isfile(filename): + g = gen_g() + bs = g.serialize(encoding='utf-8', format='turtle') + with open(filename, "wb") as f: + f.write(bs) + cli.info("Wrote %s" % filename) + else: + cli.info("Skipped writing %s, already exists" % filename) + else: + g = gen_g() + bs = g.serialize(encoding='utf-8', format='turtle') + sys.stdout.buffer.write(bs) diff --git a/trygvis/eda/__main__.py b/trygvis/eda/__main__.py new file mode 100644 index 0000000..5f641d8 --- /dev/null +++ b/trygvis/eda/__main__.py @@ -0,0 +1,3 @@ +from .cli import eda_rdf + +eda_rdf.main() diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py new file mode 100644 index 0000000..aa9e57e --- /dev/null +++ b/trygvis/eda/cli/__init__.py @@ -0,0 +1,56 @@ +import sys +import logging + +from rdflib import ConjunctiveGraph +from rdflib import Graph +from rdflib import store + +from ..digikey import rdf as digikey_rdf +from ..kicad import rdf as kicad_rdf + + +class CliException(Exception): + pass + + +def init(): + logging.basicConfig(level=logging.DEBUG) + pass + + +def info(msg=None): + if msg is not None: + sys.stderr.write(msg) + sys.stderr.write("\n") + + +def exit(msg=None): + sys.exit(msg) + + +def open_database(path): + g = ConjunctiveGraph('Sleepycat') + rt = g.open(path, create=False) + if rt == store.NO_STORE: + info("Creating store in %s" % path) + g.open(path, create=True) + elif rt != store.VALID_STORE: + raise CliException("The database is corrupt: %s" % path) + + return g + + +def create_graph(digikey=False, kicad=False): + g = Graph() + + if digikey: + g.bind("dk", digikey_rdf.DIGIKEY) + g.bind("dk-part", digikey_rdf.DIGIKEY_PART) + g.bind("dk-attr-type", digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE) + g.bind("dk-attr-value", digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE) + g.bind("dk-product-category", digikey_rdf.DIGIKEY_PRODUCT_CATEGORY) + + if kicad: + g.bind("kicad", kicad_rdf.KICAD) + g.bind("kicad-type", kicad_rdf.KICAD_TYPE) + return g diff --git a/trygvis/eda/cli/add_to_db.py b/trygvis/eda/cli/add_to_db.py new file mode 100644 index 0000000..0511850 --- /dev/null +++ b/trygvis/eda/cli/add_to_db.py @@ -0,0 +1,17 @@ +import trygvis.eda.cli as cli + + +def run(files, path, args): + g = cli.open_database(path) + + s = 0 + for f in files: + cli.info("Adding %s" % f) + pre = len(g) + g.load(f, format="turtle") + post = len(g) + diff = post - pre + s += diff + cli.info("Loaded %d tuples" % diff) + + cli.info("Done. Loaded %d tuples" % s) diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py new file mode 100644 index 0000000..aef360c --- /dev/null +++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py @@ -0,0 +1,29 @@ +from trygvis.eda import cli, write_graph + +from trygvis.eda.digikey import * + + +def run(category, sub_category, output_file, args): + client = DigikeyClient() + db = DigikeyDatabase() + + download_category_tree(db, client) + c = db.find_category(category) + + if c is None: + cli.exit("Could not find category \"%s\"" % category) + + sc = c.find_sub_category_by_label(sub_category) + + if c is None: + cli.exit("Could not find sub-category \"%s\" inside \"%s\"" % (sub_category, category)) + + attributes = download_attribute_types_from_category(sc, client) + db.merge_attribute_types(attributes) + + g = cli.create_graph() + for a in attributes: + [g.add(node) for node in a.to_nodes()] + + filename = output_file if output_file is not "-" else None + write_graph(gen_g=lambda: g, filename=filename) diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py new file mode 100755 index 0000000..99f5266 --- /dev/null +++ b/trygvis/eda/cli/digikey_download_for_schematic.py @@ -0,0 +1,133 @@ +from os.path import isfile + +from rdflib.plugins.sparql import prepareQuery +import rdflib.term + +from trygvis.eda import cli, write_graph +from trygvis.eda.digikey import * +from trygvis.eda.digikey import rdf as digikey_rdf +from trygvis.eda.kicad import rdf as kicad_rdf + +initNs = { + "rdf": RDF, + "rdfs": RDFS, + "dk": digikey_rdf.DIGIKEY, + "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE, + "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE, + "dk-part": digikey_rdf.DIGIKEY_PART, + "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY, + "kicad": kicad_rdf.KICAD, + "kicad-type": kicad_rdf.KICAD_TYPE} + + +def run(schematic_url, db_path, args): + cli.info("Schematic: %s" % schematic_url) + g = cli.open_database(db_path) + + client = DigikeyClient() + db = DigikeyDatabase() + download_category_tree(db, client) + + # Dump schematic: + # SELECT ?schematic + # WHERE { + # ?schematic a kicad-type:schematic . + # b:rateboard-2_a ? ?. + # ?s ?predicate ?object . + # } + + # allComponentsQ = prepareQuery('SELECT ?ref ?value WHERE { ?schematic_url a kicad-type:schematic . ?schematic_url kicad:component ?o . ?o rdfs:label ?ref . ?o kicad:value ?value . }', initNs = initNs) + # + # for row in g.query(allComponentsQ, initBindings={'schematic_url': schematic_url}): + # print("ref=%s, value=%s" % (row.ref, row.value)) + + res = g.query(prepareQuery("SELECT ?schematic WHERE {?schematic a kicad-type:schematic}", initNs=initNs)) + print("Found %d schematics in database" % len(res)) + for row in res: + print("schematic: %s" % row.schematic) + + res = g.query(prepareQuery("SELECT ?dk_part ?dk_part_number WHERE {?dk_part a dk:part ; dk:partNumber ?dk_part_number}", initNs=initNs)) + print("Found %d Digikey parts in database" % len(res)) + for row in res: + print("Part: url=%s, partNumber=%s" % (row.dk_part, row.dk_part_number)) + + q = prepareQuery(""" +SELECT + ?digikey_pn + (group_concat(distinct ?ref;separator=";") as ?refs) +WHERE { + ?schematic_url kicad:component ?cmp . + ?cmp rdfs:label ?ref ; + kicad:value ?value . + OPTIONAL { + ?cmp kicad:field ?d . + ?d kicad:field_name "digikey" ; + kicad:field_value ?digikey_pn . +# OPTIONAL { + ?dk_part a dk:part ; + dk:partNumber ?digikey_pn . +# } + } +} +GROUP BY ?digikey_pn +ORDER BY ?digikey_pn +""", initNs=initNs) + + res = g.query(q, initBindings={'schematic_url': rdflib.term.URIRef(schematic_url)}) + for row in res: + pn = row.digikey_pn + + if pn is None: + continue + + refs = row.refs.split(';') + + cli.info("Part \"%s\" is used by %s" % (pn, refs)) + + filename = 'ttl/digikey-part-' + normalize_filename(pn + ".ttl") + + def download_graph(): + cli.info("Downloading product: " + pn) + product = download_product(client, db, pn) + + g = cli.create_graph(digikey=True) + [g.add(node) for node in product.to_nodes()] + return g + + write_graph(download_graph, filename) + + # q = prepareQuery(""" + # SELECT + # DISTINCT ?category ?digikeyUrl + # WHERE { + # ?schematic_url kicad:component ?cmp . + # ?cmp rdfs:label ?ref . + # ?cmp kicad:value ?value . + # ?cmp kicad:field ?d . + # ?d kicad:field_name "digikey" . + # ?d kicad:field_value ?digikey . + # + # ?part a dk:part . + # ?part dk:partNumber ?digikey . + # ?part dk:category ?category . + # ?category dk:parent ?_ . + # ?category dk:url ?digikeyUrl + # }""", initNs=initNs) + # + # res = g.query(q, initBindings={'schematic_url': schematic_url}) + # + # cli.info("Found %d categories" % (len(res))) + # for row in res: + # cli.info("Category: %s" % (row.category)) + # category = db.findSubCategoryByUrl(row.category) + # if category is None: + # raise Exception('could not find category: ' + row.category) + # attributes = downloadAttributeTypesFromCategory(category, client) + # db.mergeAttributeTypes(attributes) + # + # filename = 'digikey-category-' + normalize_filename(str(category.label) + ".ttl") + # if not isfile(filename): + # tmpG = cli.create_graph() + # for a in attributes: + # [tmpG.add(node) for node in a.toNodes()] + # writeGraph(tmpG, 'ttl/' + filename) diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py new file mode 100644 index 0000000..4e80e93 --- /dev/null +++ b/trygvis/eda/cli/eda_rdf.py @@ -0,0 +1,69 @@ +import argparse +import sys +import trygvis.eda.cli as cli + + +def main(): + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(dest="cmd") + + p = subparsers.add_parser("kicad-bom-to-ttl") + p.add_argument("-o", "--output", required=False) + p.add_argument("-i", "--input", required=False) + + p = subparsers.add_parser("add-to-db") + p.add_argument("-d", "--db", required=True) + p.add_argument("files", nargs='*') + + p = subparsers.add_parser("make-bom") + p.add_argument("-d", "--db", required=True) + p.add_argument("--schematic", required=True) + + p = subparsers.add_parser("digikey-download-for-schematic") + p.add_argument("-d", "--db", required=True) + p.add_argument("--schematic", required=True) + + p = subparsers.add_parser("digikey-download-attribute-types-for-category") + p.add_argument("-c", "--category", required=True) + p.add_argument("-s", "--sub-category", required=True) + p.add_argument("-o", "--output", required=False) + + args = parser.parse_args() + + cli.init() + + if args.cmd == "kicad-bom-to-ttl": + from trygvis.eda.cli import kicad_bom_to_ttl + + if args.input is not None: + src = open(args.input, "r") + else: + src = sys.stdin + + if args.output is not None: + dst = open(args.output, "wb") + else: + dst = sys.stdout.buffer + + with src, dst: + kicad_bom_to_ttl.run(src, dst, args) + + elif args.cmd == "add-to-db": + from trygvis.eda.cli import add_to_db + + add_to_db.run(args.files, args.db, args) + + elif args.cmd == "make-bom": + from trygvis.eda.cli import make_bom + + make_bom.run(args.schematic, args.db, args) + + elif args.cmd == "digikey-download-for-schematic": + from trygvis.eda.cli import digikey_download_for_schematic + digikey_download_for_schematic.run(args.schematic, args.db, args) + elif args.cmd == "digikey-download-attribute-types-for-category": + from trygvis.eda.cli import digikey_download_attribute_types_for_category + digikey_download_attribute_types_for_category.run(args.category, args.sub_category, args.output, args) + else: + sys.exit("Unknown command: %s" % args.cmd) diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py new file mode 100755 index 0000000..c7d6899 --- /dev/null +++ b/trygvis/eda/cli/kicad_bom_to_ttl.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 + +import functools +import os.path +import re +import xml.etree.ElementTree +from urllib.parse import quote_plus + +from trygvis.eda import cli +from trygvis.eda.kicad import rdf as kicad_rdf + + +def _clean(s): + if s is None: + return None + s = s.strip() + return None if len(s) == 0 else s + + +def _cleaned_text(e, name): + child = e.find(name) + if child is None: + return None + return child.text + + +def _cleaned_attr(e, name): + value = e.get(name) + if value is None: + return None + return _clean(value) + + +def _iso_size_to_factor(size): + if size == 'k': + return 3 + if size == 'M': + return 6 + if size == 'm': + return -3 + if size == 'u' or size == '\N{GREEK SMALL LETTER MU}' or size == '\N{MICRO SIGN}': + return -6 + if size == 'n': + return -9 + if size == 'p': + return -12 + return None + + +@functools.total_ordering +class Value: + valueRe = re.compile("([0-9]+\.?[0-9]*)(.*)") + typeRe = re.compile("(.*)(H|Hz)$") + + def __init__(self, text, value, factor, type): + self.text = text + self.value = value + self.factor = factor + self.type = type + + def __eq__(self, other): + return self.text == other.text + + def __lt__(self, other): + if self.value is not None: + if other.value is None: + return 1 + else: + return (self.factor, self.value) < (other.factor, other.value) + else: + if other.value is None: + return self.text < other.text + else: + return -1 + + def __str__(self): + if self.value is not None: + s = str(self.value) + 'e' + str(self.factor) + if self.type is not None: + s = s + ' ' + self.type + return s + if self.text is not None: + return self.text + return "unknown" + + @staticmethod + def parse(text): + if text is None: + return Value(None, None, None, None) + + m = Value.valueRe.match(text) + if m: + value_int = float(m.group(1)) + suffix = m.group(2) + m = Value.typeRe.match(suffix) + if m: + suffix = m.group(1) + type = m.group(2) + else: + type = None + + factor = _iso_size_to_factor(suffix) + # cli.info(text + ': suffix:' + suffix + ' => value_int: ' + str(value_int) + ', factor: ' + str( + # factor) + ', type=' + str(type)) + + if value_int is not None and factor is not None: + return Value(text, value_int, factor, type) + + return Value(text, None, None, None) + + +class Component: + refRe = re.compile("([a-zA-Z]+)([0-9]+)") + + def __init__(self, ref, value, footprint, fields): + self.ref = ref + self.value = Value.parse(value) + self.footprint = footprint + self.fields = fields + m = Component.refRe.match(ref) + if m: + self.ref_class = m.group(1) + self.ref_instance = int(m.group(2)) + else: + self.ref_class = None + self.ref_instance = None + + def __str__(self): + s = 'ref=' + self.ref + if self.value is not None: + s += ', value=' + str(self.value) + if self.footprint is not None: + s += ', footprint=' + self.footprint + for name, value in self.fields.items(): + s += ', ' + name + '=' + value + return s + + def find_field(self, key): + return self.fields[key] if key in self.fields else None + + @staticmethod + def from_xml(e): + ref = e.get('ref') + value = _cleaned_text(e, 'value') + footprint = _cleaned_text(e, 'footprint') + fs = {} + + fields = e.find('fields') + if fields is not None: + for field in fields.findall('field'): + fs[field.get('name')] = field.text + + if ref is not None: + return Component(ref, value, footprint, fs) + + return None + + +class TitleBlock(object): + def __init__(self, title=None, rev=None, date=None, source=None): + self.title = title + self.rev = rev + self.date = date + self.source = source + + @staticmethod + def from_xml(tb): + title = _cleaned_text(tb, 'title') + rev = _cleaned_text(tb, 'rev') + date = _cleaned_text(tb, 'date') + source = _cleaned_text(tb, 'source') + return TitleBlock(title, rev, date, source) + + +class Sheet(object): + def __init__(self, number, name, title_block): + self.number = number + self.name = name + self.title_block = title_block + + @staticmethod + def from_xml(s): + number = _cleaned_attr(s, "number") + name = _cleaned_attr(s, "name") + node = s.find('title_block') + title_block = TitleBlock.from_xml(node) if node is not None else TitleBlock() + return Sheet(int(number) if number is not None else None, name, title_block) + + +class Design(object): + def __init__(self, source=None, date=None, tool=None, sheets=None): + self.source = source + self.date = date + self.tool = tool + self.sheets = sheets if sheets is not None else [] + + @staticmethod + def from_xml(d): + source = _cleaned_text(d, 'source') + date = _cleaned_text(d, 'date') + tool = _cleaned_text(d, 'tool') + sheets = [] + for s in d.iterfind('sheet'): + sheets.append(Sheet.from_xml(s)) + return Design(source, date, tool, sheets) + + +class Export(object): + def __init__(self, design, components): + self.design = design if design is not None else Design() + self.components = components + + @staticmethod + def from_xml(doc): + cs = [] + + node = doc.find('design') + design = Design.from_xml(node) if node is not None else None + + components = doc.find('components') + for e in components.iter('comp'): + c = Component.from_xml(e) + if c is not None: + cs.append(c) + + return Export(design, cs) + + def component_fields(self): + fs = set() + for c in self.components: + for key, value in c.fields.items(): + fs.add(key) + return fs + + def generate_schematic_url(self): + s = next((s for s in self.design.sheets if s.number == 1), None) + title = s.title_block.title if s is not None else None + if title is None: + if self.design.source is not None: + title = _clean(os.path.basename(self.design.source)) + + if title is None: + raise cli.CliException("Could not generate a stable, identifying URL for the schematic") + + title = quote_plus(title) + return kicad_rdf.KICAD_BOARD[title] + + +def run(src, dst, args): + from trygvis.eda import cli + + tree = xml.etree.ElementTree.parse(src) + root = tree.getroot() + + export = Export.from_xml(root) + + # print("components:") + # for c in export.components: + # print(c) + + # for name in export.component_fields(): + # cli.info(name) + + from operator import attrgetter + import itertools + + # cli.info("components:") + # fmt = '%2s%3s' + # cli.info(fmt % ('Ref', '')) + + components = sorted(export.components, key=attrgetter('ref')) + + component_count = 0 + part_count = 0 + has_part = 0 + has_digikey = 0 + + for cls, cs in itertools.groupby(components, key=attrgetter('ref_class')): + # cli.info('--- Class: %s ---' % cls) + cs = sorted(cs, key=attrgetter('value')) + for c in cs: + component_count += 1 + + value = str(c.value) + part = c.find_field("part") + digikey = c.find_field("digikey") + footprint = c.footprint + + if c.ref_class is not None: + ref_class = c.ref_class + ref_instance = c.ref_instance + else: + ref_class = c.ref + ref_instance = "" + + # cli.info(fmt % (ref_class, ref_instance)) + # cli.info(' Value: %s' % c.value.text) + # if c.footprint: + # cli.info(' Footprint: %s' % c.footprint) + + if part is not None: + if part != 'NA': + has_part += 1 if part is not None else 0 + part_count += 1 + + if digikey is not None: + has_digikey += 1 + else: + part = None + else: + part_count += 1 + part = "MISSING" + + # if part is not None: + # cli.info(' Part: %s' % part) + # if digikey is not None: + # cli.info(' Digikey: %s' % digikey) + + schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url() + + # cli.info() + # cli.info('=== Summary ===') + # cli.info('Schematic URL: %s' % schematic_url) + # cli.info('Number of components: %d' % component_count) + # cli.info('Number of parts: %d' % part_count) + # cli.info('Assigned part: %d / %2.f%%' % (has_part, (has_part / part_count) * 100)) + # cli.info('Assigned digikey: %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100)) + + from ..kicad import rdf as kicad_rdf + from rdflib import Literal, BNode, URIRef + from rdflib.namespace import RDF, RDFS + + g = cli.create_graph(kicad=True) + + schematic = URIRef(schematic_url) + g.add((schematic, RDF.type, kicad_rdf.KICAD_TYPE.schematic)) + + # componentNodes = [] + for c in export.components: + node = BNode() + g.add((schematic, kicad_rdf.KICAD.component, node)) + + g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component)) + g.add((node, RDFS.label, Literal(c.ref))) + g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text))) + + for name, value in c.fields.items(): + f = BNode() + g.add((node, kicad_rdf.KICAD.field, f)) + g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field)) + g.add((f, kicad_rdf.KICAD.field_name, Literal(name))) + g.add((f, kicad_rdf.KICAD.field_value, Literal(value))) + + # TODO: serialize the data from too + + g.serialize(destination=dst, encoding='utf-8', format='turtle') diff --git a/trygvis/eda/cli/make_bom.py b/trygvis/eda/cli/make_bom.py new file mode 100755 index 0000000..28059cb --- /dev/null +++ b/trygvis/eda/cli/make_bom.py @@ -0,0 +1,21 @@ +from trygvis.eda import cli +from trygvis.eda.digikey import * +from trygvis.eda.digikey import rdf as digikey_rdf +from trygvis.eda.kicad import rdf as kicad_rdf + +initNs = { + "rdf": RDF, + "rdfs": RDFS, + "dk": digikey_rdf.DIGIKEY, + "dk-attr-type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE, + "dk-attr-value": digikey_rdf.DIGIKEY_ATTRIBUTE_VALUE, + "dk-part": digikey_rdf.DIGIKEY_PART, + "dk-p-c": digikey_rdf.DIGIKEY_PRODUCT_CATEGORY, + "kicad": kicad_rdf.KICAD, + "kicad-type": kicad_rdf.KICAD_TYPE} + + +def run(schematic_url, db_path, args): + g = cli.open_database(db_path) + + cli.info('implement..') diff --git a/trygvis/eda/digikey/__init__.py b/trygvis/eda/digikey/__init__.py new file mode 100644 index 0000000..5f4ad8a --- /dev/null +++ b/trygvis/eda/digikey/__init__.py @@ -0,0 +1,300 @@ +import re + +import requests +from cachecontrol import CacheControl +from cachecontrol.caches.file_cache import FileCache +from cachecontrol.heuristics import ExpiresAfter +from lxml import html +from rdflib import Literal +from rdflib.namespace import RDF, RDFS + +import trygvis.eda.digikey.rdf + +def normalize_filename(part): + return part.replace('/', '_').replace(' ', '_') + +def _clean(s): + if s is None: + return None + s = s.strip() + return None if len(s) == 0 else s + + +class DigikeyDatabase(object): + def __init__(self): + self.productCategories = [] + self.attributeTypes = {} + + def add_product_category(self, pc): + self.productCategories.append(pc) + + def find_category(self, label): + return next((c for c in self.productCategories if c.label == label), None) + + def find_sub_category_by_url(self, url): + for p in self.productCategories: + for sc in p.subCategories: + if sc.url() == url: + return sc + return None + + def merge_attribute_types(self, attributeTypes): + for a in attributeTypes: + if a.id in self.attributeTypes: + # TODO: implement merging + continue + self.attributeTypes[a.id] = a + + def find_type(self, id): + return self.attributeTypes.get(id, None) + + +class DigikeyProductCategory(object): + def __init__(self, id, label, digikey_url=None, parent=None): + self.id = _clean(id) + self.label = _clean(label) + self.digikey_url = digikey_url if digikey_url is None or digikey_url.startswith("http") else \ + "http://www.digikey.com" + digikey_url + self.parent = parent + self.subCategories = [] + + assert self.id is not None + assert self.label is not None + + def add_sub_category(self, id, label, digikey_url): + sc = DigikeyProductCategory(id, label, digikey_url=digikey_url, parent=self) + self.subCategories.append(sc) + + def find_sub_category_by_label(self, label): + return next((sc for sc in self.subCategories if sc.label == label), None) + + def url(self): + return rdf.DIGIKEY_PRODUCT_CATEGORY[self.id] + + def to_nodes(self): + node = self.url() + nodes = [ + (node, RDF.type, rdf.DIGIKEY.productCategory), + (node, RDFS.label, Literal(self.label)), + ] + if self.parent is not None: + parentUrl = rdf.DIGIKEY_PRODUCT_CATEGORY[self.parent.id] + nodes.append((node, rdf.DIGIKEY.parent, parentUrl)) + if self.digikey_url is not None: + nodes.append((node, rdf.DIGIKEY.url, Literal(self.digikey_url))) + return nodes + + +class DigikeyAttributeType(object): + def __init__(self, category, id, label, options): + self.category = category + self.id = _clean(id) + self.label = _clean(label) + self.options = options + + assert self.category is not None + assert self.id is not None + assert self.label is not None + assert self.options is not None + + def to_nodes(self): + nodes = [] + node = rdf.DIGIKEY_ATTRIBUTE_TYPE[self.id] + nodes.append((node, RDF.type, rdf.DIGIKEY.attributeType)) + nodes.append((node, RDFS.label, Literal(self.label))) + + for o in self.options: + optionNode = rdf.DIGIKEY_ATTRIBUTE_VALUE[self.id + '-' + o.id] + nodes.extend([ + (optionNode, rdf.DIGIKEY.id, Literal(o.id)), + (optionNode, RDFS.label, Literal(o.label)), + (node, rdf.DIGIKEY.value, optionNode)]) + return nodes + + +class DigikeyAttributeValue(object): + def __init__(self, id, label, type=None, type_id=None, type_label=None): + self.id = _clean(id) + self.label = _clean(label) + self.type = type + self.type_id = type_id + self.type_label = type_label + + assert self.id is not None + assert self.label is not None + + +class DigikeyProduct(object): + def __init__(self, part_id, part_number, values, categories): + self.part_id = _clean(part_id) + self.part_number = _clean(part_number) + self.values = values + self.categories = categories + self.quantity_available = None + self.description = None + + assert self.part_id is not None + assert self.part_number is not None + + def to_nodes(self): + nodes = [] + node = rdf.DIGIKEY_PART[self.part_id] + nodes.append((node, RDF.type, rdf.DIGIKEY.part)) + nodes.append((node, rdf.DIGIKEY.partNumber, Literal(self.part_number))) + nodes.append((node, RDFS.label, Literal(self.description))) + for v in self.values: + typeLabel = v.type.label if v.type is not None else v.typeLabel + typeId = v.type.id if v.type is not None else v.typeId + nodes.append((node, rdf.DIGIKEY['attribute-value'], rdf.DIGIKEY_ATTRIBUTE_VALUE[typeId + '-' + v.id])) + + for c in self.categories: + nodes.append((node, rdf.DIGIKEY.category, c.url())) + return nodes + + +class DigikeyClient(object): + def __init__(self): + cache = FileCache('digikey_cache', forever=True) + self.sess = CacheControl(requests.Session(), cache=cache, heuristic=ExpiresAfter(days=1)) + + def req(self, url, params=None): + if not url.startswith("http://"): + url = "http://www.digikey.com" + url + return self.sess.get(url, params=params) + + +def _to_string(e): + s = "" + for t in e.itertext(): + s += t + return s.strip() + + +def _id_from_url(url): + if url is None: + return None + m = re.search(r".*/([0-9]+)", url) + return m.group(1) if m else None + + +def download_category_tree(database, client, baseurl="http://www.digikey.com/products/en"): + page = client.req(baseurl) + dom = html.fromstring(page.content) + + items = dom.xpath("//h2[contains(@class, 'catfiltertopitem')]") + for h2 in items: + label = _to_string(h2) + # print(h2) + pcId = None + for a in h2.getchildren(): + url = a.get('href') + pcId = _id_from_url(url) + if pcId is None: + continue + + if pcId is None: + continue + + pc = DigikeyProductCategory(pcId, label) + n = h2.getnext() + if n.tag == 'span': + n = n.getnext() + if n.tag == 'ul': + for a in n.xpath('./li/a'): + label = _to_string(a) + url = a.get('href') + id = _id_from_url(url) + if id is None: + continue + # print(' ' + toString(a) + ', id=' + str(id) + ', url=' + url) + pc.add_sub_category(id, label, url) + + database.add_product_category(pc) + + +def download_attribute_types_from_category(category, client): + page = client.req(category.digikey_url) + tree = html.fromstring(page.content) + + attributes = [] + for form in tree.xpath("//form[contains(@class, 'search-form')]"): + print('form: ' + str(form)) + headers = form.xpath(".//tr[@id='appliedFilterHeaderRow']/th/text()") + print("headers: " + str(headers)) + for select in form.xpath(".//td/select[contains(@class, 'filter-selectors')]"): + td = select.getparent() + index = td.getparent().index(td) + try: + attributeLabel = headers[index] + except: + continue + attributeId = select.get('name') + print("label: " + attributeLabel + ", id: " + attributeId) + options = [] + type = DigikeyAttributeType(category, attributeId, attributeLabel, options) + for o in select.xpath("./option"): + id = o.get('value') + label = _to_string(o) + # print("o: %s" % str(o)) + options.append(DigikeyAttributeValue(id, label, type=type)) + attributes.append(type) + + return attributes + + +def download_product(client, db, query): + # http://www.digikey.com/products/en?x=0&y=0&lang=en&site=us&keywords=553-2320-1-ND + page = client.req("http://www.digikey.com/products/en", params={'lang': 'en', 'site': 'us', 'keywords': query}) + tree = html.fromstring(page.content) + + values = [] + categories = [] + for table in tree.xpath("//table[contains(@class, 'attributes-table-main')]"): + label = None + id = None + for tr in table.xpath(".//tr"): + if tr.get("id") is not None: + continue + tds = tr.xpath("./th | ./td") + if len(tds) != 3: + continue + type_label = _to_string(tds[0]) + label = _to_string(tds[1]) + for input in tds[2].xpath("./input[@name]"): + typeId = input.get("name") + id = input.get("value") + else: + typeId = None + + if id is None or typeId is None: + continue + if typeId == "t": # categories are handled later + continue + values.append(DigikeyAttributeValue(id, label, type_id=typeId, type_label=type_label)) + + for td in table.xpath(".//td[@class='attributes-td-categories-link']"): + tr = td.getparent() + id = None + url = None + for a in td.xpath(".//a[@href]"): + url = a.get("href") + id = _id_from_url(url) + + for input in tr.xpath(".//input[@name='t' and @value]"): + categoryId = input.get("value") + + if id is None: + continue + categories.append(DigikeyProductCategory(id, label, digikey_url=url)) + + part_id = part_number = None + for n in tree.xpath("//input[@name='partid' and @value]"): + part_id = n.get("value") + for n in tree.xpath("//*[@itemprop='productID' and @content]"): + part_number = n.get("content") + part_number = part_number.replace('sku:', '') + + p = DigikeyProduct(part_id, part_number, values, categories) + for n in tree.xpath("//*[@itemprop='description']"): + p.description = _to_string(n) + return p diff --git a/trygvis/eda/digikey/__main__.py b/trygvis/eda/digikey/__main__.py new file mode 100644 index 0000000..ceb341e --- /dev/null +++ b/trygvis/eda/digikey/__main__.py @@ -0,0 +1,42 @@ +import argparse + +from .. import write_graph +from ..cli import * +from ..digikey import * + +parser = argparse.ArgumentParser() +subparsers = parser.add_subparsers(dest='cmd') # help='sub-command help' + +dct_parser = subparsers.add_parser("download-category-tree") +dct_parser.add_argument("-o", "--output", required=False) + +dp_parser = subparsers.add_parser("download-product") +dp_parser.add_argument("-p", "--product") +dp_parser.add_argument("-o", "--output", required=False) + +args = parser.parse_args() + +client = DigikeyClient() +db = DigikeyDatabase() + +if args.cmd == "download-category-tree": + download_category_tree(db, client) + if args.output is not None: + def make_graph(): + g = create_graph(digikey=True) + for pc in db.productCategories: + [g.add(node) for node in pc.to_nodes()] + + for sc in pc.subCategories: + [g.add(node) for node in sc.to_nodes()] + write_graph(make_graph, args.output) + +elif args.cmd == "download-product": + download_category_tree(db, client) + product = download_product(client, db, args.product) + + if args.output is not None: + def make_graph(): + g = create_graph(digikey=True) + [g.add(node) for node in product.to_nodes()] + write_graph(make_graph, args.output) diff --git a/trygvis/eda/digikey/rdf.py b/trygvis/eda/digikey/rdf.py new file mode 100644 index 0000000..5f1dede --- /dev/null +++ b/trygvis/eda/digikey/rdf.py @@ -0,0 +1,7 @@ +from rdflib import Namespace + +DIGIKEY = Namespace("https://trygvis.io/purl/digikey#") +DIGIKEY_ATTRIBUTE_TYPE = Namespace("https://trygvis.io/purl/digikey-attribute-type#") +DIGIKEY_ATTRIBUTE_VALUE = Namespace("https://trygvis.io/purl/digikey-attribute-value#") +DIGIKEY_PART = Namespace("https://trygvis.io/purl/digikey-part#") +DIGIKEY_PRODUCT_CATEGORY = Namespace("https://trygvis.io/purl/digikey-product-category#") diff --git a/trygvis/eda/kicad/__init__.py b/trygvis/eda/kicad/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/trygvis/eda/kicad/rdf.py b/trygvis/eda/kicad/rdf.py new file mode 100644 index 0000000..683eaad --- /dev/null +++ b/trygvis/eda/kicad/rdf.py @@ -0,0 +1,7 @@ +from rdflib import Namespace + +KICAD = Namespace("https://trygvis/purl/kicad#") +KICAD_TYPE = Namespace("https://trygvis/purl/kicad-type#") + +# Namespace for all unknown kicad boards +KICAD_BOARD = Namespace("https://trygvis/purl/kicad-board#") -- cgit v1.2.3