From 0958273a71dd19c2a90471a182ccc5b90b14e5b4 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Sat, 7 Jan 2017 14:00:46 +0100 Subject: Renaming 'schematic' to 'project'. Renaming 'kicad-bom-to-ttl' to 'kicad-import-project'. Renaming 'digikey-download-for-schematic' to 'digikey-download-for-project'. Splitting out the Export xml file code into its own module. init: putting project.url and project.file in config.ini. init: putting db.update-url in config.ini if given on the command line. kicad-import-project: by default, assume that the user want to update local database, optionally write the ttl file to disk. cli.write_graph: create any missing parent directories. --- README.md | 4 - trygvis/eda/__init__.py | 2 + trygvis/eda/cli/__init__.py | 27 +- trygvis/eda/cli/db_stats.py | 12 +- trygvis/eda/cli/digikey_download_for_project.py | 105 ++++++ trygvis/eda/cli/digikey_download_for_schematic.py | 87 ----- trygvis/eda/cli/eda_rdf.py | 64 +--- trygvis/eda/cli/init.py | 70 +++- trygvis/eda/cli/kicad_bom_to_ttl.py | 387 ---------------------- trygvis/eda/cli/kicad_import_project.py | 173 ++++++++++ trygvis/eda/cli/make_bom.py | 26 +- trygvis/eda/kicad/export/__init__.py | 250 ++++++++++++++ 12 files changed, 645 insertions(+), 562 deletions(-) create mode 100755 trygvis/eda/cli/digikey_download_for_project.py delete mode 100755 trygvis/eda/cli/digikey_download_for_schematic.py delete mode 100755 trygvis/eda/cli/kicad_bom_to_ttl.py create mode 100755 trygvis/eda/cli/kicad_import_project.py create mode 100644 trygvis/eda/kicad/export/__init__.py diff --git a/README.md b/README.md index 0440384..f92643a 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,3 @@ o Download datasheets, IBIS model, SPICE model * Change the arg parser to always allow a '--db' argument that defaults to './.eda-rdf' -* Create an 'init' command similar to git init -* Rename "schematic" to "project"? Schematic is .. schematic specific - but many of the possible tools work on the BOM - and/or on the PCB so 'project' is probably a better term. diff --git a/trygvis/eda/__init__.py b/trygvis/eda/__init__.py index e69de29..408bd7c 100644 --- a/trygvis/eda/__init__.py +++ b/trygvis/eda/__init__.py @@ -0,0 +1,2 @@ +class EdaException(Exception): + pass diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py index 308e1bd..aa61021 100644 --- a/trygvis/eda/cli/__init__.py +++ b/trygvis/eda/cli/__init__.py @@ -1,16 +1,26 @@ import sys import logging from genericpath import isfile +from os import mkdir +from os.path import dirname, isdir from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS from rdflib.plugins.sparql import prepareQuery import rdflib.plugins.stores.sparqlstore as sparqlstore import configparser +# noinspection PyUnresolvedReferences +import argparse from ..digikey import rdf as digikey_rdf from ..kicad import rdf as kicad_rdf +class CliCommand(object): + def __init__(self, key, description): + self.key = key + self.description = description + + class CliException(Exception): pass @@ -26,6 +36,12 @@ def info(msg=None): sys.stderr.write("\n") +def debug(msg=None): + if msg is not None: + sys.stderr.write('D: %s' % msg) + sys.stderr.write("\n") + + def do_exit(msg=None): sys.exit(msg) @@ -45,7 +61,7 @@ def with_database(tx): raise CliException("The database is corrupt: %s" % path) elif db_type == 'sparql': query_endpoint = config["db"]["url"] - update_endpoint = config["db"].get("update_url") + update_endpoint = config["db"].get("update-url") if update_endpoint is None: g = sparqlstore.SPARQLStore() @@ -57,7 +73,6 @@ def with_database(tx): raise CliException("Unknown db.type: %s" % db_type) try: - print("g=%s" % g) tx(g) if isinstance(g, sparqlstore.SPARQLUpdateStore): g.commit() @@ -81,9 +96,13 @@ def create_graph(digikey=False, kicad=False) -> Graph: return g -def write_graph(gen_g, filename: str = None, force_write: bool = False): +def write_graph(gen_g: Graph, filename: str = None, force_write: bool = False): if filename is not None: if force_write or not isfile(filename): + parent = dirname(filename) + if not isdir(parent): + mkdir(parent) + g = gen_g() if g is None: @@ -129,7 +148,7 @@ def write_config(config: configparser.ConfigParser): config.write(configfile) -def read_config(): +def read_config() -> configparser.ConfigParser: try: with open('.eda-rdf/config.ini', 'r') as f: config = configparser.ConfigParser() diff --git a/trygvis/eda/cli/db_stats.py b/trygvis/eda/cli/db_stats.py index aabc46a..babc53b 100755 --- a/trygvis/eda/cli/db_stats.py +++ b/trygvis/eda/cli/db_stats.py @@ -1,21 +1,21 @@ from trygvis.eda import cli -def run(): +def run(args: object): def db_stats(g): res = cli.sparql(g, """ -SELECT ?schematic ?label +SELECT ?project ?label WHERE { - ?schematic a kicad-type:schematic + ?project a kicad-type:project OPTIONAL { - ?schematic rdfs:label ?label + ?project rdfs:label ?label } } """) - cli.info("Found %d schematics in database" % len(res)) + cli.info("Found %d projects in database" % len(res)) for row in res: name = row.label if row.label is not None else "" - url = row.schematic + url = row.project cli.info("%s:" % name) cli.info(" URL: %s" % url) diff --git a/trygvis/eda/cli/digikey_download_for_project.py b/trygvis/eda/cli/digikey_download_for_project.py new file mode 100755 index 0000000..02e3cb1 --- /dev/null +++ b/trygvis/eda/cli/digikey_download_for_project.py @@ -0,0 +1,105 @@ +import argparse +import rdflib +import rdflib.term + +from trygvis.eda import cli +from trygvis.eda.digikey import * + + +class DigikeyDownloadForProjectCommand(cli.CliCommand): + def __init__(self): + super().__init__("digikey-download-for-project", "Download missing data from digikey.com") + + def run(self, argv): + p = argparse.ArgumentParser(prog=self.key, description=self.description) + p.add_argument("--project", required=False) + p.add_argument("-f", "--force", default=False, action='store_true') + args = p.parse_args(argv) + + run(args) + + +def work(project_url, force, g): + client = DigikeyClient() + db = DigikeyDatabase() + download_category_tree(db, client) + + # Dump project: + # SELECT ?project + # WHERE { + # ?project a kicad-type:project . + # b:rateboard-2_a ? ?. + # ?s ?predicate ?object . + # } + + # allComponentsQ = prepareQuery('SELECT ?ref ?value WHERE { ?project_url a kicad-type:project . ?project_url kicad:component ?o . ?o rdfs:label ?ref . ?o kicad:value ?value . }', initNs = initNs) + # + # for row in g.query(allComponentsQ, initBindings={'project_url': project_url}): + # print("ref=%s, value=%s" % (row.ref, row.value)) + + res = cli.sparql(g, "SELECT ?project WHERE {?project a kicad-type:project}") + print("Found %d projects in database" % len(res)) + for row in res: + print("project: %s" % row.project) + + res = cli.sparql(g, "SELECT ?dk_part ?dk_part_number WHERE {?dk_part a dk:part ; dk:partNumber ?dk_part_number}") + print("Found %d Digikey parts in database" % len(res)) + for row in res: + print("Part: url=%s, partNumber=%s" % (row.dk_part, row.dk_part_number)) + + res = cli.sparql(g, """ +SELECT + ?digikey_pn + (group_concat(distinct ?ref;separator=";") as ?refs) +WHERE { + ?project_url kicad:component ?cmp . + ?cmp rdfs:label ?ref ; + kicad:value ?value . + OPTIONAL { + ?cmp kicad:field ?d . + ?d kicad:field_name "digikey" ; + kicad:field_value ?digikey_pn . + OPTIONAL { + ?dk_part a dk:part ; + dk:partNumber ?digikey_pn . + } + } +} +GROUP BY ?digikey_pn +ORDER BY ?digikey_pn +""", init_bindings={'project_url': rdflib.term.URIRef(project_url)}) + size = len(res) + + if size == 0: + cli.info('Could not find an parts for the project, did you use the correct URL: %s?' % project_url) + + for row in res: + pn = row.digikey_pn + + if pn is None: + continue + + refs = row.refs.split(';') + + cli.info("Part \"%s\" is used by %s" % (pn, refs)) + + filename = 'ttl/digikey-part-' + normalize_filename(pn + ".ttl") + + def download_graph(): + cli.info("Downloading product: " + pn) + product = download_product(client, db, pn) + + g = cli.create_graph(digikey=True) + [g.add(node) for node in product.to_nodes()] + return g + + cli.write_graph(download_graph, filename, force_write=force) + + +def run(args): + config = cli.read_config() + + project_url = config['project']['url'] + + cli.info("Project: %s" % project_url) + cli.with_database(lambda g: work(project_url, args.force, g)) diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py deleted file mode 100755 index bf4f0ce..0000000 --- a/trygvis/eda/cli/digikey_download_for_schematic.py +++ /dev/null @@ -1,87 +0,0 @@ -import rdflib -import rdflib.term - -from trygvis.eda import cli -from trygvis.eda.digikey import * - - -def work(schematic_url, force, g): - client = DigikeyClient() - db = DigikeyDatabase() - download_category_tree(db, client) - - # Dump schematic: - # SELECT ?schematic - # WHERE { - # ?schematic a kicad-type:schematic . - # b:rateboard-2_a ? ?. - # ?s ?predicate ?object . - # } - - # allComponentsQ = prepareQuery('SELECT ?ref ?value WHERE { ?schematic_url a kicad-type:schematic . ?schematic_url kicad:component ?o . ?o rdfs:label ?ref . ?o kicad:value ?value . }', initNs = initNs) - # - # for row in g.query(allComponentsQ, initBindings={'schematic_url': schematic_url}): - # print("ref=%s, value=%s" % (row.ref, row.value)) - - res = cli.sparql(g, "SELECT ?schematic WHERE {?schematic a kicad-type:schematic}") - print("Found %d schematics in database" % len(res)) - for row in res: - print("schematic: %s" % row.schematic) - - res = cli.sparql(g, "SELECT ?dk_part ?dk_part_number WHERE {?dk_part a dk:part ; dk:partNumber ?dk_part_number}") - print("Found %d Digikey parts in database" % len(res)) - for row in res: - print("Part: url=%s, partNumber=%s" % (row.dk_part, row.dk_part_number)) - - res = cli.sparql(g, """ -SELECT - ?digikey_pn - (group_concat(distinct ?ref;separator=";") as ?refs) -WHERE { - ?schematic_url kicad:component ?cmp . - ?cmp rdfs:label ?ref ; - kicad:value ?value . - OPTIONAL { - ?cmp kicad:field ?d . - ?d kicad:field_name "digikey" ; - kicad:field_value ?digikey_pn . - OPTIONAL { - ?dk_part a dk:part ; - dk:partNumber ?digikey_pn . - } - } -} -GROUP BY ?digikey_pn -ORDER BY ?digikey_pn -""", init_bindings={'schematic_url': rdflib.term.URIRef(schematic_url)}) - size = len(res) - - if size == 0: - cli.info('Could not find an parts for the schematic, did you use the correct URL: %s?' % schematic_url) - - for row in res: - pn = row.digikey_pn - - if pn is None: - continue - - refs = row.refs.split(';') - - cli.info("Part \"%s\" is used by %s" % (pn, refs)) - - filename = 'ttl/digikey-part-' + normalize_filename(pn + ".ttl") - - def download_graph(): - cli.info("Downloading product: " + pn) - product = download_product(client, db, pn) - - g = cli.create_graph(digikey=True) - [g.add(node) for node in product.to_nodes()] - return g - - cli.write_graph(download_graph, filename, force_write=force) - - -def run(schematic_url, args): - cli.info("Schematic: %s" % schematic_url) - cli.with_database(lambda g: work(schematic_url, args.force, g)) diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py index ee4b0dd..0df66ac 100644 --- a/trygvis/eda/cli/eda_rdf.py +++ b/trygvis/eda/cli/eda_rdf.py @@ -1,12 +1,7 @@ -import argparse -import sys -import trygvis.eda.cli as cli - - -class CliCommand(object): - def __init__(self, key, description): - self.key = key - self.description = description +from trygvis.eda.cli.digikey_download_for_project import DigikeyDownloadForProjectCommand +from trygvis.eda.cli.init import InitCommand +from trygvis.eda.cli.kicad_import_project import KicadImportProjectCommand +from . import * # TODO: move all of the command classes to the file they delegate to. @@ -33,35 +28,7 @@ class DbStats(CliCommand): args = p.parse_args(argv) from trygvis.eda.cli import db_stats - db_stats.run() - - -class KicadBomToTtl(CliCommand): - def __init__(self): - super().__init__("kicad-bom-to-ttl", "Create RDF triples from a KiCAD BOM.xml file") - - def run(self, argv): - p = argparse.ArgumentParser(prog=self.key, description=self.description) - p.add_argument("-o", "--output", required=False) - p.add_argument("-i", "--input", required=False) - args = p.parse_args(argv) - - from trygvis.eda.cli import kicad_bom_to_ttl - kicad_bom_to_ttl.run(args) - - -class DigikeyDownloadForSchematic(CliCommand): - def __init__(self): - super().__init__("digikey-download-for-schematic", "Download missing data from digikey.com") - - def run(self, argv): - p = argparse.ArgumentParser(prog=self.key, description=self.description) - p.add_argument("--schematic", required=True) - p.add_argument("-f", "--force", default=False, action='store_true') - args = p.parse_args(argv) - - from trygvis.eda.cli import digikey_download_for_schematic - digikey_download_for_schematic.run(args.schematic, args) + db_stats.run(args) class DigikeyDownloadMetadata(CliCommand): @@ -107,29 +74,16 @@ class MakeBom(CliCommand): make_bom.run(args.schematic) -class Init(CliCommand): - def __init__(self): - super().__init__("init", "Initialize a EDA-RFD database") - - def run(self, argv): - p = argparse.ArgumentParser(prog=self.key, description=self.description) - p.add_argument("--database-url", dest="database_url") - args = p.parse_args(argv) - - from trygvis.eda.cli import init - init.run(args) - - def main(): - cli.initialize() + initialize() commands = [ AddToDb(), - Init(), + InitCommand(), DbStats(), MakeBom(), - KicadBomToTtl(), - DigikeyDownloadForSchematic(), + KicadImportProjectCommand(), + DigikeyDownloadForProjectCommand(), DigikeyDownloadAttributeTypesForCategory(), DigikeyDownloadMetadata() ] diff --git a/trygvis/eda/cli/init.py b/trygvis/eda/cli/init.py index 1ea9079..fbc7d04 100755 --- a/trygvis/eda/cli/init.py +++ b/trygvis/eda/cli/init.py @@ -1,23 +1,79 @@ -from os.path import isfile, exists, isdir +import argparse +from os.path import isdir +from os import mkdir +from sys import exit +import glob +import configparser from trygvis.eda import cli -import configparser +from trygvis.eda.cli import CliException, CliCommand +from trygvis.eda.kicad.export import * + + +class InitCommand(CliCommand): + def __init__(self): + super().__init__("init", "Initialize a EDA-RFD database") + + def run(self, argv): + p = argparse.ArgumentParser(prog=self.key, description=self.description) + p.add_argument("--database-url", dest="database_url") + p.add_argument("--database-update-url", dest="database_update_url") + p.add_argument("--project-file", dest="project_file", required=False) + p.add_argument("--project-url", dest="project_url", required=False) + args = p.parse_args(argv) + + run(args) + + +def load_from_kicad_project(args, config: configparser.ConfigParser): + project_file = args.project_file + + if project_file is None: + for path in glob.glob('*.pro'): + if project_file is not None: + raise CliException("Found more than one project file in directly, use --project to specify which one " + "you want") + + project_file = path -import os -import sys + if project_file is None: + raise CliException("Could not find any KiCAD projects (.pro files) in the current directly.") + + project_url = args.project_url + + if project_url is None: + project_url = os.path.basename(project_file) + (project_url, _) = os.path.splitext(project_url) + + if not project_url.startswith('http'): + project_url = quote_plus(project_url) + project_url = kicad_rdf.KICAD_BOARD[project_url] + + cli.info("EDA-RDF project initialized with Project url: %s" % project_url) + + config['project']['file'] = project_file + config['project']['url'] = project_url def run(args): try: - os.mkdir('.eda-rdf') + mkdir('.eda-rdf') except FileExistsError: if isdir('.eda-rdf'): cli.info('Already initialized') else: cli.info('.eda-rdf exists, but is not a directory') - sys.exit(1) + exit(1) config = configparser.ConfigParser() + + # KiCAD is all that is supported for now + config['project'] = { + "type": "kicad" + } + + load_from_kicad_project(args, config) + config['db'] = { "type": "local" } @@ -25,5 +81,7 @@ def run(args): if args.database_url is not None: config["db"]["type"] = "sparql" config["db"]["url"] = args.database_url + if args.database_update_url is not None: + config["db"]["update-url"] = args.database_update_url cli.write_config(config) diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py deleted file mode 100755 index 9cd3f8c..0000000 --- a/trygvis/eda/cli/kicad_bom_to_ttl.py +++ /dev/null @@ -1,387 +0,0 @@ -import functools -import os.path -import re -import sys -import xml.etree.ElementTree -from urllib.parse import quote_plus -from operator import attrgetter -import itertools -from typing import List - -import rdflib -from rdflib import Literal, URIRef -from rdflib.namespace import RDF, RDFS - -from .. import cli -from ..kicad import rdf as kicad_rdf - - -def _clean(s): - if s is None: - return None - s = s.strip() - return None if len(s) == 0 else s - - -def _cleaned_text(e, name): - child = e.find(name) - if child is None: - return None - return child.text - - -def _cleaned_attr(e, name): - value = e.get(name) - if value is None: - return None - return _clean(value) - - -def _iso_size_to_factor(size): - if size == 'k': - return 3 - if size == 'M': - return 6 - if size == 'm': - return -3 - if size == 'u' or size == '\N{GREEK SMALL LETTER MU}' or size == '\N{MICRO SIGN}': - return -6 - if size == 'n': - return -9 - if size == 'p': - return -12 - return None - - -@functools.total_ordering -class Value: - valueRe = re.compile('([0-9]+\.?[0-9]*)(.*)') - typeRe = re.compile('(.*)(H|Hz)$') - - def __init__(self, text, value, factor, type): - self.text = text - self.value = value - self.factor = factor - self.type = type - - def __eq__(self, other): - return self.text == other.text - - def __lt__(self, other): - if self.value is not None: - if other.value is None: - return 1 - else: - return (self.factor, self.value) < (other.factor, other.value) - else: - if other.value is None: - return self.text < other.text - else: - return -1 - - def __str__(self): - if self.value is not None: - s = str(self.value) + 'e' + str(self.factor) - if self.type is not None: - s = s + ' ' + self.type - return s - if self.text is not None: - return self.text - return 'unknown' - - @staticmethod - def parse(text): - if text is None: - return Value(None, None, None, None) - - m = Value.valueRe.match(text) - if m: - value_int = float(m.group(1)) - suffix = m.group(2) - m = Value.typeRe.match(suffix) - if m: - suffix = m.group(1) - type = m.group(2) - else: - type = None - - factor = _iso_size_to_factor(suffix) - # cli.info(text + ': suffix:' + suffix + ' => value_int: ' + str(value_int) + ', factor: ' + str( - # factor) + ', type=' + str(type)) - - if value_int is not None and factor is not None: - return Value(text, value_int, factor, type) - - return Value(text, None, None, None) - - -class Component: - refRe = re.compile('([a-zA-Z]+)([0-9]+)') - - def __init__(self, ref, value, footprint, fields): - self.ref = ref # type: str - self.value = Value.parse(value) # type: Value - self.footprint = footprint # type: str - self.fields = fields - m = Component.refRe.match(ref) - if m: - self.ref_class = m.group(1) - self.ref_instance = int(m.group(2)) - else: - self.ref_class = None - self.ref_instance = None - - def __str__(self): - s = 'ref=' + self.ref - if self.value is not None: - s += ', value=' + str(self.value) - if self.footprint is not None: - s += ', footprint=' + self.footprint - for name, value in self.fields.items(): - s += ', ' + name + '=' + value - return s - - def find_field(self, key): - return self.fields[key] if key in self.fields else None - - @staticmethod - def from_xml(e): - ref = e.get('ref') - value = _cleaned_text(e, 'value') - footprint = _cleaned_text(e, 'footprint') - fs = {} - - fields = e.find('fields') - if fields is not None: - for field in fields.findall('field'): - fs[field.get('name')] = field.text - - if ref is not None: - return Component(ref, value, footprint, fs) - - return None - - -class TitleBlock(object): - def __init__(self, title=None, rev=None, date=None, source=None): - self.title = title - self.rev = rev - self.date = date - self.source = source - - @staticmethod - def from_xml(tb): - title = _cleaned_text(tb, 'title') - rev = _cleaned_text(tb, 'rev') - date = _cleaned_text(tb, 'date') - source = _cleaned_text(tb, 'source') - return TitleBlock(title, rev, date, source) - - -class Sheet(object): - def __init__(self, number, name, title_block): - self.number = number - self.name = name - self.title_block = title_block - - @staticmethod - def from_xml(s): - number = _cleaned_attr(s, 'number') - name = _cleaned_attr(s, 'name') - node = s.find('title_block') - title_block = TitleBlock.from_xml(node) if node is not None else TitleBlock() - return Sheet(int(number) if number is not None else None, name, title_block) - - -class Design(object): - def __init__(self, source=None, date=None, tool=None, sheets=None): - self.source = source - self.date = date - self.tool = tool - self.sheets = sheets if sheets is not None else [] - - @staticmethod - def from_xml(d): - source = _cleaned_text(d, 'source') - date = _cleaned_text(d, 'date') - tool = _cleaned_text(d, 'tool') - sheets = [] - for s in d.iterfind('sheet'): - sheets.append(Sheet.from_xml(s)) - return Design(source, date, tool, sheets) - - -class Export(object): - def __init__(self, design, components): - self.design = design if design is not None else Design() # type: Design - self.components = components # type: List[Component] - - @staticmethod - def from_xml(doc): - cs = [] - - node = doc.find('design') - design = Design.from_xml(node) if node is not None else None - - components = doc.find('components') - for e in components.iter('comp'): - c = Component.from_xml(e) - if c is not None: - cs.append(c) - - return Export(design, cs) - - def component_fields(self): - fs = set() - for c in self.components: - for key, value in c.fields.items(): - fs.add(key) - return fs - - def generate_schematic_url(self): - s = next((s for s in self.design.sheets if s.number == 1), None) - title = s.title_block.title if s is not None else None - if title is None: - if self.design.source is not None: - title = _clean(os.path.basename(self.design.source)) - - if title is None: - raise cli.CliException('Could not generate a stable, identifying URL for the schematic') - - title = quote_plus(title) - return kicad_rdf.KICAD_BOARD[title] - - -def run(args): - if args.input is not None: - src = open(args.input, 'r') - else: - src = sys.stdin - - if args.output is not None: - parent = os.path.dirname(args.output) - if not os.path.exists(parent): - os.mkdir(parent) - dst = open(args.output, 'wb') - else: - dst = sys.stdout.buffer - - schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else None - - with src, dst: - process(src, dst, schematic_url) - - -def process(src, dst, schematic_url): - tree = xml.etree.ElementTree.parse(src) - root = tree.getroot() - - export = Export.from_xml(root) - - # print('components:') - # for c in export.components: - # print(c) - - # for name in export.component_fields(): - # cli.info(name) - - # cli.info('components:') - # fmt = '%2s%3s' - # cli.info(fmt % ('Ref', '')) - - components = sorted(export.components, key=attrgetter('ref')) - - component_count = 0 - part_count = 0 - has_part = 0 - has_digikey = 0 - - for cls, cs in itertools.groupby(components, key=attrgetter('ref_class')): - # cli.info('--- Class: %s ---' % cls) - cs = sorted(cs, key=attrgetter('value')) - for c in cs: - component_count += 1 - - value = str(c.value) - part = c.find_field('part') - digikey = c.find_field('digikey') - footprint = c.footprint - - if c.ref_class is not None: - ref_class = c.ref_class - ref_instance = c.ref_instance - else: - ref_class = c.ref - ref_instance = '' - - # cli.info(fmt % (ref_class, ref_instance)) - # cli.info(' Value: %s' % c.value.text) - # if c.footprint: - # cli.info(' Footprint: %s' % c.footprint) - - if part is not None: - if part != 'NA': - has_part += 1 if part is not None else 0 - part_count += 1 - - if digikey is not None: - has_digikey += 1 - else: - part = None - else: - part_count += 1 - part = 'MISSING' - - # if part is not None: - # cli.info(' Part: %s' % part) - # if digikey is not None: - # cli.info(' Digikey: %s' % digikey) - - schematic_url = schematic_url if schematic_url is not None else export.generate_schematic_url() - - # cli.info() - # cli.info('=== Summary ===') - # cli.info('Schematic URL: %s' % schematic_url) - # cli.info('Number of components: %d' % component_count) - # cli.info('Number of parts: %d' % part_count) - # cli.info('Assigned part: %d / %2.f%%' % (has_part, (has_part / part_count) * 100)) - # cli.info('Assigned digikey: %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100)) - - g = cli.create_graph(kicad=True) - - schematic = URIRef(schematic_url) - g.add((schematic, RDF.type, kicad_rdf.KICAD_TYPE.schematic)) - - # The components and fields could/should have been a BNodes. Their generated names are not very nice. - # TODO: try using a hash of the current value and put that under a special generated namespace. - # 'http://example.org/my-board' + ref='C10' => 'http://example.org/my-boardC10' - # hash('http://example.org/my-boardC10') => 123456 - # add_prefix(hash) => 'https://trygvis.io/purl/kicad/generated#123456' - footprints = set() - for c in export.components: - ns = rdflib.Namespace(schematic_url) - node = ns[c.ref] - g.add((schematic, kicad_rdf.KICAD.component, node)) - - g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component)) - g.add((node, RDFS.label, Literal(c.ref))) - g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text))) - - footprint_uri = URIRef(kicad_rdf.KICAD_FOOTPRINT[quote_plus(c.footprint)]) - if not footprint_uri in footprints: - g.add((footprint_uri, RDF.type, kicad_rdf.KICAD_TYPE.footprint)) - g.add((footprint_uri, RDFS.label, Literal(c.footprint))) - footprints.add(footprint_uri) - - g.add((node, kicad_rdf.KICAD.footprint, footprint_uri)) - - for name, value in c.fields.items(): - f = ns['%s-%s' % (c.ref, name)] - g.add((node, kicad_rdf.KICAD.field, f)) - g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field)) - g.add((f, kicad_rdf.KICAD.field_name, Literal(name))) - g.add((f, kicad_rdf.KICAD.field_value, Literal(value))) - - # TODO: serialize the data from too - - g.serialize(destination=dst, encoding='utf-8', format='turtle') diff --git a/trygvis/eda/cli/kicad_import_project.py b/trygvis/eda/cli/kicad_import_project.py new file mode 100755 index 0000000..288b7fc --- /dev/null +++ b/trygvis/eda/cli/kicad_import_project.py @@ -0,0 +1,173 @@ +import os.path +from operator import attrgetter +import itertools + +import rdflib +from rdflib import Literal, URIRef +from rdflib.namespace import RDF, RDFS + +# from ..kicad import rdf as kicad_rdf +from . import * +from ..kicad.export import * + + +class KicadImportProjectCommand(CliCommand): + def __init__(self): + super().__init__("kicad-import-project", "Import a KiCAD project") + + def run(self, argv): + p = argparse.ArgumentParser(prog=self.key, description=self.description) + p.add_argument("-o", "--output-dir", dest="output_dir", required=False) + p.add_argument("-i", "--input", required=False) + args = p.parse_args(argv) + + run(args) + + +def run(args): + config = read_config() + + if args.input is "-": + src = sys.stdin + else: + if args.input is None: + (filename, _) = os.path.splitext(config['project']['file']) + filename += '.xml' + else: + filename = args.input + + if not isfile(filename): + raise CliException("No such file: %s. Did you export the BOM?" % filename) + + src = open(filename, 'r') + + project_url = config['project']['url'] + + with src: + project = export_to_graph(src, project_url) + + debug('Loaded %s tuples' % len(project)) + + if args.output_dir is not None: + parent = args.output_dir + if not os.path.exists(parent): + os.mkdir(parent) + output_file = os.path.join(parent, "project.ttl") + with open(output_file, 'wb') as dst: + project.serialize(destination=dst, encoding='utf-8', format='turtle') + + def import_project(g): + for idx, t in enumerate(project.triples((None, None, None))): + g.add(t) + + with_database(import_project) + + +def export_to_graph(src: object, project_url: str) -> rdflib.Graph: + export = Export.from_xml_file(src) + + # print('components:') + # for c in export.components: + # print(c) + + # for name in export.component_fields(): + # cli.info(name) + + # cli.info('components:') + # fmt = '%2s%3s' + # cli.info(fmt % ('Ref', '')) + + components = sorted(export.components, key=attrgetter('ref')) + + component_count = 0 + part_count = 0 + has_part = 0 + has_digikey = 0 + + for cls, cs in itertools.groupby(components, key=attrgetter('ref_class')): + # cli.info('--- Class: %s ---' % cls) + cs = sorted(cs, key=attrgetter('value')) + for c in cs: + component_count += 1 + + value = str(c.value) + part = c.find_field('part') + digikey = c.find_field('digikey') + footprint = c.footprint + + if c.ref_class is not None: + ref_class = c.ref_class + ref_instance = c.ref_instance + else: + ref_class = c.ref + ref_instance = '' + + # cli.info(fmt % (ref_class, ref_instance)) + # cli.info(' Value: %s' % c.value.text) + # if c.footprint: + # cli.info(' Footprint: %s' % c.footprint) + + if part is not None: + if part != 'NA': + has_part += 1 if part is not None else 0 + part_count += 1 + + if digikey is not None: + has_digikey += 1 + else: + part = None + else: + part_count += 1 + part = 'MISSING' + + # if part is not None: + # cli.info(' Part: %s' % part) + # if digikey is not None: + # cli.info(' Digikey: %s' % digikey) + + # cli.info() + # cli.info('=== Summary ===') + # cli.info('Project URL: %s' % project_url) + # cli.info('Number of components: %d' % component_count) + # cli.info('Number of parts: %d' % part_count) + # cli.info('Assigned part: %d / %2.f%%' % (has_part, (has_part / part_count) * 100)) + # cli.info('Assigned digikey: %d / %2.f%%' % (has_digikey, (has_digikey / part_count) * 100)) + + g = create_graph(kicad=True) + + project = URIRef(project_url) + g.add((project, RDF.type, kicad_rdf.KICAD_TYPE.project)) + + # The components and fields could/should have been a BNodes. Their generated names are not very nice. + # TODO: try using a hash of the current value and put that under a special generated namespace. + # 'http://example.org/my-board' + ref='C10' => 'http://example.org/my-boardC10' + # hash('http://example.org/my-boardC10') => 123456 + # add_prefix(hash) => 'https://trygvis.io/purl/kicad/generated#123456' + footprints = set() + for c in export.components: + ns = rdflib.Namespace(project_url) + node = ns[c.ref] + g.add((project, kicad_rdf.KICAD.component, node)) + + g.add((node, RDF.type, kicad_rdf.KICAD_TYPE.schematic_component)) + g.add((node, RDFS.label, Literal(c.ref))) + g.add((node, kicad_rdf.KICAD.value, Literal(c.value.text))) + + footprint_uri = URIRef(kicad_rdf.KICAD_FOOTPRINT[quote_plus(c.footprint)]) + if not footprint_uri in footprints: + g.add((footprint_uri, RDF.type, kicad_rdf.KICAD_TYPE.footprint)) + g.add((footprint_uri, RDFS.label, Literal(c.footprint))) + footprints.add(footprint_uri) + + g.add((node, kicad_rdf.KICAD.footprint, footprint_uri)) + + for name, value in c.fields.items(): + f = ns['%s-%s' % (c.ref, name)] + g.add((node, kicad_rdf.KICAD.field, f)) + g.add((f, RDF.type, kicad_rdf.KICAD_TYPE.field)) + g.add((f, kicad_rdf.KICAD.field_name, Literal(name))) + g.add((f, kicad_rdf.KICAD.field_value, Literal(value))) + + # TODO: serialize the data from too + + return g diff --git a/trygvis/eda/cli/make_bom.py b/trygvis/eda/cli/make_bom.py index 7805ead..a47e04a 100755 --- a/trygvis/eda/cli/make_bom.py +++ b/trygvis/eda/cli/make_bom.py @@ -22,11 +22,11 @@ class Component(object): self.fields[field_name] = field_value -def run(schematic_url): - cli.with_database(lambda g: work(schematic_url, g)) +def run(project_url): + cli.with_database(lambda g: work(project_url, g)) -def work(schematic_url, g): +def work(project_url, g): components = {} dk_parts = {} @@ -36,14 +36,14 @@ def work(schematic_url, g): SELECT ?ref ?value WHERE { - ?schematic a kicad-type:schematic ; + ?project a kicad-type:project ; kicad:component ?cmp . ?cmp a kicad-type:schematic_component ; kicad:value ?value ; rdfs:label ?ref . } ORDER BY ?ref -""", init_bindings={"schematic": rdflib.URIRef(schematic_url)}) +""", init_bindings={"project": rdflib.URIRef(project_url)}) for row in res: c = Component(row.ref, row.value) @@ -55,7 +55,7 @@ ORDER BY ?ref SELECT ?ref ?field ?field_name ?field_value WHERE { - ?schematic a kicad-type:schematic ; + ?project a kicad-type:project ; kicad:component ?cmp . ?cmp a kicad-type:schematic_component ; rdfs:label ?ref ; @@ -64,7 +64,7 @@ WHERE { ?field a kicad-type:field ; kicad:field_value ?field_value . } ORDER BY ?ref ?field_name -""", init_bindings={"schematic": rdflib.URIRef(schematic_url)}) +""", init_bindings={"project": rdflib.URIRef(project_url)}) for row in res: c = components[row.ref] @@ -72,12 +72,12 @@ ORDER BY ?ref ?field_name c.set_field(row.field_name, row.field_value) cli.info('%5s: %-20s %s' % (c.ref, row.field_name + ':', row.field_value)) - cli.info("Loading Digi-Key parts for schematic") + cli.info("Loading Digi-Key parts for project") res = cli.sparql(g, """ SELECT ?ref ?part_number ?type ?value ?attr_type WHERE { - ?schematic kicad:component ?cmp . + ?project kicad:component ?cmp . ?cmp a kicad-type:schematic_component ; rdfs:label ?ref ; kicad:field ?d . @@ -93,7 +93,7 @@ WHERE { dk:value ?attr_value . } ORDER BY ?ref ?attr_type ?attr_value -""", init_bindings={"schematic": rdflib.URIRef(schematic_url)}) +""", init_bindings={"project": rdflib.URIRef(project_url)}) for row in res: pn = row.part_number @@ -113,7 +113,7 @@ SELECT ?ref ?footprint ?part_number ?package_value ?case_value # * WHERE { - ?schematic kicad:component ?cmp . + ?project kicad:component ?cmp . ?cmp a kicad-type:schematic_component ; rdfs:label ?ref ; kicad:field ?dk_field . @@ -138,12 +138,12 @@ WHERE { # ?case_type a dk:attributeType ; dk:value ?case_url . # } . -# VALUES (?schematic_url) { () } . +# VALUES (?project_url) { () } . # VALUES (?package_type) { (dk-attr-type:pv16) } . # VALUES (?case_type) { (dk-attr-type:pv1291) } . } ORDER BY ?ref -""", init_bindings={"schematic": rdflib.URIRef(schematic_url), +""", init_bindings={"project": rdflib.URIRef(project_url), "package_type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE["pv16"], "cast_type": digikey_rdf.DIGIKEY_ATTRIBUTE_TYPE["pv1291"]}) diff --git a/trygvis/eda/kicad/export/__init__.py b/trygvis/eda/kicad/export/__init__.py new file mode 100644 index 0000000..64b82d2 --- /dev/null +++ b/trygvis/eda/kicad/export/__init__.py @@ -0,0 +1,250 @@ +from typing import List +import functools +import re +import os.path +from trygvis.eda import EdaException +from trygvis.eda.kicad import rdf as kicad_rdf +from urllib.parse import quote_plus +import xml.etree.ElementTree + + +def _clean(s): + if s is None: + return None + s = s.strip() + return None if len(s) == 0 else s + + +def _cleaned_text(e, name): + child = e.find(name) + if child is None: + return None + return child.text + + +def _cleaned_attr(e, name): + value = e.get(name) + if value is None: + return None + return _clean(value) + + +def _iso_size_to_factor(size): + if size == 'k': + return 3 + if size == 'M': + return 6 + if size == 'm': + return -3 + if size == 'u' or size == '\N{GREEK SMALL LETTER MU}' or size == '\N{MICRO SIGN}': + return -6 + if size == 'n': + return -9 + if size == 'p': + return -12 + return None + + +@functools.total_ordering +class Value: + valueRe = re.compile('([0-9]+\.?[0-9]*)(.*)') + typeRe = re.compile('(.*)(H|Hz)$') + + def __init__(self, text, value, factor, typ): + self.text = text + self.value = value + self.factor = factor + self.type = typ + + def __eq__(self, other): + return self.text == other.text + + def __lt__(self, other): + if self.value is not None: + if other.value is None: + return 1 + else: + return (self.factor, self.value) < (other.factor, other.value) + else: + if other.value is None: + return self.text < other.text + else: + return -1 + + def __str__(self): + if self.value is not None: + s = str(self.value) + 'e' + str(self.factor) + if self.type is not None: + s = s + ' ' + self.type + return s + if self.text is not None: + return self.text + return 'unknown' + + @staticmethod + def parse(text): + if text is None: + return Value(None, None, None, None) + + m = Value.valueRe.match(text) + if m: + value_int = float(m.group(1)) + suffix = m.group(2) + m = Value.typeRe.match(suffix) + if m: + suffix = m.group(1) + typ = m.group(2) + else: + typ = None + + factor = _iso_size_to_factor(suffix) + # cli.info(text + ': suffix:' + suffix + ' => value_int: ' + str(value_int) + ', factor: ' + str( + # factor) + ', type=' + str(type)) + + if value_int is not None and factor is not None: + return Value(text, value_int, factor, typ) + + return Value(text, None, None, None) + + +class Component: + refRe = re.compile('([a-zA-Z]+)([0-9]+)') + + def __init__(self, ref, value, footprint, fields): + self.ref = ref # type: str + self.value = Value.parse(value) # type: Value + self.footprint = footprint # type: str + self.fields = fields + m = Component.refRe.match(ref) + if m: + self.ref_class = m.group(1) + self.ref_instance = int(m.group(2)) + else: + self.ref_class = None + self.ref_instance = None + + def __str__(self): + s = 'ref=' + self.ref + if self.value is not None: + s += ', value=' + str(self.value) + if self.footprint is not None: + s += ', footprint=' + self.footprint + for name, value in self.fields.items(): + s += ', ' + name + '=' + value + return s + + def find_field(self, key): + return self.fields[key] if key in self.fields else None + + @staticmethod + def from_xml(e): + ref = e.get('ref') + value = _cleaned_text(e, 'value') + footprint = _cleaned_text(e, 'footprint') + fs = {} + + fields = e.find('fields') + if fields is not None: + for field in fields.findall('field'): + fs[field.get('name')] = field.text + + if ref is not None: + return Component(ref, value, footprint, fs) + + return None + + +class TitleBlock(object): + def __init__(self, title=None, rev=None, date=None, source=None): + self.title = title + self.rev = rev + self.date = date + self.source = source + + @staticmethod + def from_xml(tb): + title = _cleaned_text(tb, 'title') + rev = _cleaned_text(tb, 'rev') + date = _cleaned_text(tb, 'date') + source = _cleaned_text(tb, 'source') + return TitleBlock(title, rev, date, source) + + +class Sheet(object): + def __init__(self, number, name, title_block): + self.number = number + self.name = name + self.title_block = title_block + + @staticmethod + def from_xml(s): + number = _cleaned_attr(s, 'number') + name = _cleaned_attr(s, 'name') + node = s.find('title_block') + title_block = TitleBlock.from_xml(node) if node is not None else TitleBlock() + return Sheet(int(number) if number is not None else None, name, title_block) + + +class Design(object): + def __init__(self, source=None, date=None, tool=None, sheets=None): + self.source = source + self.date = date + self.tool = tool + self.sheets = sheets if sheets is not None else [] + + @staticmethod + def from_xml(d): + source = _cleaned_text(d, 'source') + date = _cleaned_text(d, 'date') + tool = _cleaned_text(d, 'tool') + sheets = [] + for s in d.iterfind('sheet'): + sheets.append(Sheet.from_xml(s)) + return Design(source, date, tool, sheets) + + +class Export(object): + def __init__(self, design, components): + self.design = design if design is not None else Design() # type: Design + self.components = components # type: List[Component] + + @staticmethod + def from_xml_document(doc) -> 'Export': + cs = [] + + node = doc.find('design') + design = Design.from_xml(node) if node is not None else None + + components = doc.find('components') + for e in components.iter('comp'): + c = Component.from_xml(e) + if c is not None: + cs.append(c) + + return Export(design, cs) + + @staticmethod + def from_xml_file(src) -> 'Export': + tree = xml.etree.ElementTree.parse(src) + root = tree.getroot() + return Export.from_xml_document(root) + + def component_fields(self): + fs = set() + for c in self.components: + for key, value in c.fields.items(): + fs.add(key) + return fs + + # def generate_project_url(self): + # s = next((s for s in self.design.sheets if s.number == 1), None) + # title = s.title_block.title if s is not None else None + # if title is None: + # if self.design.source is not None: + # title = _clean(os.path.basename(self.design.source)) + # + # if title is None: + # raise EdaException('Could not generate a stable, identifying URL for the project') + # + # title = quote_plus(title) + # return kicad_rdf.KICAD_BOARD[title] -- cgit v1.2.3