From 117431b9511be07db8ce53526dbb985b5fad00a2 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Wed, 28 Dec 2016 14:08:52 +0100 Subject: o Adding 'digikey-download-metadata' tool that downloads everything from Digi-Key. Should support writing directly to database. --- trygvis/eda/cli/__init__.py | 34 +++++++++++++--- ...igikey_download_attribute_types_for_category.py | 4 +- trygvis/eda/cli/digikey_download_for_schematic.py | 4 +- trygvis/eda/cli/digikey_download_metadata.py | 45 ++++++++++++++++++++++ trygvis/eda/cli/eda_rdf.py | 28 +++++++------- trygvis/eda/cli/kicad_bom_to_ttl.py | 25 ++++++++++-- 6 files changed, 115 insertions(+), 25 deletions(-) create mode 100755 trygvis/eda/cli/digikey_download_metadata.py (limited to 'trygvis/eda/cli') diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py index f300f86..8050d6d 100644 --- a/trygvis/eda/cli/__init__.py +++ b/trygvis/eda/cli/__init__.py @@ -1,5 +1,7 @@ import sys import logging +from genericpath import isfile + from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS from rdflib.plugins.sparql import prepareQuery import rdflib.plugins.stores.sparqlstore as sparqlstore @@ -70,6 +72,26 @@ def create_graph(digikey=False, kicad=False): return g +def write_graph(gen_g, filename: str = None, force_write: bool = False): + if filename is not None: + if force_write or not isfile(filename): + g = gen_g() + + if g is None: + raise CliException("internal error: graph generator returned None") + + bs = g.serialize(encoding='utf-8', format='turtle') + with open(filename, "wb") as f: + f.write(bs) + info("Wrote %s" % filename) + else: + info("Skipped writing %s, already exists" % filename) + else: + g = gen_g() + bs = g.serialize(encoding='utf-8', format='turtle') + sys.stdout.buffer.write(bs) + + _initNs = { "rdf": RDF, "rdfs": RDFS, @@ -83,7 +105,6 @@ _initNs = { def sparql(g: Graph, query: str, init_bindings=None): - if isinstance(g, sparqlstore.SPARQLStore): return g.query(query, initNs=_initNs, initBindings=init_bindings) else: @@ -97,7 +118,10 @@ def write_config(config: configparser.ConfigParser): def read_config(): - with open('.eda-rdf/config.ini', 'r') as f: - config = configparser.ConfigParser() - config.read_file(f) - return config + try: + with open('.eda-rdf/config.ini', 'r') as f: + config = configparser.ConfigParser() + config.read_file(f) + return config + except FileNotFoundError: + raise CliException("Not a EDA-RDF project. Run eda-rdf init first.") diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py index 1a33ab1..9839c1b 100644 --- a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py +++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py @@ -1,4 +1,4 @@ -from trygvis.eda import cli, write_graph +from trygvis.eda import cli from trygvis.eda.digikey import * @@ -26,4 +26,4 @@ def run(category, sub_category, output_file, args): [g.add(node) for node in a.to_nodes()] filename = output_file if output_file is not "-" else None - write_graph(gen_g=lambda: g, filename=filename) + cli.write_graph(gen_g=lambda: g, filename=filename) diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py index 0c21b3a..0cb5baa 100755 --- a/trygvis/eda/cli/digikey_download_for_schematic.py +++ b/trygvis/eda/cli/digikey_download_for_schematic.py @@ -2,7 +2,7 @@ from rdflib.plugins.sparql import prepareQuery import rdflib import rdflib.term -from trygvis.eda import cli, write_graph +from trygvis.eda import cli from trygvis.eda.digikey import * from trygvis.eda.digikey import rdf as digikey_rdf from trygvis.eda.kicad import rdf as kicad_rdf @@ -90,7 +90,7 @@ ORDER BY ?digikey_pn [g.add(node) for node in product.to_nodes()] return g - write_graph(download_graph, filename) + cli.write_graph(download_graph, filename) # q = prepareQuery(""" # SELECT diff --git a/trygvis/eda/cli/digikey_download_metadata.py b/trygvis/eda/cli/digikey_download_metadata.py new file mode 100755 index 0000000..d807c1a --- /dev/null +++ b/trygvis/eda/cli/digikey_download_metadata.py @@ -0,0 +1,45 @@ +from trygvis.eda import cli +from trygvis.eda.digikey import * + + +def do_categories(db: DigikeyDatabase, client: DigikeyClient, output: str): + cli.info("Downloading category tree") + download_category_tree(db, client) + + def make_graph(): + g = cli.create_graph(digikey=True) + for pc in db.productCategories: + [g.add(node) for node in pc.to_nodes()] + + for sc in pc.subCategories: + [g.add(node) for node in sc.to_nodes()] + return g + + cli.write_graph(make_graph, output, force_write=True) + + +def do_category(client: DigikeyClient, sc: DigikeyProductCategory, output: str): + cli.info('Downloading categories for %s' % sc.label) + + def make_graph(): + attributes = download_attribute_types_from_category(sc, client) + g = cli.create_graph() + for a in attributes: + [g.add(node) for node in a.to_nodes()] + + return g + + cli.write_graph(make_graph, output, force_write=True) + +def run(args): + db = DigikeyDatabase() + client = DigikeyClient() + + # TODO: store in database too, not sure write to disk + output = "%s/digikey-categories.ttl" % args.output_dir + do_categories(db, client, output) + + for pc in db.productCategories: + for sc in pc.subCategories: + output = "%s/digikey-%s.ttl" % (args.output_dir, normalize_filename(sc.label)) + do_category(client, sc, output) diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py index 33018ec..ecd4ba6 100644 --- a/trygvis/eda/cli/eda_rdf.py +++ b/trygvis/eda/cli/eda_rdf.py @@ -45,19 +45,7 @@ class KicadBomToTtl(CliCommand): args = p.parse_args(argv) from trygvis.eda.cli import kicad_bom_to_ttl - - if args.input is not None: - src = open(args.input, "r") - else: - src = sys.stdin - - if args.output is not None: - dst = open(args.output, "wb") - else: - dst = sys.stdout.buffer - - with src, dst: - kicad_bom_to_ttl.run(src, dst, args) + kicad_bom_to_ttl.run(args) class DigikeyDownloadForSchematic(CliCommand): @@ -73,6 +61,19 @@ class DigikeyDownloadForSchematic(CliCommand): digikey_download_for_schematic.run(args.schematic, args) +class DigikeyDownloadMetadata(CliCommand): + def __init__(self): + super().__init__("digikey-download-metadata", "Download category tree and attribute types and values from digikey.com") + + def run(self, argv): + p = argparse.ArgumentParser(prog=self.key, description=self.description) + p.add_argument("--output-dir", dest="output_dir", required=True) + args = p.parse_args(argv) + + from trygvis.eda.cli import digikey_download_metadata + digikey_download_metadata.run(args) + + class DigikeyDownloadAttributeTypesForCategory(CliCommand): def __init__(self): super().__init__("digikey-download-attribute-types-for-category", "Download the attribute types for a category from digikey.com") @@ -124,6 +125,7 @@ def main(): KicadBomToTtl(), DigikeyDownloadForSchematic(), DigikeyDownloadAttributeTypesForCategory(), + DigikeyDownloadMetadata() ] parser = argparse.ArgumentParser( diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py index c7d6899..b017767 100755 --- a/trygvis/eda/cli/kicad_bom_to_ttl.py +++ b/trygvis/eda/cli/kicad_bom_to_ttl.py @@ -3,6 +3,7 @@ import functools import os.path import re +import sys import xml.etree.ElementTree from urllib.parse import quote_plus @@ -246,9 +247,27 @@ class Export(object): return kicad_rdf.KICAD_BOARD[title] -def run(src, dst, args): - from trygvis.eda import cli +def run(args): + if args.input is not None: + src = open(args.input, "r") + else: + src = sys.stdin + if args.output is not None: + parent = os.path.dirname(args.output) + if not os.path.exists(parent): + os.mkdir(parent) + dst = open(args.output, "wb") + else: + dst = sys.stdout.buffer + + schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else None + + with src, dst: + process(src, dst, schematic_url) + + +def process(src, dst, schematic_url): tree = xml.etree.ElementTree.parse(src) root = tree.getroot() @@ -316,7 +335,7 @@ def run(src, dst, args): # if digikey is not None: # cli.info(' Digikey: %s' % digikey) - schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url() + schematic_url = schematic_url if schematic_url is not None else export.generate_schematic_url() # cli.info() # cli.info('=== Summary ===') -- cgit v1.2.3