diff options
Diffstat (limited to 'trygvis')
| -rw-r--r-- | trygvis/eda/__init__.py | 20 | ||||
| -rw-r--r-- | trygvis/eda/cli/__init__.py | 34 | ||||
| -rw-r--r-- | trygvis/eda/cli/digikey_download_attribute_types_for_category.py | 4 | ||||
| -rwxr-xr-x | trygvis/eda/cli/digikey_download_for_schematic.py | 4 | ||||
| -rwxr-xr-x | trygvis/eda/cli/digikey_download_metadata.py | 45 | ||||
| -rw-r--r-- | trygvis/eda/cli/eda_rdf.py | 28 | ||||
| -rwxr-xr-x | trygvis/eda/cli/kicad_bom_to_ttl.py | 25 | ||||
| -rw-r--r-- | trygvis/eda/digikey/__init__.py | 17 | ||||
| -rw-r--r-- | trygvis/eda/digikey/__main__.py | 42 | ||||
| -rw-r--r-- | trygvis/eda/kicad/rdf.py | 8 |
10 files changed, 130 insertions, 97 deletions
diff --git a/trygvis/eda/__init__.py b/trygvis/eda/__init__.py index e6ccb44..e69de29 100644 --- a/trygvis/eda/__init__.py +++ b/trygvis/eda/__init__.py @@ -1,20 +0,0 @@ -import sys - -from os.path import isfile -from . import cli - - -def write_graph(gen_g, filename=None, force_write=False): - if filename is not None: - if force_write or not isfile(filename): - g = gen_g() - bs = g.serialize(encoding='utf-8', format='turtle') - with open(filename, "wb") as f: - f.write(bs) - cli.info("Wrote %s" % filename) - else: - cli.info("Skipped writing %s, already exists" % filename) - else: - g = gen_g() - bs = g.serialize(encoding='utf-8', format='turtle') - sys.stdout.buffer.write(bs) diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py index f300f86..8050d6d 100644 --- a/trygvis/eda/cli/__init__.py +++ b/trygvis/eda/cli/__init__.py @@ -1,5 +1,7 @@ import sys import logging +from genericpath import isfile + from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS from rdflib.plugins.sparql import prepareQuery import rdflib.plugins.stores.sparqlstore as sparqlstore @@ -70,6 +72,26 @@ def create_graph(digikey=False, kicad=False): return g +def write_graph(gen_g, filename: str = None, force_write: bool = False): + if filename is not None: + if force_write or not isfile(filename): + g = gen_g() + + if g is None: + raise CliException("internal error: graph generator returned None") + + bs = g.serialize(encoding='utf-8', format='turtle') + with open(filename, "wb") as f: + f.write(bs) + info("Wrote %s" % filename) + else: + info("Skipped writing %s, already exists" % filename) + else: + g = gen_g() + bs = g.serialize(encoding='utf-8', format='turtle') + sys.stdout.buffer.write(bs) + + _initNs = { "rdf": RDF, "rdfs": RDFS, @@ -83,7 +105,6 @@ _initNs = { def sparql(g: Graph, query: str, init_bindings=None): - if isinstance(g, sparqlstore.SPARQLStore): return g.query(query, initNs=_initNs, initBindings=init_bindings) else: @@ -97,7 +118,10 @@ def write_config(config: configparser.ConfigParser): def read_config(): - with open('.eda-rdf/config.ini', 'r') as f: - config = configparser.ConfigParser() - config.read_file(f) - return config + try: + with open('.eda-rdf/config.ini', 'r') as f: + config = configparser.ConfigParser() + config.read_file(f) + return config + except FileNotFoundError: + raise CliException("Not a EDA-RDF project. Run eda-rdf init first.") diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py index 1a33ab1..9839c1b 100644 --- a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py +++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py @@ -1,4 +1,4 @@ -from trygvis.eda import cli, write_graph +from trygvis.eda import cli from trygvis.eda.digikey import * @@ -26,4 +26,4 @@ def run(category, sub_category, output_file, args): [g.add(node) for node in a.to_nodes()] filename = output_file if output_file is not "-" else None - write_graph(gen_g=lambda: g, filename=filename) + cli.write_graph(gen_g=lambda: g, filename=filename) diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py index 0c21b3a..0cb5baa 100755 --- a/trygvis/eda/cli/digikey_download_for_schematic.py +++ b/trygvis/eda/cli/digikey_download_for_schematic.py @@ -2,7 +2,7 @@ from rdflib.plugins.sparql import prepareQuery import rdflib import rdflib.term -from trygvis.eda import cli, write_graph +from trygvis.eda import cli from trygvis.eda.digikey import * from trygvis.eda.digikey import rdf as digikey_rdf from trygvis.eda.kicad import rdf as kicad_rdf @@ -90,7 +90,7 @@ ORDER BY ?digikey_pn [g.add(node) for node in product.to_nodes()] return g - write_graph(download_graph, filename) + cli.write_graph(download_graph, filename) # q = prepareQuery(""" # SELECT diff --git a/trygvis/eda/cli/digikey_download_metadata.py b/trygvis/eda/cli/digikey_download_metadata.py new file mode 100755 index 0000000..d807c1a --- /dev/null +++ b/trygvis/eda/cli/digikey_download_metadata.py @@ -0,0 +1,45 @@ +from trygvis.eda import cli +from trygvis.eda.digikey import * + + +def do_categories(db: DigikeyDatabase, client: DigikeyClient, output: str): + cli.info("Downloading category tree") + download_category_tree(db, client) + + def make_graph(): + g = cli.create_graph(digikey=True) + for pc in db.productCategories: + [g.add(node) for node in pc.to_nodes()] + + for sc in pc.subCategories: + [g.add(node) for node in sc.to_nodes()] + return g + + cli.write_graph(make_graph, output, force_write=True) + + +def do_category(client: DigikeyClient, sc: DigikeyProductCategory, output: str): + cli.info('Downloading categories for %s' % sc.label) + + def make_graph(): + attributes = download_attribute_types_from_category(sc, client) + g = cli.create_graph() + for a in attributes: + [g.add(node) for node in a.to_nodes()] + + return g + + cli.write_graph(make_graph, output, force_write=True) + +def run(args): + db = DigikeyDatabase() + client = DigikeyClient() + + # TODO: store in database too, not sure write to disk + output = "%s/digikey-categories.ttl" % args.output_dir + do_categories(db, client, output) + + for pc in db.productCategories: + for sc in pc.subCategories: + output = "%s/digikey-%s.ttl" % (args.output_dir, normalize_filename(sc.label)) + do_category(client, sc, output) diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py index 33018ec..ecd4ba6 100644 --- a/trygvis/eda/cli/eda_rdf.py +++ b/trygvis/eda/cli/eda_rdf.py @@ -45,19 +45,7 @@ class KicadBomToTtl(CliCommand): args = p.parse_args(argv) from trygvis.eda.cli import kicad_bom_to_ttl - - if args.input is not None: - src = open(args.input, "r") - else: - src = sys.stdin - - if args.output is not None: - dst = open(args.output, "wb") - else: - dst = sys.stdout.buffer - - with src, dst: - kicad_bom_to_ttl.run(src, dst, args) + kicad_bom_to_ttl.run(args) class DigikeyDownloadForSchematic(CliCommand): @@ -73,6 +61,19 @@ class DigikeyDownloadForSchematic(CliCommand): digikey_download_for_schematic.run(args.schematic, args) +class DigikeyDownloadMetadata(CliCommand): + def __init__(self): + super().__init__("digikey-download-metadata", "Download category tree and attribute types and values from digikey.com") + + def run(self, argv): + p = argparse.ArgumentParser(prog=self.key, description=self.description) + p.add_argument("--output-dir", dest="output_dir", required=True) + args = p.parse_args(argv) + + from trygvis.eda.cli import digikey_download_metadata + digikey_download_metadata.run(args) + + class DigikeyDownloadAttributeTypesForCategory(CliCommand): def __init__(self): super().__init__("digikey-download-attribute-types-for-category", "Download the attribute types for a category from digikey.com") @@ -124,6 +125,7 @@ def main(): KicadBomToTtl(), DigikeyDownloadForSchematic(), DigikeyDownloadAttributeTypesForCategory(), + DigikeyDownloadMetadata() ] parser = argparse.ArgumentParser( diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py index c7d6899..b017767 100755 --- a/trygvis/eda/cli/kicad_bom_to_ttl.py +++ b/trygvis/eda/cli/kicad_bom_to_ttl.py @@ -3,6 +3,7 @@ import functools import os.path import re +import sys import xml.etree.ElementTree from urllib.parse import quote_plus @@ -246,9 +247,27 @@ class Export(object): return kicad_rdf.KICAD_BOARD[title] -def run(src, dst, args): - from trygvis.eda import cli +def run(args): + if args.input is not None: + src = open(args.input, "r") + else: + src = sys.stdin + if args.output is not None: + parent = os.path.dirname(args.output) + if not os.path.exists(parent): + os.mkdir(parent) + dst = open(args.output, "wb") + else: + dst = sys.stdout.buffer + + schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else None + + with src, dst: + process(src, dst, schematic_url) + + +def process(src, dst, schematic_url): tree = xml.etree.ElementTree.parse(src) root = tree.getroot() @@ -316,7 +335,7 @@ def run(src, dst, args): # if digikey is not None: # cli.info(' Digikey: %s' % digikey) - schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url() + schematic_url = schematic_url if schematic_url is not None else export.generate_schematic_url() # cli.info() # cli.info('=== Summary ===') diff --git a/trygvis/eda/digikey/__init__.py b/trygvis/eda/digikey/__init__.py index 5f4ad8a..ec1167d 100644 --- a/trygvis/eda/digikey/__init__.py +++ b/trygvis/eda/digikey/__init__.py @@ -1,5 +1,6 @@ import re +from typing import List import requests from cachecontrol import CacheControl from cachecontrol.caches.file_cache import FileCache @@ -10,9 +11,11 @@ from rdflib.namespace import RDF, RDFS import trygvis.eda.digikey.rdf + def normalize_filename(part): return part.replace('/', '_').replace(' ', '_') + def _clean(s): if s is None: return None @@ -22,7 +25,7 @@ def _clean(s): class DigikeyDatabase(object): def __init__(self): - self.productCategories = [] + self.productCategories = [] # type: List[DigikeyProductCategory] self.attributeTypes = {} def add_product_category(self, pc): @@ -55,8 +58,8 @@ class DigikeyProductCategory(object): self.label = _clean(label) self.digikey_url = digikey_url if digikey_url is None or digikey_url.startswith("http") else \ "http://www.digikey.com" + digikey_url - self.parent = parent - self.subCategories = [] + self.parent = parent # type: DigikeyProductCategory + self.subCategories = [] # type: List[DigikeyProductCategory assert self.id is not None assert self.label is not None @@ -154,6 +157,7 @@ class DigikeyProduct(object): class DigikeyClient(object): def __init__(self): + # TODO: this should be put under .eda-rdf/ cache = FileCache('digikey_cache', forever=True) self.sess = CacheControl(requests.Session(), cache=cache, heuristic=ExpiresAfter(days=1)) @@ -177,7 +181,8 @@ def _id_from_url(url): return m.group(1) if m else None -def download_category_tree(database, client, baseurl="http://www.digikey.com/products/en"): +def download_category_tree(database: DigikeyDatabase, client: DigikeyClient, + baseurl="http://www.digikey.com/products/en"): page = client.req(baseurl) dom = html.fromstring(page.content) @@ -212,7 +217,7 @@ def download_category_tree(database, client, baseurl="http://www.digikey.com/pro database.add_product_category(pc) -def download_attribute_types_from_category(category, client): +def download_attribute_types_from_category(category: DigikeyProductCategory, client: DigikeyClient) -> List[DigikeyAttributeType]: page = client.req(category.digikey_url) tree = html.fromstring(page.content) @@ -242,7 +247,7 @@ def download_attribute_types_from_category(category, client): return attributes -def download_product(client, db, query): +def download_product(client: DigikeyClient, db, query): # http://www.digikey.com/products/en?x=0&y=0&lang=en&site=us&keywords=553-2320-1-ND page = client.req("http://www.digikey.com/products/en", params={'lang': 'en', 'site': 'us', 'keywords': query}) tree = html.fromstring(page.content) diff --git a/trygvis/eda/digikey/__main__.py b/trygvis/eda/digikey/__main__.py deleted file mode 100644 index ceb341e..0000000 --- a/trygvis/eda/digikey/__main__.py +++ /dev/null @@ -1,42 +0,0 @@ -import argparse - -from .. import write_graph -from ..cli import * -from ..digikey import * - -parser = argparse.ArgumentParser() -subparsers = parser.add_subparsers(dest='cmd') # help='sub-command help' - -dct_parser = subparsers.add_parser("download-category-tree") -dct_parser.add_argument("-o", "--output", required=False) - -dp_parser = subparsers.add_parser("download-product") -dp_parser.add_argument("-p", "--product") -dp_parser.add_argument("-o", "--output", required=False) - -args = parser.parse_args() - -client = DigikeyClient() -db = DigikeyDatabase() - -if args.cmd == "download-category-tree": - download_category_tree(db, client) - if args.output is not None: - def make_graph(): - g = create_graph(digikey=True) - for pc in db.productCategories: - [g.add(node) for node in pc.to_nodes()] - - for sc in pc.subCategories: - [g.add(node) for node in sc.to_nodes()] - write_graph(make_graph, args.output) - -elif args.cmd == "download-product": - download_category_tree(db, client) - product = download_product(client, db, args.product) - - if args.output is not None: - def make_graph(): - g = create_graph(digikey=True) - [g.add(node) for node in product.to_nodes()] - write_graph(make_graph, args.output) diff --git a/trygvis/eda/kicad/rdf.py b/trygvis/eda/kicad/rdf.py index 683eaad..24cba00 100644 --- a/trygvis/eda/kicad/rdf.py +++ b/trygvis/eda/kicad/rdf.py @@ -1,7 +1,7 @@ -from rdflib import Namespace +import rdflib -KICAD = Namespace("https://trygvis/purl/kicad#") -KICAD_TYPE = Namespace("https://trygvis/purl/kicad-type#") +KICAD = rdflib.Namespace("https://trygvis/purl/kicad#") +KICAD_TYPE = rdflib.Namespace("https://trygvis/purl/kicad-type#") # Namespace for all unknown kicad boards -KICAD_BOARD = Namespace("https://trygvis/purl/kicad-board#") +KICAD_BOARD = rdflib.Namespace("https://trygvis/purl/kicad-board#") |