aboutsummaryrefslogtreecommitdiff
path: root/trygvis
diff options
context:
space:
mode:
Diffstat (limited to 'trygvis')
-rw-r--r--trygvis/eda/__init__.py20
-rw-r--r--trygvis/eda/cli/__init__.py34
-rw-r--r--trygvis/eda/cli/digikey_download_attribute_types_for_category.py4
-rwxr-xr-xtrygvis/eda/cli/digikey_download_for_schematic.py4
-rwxr-xr-xtrygvis/eda/cli/digikey_download_metadata.py45
-rw-r--r--trygvis/eda/cli/eda_rdf.py28
-rwxr-xr-xtrygvis/eda/cli/kicad_bom_to_ttl.py25
-rw-r--r--trygvis/eda/digikey/__init__.py17
-rw-r--r--trygvis/eda/digikey/__main__.py42
-rw-r--r--trygvis/eda/kicad/rdf.py8
10 files changed, 130 insertions, 97 deletions
diff --git a/trygvis/eda/__init__.py b/trygvis/eda/__init__.py
index e6ccb44..e69de29 100644
--- a/trygvis/eda/__init__.py
+++ b/trygvis/eda/__init__.py
@@ -1,20 +0,0 @@
-import sys
-
-from os.path import isfile
-from . import cli
-
-
-def write_graph(gen_g, filename=None, force_write=False):
- if filename is not None:
- if force_write or not isfile(filename):
- g = gen_g()
- bs = g.serialize(encoding='utf-8', format='turtle')
- with open(filename, "wb") as f:
- f.write(bs)
- cli.info("Wrote %s" % filename)
- else:
- cli.info("Skipped writing %s, already exists" % filename)
- else:
- g = gen_g()
- bs = g.serialize(encoding='utf-8', format='turtle')
- sys.stdout.buffer.write(bs)
diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py
index f300f86..8050d6d 100644
--- a/trygvis/eda/cli/__init__.py
+++ b/trygvis/eda/cli/__init__.py
@@ -1,5 +1,7 @@
import sys
import logging
+from genericpath import isfile
+
from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS
from rdflib.plugins.sparql import prepareQuery
import rdflib.plugins.stores.sparqlstore as sparqlstore
@@ -70,6 +72,26 @@ def create_graph(digikey=False, kicad=False):
return g
+def write_graph(gen_g, filename: str = None, force_write: bool = False):
+ if filename is not None:
+ if force_write or not isfile(filename):
+ g = gen_g()
+
+ if g is None:
+ raise CliException("internal error: graph generator returned None")
+
+ bs = g.serialize(encoding='utf-8', format='turtle')
+ with open(filename, "wb") as f:
+ f.write(bs)
+ info("Wrote %s" % filename)
+ else:
+ info("Skipped writing %s, already exists" % filename)
+ else:
+ g = gen_g()
+ bs = g.serialize(encoding='utf-8', format='turtle')
+ sys.stdout.buffer.write(bs)
+
+
_initNs = {
"rdf": RDF,
"rdfs": RDFS,
@@ -83,7 +105,6 @@ _initNs = {
def sparql(g: Graph, query: str, init_bindings=None):
-
if isinstance(g, sparqlstore.SPARQLStore):
return g.query(query, initNs=_initNs, initBindings=init_bindings)
else:
@@ -97,7 +118,10 @@ def write_config(config: configparser.ConfigParser):
def read_config():
- with open('.eda-rdf/config.ini', 'r') as f:
- config = configparser.ConfigParser()
- config.read_file(f)
- return config
+ try:
+ with open('.eda-rdf/config.ini', 'r') as f:
+ config = configparser.ConfigParser()
+ config.read_file(f)
+ return config
+ except FileNotFoundError:
+ raise CliException("Not a EDA-RDF project. Run eda-rdf init first.")
diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
index 1a33ab1..9839c1b 100644
--- a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
+++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
@@ -1,4 +1,4 @@
-from trygvis.eda import cli, write_graph
+from trygvis.eda import cli
from trygvis.eda.digikey import *
@@ -26,4 +26,4 @@ def run(category, sub_category, output_file, args):
[g.add(node) for node in a.to_nodes()]
filename = output_file if output_file is not "-" else None
- write_graph(gen_g=lambda: g, filename=filename)
+ cli.write_graph(gen_g=lambda: g, filename=filename)
diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py
index 0c21b3a..0cb5baa 100755
--- a/trygvis/eda/cli/digikey_download_for_schematic.py
+++ b/trygvis/eda/cli/digikey_download_for_schematic.py
@@ -2,7 +2,7 @@ from rdflib.plugins.sparql import prepareQuery
import rdflib
import rdflib.term
-from trygvis.eda import cli, write_graph
+from trygvis.eda import cli
from trygvis.eda.digikey import *
from trygvis.eda.digikey import rdf as digikey_rdf
from trygvis.eda.kicad import rdf as kicad_rdf
@@ -90,7 +90,7 @@ ORDER BY ?digikey_pn
[g.add(node) for node in product.to_nodes()]
return g
- write_graph(download_graph, filename)
+ cli.write_graph(download_graph, filename)
# q = prepareQuery("""
# SELECT
diff --git a/trygvis/eda/cli/digikey_download_metadata.py b/trygvis/eda/cli/digikey_download_metadata.py
new file mode 100755
index 0000000..d807c1a
--- /dev/null
+++ b/trygvis/eda/cli/digikey_download_metadata.py
@@ -0,0 +1,45 @@
+from trygvis.eda import cli
+from trygvis.eda.digikey import *
+
+
+def do_categories(db: DigikeyDatabase, client: DigikeyClient, output: str):
+ cli.info("Downloading category tree")
+ download_category_tree(db, client)
+
+ def make_graph():
+ g = cli.create_graph(digikey=True)
+ for pc in db.productCategories:
+ [g.add(node) for node in pc.to_nodes()]
+
+ for sc in pc.subCategories:
+ [g.add(node) for node in sc.to_nodes()]
+ return g
+
+ cli.write_graph(make_graph, output, force_write=True)
+
+
+def do_category(client: DigikeyClient, sc: DigikeyProductCategory, output: str):
+ cli.info('Downloading categories for %s' % sc.label)
+
+ def make_graph():
+ attributes = download_attribute_types_from_category(sc, client)
+ g = cli.create_graph()
+ for a in attributes:
+ [g.add(node) for node in a.to_nodes()]
+
+ return g
+
+ cli.write_graph(make_graph, output, force_write=True)
+
+def run(args):
+ db = DigikeyDatabase()
+ client = DigikeyClient()
+
+ # TODO: store in database too, not sure write to disk
+ output = "%s/digikey-categories.ttl" % args.output_dir
+ do_categories(db, client, output)
+
+ for pc in db.productCategories:
+ for sc in pc.subCategories:
+ output = "%s/digikey-%s.ttl" % (args.output_dir, normalize_filename(sc.label))
+ do_category(client, sc, output)
diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py
index 33018ec..ecd4ba6 100644
--- a/trygvis/eda/cli/eda_rdf.py
+++ b/trygvis/eda/cli/eda_rdf.py
@@ -45,19 +45,7 @@ class KicadBomToTtl(CliCommand):
args = p.parse_args(argv)
from trygvis.eda.cli import kicad_bom_to_ttl
-
- if args.input is not None:
- src = open(args.input, "r")
- else:
- src = sys.stdin
-
- if args.output is not None:
- dst = open(args.output, "wb")
- else:
- dst = sys.stdout.buffer
-
- with src, dst:
- kicad_bom_to_ttl.run(src, dst, args)
+ kicad_bom_to_ttl.run(args)
class DigikeyDownloadForSchematic(CliCommand):
@@ -73,6 +61,19 @@ class DigikeyDownloadForSchematic(CliCommand):
digikey_download_for_schematic.run(args.schematic, args)
+class DigikeyDownloadMetadata(CliCommand):
+ def __init__(self):
+ super().__init__("digikey-download-metadata", "Download category tree and attribute types and values from digikey.com")
+
+ def run(self, argv):
+ p = argparse.ArgumentParser(prog=self.key, description=self.description)
+ p.add_argument("--output-dir", dest="output_dir", required=True)
+ args = p.parse_args(argv)
+
+ from trygvis.eda.cli import digikey_download_metadata
+ digikey_download_metadata.run(args)
+
+
class DigikeyDownloadAttributeTypesForCategory(CliCommand):
def __init__(self):
super().__init__("digikey-download-attribute-types-for-category", "Download the attribute types for a category from digikey.com")
@@ -124,6 +125,7 @@ def main():
KicadBomToTtl(),
DigikeyDownloadForSchematic(),
DigikeyDownloadAttributeTypesForCategory(),
+ DigikeyDownloadMetadata()
]
parser = argparse.ArgumentParser(
diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py
index c7d6899..b017767 100755
--- a/trygvis/eda/cli/kicad_bom_to_ttl.py
+++ b/trygvis/eda/cli/kicad_bom_to_ttl.py
@@ -3,6 +3,7 @@
import functools
import os.path
import re
+import sys
import xml.etree.ElementTree
from urllib.parse import quote_plus
@@ -246,9 +247,27 @@ class Export(object):
return kicad_rdf.KICAD_BOARD[title]
-def run(src, dst, args):
- from trygvis.eda import cli
+def run(args):
+ if args.input is not None:
+ src = open(args.input, "r")
+ else:
+ src = sys.stdin
+ if args.output is not None:
+ parent = os.path.dirname(args.output)
+ if not os.path.exists(parent):
+ os.mkdir(parent)
+ dst = open(args.output, "wb")
+ else:
+ dst = sys.stdout.buffer
+
+ schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else None
+
+ with src, dst:
+ process(src, dst, schematic_url)
+
+
+def process(src, dst, schematic_url):
tree = xml.etree.ElementTree.parse(src)
root = tree.getroot()
@@ -316,7 +335,7 @@ def run(src, dst, args):
# if digikey is not None:
# cli.info(' Digikey: %s' % digikey)
- schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url()
+ schematic_url = schematic_url if schematic_url is not None else export.generate_schematic_url()
# cli.info()
# cli.info('=== Summary ===')
diff --git a/trygvis/eda/digikey/__init__.py b/trygvis/eda/digikey/__init__.py
index 5f4ad8a..ec1167d 100644
--- a/trygvis/eda/digikey/__init__.py
+++ b/trygvis/eda/digikey/__init__.py
@@ -1,5 +1,6 @@
import re
+from typing import List
import requests
from cachecontrol import CacheControl
from cachecontrol.caches.file_cache import FileCache
@@ -10,9 +11,11 @@ from rdflib.namespace import RDF, RDFS
import trygvis.eda.digikey.rdf
+
def normalize_filename(part):
return part.replace('/', '_').replace(' ', '_')
+
def _clean(s):
if s is None:
return None
@@ -22,7 +25,7 @@ def _clean(s):
class DigikeyDatabase(object):
def __init__(self):
- self.productCategories = []
+ self.productCategories = [] # type: List[DigikeyProductCategory]
self.attributeTypes = {}
def add_product_category(self, pc):
@@ -55,8 +58,8 @@ class DigikeyProductCategory(object):
self.label = _clean(label)
self.digikey_url = digikey_url if digikey_url is None or digikey_url.startswith("http") else \
"http://www.digikey.com" + digikey_url
- self.parent = parent
- self.subCategories = []
+ self.parent = parent # type: DigikeyProductCategory
+ self.subCategories = [] # type: List[DigikeyProductCategory
assert self.id is not None
assert self.label is not None
@@ -154,6 +157,7 @@ class DigikeyProduct(object):
class DigikeyClient(object):
def __init__(self):
+ # TODO: this should be put under .eda-rdf/
cache = FileCache('digikey_cache', forever=True)
self.sess = CacheControl(requests.Session(), cache=cache, heuristic=ExpiresAfter(days=1))
@@ -177,7 +181,8 @@ def _id_from_url(url):
return m.group(1) if m else None
-def download_category_tree(database, client, baseurl="http://www.digikey.com/products/en"):
+def download_category_tree(database: DigikeyDatabase, client: DigikeyClient,
+ baseurl="http://www.digikey.com/products/en"):
page = client.req(baseurl)
dom = html.fromstring(page.content)
@@ -212,7 +217,7 @@ def download_category_tree(database, client, baseurl="http://www.digikey.com/pro
database.add_product_category(pc)
-def download_attribute_types_from_category(category, client):
+def download_attribute_types_from_category(category: DigikeyProductCategory, client: DigikeyClient) -> List[DigikeyAttributeType]:
page = client.req(category.digikey_url)
tree = html.fromstring(page.content)
@@ -242,7 +247,7 @@ def download_attribute_types_from_category(category, client):
return attributes
-def download_product(client, db, query):
+def download_product(client: DigikeyClient, db, query):
# http://www.digikey.com/products/en?x=0&y=0&lang=en&site=us&keywords=553-2320-1-ND
page = client.req("http://www.digikey.com/products/en", params={'lang': 'en', 'site': 'us', 'keywords': query})
tree = html.fromstring(page.content)
diff --git a/trygvis/eda/digikey/__main__.py b/trygvis/eda/digikey/__main__.py
deleted file mode 100644
index ceb341e..0000000
--- a/trygvis/eda/digikey/__main__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import argparse
-
-from .. import write_graph
-from ..cli import *
-from ..digikey import *
-
-parser = argparse.ArgumentParser()
-subparsers = parser.add_subparsers(dest='cmd') # help='sub-command help'
-
-dct_parser = subparsers.add_parser("download-category-tree")
-dct_parser.add_argument("-o", "--output", required=False)
-
-dp_parser = subparsers.add_parser("download-product")
-dp_parser.add_argument("-p", "--product")
-dp_parser.add_argument("-o", "--output", required=False)
-
-args = parser.parse_args()
-
-client = DigikeyClient()
-db = DigikeyDatabase()
-
-if args.cmd == "download-category-tree":
- download_category_tree(db, client)
- if args.output is not None:
- def make_graph():
- g = create_graph(digikey=True)
- for pc in db.productCategories:
- [g.add(node) for node in pc.to_nodes()]
-
- for sc in pc.subCategories:
- [g.add(node) for node in sc.to_nodes()]
- write_graph(make_graph, args.output)
-
-elif args.cmd == "download-product":
- download_category_tree(db, client)
- product = download_product(client, db, args.product)
-
- if args.output is not None:
- def make_graph():
- g = create_graph(digikey=True)
- [g.add(node) for node in product.to_nodes()]
- write_graph(make_graph, args.output)
diff --git a/trygvis/eda/kicad/rdf.py b/trygvis/eda/kicad/rdf.py
index 683eaad..24cba00 100644
--- a/trygvis/eda/kicad/rdf.py
+++ b/trygvis/eda/kicad/rdf.py
@@ -1,7 +1,7 @@
-from rdflib import Namespace
+import rdflib
-KICAD = Namespace("https://trygvis/purl/kicad#")
-KICAD_TYPE = Namespace("https://trygvis/purl/kicad-type#")
+KICAD = rdflib.Namespace("https://trygvis/purl/kicad#")
+KICAD_TYPE = rdflib.Namespace("https://trygvis/purl/kicad-type#")
# Namespace for all unknown kicad boards
-KICAD_BOARD = Namespace("https://trygvis/purl/kicad-board#")
+KICAD_BOARD = rdflib.Namespace("https://trygvis/purl/kicad-board#")