o Adding 'digikey-download-metadata' tool that downloads everything from Digi-Key. Should support writing directly to database.

12 files changed, 155 insertions, 99 deletions

diff --git a/.gitignore b/.gitignore
index a4f7607..57eb9be 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 env
 *.db
 .eda-rdf
+apache-jena-*/
 *.pyc
 
 digikey_cache
diff --git a/README.md b/README.md
index de1d761..20b2fc8 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,23 @@
+# Installing
+
+## From Git
+
+For development, set `dir` to `env`.
+
+    virtualenv -p python3 $dir
+    env/bin/pip install -r requirement.txt
+
+On Debian-based distributions you might need to have the `libdb-dev`
+package installed.
+
+To test the pip installation, use
+
+    env/bin/pip install . --upgrade --prefix $dir
+
+Then you can use this to play around with the application:
+
+    env/bin/eda-rdf
+
 # Applications to implement
 
 ## Rules
@@ -21,7 +41,9 @@ o Download datasheets, IBIS model, SPICE model
 
 # TODOs
 
-* Change the arg parser to always allow a '--db' argument that defaults to './.eda-rdf'
+* Change the arg parser to always allow a '--db' argument that
+  defaults to './.eda-rdf'
 * Create an 'init' command similar to git init
-* Rename "schematic" to "project"? Schematic is .. schematic specific but many of the possible tools work on the BOM
+* Rename "schematic" to "project"? Schematic is .. schematic specific
+  but many of the possible tools work on the BOM
   and/or on the PCB so 'project' is probably a better term.
diff --git a/trygvis/eda/__init__.py b/trygvis/eda/__init__.py
index e6ccb44..e69de29 100644
--- a/trygvis/eda/__init__.py
+++ b/trygvis/eda/__init__.py
@@ -1,20 +0,0 @@
-import sys
-
-from os.path import isfile
-from . import cli
-
-
-def write_graph(gen_g, filename=None, force_write=False):
-    if filename is not None:
-        if force_write or not isfile(filename):
-            g = gen_g()
-            bs = g.serialize(encoding='utf-8', format='turtle')
-            with open(filename, "wb") as f:
-                f.write(bs)
-            cli.info("Wrote %s" % filename)
-        else:
-            cli.info("Skipped writing %s, already exists" % filename)
-    else:
-        g = gen_g()
-        bs = g.serialize(encoding='utf-8', format='turtle')
-        sys.stdout.buffer.write(bs)
diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py
index f300f86..8050d6d 100644
--- a/trygvis/eda/cli/__init__.py
+++ b/trygvis/eda/cli/__init__.py
@@ -1,5 +1,7 @@
 import sys
 import logging
+from genericpath import isfile
+
 from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS
 from rdflib.plugins.sparql import prepareQuery
 import rdflib.plugins.stores.sparqlstore as sparqlstore
@@ -70,6 +72,26 @@ def create_graph(digikey=False, kicad=False):
     return g
 
 
+def write_graph(gen_g, filename: str = None, force_write: bool = False):
+    if filename is not None:
+        if force_write or not isfile(filename):
+            g = gen_g()
+
+            if g is None:
+                raise CliException("internal error: graph generator returned None")
+
+            bs = g.serialize(encoding='utf-8', format='turtle')
+            with open(filename, "wb") as f:
+                f.write(bs)
+            info("Wrote %s" % filename)
+        else:
+            info("Skipped writing %s, already exists" % filename)
+    else:
+        g = gen_g()
+        bs = g.serialize(encoding='utf-8', format='turtle')
+        sys.stdout.buffer.write(bs)
+
+
 _initNs = {
     "rdf": RDF,
     "rdfs": RDFS,
@@ -83,7 +105,6 @@ _initNs = {
 
 
 def sparql(g: Graph, query: str, init_bindings=None):
-
     if isinstance(g, sparqlstore.SPARQLStore):
         return g.query(query, initNs=_initNs, initBindings=init_bindings)
     else:
@@ -97,7 +118,10 @@ def write_config(config: configparser.ConfigParser):
 
 
 def read_config():
-    with open('.eda-rdf/config.ini', 'r') as f:
-        config = configparser.ConfigParser()
-        config.read_file(f)
-        return config
+    try:
+        with open('.eda-rdf/config.ini', 'r') as f:
+            config = configparser.ConfigParser()
+            config.read_file(f)
+            return config
+    except FileNotFoundError:
+        raise CliException("Not a EDA-RDF project. Run eda-rdf init first.")
diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
index 1a33ab1..9839c1b 100644
--- a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
+++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
@@ -1,4 +1,4 @@
-from trygvis.eda import cli, write_graph
+from trygvis.eda import cli
 
 from trygvis.eda.digikey import *
 
@@ -26,4 +26,4 @@ def run(category, sub_category, output_file, args):
         [g.add(node) for node in a.to_nodes()]
 
     filename = output_file if output_file is not "-" else None
-    write_graph(gen_g=lambda: g, filename=filename)
+    cli.write_graph(gen_g=lambda: g, filename=filename)
diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py
index 0c21b3a..0cb5baa 100755
--- a/trygvis/eda/cli/digikey_download_for_schematic.py
+++ b/trygvis/eda/cli/digikey_download_for_schematic.py
@@ -2,7 +2,7 @@ from rdflib.plugins.sparql import prepareQuery
 import rdflib
 import rdflib.term
 
-from trygvis.eda import cli, write_graph
+from trygvis.eda import cli
 from trygvis.eda.digikey import *
 from trygvis.eda.digikey import rdf as digikey_rdf
 from trygvis.eda.kicad import rdf as kicad_rdf
@@ -90,7 +90,7 @@ ORDER BY ?digikey_pn
             [g.add(node) for node in product.to_nodes()]
             return g
 
-        write_graph(download_graph, filename)
+        cli.write_graph(download_graph, filename)
 
             # q = prepareQuery("""
     # SELECT
diff --git a/trygvis/eda/cli/digikey_download_metadata.py b/trygvis/eda/cli/digikey_download_metadata.py
new file mode 100755
index 0000000..d807c1a
--- /dev/null
+++ b/trygvis/eda/cli/digikey_download_metadata.py
@@ -0,0 +1,45 @@
+from trygvis.eda import cli
+from trygvis.eda.digikey import *
+
+
+def do_categories(db: DigikeyDatabase, client: DigikeyClient, output: str):
+    cli.info("Downloading category tree")
+    download_category_tree(db, client)
+
+    def make_graph():
+        g = cli.create_graph(digikey=True)
+        for pc in db.productCategories:
+            [g.add(node) for node in pc.to_nodes()]
+
+            for sc in pc.subCategories:
+                [g.add(node) for node in sc.to_nodes()]
+        return g
+
+    cli.write_graph(make_graph, output, force_write=True)
+
+
+def do_category(client: DigikeyClient, sc: DigikeyProductCategory, output: str):
+    cli.info('Downloading categories for %s' % sc.label)
+
+    def make_graph():
+        attributes = download_attribute_types_from_category(sc, client)
+        g = cli.create_graph()
+        for a in attributes:
+            [g.add(node) for node in a.to_nodes()]
+
+        return g
+
+    cli.write_graph(make_graph, output, force_write=True)
+
+def run(args):
+    db = DigikeyDatabase()
+    client = DigikeyClient()
+
+    # TODO: store in database too, not sure write to disk
+    output = "%s/digikey-categories.ttl" % args.output_dir
+    do_categories(db, client, output)
+
+    for pc in db.productCategories:
+        for sc in pc.subCategories:
+            output = "%s/digikey-%s.ttl" % (args.output_dir, normalize_filename(sc.label))
+            do_category(client, sc, output)
diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py
index 33018ec..ecd4ba6 100644
--- a/trygvis/eda/cli/eda_rdf.py
+++ b/trygvis/eda/cli/eda_rdf.py
@@ -45,19 +45,7 @@ class KicadBomToTtl(CliCommand):
         args = p.parse_args(argv)
 
         from trygvis.eda.cli import kicad_bom_to_ttl
-
-        if args.input is not None:
-            src = open(args.input, "r")
-        else:
-            src = sys.stdin
-
-        if args.output is not None:
-            dst = open(args.output, "wb")
-        else:
-            dst = sys.stdout.buffer
-
-        with src, dst:
-            kicad_bom_to_ttl.run(src, dst, args)
+        kicad_bom_to_ttl.run(args)
 
 
 class DigikeyDownloadForSchematic(CliCommand):
@@ -73,6 +61,19 @@ class DigikeyDownloadForSchematic(CliCommand):
         digikey_download_for_schematic.run(args.schematic, args)
 
 
+class DigikeyDownloadMetadata(CliCommand):
+    def __init__(self):
+        super().__init__("digikey-download-metadata", "Download category tree and attribute types and values from digikey.com")
+
+    def run(self, argv):
+        p = argparse.ArgumentParser(prog=self.key, description=self.description)
+        p.add_argument("--output-dir", dest="output_dir", required=True)
+        args = p.parse_args(argv)
+
+        from trygvis.eda.cli import digikey_download_metadata
+        digikey_download_metadata.run(args)
+
+
 class DigikeyDownloadAttributeTypesForCategory(CliCommand):
     def __init__(self):
         super().__init__("digikey-download-attribute-types-for-category", "Download the attribute types for a category from digikey.com")
@@ -124,6 +125,7 @@ def main():
         KicadBomToTtl(),
         DigikeyDownloadForSchematic(),
         DigikeyDownloadAttributeTypesForCategory(),
+        DigikeyDownloadMetadata()
     ]
 
     parser = argparse.ArgumentParser(
diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py
index c7d6899..b017767 100755
--- a/trygvis/eda/cli/kicad_bom_to_ttl.py
+++ b/trygvis/eda/cli/kicad_bom_to_ttl.py
@@ -3,6 +3,7 @@
 import functools
 import os.path
 import re
+import sys
 import xml.etree.ElementTree
 from urllib.parse import quote_plus
 
@@ -246,9 +247,27 @@ class Export(object):
         return kicad_rdf.KICAD_BOARD[title]
 
 
-def run(src, dst, args):
-    from trygvis.eda import cli
+def run(args):
+    if args.input is not None:
+        src = open(args.input, "r")
+    else:
+        src = sys.stdin
 
+    if args.output is not None:
+        parent = os.path.dirname(args.output)
+        if not os.path.exists(parent):
+            os.mkdir(parent)
+        dst = open(args.output, "wb")
+    else:
+        dst = sys.stdout.buffer
+
+    schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else None
+
+    with src, dst:
+        process(src, dst, schematic_url)
+
+
+def process(src, dst, schematic_url):
     tree = xml.etree.ElementTree.parse(src)
     root = tree.getroot()
 
@@ -316,7 +335,7 @@ def run(src, dst, args):
                 #     if digikey is not None:
                 #         cli.info('    Digikey: %s' % digikey)
 
-    schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url()
+    schematic_url = schematic_url if schematic_url is not None else export.generate_schematic_url()
 
     # cli.info()
     # cli.info('=== Summary ===')
diff --git a/trygvis/eda/digikey/__init__.py b/trygvis/eda/digikey/__init__.py
index 5f4ad8a..ec1167d 100644
--- a/trygvis/eda/digikey/__init__.py
+++ b/trygvis/eda/digikey/__init__.py
@@ -1,5 +1,6 @@
 import re
 
+from typing import List
 import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
@@ -10,9 +11,11 @@ from rdflib.namespace import RDF, RDFS
 
 import trygvis.eda.digikey.rdf
 
+
 def normalize_filename(part):
     return part.replace('/', '_').replace(' ', '_')
 
+
 def _clean(s):
     if s is None:
         return None
@@ -22,7 +25,7 @@ def _clean(s):
 
 class DigikeyDatabase(object):
     def __init__(self):
-        self.productCategories = []
+        self.productCategories = []  # type: List[DigikeyProductCategory]
         self.attributeTypes = {}
 
     def add_product_category(self, pc):
@@ -55,8 +58,8 @@ class DigikeyProductCategory(object):
         self.label = _clean(label)
         self.digikey_url = digikey_url if digikey_url is None or digikey_url.startswith("http") else \
             "http://www.digikey.com" + digikey_url
-        self.parent = parent
-        self.subCategories = []
+        self.parent = parent  # type: DigikeyProductCategory
+        self.subCategories = []  # type: List[DigikeyProductCategory
 
         assert self.id is not None
         assert self.label is not None
@@ -154,6 +157,7 @@ class DigikeyProduct(object):
 
 class DigikeyClient(object):
     def __init__(self):
+        # TODO: this should be put under .eda-rdf/
         cache = FileCache('digikey_cache', forever=True)
         self.sess = CacheControl(requests.Session(), cache=cache, heuristic=ExpiresAfter(days=1))
 
@@ -177,7 +181,8 @@ def _id_from_url(url):
     return m.group(1) if m else None
 
 
-def download_category_tree(database, client, baseurl="http://www.digikey.com/products/en"):
+def download_category_tree(database: DigikeyDatabase, client: DigikeyClient,
+                           baseurl="http://www.digikey.com/products/en"):
     page = client.req(baseurl)
     dom = html.fromstring(page.content)
 
@@ -212,7 +217,7 @@ def download_category_tree(database, client, baseurl="http://www.digikey.com/pro
         database.add_product_category(pc)
 
 
-def download_attribute_types_from_category(category, client):
+def download_attribute_types_from_category(category: DigikeyProductCategory, client: DigikeyClient) -> List[DigikeyAttributeType]:
     page = client.req(category.digikey_url)
     tree = html.fromstring(page.content)
 
@@ -242,7 +247,7 @@ def download_attribute_types_from_category(category, client):
     return attributes
 
 
-def download_product(client, db, query):
+def download_product(client: DigikeyClient, db, query):
     # http://www.digikey.com/products/en?x=0&y=0&lang=en&site=us&keywords=553-2320-1-ND
     page = client.req("http://www.digikey.com/products/en", params={'lang': 'en', 'site': 'us', 'keywords': query})
     tree = html.fromstring(page.content)
diff --git a/trygvis/eda/digikey/__main__.py b/trygvis/eda/digikey/__main__.py
deleted file mode 100644
index ceb341e..0000000
--- a/trygvis/eda/digikey/__main__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import argparse
-
-from .. import write_graph
-from ..cli import *
-from ..digikey import *
-
-parser = argparse.ArgumentParser()
-subparsers = parser.add_subparsers(dest='cmd') # help='sub-command help'
-
-dct_parser = subparsers.add_parser("download-category-tree")
-dct_parser.add_argument("-o", "--output", required=False)
-
-dp_parser = subparsers.add_parser("download-product")
-dp_parser.add_argument("-p", "--product")
-dp_parser.add_argument("-o", "--output", required=False)
-
-args = parser.parse_args()
-
-client = DigikeyClient()
-db = DigikeyDatabase()
-
-if args.cmd == "download-category-tree":
-    download_category_tree(db, client)
-    if args.output is not None:
-        def make_graph():
-            g = create_graph(digikey=True)
-            for pc in db.productCategories:
-                [g.add(node) for node in pc.to_nodes()]
-
-                for sc in pc.subCategories:
-                    [g.add(node) for node in sc.to_nodes()]
-        write_graph(make_graph, args.output)
-
-elif args.cmd == "download-product":
-    download_category_tree(db, client)
-    product = download_product(client, db, args.product)
-
-    if args.output is not None:
-        def make_graph():
-            g = create_graph(digikey=True)
-            [g.add(node) for node in product.to_nodes()]
-        write_graph(make_graph, args.output)
diff --git a/trygvis/eda/kicad/rdf.py b/trygvis/eda/kicad/rdf.py
index 683eaad..24cba00 100644
--- a/trygvis/eda/kicad/rdf.py
+++ b/trygvis/eda/kicad/rdf.py
@@ -1,7 +1,7 @@
-from rdflib import Namespace
+import rdflib
 
-KICAD = Namespace("https://trygvis/purl/kicad#")
-KICAD_TYPE = Namespace("https://trygvis/purl/kicad-type#")
+KICAD = rdflib.Namespace("https://trygvis/purl/kicad#")
+KICAD_TYPE = rdflib.Namespace("https://trygvis/purl/kicad-type#")
 
 # Namespace for all unknown kicad boards
-KICAD_BOARD = Namespace("https://trygvis/purl/kicad-board#")
+KICAD_BOARD = rdflib.Namespace("https://trygvis/purl/kicad-board#")