o Adding 'digikey-download-metadata' tool that downloads everything from Digi-Key. Should support writing directly to database.

2 files changed, 11 insertions, 48 deletions

diff --git a/trygvis/eda/digikey/__init__.py b/trygvis/eda/digikey/__init__.py
index 5f4ad8a..ec1167d 100644
--- a/trygvis/eda/digikey/__init__.py
+++ b/trygvis/eda/digikey/__init__.py
@@ -1,5 +1,6 @@
 import re
 
+from typing import List
 import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
@@ -10,9 +11,11 @@ from rdflib.namespace import RDF, RDFS
 
 import trygvis.eda.digikey.rdf
 
+
 def normalize_filename(part):
     return part.replace('/', '_').replace(' ', '_')
 
+
 def _clean(s):
     if s is None:
         return None
@@ -22,7 +25,7 @@ def _clean(s):
 
 class DigikeyDatabase(object):
     def __init__(self):
-        self.productCategories = []
+        self.productCategories = []  # type: List[DigikeyProductCategory]
         self.attributeTypes = {}
 
     def add_product_category(self, pc):
@@ -55,8 +58,8 @@ class DigikeyProductCategory(object):
         self.label = _clean(label)
         self.digikey_url = digikey_url if digikey_url is None or digikey_url.startswith("http") else \
             "http://www.digikey.com" + digikey_url
-        self.parent = parent
-        self.subCategories = []
+        self.parent = parent  # type: DigikeyProductCategory
+        self.subCategories = []  # type: List[DigikeyProductCategory
 
         assert self.id is not None
         assert self.label is not None
@@ -154,6 +157,7 @@ class DigikeyProduct(object):
 
 class DigikeyClient(object):
     def __init__(self):
+        # TODO: this should be put under .eda-rdf/
         cache = FileCache('digikey_cache', forever=True)
         self.sess = CacheControl(requests.Session(), cache=cache, heuristic=ExpiresAfter(days=1))
 
@@ -177,7 +181,8 @@ def _id_from_url(url):
     return m.group(1) if m else None
 
 
-def download_category_tree(database, client, baseurl="http://www.digikey.com/products/en"):
+def download_category_tree(database: DigikeyDatabase, client: DigikeyClient,
+                           baseurl="http://www.digikey.com/products/en"):
     page = client.req(baseurl)
     dom = html.fromstring(page.content)
 
@@ -212,7 +217,7 @@ def download_category_tree(database, client, baseurl="http://www.digikey.com/pro
         database.add_product_category(pc)
 
 
-def download_attribute_types_from_category(category, client):
+def download_attribute_types_from_category(category: DigikeyProductCategory, client: DigikeyClient) -> List[DigikeyAttributeType]:
     page = client.req(category.digikey_url)
     tree = html.fromstring(page.content)
 
@@ -242,7 +247,7 @@ def download_attribute_types_from_category(category, client):
     return attributes
 
 
-def download_product(client, db, query):
+def download_product(client: DigikeyClient, db, query):
     # http://www.digikey.com/products/en?x=0&y=0&lang=en&site=us&keywords=553-2320-1-ND
     page = client.req("http://www.digikey.com/products/en", params={'lang': 'en', 'site': 'us', 'keywords': query})
     tree = html.fromstring(page.content)
diff --git a/trygvis/eda/digikey/__main__.py b/trygvis/eda/digikey/__main__.py
deleted file mode 100644
index ceb341e..0000000
--- a/trygvis/eda/digikey/__main__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import argparse
-
-from .. import write_graph
-from ..cli import *
-from ..digikey import *
-
-parser = argparse.ArgumentParser()
-subparsers = parser.add_subparsers(dest='cmd') # help='sub-command help'
-
-dct_parser = subparsers.add_parser("download-category-tree")
-dct_parser.add_argument("-o", "--output", required=False)
-
-dp_parser = subparsers.add_parser("download-product")
-dp_parser.add_argument("-p", "--product")
-dp_parser.add_argument("-o", "--output", required=False)
-
-args = parser.parse_args()
-
-client = DigikeyClient()
-db = DigikeyDatabase()
-
-if args.cmd == "download-category-tree":
-    download_category_tree(db, client)
-    if args.output is not None:
-        def make_graph():
-            g = create_graph(digikey=True)
-            for pc in db.productCategories:
-                [g.add(node) for node in pc.to_nodes()]
-
-                for sc in pc.subCategories:
-                    [g.add(node) for node in sc.to_nodes()]
-        write_graph(make_graph, args.output)
-
-elif args.cmd == "download-product":
-    download_category_tree(db, client)
-    product = download_product(client, db, args.product)
-
-    if args.output is not None:
-        def make_graph():
-            g = create_graph(digikey=True)
-            [g.add(node) for node in product.to_nodes()]
-        write_graph(make_graph, args.output)