o Adding 'digikey-download-metadata' tool that downloads everything from Digi-Key. Should support writing directly to database.

6 files changed, 115 insertions, 25 deletions

diff --git a/trygvis/eda/cli/__init__.py b/trygvis/eda/cli/__init__.py
index f300f86..8050d6d 100644
--- a/trygvis/eda/cli/__init__.py
+++ b/trygvis/eda/cli/__init__.py
@@ -1,5 +1,7 @@
 import sys
 import logging
+from genericpath import isfile
+
 from rdflib import store, ConjunctiveGraph, Graph, RDF, RDFS
 from rdflib.plugins.sparql import prepareQuery
 import rdflib.plugins.stores.sparqlstore as sparqlstore
@@ -70,6 +72,26 @@ def create_graph(digikey=False, kicad=False):
     return g
 
 
+def write_graph(gen_g, filename: str = None, force_write: bool = False):
+    if filename is not None:
+        if force_write or not isfile(filename):
+            g = gen_g()
+
+            if g is None:
+                raise CliException("internal error: graph generator returned None")
+
+            bs = g.serialize(encoding='utf-8', format='turtle')
+            with open(filename, "wb") as f:
+                f.write(bs)
+            info("Wrote %s" % filename)
+        else:
+            info("Skipped writing %s, already exists" % filename)
+    else:
+        g = gen_g()
+        bs = g.serialize(encoding='utf-8', format='turtle')
+        sys.stdout.buffer.write(bs)
+
+
 _initNs = {
     "rdf": RDF,
     "rdfs": RDFS,
@@ -83,7 +105,6 @@ _initNs = {
 
 
 def sparql(g: Graph, query: str, init_bindings=None):
-
     if isinstance(g, sparqlstore.SPARQLStore):
         return g.query(query, initNs=_initNs, initBindings=init_bindings)
     else:
@@ -97,7 +118,10 @@ def write_config(config: configparser.ConfigParser):
 
 
 def read_config():
-    with open('.eda-rdf/config.ini', 'r') as f:
-        config = configparser.ConfigParser()
-        config.read_file(f)
-        return config
+    try:
+        with open('.eda-rdf/config.ini', 'r') as f:
+            config = configparser.ConfigParser()
+            config.read_file(f)
+            return config
+    except FileNotFoundError:
+        raise CliException("Not a EDA-RDF project. Run eda-rdf init first.")
diff --git a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
index 1a33ab1..9839c1b 100644
--- a/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
+++ b/trygvis/eda/cli/digikey_download_attribute_types_for_category.py
@@ -1,4 +1,4 @@
-from trygvis.eda import cli, write_graph
+from trygvis.eda import cli
 
 from trygvis.eda.digikey import *
 
@@ -26,4 +26,4 @@ def run(category, sub_category, output_file, args):
         [g.add(node) for node in a.to_nodes()]
 
     filename = output_file if output_file is not "-" else None
-    write_graph(gen_g=lambda: g, filename=filename)
+    cli.write_graph(gen_g=lambda: g, filename=filename)
diff --git a/trygvis/eda/cli/digikey_download_for_schematic.py b/trygvis/eda/cli/digikey_download_for_schematic.py
index 0c21b3a..0cb5baa 100755
--- a/trygvis/eda/cli/digikey_download_for_schematic.py
+++ b/trygvis/eda/cli/digikey_download_for_schematic.py
@@ -2,7 +2,7 @@ from rdflib.plugins.sparql import prepareQuery
 import rdflib
 import rdflib.term
 
-from trygvis.eda import cli, write_graph
+from trygvis.eda import cli
 from trygvis.eda.digikey import *
 from trygvis.eda.digikey import rdf as digikey_rdf
 from trygvis.eda.kicad import rdf as kicad_rdf
@@ -90,7 +90,7 @@ ORDER BY ?digikey_pn
             [g.add(node) for node in product.to_nodes()]
             return g
 
-        write_graph(download_graph, filename)
+        cli.write_graph(download_graph, filename)
 
             # q = prepareQuery("""
     # SELECT
diff --git a/trygvis/eda/cli/digikey_download_metadata.py b/trygvis/eda/cli/digikey_download_metadata.py
new file mode 100755
index 0000000..d807c1a
--- /dev/null
+++ b/trygvis/eda/cli/digikey_download_metadata.py
@@ -0,0 +1,45 @@
+from trygvis.eda import cli
+from trygvis.eda.digikey import *
+
+
+def do_categories(db: DigikeyDatabase, client: DigikeyClient, output: str):
+    cli.info("Downloading category tree")
+    download_category_tree(db, client)
+
+    def make_graph():
+        g = cli.create_graph(digikey=True)
+        for pc in db.productCategories:
+            [g.add(node) for node in pc.to_nodes()]
+
+            for sc in pc.subCategories:
+                [g.add(node) for node in sc.to_nodes()]
+        return g
+
+    cli.write_graph(make_graph, output, force_write=True)
+
+
+def do_category(client: DigikeyClient, sc: DigikeyProductCategory, output: str):
+    cli.info('Downloading categories for %s' % sc.label)
+
+    def make_graph():
+        attributes = download_attribute_types_from_category(sc, client)
+        g = cli.create_graph()
+        for a in attributes:
+            [g.add(node) for node in a.to_nodes()]
+
+        return g
+
+    cli.write_graph(make_graph, output, force_write=True)
+
+def run(args):
+    db = DigikeyDatabase()
+    client = DigikeyClient()
+
+    # TODO: store in database too, not sure write to disk
+    output = "%s/digikey-categories.ttl" % args.output_dir
+    do_categories(db, client, output)
+
+    for pc in db.productCategories:
+        for sc in pc.subCategories:
+            output = "%s/digikey-%s.ttl" % (args.output_dir, normalize_filename(sc.label))
+            do_category(client, sc, output)
diff --git a/trygvis/eda/cli/eda_rdf.py b/trygvis/eda/cli/eda_rdf.py
index 33018ec..ecd4ba6 100644
--- a/trygvis/eda/cli/eda_rdf.py
+++ b/trygvis/eda/cli/eda_rdf.py
@@ -45,19 +45,7 @@ class KicadBomToTtl(CliCommand):
         args = p.parse_args(argv)
 
         from trygvis.eda.cli import kicad_bom_to_ttl
-
-        if args.input is not None:
-            src = open(args.input, "r")
-        else:
-            src = sys.stdin
-
-        if args.output is not None:
-            dst = open(args.output, "wb")
-        else:
-            dst = sys.stdout.buffer
-
-        with src, dst:
-            kicad_bom_to_ttl.run(src, dst, args)
+        kicad_bom_to_ttl.run(args)
 
 
 class DigikeyDownloadForSchematic(CliCommand):
@@ -73,6 +61,19 @@ class DigikeyDownloadForSchematic(CliCommand):
         digikey_download_for_schematic.run(args.schematic, args)
 
 
+class DigikeyDownloadMetadata(CliCommand):
+    def __init__(self):
+        super().__init__("digikey-download-metadata", "Download category tree and attribute types and values from digikey.com")
+
+    def run(self, argv):
+        p = argparse.ArgumentParser(prog=self.key, description=self.description)
+        p.add_argument("--output-dir", dest="output_dir", required=True)
+        args = p.parse_args(argv)
+
+        from trygvis.eda.cli import digikey_download_metadata
+        digikey_download_metadata.run(args)
+
+
 class DigikeyDownloadAttributeTypesForCategory(CliCommand):
     def __init__(self):
         super().__init__("digikey-download-attribute-types-for-category", "Download the attribute types for a category from digikey.com")
@@ -124,6 +125,7 @@ def main():
         KicadBomToTtl(),
         DigikeyDownloadForSchematic(),
         DigikeyDownloadAttributeTypesForCategory(),
+        DigikeyDownloadMetadata()
     ]
 
     parser = argparse.ArgumentParser(
diff --git a/trygvis/eda/cli/kicad_bom_to_ttl.py b/trygvis/eda/cli/kicad_bom_to_ttl.py
index c7d6899..b017767 100755
--- a/trygvis/eda/cli/kicad_bom_to_ttl.py
+++ b/trygvis/eda/cli/kicad_bom_to_ttl.py
@@ -3,6 +3,7 @@
 import functools
 import os.path
 import re
+import sys
 import xml.etree.ElementTree
 from urllib.parse import quote_plus
 
@@ -246,9 +247,27 @@ class Export(object):
         return kicad_rdf.KICAD_BOARD[title]
 
 
-def run(src, dst, args):
-    from trygvis.eda import cli
+def run(args):
+    if args.input is not None:
+        src = open(args.input, "r")
+    else:
+        src = sys.stdin
 
+    if args.output is not None:
+        parent = os.path.dirname(args.output)
+        if not os.path.exists(parent):
+            os.mkdir(parent)
+        dst = open(args.output, "wb")
+    else:
+        dst = sys.stdout.buffer
+
+    schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else None
+
+    with src, dst:
+        process(src, dst, schematic_url)
+
+
+def process(src, dst, schematic_url):
     tree = xml.etree.ElementTree.parse(src)
     root = tree.getroot()
 
@@ -316,7 +335,7 @@ def run(src, dst, args):
                 #     if digikey is not None:
                 #         cli.info('    Digikey: %s' % digikey)
 
-    schematic_url = args.schematic_url if hasattr(args, 'schematic_url') else export.generate_schematic_url()
+    schematic_url = schematic_url if schematic_url is not None else export.generate_schematic_url()
 
     # cli.info()
     # cli.info('=== Summary ===')