o Switching from YAML to INI files for downloaded facts.

o Improved fact downloader.

4 files changed, 144 insertions, 43 deletions

diff --git a/requirements.txt b/requirements.txt
index 4278b1f..05d8a6b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,6 @@ pandas==0.20.3
 parsec==3.3
 Pillow==4.2.1
 pytest==3.2.0
-pyyaml==3.12
 requests==2.18.4
 sympy==1.0
 typing==3.6.2; python_version < '3.0'
diff --git a/src/ee/digikey/__init__.py b/src/ee/digikey/__init__.py
index af5977b..1315f96 100644
--- a/src/ee/digikey/__init__.py
+++ b/src/ee/digikey/__init__.py
@@ -6,7 +6,8 @@ import re
 import requests
 import os
 import os.path
-import yaml
+import configparser
+import glob
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
 from cachecontrol.heuristics import ExpiresAfter
@@ -90,13 +91,32 @@ class DigikeyProduct(object):
     def __hash__(self):
         return self.part_number.__hash__()
 
-    def to_yaml(self):
-        yaml = {"part_number": self.part_number}
+    def to_ini(self):
+        c = configparser.ConfigParser()
+        c["overview"] = {}; overview = c["overview"]
+        overview["part_number"] = self.part_number
         if self.mpn:
-            yaml["mpn"] = self.mpn
-        yaml["attributes"] = [{"type": {"id": a.attribute_type.id, "label": a.attribute_type.label}, "value": a.value}
-                              for a in self.attributes]
-        return yaml
+            overview["mpn"] = self.mpn
+        c["attributes"] = {}; attributes = c["attributes"]
+        for a in self.attributes:
+            key = "{}/{}".format(a.attribute_type.id, a.attribute_type.label)
+            key = key.replace("%", "_")
+            value = a.value.replace("%", "%%")
+            attributes[key] = value
+        return c
+
+    from_ini_r = re.compile("([^/]*)/(.*)")
+
+    @staticmethod
+    def from_ini(digikey, c):
+        overview = c["overview"]
+        attributes = []
+        for k, value in c.items("attributes"):
+#            print("k={}".format(k))
+            (type_id, label) = DigikeyProduct.from_ini_r.match(k).groups()
+            a_type = digikey.get_attribute_type(type_id, label)
+            attributes.append(DigikeyAttributeValue(value, a_type))
+        return DigikeyProduct(overview["part_number"], overview["mpn"], attributes)
 
 
 class DigikeyAttributeType(object):
@@ -228,8 +248,8 @@ class DigikeyClient(object):
         products = tree.xpath("//*[@itemtype='http://schema.org/Product']")
 
         for product in products:
-            part_number = _first(product.xpath("//*[@itemprop='productid' and @content]"))
-            mpn = _first(product.xpath("//*[@itemprop='name']"))
+            part_number = _first(product.xpath(".//*[@itemprop='productid' and @content]"))
+            mpn = _first(product.xpath(".//*[@itemprop='name']"))
 
             if part_number is not None and mpn is not None:
                 res.append(DigikeyProduct(
@@ -272,19 +292,45 @@ class DigikeyClient(object):
                 return DigikeySearchResponse(1, SearchResponseTypes.NO_MATCHES)
 
 class DigikeyRepository(object):
-    def __init__(self, path):
+    def __init__(self, digikey, path):
+        self._digikey = digikey
         self._path = path
+        self._products = {}
 
     def mpn_to_path(self, mpn):
-        return "{}/{}.yaml".format(self._path, mpn)
-
-    def has_product(self, mpn):
-        filename = self.mpn_to_path(mpn)
-        return os.path.isfile(filename)
+        mpn = mpn.replace("/", "_").replace(" ", "_")
+        return "{}/{}.ini".format(self._path, mpn)
+
+    def has_product(self, mpn=None, dpn=None):
+        if mpn is not None:
+            filename = self.mpn_to_path(mpn)
+            return os.path.isfile(filename)
+        if dpn is not None:
+            for p in self.products:
+                if p.part_number == dpn:
+                    return p
 
     def save(self, product: DigikeyProduct):
-        y = product.to_yaml()
+        y = product.to_ini()
         filename = self.mpn_to_path(product.mpn)
         mk_parents(filename)
         with open(filename, "w") as f:
-            yaml.dump(y, f, encoding="utf-8", allow_unicode=True)
+            y.write(f)
+
+    def load_all(self):
+        [self._load(path) for path in glob.glob(self._path + "/*.ini")]
+
+    def _load(self, path):
+        c = configparser.ConfigParser()
+        c.read(path)
+        p = DigikeyProduct.from_ini(self._digikey, c)
+        self._products[p.mpn] = p
+        return p
+
+    @property
+    def products(self):
+        self.load_all()
+        return self._products.values()
+
+    def find_by_mpn(self, mpn):
+        return [p for p in self.products if p.mpn == mpn]
diff --git a/src/ee/tools/digikey_download_facts.py b/src/ee/tools/digikey_download_facts.py
index 0550c5f..950623f 100644
--- a/src/ee/tools/digikey_download_facts.py
+++ b/src/ee/tools/digikey_download_facts.py
@@ -1,5 +1,6 @@
 import argparse
 from itertools import *
+from functools import total_ordering
 
 from colors import color
 
@@ -7,6 +8,22 @@ import ee.digikey as dk
 from ee.digikey import SearchResponseTypes, DigikeyProduct
 from ee.tools import mk_parents
 
+
+@total_ordering
+class Query(object):
+    def __init__(self, is_mpn, query):
+        self.is_mpn = is_mpn
+        self.query = query
+
+    def __eq__(self, other):
+        return (self.is_mpn, self.query) == (other.is_mpn, other.query)
+
+    def __lt__(self, other):
+        return (self.is_mpn, self.query) < (other.is_mpn, other.query)
+
+    def __hash__(self):
+        return hash((self.is_mpn, self.query))
+
 parser = argparse.ArgumentParser(description="Download facts about parts from Digi-Key")
 
 parser.add_argument("--out",
@@ -31,7 +48,7 @@ args = parser.parse_args()
 
 digikey = dk.Digikey()
 client = dk.DigikeyClient(digikey, on_download=lambda s: print(color(s, 'grey')))
-repo = dk.DigikeyRepository(args.out)
+repo = dk.DigikeyRepository(digikey, args.out)
 
 
 def on_product(product: DigikeyProduct):
@@ -41,7 +58,7 @@ def on_product(product: DigikeyProduct):
 parts = []
 
 if args.part:
-    parts.append(args.part)
+    [parts.append(Query(True, p)) for p in args.part]
 
 if args.sch:
     from ee.kicad import read_schematic, to_bom
@@ -49,39 +66,52 @@ if args.sch:
     for c in to_bom(sch):
         digikey = c.get_field("digikey")
         if digikey:
-            parts.append(digikey.value)
+            parts.append(Query(False, digikey.value))
         mpn = c.get_field("mpn")
         if mpn:
-            parts.append(mpn.value)
+            parts.append(Query(True, mpn.value))
 
-for p in parts:
-    print(color("Searching for {}".format(p), "white"))
+parts = sorted(set(parts))
 
-    if repo.has_product(p) and not args.force:
+for q in parts:
+    p = q.query
+    if repo.has_product(mpn = p if q.is_mpn else None, dpn = p if not q.is_mpn else None) and not args.force:
+        print(color("Already have facts for {}".format(p), "white"))
         continue
 
+    print(color("Searching for {}".format(p), "white"))
     response = client.search(p)
 
+    todos = []
+
     if response.response_type == SearchResponseTypes.SINGLE:
         p = response.products[0]
-        print(color("Direct match {}".format(p.mpn), "white"))
+        print(color("Direct match mpn/dpn: {}/{}".format(p.mpn, p.part_number), "white"))
         on_product(p)
     elif response.response_type == SearchResponseTypes.MANY:
-        hits = list(groupby(sorted(response.products), lambda p: p.mpn))
+        hits = [(mpn, list(products)) for mpn, products in groupby(sorted(response.products, key=lambda p: p.mpn), lambda p: p.mpn)]
 
         if len(hits) == 1:
             (mpn, products) = hits[0]
-            products = list(products)
-
-            if len(products) == 1:
-                print(color("Got many results, but they all point to the same part: {}".format(mpn), "white"))
-                on_product(products[0])
-                continue
 
-        for k, g in hits:
-            print(color("Got many results with many parts: {}: {}".format(k, list(g)), "white"))
-            on_product(list(g)[0])
+            part_number = products[0].part_number
+            print(color("Got many results, but they all point to the same part: {}. Will use {} for downloading attributes.".format(mpn, ", ".join([p.part_number for p in products])), "white"))
+            todos.append(part_number)
+        else:
+            for k, g in hits:
+                print(color("Got many results with many parts: {}: {}".format(k, list(g)), "white"))
+                on_product(list(g)[0])
     elif response.response_type == SearchResponseTypes.TOO_MANY:
         print(color("Too many results ({}), select a category first".format(response.count), 'red'))
     elif response.response_type == SearchResponseTypes.NO_MATCHES:
         print(color("Part not found", "orange"))
+
+    for part_number in todos:
+        response = client.search(part_number)
+
+        if response.response_type == SearchResponseTypes.SINGLE:
+            p = response.products[0]
+            print(color("Downloaded {}".format(p.mpn), "white"))
+            on_product(p)
+        else:
+            print("Got response of type {} when really expecting a single product.".format(response.response_type))
diff --git a/test/test_digikey.py b/test/test_digikey.py
index 7509e2b..04b4642 100644
--- a/test/test_digikey.py
+++ b/test/test_digikey.py
@@ -1,7 +1,8 @@
 import ee.digikey as dk
 import os.path
-import yaml
 import sys
+import io
+from itertools import groupby
 
 basedir = os.path.dirname(os.path.abspath(__file__))
 
@@ -9,20 +10,45 @@ digikey = dk.Digikey()
 client = dk.DigikeyClient(digikey, on_download=print)
 
 
-def test_digikey_1():
+def test_digikey_1(tmpdir):
     res = client.search("TCR2LF18LM(CTTR-ND")
     assert res.response_type == dk.SearchResponseTypes.SINGLE
     p = res.products[0]
     assert p.part_number == "TCR2LF18LM(CTTR-ND"
     assert p.mpn == "TCR2LF18,LM(CT"
     assert len(p.attributes) > 5
-    x = p.to_yaml()
-    print(str(x))
-    yaml.dump(x, sys.stdout)
 
+    x = io.StringIO()
+    p.to_ini().write(x)
+    print(x.getvalue())
+    repo = dk.DigikeyRepository(digikey, str(tmpdir))
+    repo.save(p)
+    repo = dk.DigikeyRepository(digikey, str(tmpdir))
+    repo.load_all()
+    ps = repo.find_by_mpn(p.mpn)
+    assert len(ps) == 1
+    y = io.StringIO()
+    p.to_ini().write(y)
+    assert x.getvalue() == y.getvalue()
 
-def test_digikey_2():
+
+def test_digikey_2(tmpdir):
     res = client.search("TCR2LF", page_size=500)
     assert res.response_type == dk.SearchResponseTypes.MANY
-    [print(p.part_number) for p in res.products]
+    [print("dpn={}, mpn={}".format(p.part_number, p.mpn)) for p in res.products]
     assert len(res.products) == 28
+
+    print("path={}".format(tmpdir))
+    repo = dk.DigikeyRepository(digikey, tmpdir)
+
+    for mpn, parts in groupby(sorted(res.products, key=lambda p: p.mpn), lambda p: p.mpn):
+        parts = list(parts)
+        print("mpn={}, parts={}".format(mpn, [p.part_number for p in parts]))
+
+        dpn = parts[0].part_number
+        print("Downloading {} as {}".format(mpn, dpn))
+        res = client.search(dpn)
+
+        assert res.response_type == dk.SearchResponseTypes.SINGLE
+        p = res.products[0]
+        repo.save(p)