o Changing resolve-schematic-components to create generic

  'component-to-part-mapping' objects with seller=digikey
o Implementing 'task_digikey_fetch_full_part_facts' that downloads info
  about each mapped parts with seller=digikey.

1 files changed, 75 insertions, 16 deletions

diff --git a/src/ee/digikey/doit.py b/src/ee/digikey/doit.py
index 74259bb..2b30d9b 100644
--- a/src/ee/digikey/doit.py
+++ b/src/ee/digikey/doit.py
@@ -1,4 +1,6 @@
 import logging
+from itertools import groupby
+from typing import List
 
 import ee.digikey as dk
 from ee.doit import DoItConfig
@@ -11,38 +13,48 @@ doit_config = DoItConfig()
 
 def resolve_schematic_components(output: DataSet, in_ds: DataSet):
     def find(field, value):
-        return [o for o in output.items() if o.object_type.name == "digikey-part-stub" and o.get(field) == value]
-
-    def save(p: dk.DigikeyProduct):
-        logger.info("Found part, dpn={}, mpn={}".format(p.part_number, p.mpn))
-
-        return output.create_object("digikey-part-stub", p.part_number, replace=True). \
-            set("part-number", p.part_number). \
-            set("mpn", p.mpn). \
-            set("description", p.description). \
-            set("quantity-available", p.quantity_available). \
-            set("url", p.url)
+        return [o for o in output.items() if
+                o.object_type.name == "component-to-part-mapping" and o.get(field) == value]
+
+    def save(refs: List[str], p: dk.DigikeyProduct):
+        logger.info("Found part, dpn={}, mpn={}, refs={}".format(p.part_number, p.mpn, ", ".join(refs)))
+
+        for ref in refs:
+            # Make the key a bit long so we can have several parts that matches the MPN from a single seller
+            key = "digikey-{}-{}".format(ref, p.part_number)
+            output.create_object("component-to-part-mapping", key, replace=True). \
+                set("seller", "digikey"). \
+                set("ref", ref). \
+                set("part-number", p.part_number). \
+                set("mpn", p.mpn). \
+                set("description", p.description). \
+                set("quantity-available", p.quantity_available). \
+                set("url", p.url)
 
     digikey = dk.Digikey()
     client = dk.DigikeyClient(digikey, on_download=logger.info)
 
-    components = [o for o in in_ds.items() if o.object_type.name == "component"]
+    components = [(o.get_req("mpn"), o.get_req("ref")) for o in in_ds.items() if
+                  o.object_type.name == "component" and o.has("mpn", "ref")]
+    components = sorted(components, key=lambda c: c[0])
+
+    for mpn, components in groupby(components, key=lambda c: c[0]):
+        references = [c[1] for c in components]
 
-    for mpn in sorted({c.get("mpn") for c in components if c.get("mpn")}):
         # TODO: support searching by value and digikey part number directly. Priority should be "digikey", "mpn" and
         # "value", first field present should be used.
 
         dk_components = find("mpn", mpn)
 
         if len(dk_components):
-            logger.info("Already resolved {} to {}".format(mpn, ", ".join([c.get("mpn") for c in dk_components])))
+            logger.info("Already resolved {} to {}".format(mpn, ", ".join(sorted(set([c.get("part-number") for c in dk_components])))))
             continue
 
         logger.info("Looking up {}".format(mpn))
         response = client.search(mpn)
 
         if response.response_type == dk.SearchResponseTypes.SINGLE:
-            save(response.products[0])
+            save(references, response.products[0])
         elif response.response_type == dk.SearchResponseTypes.MANY:
             # A search for "FOO" might return products "FOO" and "FOOZ" so we pick out the ones with the matching mpn
             # This will often be more than one product, but digikey shows the same part in different packagings.
@@ -60,7 +72,7 @@ def resolve_schematic_components(output: DataSet, in_ds: DataSet):
 
                     logger.info("Got multiple hits ({})".format(len(viable_products)))
 
-                save(part)
+                save(references, part)
         elif response.response_type == dk.SearchResponseTypes.TOO_MANY:
             logger.warning("to many matches")
         elif response.response_type == dk.SearchResponseTypes.NO_MATCHES:
@@ -85,3 +97,50 @@ def task_digikey_resolve_schematic_components():
 
 doit_config.set_data_sets_for(task_digikey_resolve_schematic_components,
                               "digikey-resolved-parts", "components")
+
+
+def download_part_facts(output: DataSet, in_ds: DataSet):
+    digikey = dk.Digikey()
+    client = dk.DigikeyClient(digikey, on_download=logger.info)
+
+    parts = [o for o in in_ds.items()
+             if o.object_type.name == "component-to-part-mapping" and o.get("seller") == "digikey"]
+
+    for pn in sorted({part.get("part-number") for part in parts}):
+        logger.info("Downloading facts for {}".format(pn))
+
+        response = client.search(pn)
+
+        if response.response_type == dk.SearchResponseTypes.SINGLE:
+            product = response.products[0]
+
+            o = output.create_object("digikey-part", pn, replace=True). \
+                set("part-number", product.part_number). \
+                set("url", product.url). \
+                set("mpn", product.mpn)
+
+            for a in product.attributes:
+                key = "{}/{}".format(a.attribute_type.id, a.attribute_type.label)
+                key = key.replace("%", "_")
+                value = a.value.replace("%", "%%")
+                o.set(key, value)
+
+
+def task_digikey_fetch_full_part_facts():
+    out_data_set, in_data_sets = doit_config.data_sets_for(task_digikey_fetch_full_part_facts)
+
+    def action():
+        in_ds = doit_config.dsm.load_data_sets(in_data_sets)
+
+        with doit_config.dsm.create_rw(out_data_set, clean=False) as output:
+            download_part_facts(output, in_ds)
+
+    return dict(
+        file_dep=[doit_config.dsm.cookie_for_ds(ds) for ds in in_data_sets],
+        actions=[action],
+        targets=[doit_config.dsm.cookie_for_ds(out_data_set)],
+    )
+
+
+doit_config.set_data_sets_for(task_digikey_fetch_full_part_facts,
+                              "digikey-parts", "digikey-resolved-parts")