o Support for resolving 'mpn' fields from digikey. Creates

  'digikey-part-stub', can be used to download the entire part later on.

5 files changed, 126 insertions, 11 deletions

diff --git a/src/ee/digikey/__init__.py b/src/ee/digikey/__init__.py
index f607408..6fa3161 100644
--- a/src/ee/digikey/__init__.py
+++ b/src/ee/digikey/__init__.py
@@ -203,9 +203,9 @@ class DigikeySearchResponse(object):
         self.count = count
         self.response_type = response_type
 
-        self.products = list()
+        self.products = list()  # type: List[DigikeyProduct]
 
-    def append(self, product):
+    def append(self, product: DigikeyProduct):
         self.products.append(product)
 
 
diff --git a/src/ee/digikey/doit.py b/src/ee/digikey/doit.py
new file mode 100644
index 0000000..74259bb
--- /dev/null
+++ b/src/ee/digikey/doit.py
@@ -0,0 +1,87 @@
+import logging
+
+import ee.digikey as dk
+from ee.doit import DoItConfig
+from ee.ds import DataSet
+
+logger = logging.getLogger(__name__)
+
+doit_config = DoItConfig()
+
+
+def resolve_schematic_components(output: DataSet, in_ds: DataSet):
+    def find(field, value):
+        return [o for o in output.items() if o.object_type.name == "digikey-part-stub" and o.get(field) == value]
+
+    def save(p: dk.DigikeyProduct):
+        logger.info("Found part, dpn={}, mpn={}".format(p.part_number, p.mpn))
+
+        return output.create_object("digikey-part-stub", p.part_number, replace=True). \
+            set("part-number", p.part_number). \
+            set("mpn", p.mpn). \
+            set("description", p.description). \
+            set("quantity-available", p.quantity_available). \
+            set("url", p.url)
+
+    digikey = dk.Digikey()
+    client = dk.DigikeyClient(digikey, on_download=logger.info)
+
+    components = [o for o in in_ds.items() if o.object_type.name == "component"]
+
+    for mpn in sorted({c.get("mpn") for c in components if c.get("mpn")}):
+        # TODO: support searching by value and digikey part number directly. Priority should be "digikey", "mpn" and
+        # "value", first field present should be used.
+
+        dk_components = find("mpn", mpn)
+
+        if len(dk_components):
+            logger.info("Already resolved {} to {}".format(mpn, ", ".join([c.get("mpn") for c in dk_components])))
+            continue
+
+        logger.info("Looking up {}".format(mpn))
+        response = client.search(mpn)
+
+        if response.response_type == dk.SearchResponseTypes.SINGLE:
+            save(response.products[0])
+        elif response.response_type == dk.SearchResponseTypes.MANY:
+            # A search for "FOO" might return products "FOO" and "FOOZ" so we pick out the ones with the matching mpn
+            # This will often be more than one product, but digikey shows the same part in different packagings.
+            viable_products = [p for p in response.products if p.mpn == mpn]
+
+            if len(viable_products) == 0:
+                logger.warning("BUG: Got multiple hits ({}) but didn't find anyone that matched the MPN. Strange!".
+                               format(len(response.products)))
+            else:
+                if len(viable_products) == 1:
+                    part = viable_products[0]
+                else:
+                    # Pick the first one, should be as good as any
+                    part = sorted(viable_products, key=lambda x: x.part_number)[0]
+
+                    logger.info("Got multiple hits ({})".format(len(viable_products)))
+
+                save(part)
+        elif response.response_type == dk.SearchResponseTypes.TOO_MANY:
+            logger.warning("to many matches")
+        elif response.response_type == dk.SearchResponseTypes.NO_MATCHES:
+            logger.warning("no matches")
+
+
+def task_digikey_resolve_schematic_components():
+    out_data_set, in_data_sets = doit_config.data_sets_for(task_digikey_resolve_schematic_components)
+
+    def action():
+        in_ds = doit_config.dsm.load_data_sets(in_data_sets)
+
+        with doit_config.dsm.create_rw(out_data_set, clean=False) as output:
+            resolve_schematic_components(output, in_ds)
+
+    return dict(
+        file_dep=[doit_config.dsm.cookie_for_ds(ds) for ds in in_data_sets],
+        actions=[action],
+        targets=[doit_config.dsm.cookie_for_ds(out_data_set)],
+    )
+
+
+doit_config.set_data_sets_for(task_digikey_resolve_schematic_components,
+                              "digikey-resolved-parts", "components")
diff --git a/src/ee/doit.py b/src/ee/doit.py
index 87a6601..014af05 100644
--- a/src/ee/doit.py
+++ b/src/ee/doit.py
@@ -2,6 +2,8 @@ import logging
 
 from doit import get_var
 
+from ee.ds import DataSetManager
+
 
 def configure_logging():
     log_level = get_var("log-level", None)
@@ -14,3 +16,24 @@ def configure_logging():
         ee_logger.addHandler(ch)
 
         ee_logger.setLevel(log_level)
+
+
+class DoItConfig(object):
+    def __init__(self):
+        self._dsm = None  # type: DataSetManager
+        self._data_sets = {}
+
+    def configure(self, *, data_set_manager: DataSetManager):
+        self._dsm = data_set_manager
+
+    @property
+    def dsm(self):
+        if not self._dsm:
+            raise Exception("The DataSetManager has not been set")
+        return self._dsm
+
+    def data_sets_for(self, task):
+        return self._data_sets[task]
+
+    def set_data_sets_for(self, task, out_dataset: str, *in_datasets: str):
+        self._data_sets[task] = [out_dataset, in_datasets]
diff --git a/src/ee/ds/__init__.py b/src/ee/ds/__init__.py
index f71a7c6..14bbb91 100644
--- a/src/ee/ds/__init__.py
+++ b/src/ee/ds/__init__.py
@@ -144,7 +144,7 @@ class DataSet(object):
             self._assert_not_frozen()
 
             if not create:
-                raise Exception("No such type: {}:{}".format(object_type, key))
+                raise Exception("No such object: {}:{}".format(object_type, key))
 
             o = Object(self, ot, key)
             objects[key] = o
@@ -179,11 +179,15 @@ class DataSet(object):
     def get_or_create_object(self, object_type: str, key: str) -> Object:
         return self._check_object(object_type, key, True)
 
-    def create_object(self, object_type: str, key: str) -> Object:
+    def create_object(self, object_type: str, key: str, replace=False) -> Object:
         self._assert_not_frozen()
 
         if self.has_object(object_type, key):
-            raise Exception("Object already exist: {}:{}".format(object_type, key))
+            if not replace:
+                raise Exception("Object already exist: {}:{}".format(object_type, key))
+
+            ot, objects = self._check_object_type(object_type, False)
+            del self._objects_by_type[ot][key]
 
         return self._check_object(object_type, key, True)
 
@@ -410,11 +414,14 @@ class LazyRwDataSet(object):
         cookie = self._dsm.cookie_for_ds(self._name)
 
         if cookie.exists():
-            if not self._clean:
-                raise IOError("DataSet already exists: {}, cookie={}".format(self._name, cookie))
-            self._dsm.remove(self._name)
+            if self._clean:
+                self._dsm.remove(self._name)
+                ds = DataSet()
+            else:
+                ds = self._dsm.load(self._name)
+        else:
+            ds = DataSet()
 
-        ds = DataSet()
         self._ds = ds
         return ds
 
diff --git a/src/ee/kicad/doit.py b/src/ee/kicad/doit.py
index bfbf0a1..eff6a03 100644
--- a/src/ee/kicad/doit.py
+++ b/src/ee/kicad/doit.py
@@ -159,8 +159,6 @@ def task_kicad_create_component_data_set():
     out_data_set, in_data_sets = _data_sets[task_kicad_create_component_data_set]
 
     def action():
-        logger.info("in_data_sets={}, out_data_set={}".format(in_data_sets, out_data_set))
-
         in_ds = _dsm.load_data_sets(in_data_sets)
         # for o in in_ds.items():
         #     logger.info("item: {}/{}".format(o.object_type.name, o.key))