o Ignore power flags too, they're not real components.

o Fixing digikey product lookup.

4 files changed, 104 insertions, 41 deletions

diff --git a/src/ee/bom/doit.py b/src/ee/bom/doit.py
index 521b8dd..7c8f63a 100644
--- a/src/ee/bom/doit.py
+++ b/src/ee/bom/doit.py
@@ -84,7 +84,7 @@ def task_bom():
                     raise Exception("Missing ref")
 
                 if not mpn:
-                    create_message(output, "Missing required field 'mpn'", "error")
+                    create_message(output, "Missing required field 'mpn' on component ref={}".format(ref), "error")
                     continue
 
                 if ref in bom_components:
@@ -104,61 +104,73 @@ def task_bom():
 _data_sets[task_bom] = ["bom", ["components"]]
 
 
-def order_csv(count: int, style: str, output_file: Path, data_sets) -> DataSet:
+def order_csv(count: int, style: str, output_file: Path, out_ds: DataSet, data_sets):
     ds = _dsm.load_data_sets(data_sets)
 
-    out = DataSet()
+    csv_ds = DataSet()
 
     parts = {}
 
     # noinspection PyPep8Naming
     Part = namedlist("Part", "mpn, cnt, refs, digikey_pn")
 
+    digikey_parts = [o for o in ds.items() if
+                     o.object_type.name == "component-to-part-mapping" and
+                     o.get("seller") == "digikey"]
+
     for c in [o for o in ds.items() if o.object_type.name == "bom-component"]:
         ref = c.get("ref")
         mpn = c.get("mpn")
 
-        dk_part = None
+        digikey_pn = None
 
         if style == "digikey":
-            dk_parts = [o.get_req("part-number") for o in ds.items() if
-                        o.object_type.name == "component-to-part-mapping" and
-                        o.has("seller", "ref", "part-number") and
-                        o.get("seller") == "digikey" and
-                        o.get("ref") == ref]
+
+            digikey_pns = []
+            for o in (o for o in digikey_parts if o.get("ref") == ref):
+
+                t = o.values("part-number", strip=True, required=True)
+
+                if not t:
+                    create_message(out_ds, "Missing required field value for field part-number, object key={}".
+                                   format(o.key), level="error")
+
+                part_number = t[0]
+
+                digikey_pns.append(part_number)
 
             # The actual DK part should depend on the price breaks for the different packagings, but we don't have that
             # info here. Luckily the DK store proposes cheaper packagings when ordering so that is something.
 
-            dk_parts = set(dk_parts)
+            digikey_pns = set(digikey_pns)
 
-            if len(dk_parts) == 0:
-                create_message(out, "No part for component: ref={}, mpn={}".format(ref, mpn), "error")
+            if len(digikey_pns) == 0:
+                create_message(out_ds, "No part for component: ref={}, mpn={}".format(ref, mpn), "error")
                 continue
-            elif len(dk_parts) > 1:
-                create_message(out, "Multiple parts for component: ref={}, mpn={}. Don't know which one to select.".
+            elif len(digikey_pns) > 1:
+                create_message(out_ds, "Multiple parts for component: ref={}, mpn={}. Don't know which one to select.".
                                format(ref, mpn), "error")
                 continue
             else:
-                dk_part = next(iter(dk_parts))
+                digikey_pn = next(iter(digikey_pns))
 
-            dk_part = dk_part.strip()
+            digikey_pn = digikey_pn.strip()
 
-            if len(dk_part) == 0:
+            if len(digikey_pn) == 0:
                 raise Exception("Missing digikey part number for ref={}".format(ref))
 
         if mpn in parts:
             part = parts[mpn]
 
-            if dk_part:
-                if part.digikey_pn != dk_part:
+            if digikey_pn:
+                if part.digikey_pn != digikey_pn:
                     raise Exception("Bad data, inconsistent digikey-pn for mpn '{}'. First digikey part number='{}', "
-                                    "new digikey part number='{}'".format(mpn, part.digikey_pn, dk_part))
+                                    "new digikey part number='{}'".format(mpn, part.digikey_pn, digikey_pn))
 
             part.cnt += 1
             part.refs.append(ref)
         else:
-            parts[mpn] = Part(mpn=mpn, cnt=1, refs=[ref], digikey_pn=dk_part)
+            parts[mpn] = Part(mpn=mpn, cnt=1, refs=[ref], digikey_pn=digikey_pn)
 
     mpn_field = "MPN"
     count_field = "Count"
@@ -169,7 +181,7 @@ def order_csv(count: int, style: str, output_file: Path, data_sets) -> DataSet:
         refs_field = "Customer Reference"
 
     for part in sorted(parts.values(), key=lambda p: p.mpn):
-        o = out.create_object("row", part.mpn). \
+        o = csv_ds.create_object("row", part.mpn). \
             set(mpn_field, part.mpn). \
             set(count_field, part.cnt * count). \
             set(refs_field, ",".join(part.refs))
@@ -184,14 +196,14 @@ def order_csv(count: int, style: str, output_file: Path, data_sets) -> DataSet:
         fields = ["Digi-Key Part Number", refs_field, count_field, mpn_field]
         include_extra_fields = False
 
-    _dsm.store_csv(output_file, out, "row", order_by=mpn_field, fields=fields,
+    _dsm.store_csv(output_file, csv_ds, "row", order_by=mpn_field, fields=fields,
                    include_extra_fields=include_extra_fields)
 
 
 def create_task_order_csv(*, style: str = None, output_file: Union[str, Path], out_data_set, data_sets, count: int = 1):
     def action():
-        with _dsm.create_rw(out_data_set, clean=True):
-            order_csv(count, style, Path(output_file), data_sets)
+        with _dsm.create_rw(out_data_set, clean=True) as out:
+            order_csv(count, style, Path(output_file), out, data_sets)
 
     return {
         "name": "order-{}".format(count) if not style else "order-{}-{}".format(style, count),
diff --git a/src/ee/digikey/doit.py b/src/ee/digikey/doit.py
index 2b30d9b..6f7c2d5 100644
--- a/src/ee/digikey/doit.py
+++ b/src/ee/digikey/doit.py
@@ -1,5 +1,6 @@
 import logging
 from itertools import groupby
+from operator import itemgetter
 from typing import List
 
 import ee.digikey as dk
@@ -34,23 +35,42 @@ def resolve_schematic_components(output: DataSet, in_ds: DataSet):
     digikey = dk.Digikey()
     client = dk.DigikeyClient(digikey, on_download=logger.info)
 
-    components = [(o.get_req("mpn"), o.get_req("ref")) for o in in_ds.items() if
-                  o.object_type.name == "component" and o.has("mpn", "ref")]
-    components = sorted(components, key=lambda c: c[0])
+    components = []
 
-    for mpn, components in groupby(components, key=lambda c: c[0]):
-        references = [c[1] for c in components]
+    # TODO: support searching by value and digikey part number directly. Priority should be "digikey", "mpn" and
+    # "value", first field present should be used.
+
+    for o in in_ds.items():
+        if o.object_type.name != "component":
+            continue
+
+        ref, = o.values("ref", required=True)
+
+        if not ref:
+            raise Exception("Bad component: object key={}, missing required field 'ref'".format(o.key))
+
+        mpn, = o.values("mpn", strip=True)
 
-        # TODO: support searching by value and digikey part number directly. Priority should be "digikey", "mpn" and
-        # "value", first field present should be used.
+        # We ignore components without mpn.
+        if not mpn:
+            logger.debug("Skipping component without")
+            continue
+
+        components.append([mpn, ref])
+
+    components = sorted(components, key=itemgetter(0))
+
+    for mpn, components in groupby(components, key=itemgetter(0)):
+        references = [c[1] for c in components]
 
         dk_components = find("mpn", mpn)
 
         if len(dk_components):
-            logger.info("Already resolved {} to {}".format(mpn, ", ".join(sorted(set([c.get("part-number") for c in dk_components])))))
+            logger.info("Already resolved {} to {}".format(mpn, ", ".join(
+                sorted(set([c.get("part-number") for c in dk_components])))))
             continue
 
-        logger.info("Looking up {}".format(mpn))
+        logger.info("Looking up MPN: {}, used by {}".format(mpn, ", ".join(sorted(references))))
         response = client.search(mpn)
 
         if response.response_type == dk.SearchResponseTypes.SINGLE:
@@ -101,10 +121,11 @@ doit_config.set_data_sets_for(task_digikey_resolve_schematic_components,
 
 def download_part_facts(output: DataSet, in_ds: DataSet):
     digikey = dk.Digikey()
-    client = dk.DigikeyClient(digikey, on_download=logger.info)
+    client = dk.DigikeyClient(digikey, on_download=logger.debug)
 
-    parts = [o for o in in_ds.items()
-             if o.object_type.name == "component-to-part-mapping" and o.get("seller") == "digikey"]
+    parts = [o for o in in_ds.items() if
+             o.object_type.name == "component-to-part-mapping" and
+             o.get("seller") == "digikey"]
 
     for pn in sorted({part.get("part-number") for part in parts}):
         logger.info("Downloading facts for {}".format(pn))
diff --git a/src/ee/ds/__init__.py b/src/ee/ds/__init__.py
index e48cab6..adf741f 100644
--- a/src/ee/ds/__init__.py
+++ b/src/ee/ds/__init__.py
@@ -80,8 +80,38 @@ class Object(object):
         for k in other._ot.fields:
             self.set(k, other.get(k))
 
-    def has(self, *keys: str):
-        return self.object_type.has(*keys)
+    def has_values(self, *keys: str) -> bool:
+        return all([len(value) > 0 for value in [self.get(key) for key in keys] if value is not None])
+
+    def values(self, *keys: str, strip: bool = False, required: bool = True) -> List[Optional[str]]:
+        """Looks up all values for all keys.
+
+        If required=True, strip is also set to True
+
+
+        If strip is True, all values are stripped with str.strip(). None values are preserved.
+
+        If required=True, all values has to have a len() > 0. If any fails the requirement, a list with only None values
+        is returned.
+        """
+
+        values = []
+
+        strip = True if required else strip
+
+        for key in keys:
+            v = self.get(key)
+
+            if strip:
+                v = v.strip() if v else v
+
+            if required:
+                if v is None or len(v) == 0:
+                    return [None] * len(keys)
+
+            values.append(v)
+
+        return values
 
     def get(self, key: str) -> Optional[str]:
         idx = self._ot.index_of(key)
@@ -437,6 +467,6 @@ class LazyRwDataSet(object):
 
 
 def create_message(data_set: DataSet, message: str, level: str):
-    return data_set.create_object("message", "message-{}".format(str(hash(message)))). \
+    return data_set.create_object("message", "message-{}".format(str(abs(hash(message))))). \
         set("message", message). \
         set("level", level)
diff --git a/src/ee/kicad/doit.py b/src/ee/kicad/doit.py
index eff6a03..bf0ee3a 100644
--- a/src/ee/kicad/doit.py
+++ b/src/ee/kicad/doit.py
@@ -179,7 +179,7 @@ def task_kicad_create_component_data_set():
 
             logger.info("processing {} kicad-sch".format(len(kicad_sch)))
 
-            ignored_ref_types = {"#PWR", }
+            ignored_ref_types = {"#PWR", "#FLG"}
 
             for sch in kicad_sch:
                 ref = sch.get("ref")