split-parts-by-supplier: rename to split-bom-by-supplier.

digikey-create-bom: Implementing CSV generation for Digi-Key.

12 files changed, 342 insertions, 85 deletions

diff --git a/src/ee/digikey/bom.py b/src/ee/digikey/bom.py
new file mode 100644
index 0000000..36b0143
--- /dev/null
+++ b/src/ee/digikey/bom.py
@@ -0,0 +1,72 @@
+import csv
+from pathlib import Path
+from typing import List, MutableMapping, Optional
+
+from ee.db import ObjDb
+from ee.digikey import DigikeyStore
+from ee.logging import log
+from ee.part import Part, load_db
+from ee.part.bom import load_bom, check_bom, join_refs
+
+__all__ = ["create_bom"]
+
+
+class BomLine(object):
+    def __init__(self, uri, part: Part):
+        self.uri = uri
+        self.part = part
+        self.refs = []
+
+    def add_ref(self, ref):
+        self.refs.append(ref)
+
+    @classmethod
+    def header(cls) -> List[str]:
+        return ["Quantity", "Digi-Key Part Number", "Customer Reference"]
+
+    def to_rows(self) -> List[str]:
+        return [str(len(self.refs)), self.part.get_exactly_one_spn().valueProp, join_refs(self.refs)]
+
+
+def gen_bom(allow_incomplete, bom_parts: ObjDb[Part], supplier_parts: ObjDb[Part]) -> Optional[List[List[str]]]:
+    lines: MutableMapping[str, BomLine] = {}
+
+    uri_idx = supplier_parts.index("uri")
+
+    for part in bom_parts.values:
+        pr_obj = part.get_only_part_reference()
+
+        if allow_incomplete and pr_obj is None:
+            log.debug("Skipping part without part reference: {}".format(part.printable_reference))
+            continue
+        elif pr_obj is None:
+            log.warn("Unresolved part: {}".format(part.printable_reference))
+            return
+
+        part_uri = pr_obj.part_uriProp
+
+        if part_uri not in lines:
+            lines[part_uri] = line = BomLine(part_uri, uri_idx.get_single(part_uri))
+        else:
+            line = lines[part_uri]
+
+        line.add_ref(part.get_exactly_one_schematic_reference().referenceProp)
+
+    return sorted(line.to_rows() for line in lines.values())
+
+
+def create_bom(bom_path: Path, part_files: List[Path], out_path: Path, store_code, allow_incomplete):
+    store = DigikeyStore.from_store_code(store_code)
+
+    bom_parts, supplier_parts = load_bom(bom_path, part_files)
+    check_bom(bom_parts, supplier_parts)
+
+    bom = gen_bom(allow_incomplete, bom_parts, supplier_parts)
+
+    if bom is None:
+        return
+
+    with out_path.open("w") as f:
+        w = csv.writer(f)
+        w.writerow(BomLine.header())
+        w.writerows(bom)
diff --git a/src/ee/kicad/bom/__init__.py b/src/ee/kicad/bom/__init__.py
index 0392d66..e993adb 100644
--- a/src/ee/kicad/bom/__init__.py
+++ b/src/ee/kicad/bom/__init__.py
@@ -10,6 +10,7 @@ __all__ = [
 ]
 
 
+# TODO: use ee.part.bom.split_ref
 def split_ref(ref):
     """Split "C12" into a tuple that's useful for sorting by component reference.
 
@@ -23,11 +24,11 @@ def split_ref(ref):
     val = groups[1]
     rest = groups[2]
     try:
-        return (ref, int(val), rest)
+        return ref, int(val), rest
     except ValueError:
         pass
 
-        return (ref, val, rest)
+        return ref, val, rest
 
 
 split_ref.r = re.compile("([A-Za-z]+)([0-9]+)(.*)")
diff --git a/src/ee/part/__init__.py b/src/ee/part/__init__.py
index c18dd02..deeab95 100644
--- a/src/ee/part/__init__.py
+++ b/src/ee/part/__init__.py
@@ -309,7 +309,7 @@ class Part(object):
     def get_exactly_one_spn(self) -> types.SupplierPartNumber:
         spns = self.get_spns()
         if len(spns) == 0:
-            raise EeException("This part does not contain any manufacturer part numbers{}".
+            raise EeException("This part does not contain any supplier part numbers{}".
                               format(", uri=" + self.uri if self.uri else ""))
         if len(spns) != 1:
             raise EeException("This part does not contain exactly one spn: {}".
diff --git a/src/ee/part/bom.py b/src/ee/part/bom.py
new file mode 100644
index 0000000..83ae348
--- /dev/null
+++ b/src/ee/part/bom.py
@@ -0,0 +1,110 @@
+import re
+import sys
+from itertools import groupby
+from pathlib import Path
+from typing import List
+
+from ee import EeException
+from ee.db import ObjDb
+from ee.part import Part, load_db
+
+__all__ = ["load_bom"]
+
+
+def uri_fn(part: Part):
+    return part.uri
+
+
+def supplier_fn(part: Part):
+    return part.supplier
+
+
+class BomItem(object):
+    def __init__(self, references, part: Part):
+        self.references = references
+        self.part = part
+
+
+def load_bom(bom_path: Path, part_files: List[Path]) -> (ObjDb[Part](), ObjDb[Part]()):
+    bom: ObjDb[Part] = ObjDb[Part]()
+
+    for xml in load_db(bom_path).iterparts():
+        bom.add(Part(xml))
+
+    parts: ObjDb[Part] = ObjDb[Part]()
+    parts.add_unique_index("uri", uri_fn)
+    parts.add_index("supplier", supplier_fn)
+
+    for part_file in part_files:
+        for xml in load_db(part_file).iterparts():
+            parts.add(Part(xml))
+
+    return bom, parts
+
+
+def check_bom(bom: ObjDb[Part](), parts: ObjDb[Part]()):
+    pass
+
+
+def split_ref(ref):
+    """Split "C12" into a tuple that's useful for sorting by component reference.
+
+    For example: "C12" => ("C", 12, None). "Cfoo" => ("C", sys.maxsize, "").
+    """
+    m = split_ref.r.match(ref)
+    if not m:
+        return ref, sys.maxsize, ""
+    groups = m.groups()
+    ref = groups[0]
+    val = groups[1]
+    rest = groups[2]
+    try:
+        return ref, int(val), rest
+    except ValueError:
+        pass
+
+        return ref, val, rest
+
+
+split_ref.r = re.compile("([A-Za-z]+)([0-9]+)(.*)")
+
+
+def join_refs(refs: List[str], max=None) -> str:
+    refs = [ref.strip() for ref in refs]
+
+    rs = sorted([split_ref(ref) for ref in refs])
+
+    for r in rs:
+        if r[1] is None:
+            raise EeException("Bad ref, must match '[A-Z]+[0-9]+'.")
+
+    s = ""
+    for kind, items in groupby(rs, lambda x: x[0]):
+        items = list(items)
+
+        groups = []
+        start = cur = items[0][1]
+
+        for r in items[1:]:
+            num = r[1]
+
+            if cur + 1 != num:
+                groups.append([start, cur])
+                start = cur = num
+            else:
+                cur = num
+
+        groups.append([start, cur])
+
+        parts = []
+        for start, end in groups:
+            if start == end:
+                parts.append(str(start))
+            elif start + 1 == end:
+                parts.append("{},{}".format(start, end))
+            else:
+                parts.append("{}-{}".format(start, end))
+
+        s += kind + ",".join(parts)
+
+    return s
diff --git a/src/ee/tools/digikey_create_bom.py b/src/ee/tools/digikey_create_bom.py
new file mode 100644
index 0000000..a83da83
--- /dev/null
+++ b/src/ee/tools/digikey_create_bom.py
@@ -0,0 +1,33 @@
+import argparse
+from pathlib import Path
+
+import ee.tools
+from ee.digikey.bom import create_bom
+
+parser = argparse.ArgumentParser()
+ee.tools.add_default_argparse_group(parser)
+
+parser.add_argument("--bom",
+                    required=True,
+                    metavar="PART DB")
+
+parser.add_argument("--out",
+                    required=True,
+                    metavar="CSV")
+
+parser.add_argument("--part-db",
+                    nargs="*",
+                    required=True,
+                    metavar="PART DB")
+
+parser.add_argument("--store",
+                    default="us",
+                    metavar="STORE CODE")
+
+parser.add_argument("--allow-incomplete",
+                    action="store_true")
+
+args = parser.parse_args()
+ee.tools.process_default_argparse_group(args)
+
+create_bom(Path(args.bom), [Path(p) for p in args.part_db], Path(args.out), args.store, args.allow_incomplete)
diff --git a/src/ee/tools/ninja.py b/src/ee/tools/ninja.py
index aa6952b..7d73d8f 100644
--- a/src/ee/tools/ninja.py
+++ b/src/ee/tools/ninja.py
@@ -21,6 +21,15 @@ class NinjaSupplier(object):
     def part_db(self):
         return "$public_dir/{}/parts.xml".format(self.key)
 
+    @property
+    def bom_input(self):
+        return "$public_dir/{}/bom.xml".format(self.key)
+
+    @property
+    def bom_output(self):
+        # This needs to be configurable per supplier
+        return "$public_dir/{}/bom.csv".format(self.key)
+
 
 def ninja_path_filter(s: Union[Path, str, List[str]]) -> str:
     if isinstance(s, Path):
diff --git a/src/ee/tools/split_bom_by_supplier.py b/src/ee/tools/split_bom_by_supplier.py
new file mode 100644
index 0000000..01a6ef3
--- /dev/null
+++ b/src/ee/tools/split_bom_by_supplier.py
@@ -0,0 +1,86 @@
+import argparse
+from pathlib import Path
+from typing import List, Tuple
+
+import ee.tools
+from ee.db import ObjDb
+from ee.logging import log
+from ee.part import Part, load_db, save_db, PartDb
+from ee.part.bom import load_bom, check_bom
+
+
+class OrderPart(object):
+    def __init__(self, order_part: Part, part: Part):
+        self.order_part = order_part
+        self.part = part
+
+
+def split_bom_by_supplier(bom_path: Path, part_files: List[Path], supplier_list: List[Tuple[str, str, Path]]):
+    # key, url, file
+
+    bom_parts, supplier_parts = load_bom(bom_path, part_files)
+    check_bom(bom_parts, supplier_parts)
+
+    uri_idx = supplier_parts.index("uri")
+
+    for supplier, uri, path in supplier_list:
+        print(supplier, uri, path)
+
+    supplier_info = {uri: path for _, uri, path in supplier_list}
+    part_dbs = {uri: PartDb() for _, uri, _ in supplier_list}
+
+    for bom_part in bom_parts.values:
+        sch = bom_part.get_exactly_one_schematic_reference().referenceProp
+        part_reference = bom_part.get_only_part_reference()
+
+        if not part_reference:
+            # log.warn("Missing part for {}, allowing for now".format(sch))
+            continue
+
+        # print("{}: {}".format(sch, part_reference.part_uriProp))
+        part = uri_idx.get_single(part_reference.part_uriProp)
+        print("{}: {}".format(sch, part.uri))
+        print("{}: {}".format(sch, part.supplier))
+
+        part_dbs[part.supplier].add_entry(bom_part, new=False)
+
+    for uri, path in supplier_info.items():
+        part_db = part_dbs[uri]
+        save_db(path, part_db)
+
+
+parser = argparse.ArgumentParser()
+ee.tools.add_default_argparse_group(parser)
+
+parser.add_argument("--bom",
+                    required=True,
+                    metavar="PART DB")
+
+parser.add_argument("--part-db",
+                    nargs="*",
+                    required=True,
+                    metavar="PART DB")
+
+parser.add_argument("--supplier",
+                    required=True,
+                    nargs="*",
+                    metavar="SUPPLIER")
+
+parser.add_argument("--order",
+                    required=True,
+                    nargs="*",
+                    metavar="PART DB")
+
+args = parser.parse_args()
+ee.tools.process_default_argparse_group(args)
+
+ss = []
+
+order_files = {o.split("=", maxsplit=1)[0]: o.split("=", maxsplit=1)[1] for o in args.order}
+
+for s in args.supplier:
+    key, url = s.split("=", maxsplit=1)
+    file = order_files[key]
+    ss.append((key, url, Path(file)))
+
+split_bom_by_supplier(Path(args.bom), [Path(path) for path in args.part_db], ss)
diff --git a/src/ee/tools/split_parts_by_supplier.py b/src/ee/tools/split_parts_by_supplier.py
deleted file mode 100644
index 9202f0d..0000000
--- a/src/ee/tools/split_parts_by_supplier.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import argparse
-from pathlib import Path
-from typing import List
-
-import ee.tools
-from ee.db import ObjDb
-from ee.part import Part, load_db, save_db, PartDb
-from ee.project import Project
-
-
-class OrderPart(object):
-    def __init__(self, order_part: Part, part: Part):
-        self.order_part = order_part
-        self.part = part
-
-
-def uri_fn(part: Part):
-    return part.uri
-
-
-def split_parts_by_supplier(project: Project, order_file: Path, part_dbs: List[Path], out_dir: Path):
-    parts: ObjDb[Part] = ObjDb[Part]()
-    part_by_uri = parts.add_unique_index("uri", uri_fn)
-
-    for part_db in part_dbs:
-        for xml in load_db(part_db).iterparts():
-            parts.add(Part(xml))
-
-    order_parts: ObjDb[OrderPart] = ObjDb()
-    supplier_idx = order_parts.add_index("supplier", lambda op: op.part.supplier)
-    for xml in load_db(order_file).iterparts():
-        order_part = Part(xml)
-        part = part_by_uri.get_single(order_part.get_exactly_one_part_reference().part_uriProp)
-        order_parts.add(OrderPart(order_part, part))
-
-    for supplier, parts_for_supplier in supplier_idx.items():
-        desc = project.get_supplier_by_uri(supplier)
-
-        print("{}: {}".format(desc.name, len(parts)))
-        # supplier_db: ObjDb[Part] = ObjDb[Part]()
-        # supplier_db.add_unique_index("uri", uri_fn)
-
-        supplier_descriptor = project.get_supplier_by_uri(supplier)
-
-        db = PartDb()
-        for part_for_supplier in parts_for_supplier:
-            db.add_entry(part_for_supplier.part.underlying, False)
-
-        save_db(out_dir / "{}.xml".format(supplier_descriptor.key), db, sort=True)
-
-
-parser = argparse.ArgumentParser()
-ee.tools.add_default_argparse_group(parser)
-
-parser.add_argument("--parts",
-                    required=True,
-                    metavar="PART DB")
-
-parser.add_argument("--part-db",
-                    nargs="*",
-                    required=True,
-                    metavar="PART DB")
-
-parser.add_argument("--out-dir",
-                    metavar="DIR FOR PART DBS")
-
-args = parser.parse_args()
-ee.tools.process_default_argparse_group(args)
-
-part_db_dirs = [Path(part_db) for part_db in args.part_db]
-
-split_parts_by_supplier(Project.load(), Path(args.parts), part_db_dirs, Path(args.out_dir))
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index 9fff519..1ab6102 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -56,8 +56,8 @@ rule element14-search-parts
 rule create-bom
     command = $ee create-bom {{ log }} --schematic $schematic --part-db $part_dbs --out $out $strategy
 
-rule split-parts-by-supplier
-    command = $ee split-parts-by-supplier {{ log }} --parts $in --out $out $part_dbs
+rule split-bom-by-supplier
+    command = $ee split-bom-by-supplier {{ log }} --bom $in $part_dbs $suppliers $orders
 
 rule import-parts-yaml
     description = import-parts-yaml $in
@@ -151,6 +151,12 @@ build $public_dir/{{ s.key }}/souffle/out/fact.csv: souffle {{ alt if is_file(al
 build {{ s.part_db }}: part-apply-souffle-post $public_dir/{{ s.key }}/souffle/out/fact.csv
     in_sch = $public_dir/{{ s.key }}/downloaded.xml
     work = $public_dir/{{ s.key }}/souffle
+
+rule {{ s.key }}-create-bom
+    command = $ee {{ s.key }}-create-bom {{ log }} --bom $in --out $out $part_dbs --allow-incomplete
+
+build {{ s.bom_output }}: {{ s.key }}-create-bom {{ s.bom_input }}
+    part_dbs = --part-db {{ s.part_db }}
 {%- endfor %}
 
 {%- for f in parts_yaml_files %}
@@ -167,11 +173,10 @@ build $public_dir/bom.xml | $report_dir/bom.rst: create-bom $public_dir/sch.xml
 {%- endif %}
 {%- set reports=reports+["$report_dir/bom.rst"] %}
 
-{#
-build $public_dir/orders/index.xml: split-parts-by-supplier $public_dir/bom.xml | {%- for s in suppliers %} {{ s.part_db }}{% endfor %}
-    part_dbs ={%- for s in suppliers %} {{ s.part_db }}{% endfor %}
-    suppliers = {% s for s in suppliers %}
-#}
+build {%- for s in suppliers %} {{ s.bom_input }}{% endfor %}: split-bom-by-supplier $public_dir/bom.xml | {%- for s in suppliers %} {{ s.part_db }}{% endfor %}
+    part_dbs = {%- for s in suppliers %} --part-db {{ s.part_db }}{% endfor %}
+    suppliers = {%- for s in suppliers %} --supplier {{ s.key }}={{ s.supplier.uri }}{% endfor %}
+    orders = {%- for s in suppliers %} --order {{ s.key }}={{ s.bom_input }}{% endfor %}
 
 rule seeed-download-opl
     description = seeed-download-opl $opl
@@ -190,5 +195,6 @@ build $public_dir/seeed/opl/{{ opl }}.xml: seeed-download-opl
 
 # Reports
 build ee-reports: phony {{ " ".join(reports) }}
-build part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %}
-build ee-all: phony ee-reports part-dbs
+build ee-part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %}
+build ee-orders: phony {%- for s in suppliers %} {{ s.bom_output }}{% endfor %}
+build ee-all: phony ee-reports ee-orders
diff --git a/test/part/test_bom.py b/test/part/test_bom.py
new file mode 100644
index 0000000..85eb932
--- /dev/null
+++ b/test/part/test_bom.py
@@ -0,0 +1,12 @@
+import pytest
+from ee.part.bom import join_refs
+
+
+@pytest.mark.parametrize("inputs, output", [
+    [[], ""],
+    [["C1"], "C1"],
+    [["C1", "C2"], "C1,2"],
+    [["C1", "C2", "C3", "C5", "C6", "C9"], "C1-3,5,6,9"],
+])
+def test_basic(inputs, output):
+    assert output == join_refs(inputs)
diff --git a/test/test_digikey.py b/test/test_digikey.py
index f1b010b..4961a9d 100644
--- a/test/test_digikey.py
+++ b/test/test_digikey.py
@@ -83,7 +83,7 @@ def test_digikey_3():
 
     p = next((p for p in res.products if p.part_number == "1655-1501-1-ND"), None)
     assert p.mpn == "RS1MTR"
-    assert p.url == "/product-detail/en/smc-diode-solutions/RS1MTR/1655-1501-1-ND/6022946"
+    assert p.url == "https://www.digikey.com/product-detail/en/smc-diode-solutions/RS1MTR/1655-1501-1-ND/6022946"
 
 
 @pytest.mark.digikey
diff --git a/test/test_bom.py b/test/test_kicad_bom.py
index 50cae9f..50cae9f 100644
--- a/test/test_bom.py
+++ b/test/test_kicad_bom.py