From 89197dad4f5f427faa7fba12971b20037ad5ba71 Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Fri, 14 Jun 2019 10:40:31 +0200 Subject: split-parts-by-supplier: rename to split-bom-by-supplier. digikey-create-bom: Implementing CSV generation for Digi-Key. --- src/ee/digikey/bom.py | 72 +++++++++++++++++++++ src/ee/kicad/bom/__init__.py | 5 +- src/ee/part/__init__.py | 2 +- src/ee/part/bom.py | 110 ++++++++++++++++++++++++++++++++ src/ee/tools/digikey_create_bom.py | 33 ++++++++++ src/ee/tools/ninja.py | 9 +++ src/ee/tools/split_bom_by_supplier.py | 86 +++++++++++++++++++++++++ src/ee/tools/split_parts_by_supplier.py | 72 --------------------- src/ee/tools/templates/build.ninja.j2 | 24 ++++--- test/part/test_bom.py | 12 ++++ test/test_bom.py | 66 ------------------- test/test_digikey.py | 2 +- test/test_kicad_bom.py | 66 +++++++++++++++++++ 13 files changed, 408 insertions(+), 151 deletions(-) create mode 100644 src/ee/digikey/bom.py create mode 100644 src/ee/part/bom.py create mode 100644 src/ee/tools/digikey_create_bom.py create mode 100644 src/ee/tools/split_bom_by_supplier.py delete mode 100644 src/ee/tools/split_parts_by_supplier.py create mode 100644 test/part/test_bom.py delete mode 100644 test/test_bom.py create mode 100644 test/test_kicad_bom.py diff --git a/src/ee/digikey/bom.py b/src/ee/digikey/bom.py new file mode 100644 index 0000000..36b0143 --- /dev/null +++ b/src/ee/digikey/bom.py @@ -0,0 +1,72 @@ +import csv +from pathlib import Path +from typing import List, MutableMapping, Optional + +from ee.db import ObjDb +from ee.digikey import DigikeyStore +from ee.logging import log +from ee.part import Part, load_db +from ee.part.bom import load_bom, check_bom, join_refs + +__all__ = ["create_bom"] + + +class BomLine(object): + def __init__(self, uri, part: Part): + self.uri = uri + self.part = part + self.refs = [] + + def add_ref(self, ref): + self.refs.append(ref) + + @classmethod + def header(cls) -> List[str]: + return ["Quantity", "Digi-Key Part Number", "Customer Reference"] + + def to_rows(self) -> List[str]: + return [str(len(self.refs)), self.part.get_exactly_one_spn().valueProp, join_refs(self.refs)] + + +def gen_bom(allow_incomplete, bom_parts: ObjDb[Part], supplier_parts: ObjDb[Part]) -> Optional[List[List[str]]]: + lines: MutableMapping[str, BomLine] = {} + + uri_idx = supplier_parts.index("uri") + + for part in bom_parts.values: + pr_obj = part.get_only_part_reference() + + if allow_incomplete and pr_obj is None: + log.debug("Skipping part without part reference: {}".format(part.printable_reference)) + continue + elif pr_obj is None: + log.warn("Unresolved part: {}".format(part.printable_reference)) + return + + part_uri = pr_obj.part_uriProp + + if part_uri not in lines: + lines[part_uri] = line = BomLine(part_uri, uri_idx.get_single(part_uri)) + else: + line = lines[part_uri] + + line.add_ref(part.get_exactly_one_schematic_reference().referenceProp) + + return sorted(line.to_rows() for line in lines.values()) + + +def create_bom(bom_path: Path, part_files: List[Path], out_path: Path, store_code, allow_incomplete): + store = DigikeyStore.from_store_code(store_code) + + bom_parts, supplier_parts = load_bom(bom_path, part_files) + check_bom(bom_parts, supplier_parts) + + bom = gen_bom(allow_incomplete, bom_parts, supplier_parts) + + if bom is None: + return + + with out_path.open("w") as f: + w = csv.writer(f) + w.writerow(BomLine.header()) + w.writerows(bom) diff --git a/src/ee/kicad/bom/__init__.py b/src/ee/kicad/bom/__init__.py index 0392d66..e993adb 100644 --- a/src/ee/kicad/bom/__init__.py +++ b/src/ee/kicad/bom/__init__.py @@ -10,6 +10,7 @@ __all__ = [ ] +# TODO: use ee.part.bom.split_ref def split_ref(ref): """Split "C12" into a tuple that's useful for sorting by component reference. @@ -23,11 +24,11 @@ def split_ref(ref): val = groups[1] rest = groups[2] try: - return (ref, int(val), rest) + return ref, int(val), rest except ValueError: pass - return (ref, val, rest) + return ref, val, rest split_ref.r = re.compile("([A-Za-z]+)([0-9]+)(.*)") diff --git a/src/ee/part/__init__.py b/src/ee/part/__init__.py index c18dd02..deeab95 100644 --- a/src/ee/part/__init__.py +++ b/src/ee/part/__init__.py @@ -309,7 +309,7 @@ class Part(object): def get_exactly_one_spn(self) -> types.SupplierPartNumber: spns = self.get_spns() if len(spns) == 0: - raise EeException("This part does not contain any manufacturer part numbers{}". + raise EeException("This part does not contain any supplier part numbers{}". format(", uri=" + self.uri if self.uri else "")) if len(spns) != 1: raise EeException("This part does not contain exactly one spn: {}". diff --git a/src/ee/part/bom.py b/src/ee/part/bom.py new file mode 100644 index 0000000..83ae348 --- /dev/null +++ b/src/ee/part/bom.py @@ -0,0 +1,110 @@ +import re +import sys +from itertools import groupby +from pathlib import Path +from typing import List + +from ee import EeException +from ee.db import ObjDb +from ee.part import Part, load_db + +__all__ = ["load_bom"] + + +def uri_fn(part: Part): + return part.uri + + +def supplier_fn(part: Part): + return part.supplier + + +class BomItem(object): + def __init__(self, references, part: Part): + self.references = references + self.part = part + + +def load_bom(bom_path: Path, part_files: List[Path]) -> (ObjDb[Part](), ObjDb[Part]()): + bom: ObjDb[Part] = ObjDb[Part]() + + for xml in load_db(bom_path).iterparts(): + bom.add(Part(xml)) + + parts: ObjDb[Part] = ObjDb[Part]() + parts.add_unique_index("uri", uri_fn) + parts.add_index("supplier", supplier_fn) + + for part_file in part_files: + for xml in load_db(part_file).iterparts(): + parts.add(Part(xml)) + + return bom, parts + + +def check_bom(bom: ObjDb[Part](), parts: ObjDb[Part]()): + pass + + +def split_ref(ref): + """Split "C12" into a tuple that's useful for sorting by component reference. + + For example: "C12" => ("C", 12, None). "Cfoo" => ("C", sys.maxsize, ""). + """ + m = split_ref.r.match(ref) + if not m: + return ref, sys.maxsize, "" + groups = m.groups() + ref = groups[0] + val = groups[1] + rest = groups[2] + try: + return ref, int(val), rest + except ValueError: + pass + + return ref, val, rest + + +split_ref.r = re.compile("([A-Za-z]+)([0-9]+)(.*)") + + +def join_refs(refs: List[str], max=None) -> str: + refs = [ref.strip() for ref in refs] + + rs = sorted([split_ref(ref) for ref in refs]) + + for r in rs: + if r[1] is None: + raise EeException("Bad ref, must match '[A-Z]+[0-9]+'.") + + s = "" + for kind, items in groupby(rs, lambda x: x[0]): + items = list(items) + + groups = [] + start = cur = items[0][1] + + for r in items[1:]: + num = r[1] + + if cur + 1 != num: + groups.append([start, cur]) + start = cur = num + else: + cur = num + + groups.append([start, cur]) + + parts = [] + for start, end in groups: + if start == end: + parts.append(str(start)) + elif start + 1 == end: + parts.append("{},{}".format(start, end)) + else: + parts.append("{}-{}".format(start, end)) + + s += kind + ",".join(parts) + + return s diff --git a/src/ee/tools/digikey_create_bom.py b/src/ee/tools/digikey_create_bom.py new file mode 100644 index 0000000..a83da83 --- /dev/null +++ b/src/ee/tools/digikey_create_bom.py @@ -0,0 +1,33 @@ +import argparse +from pathlib import Path + +import ee.tools +from ee.digikey.bom import create_bom + +parser = argparse.ArgumentParser() +ee.tools.add_default_argparse_group(parser) + +parser.add_argument("--bom", + required=True, + metavar="PART DB") + +parser.add_argument("--out", + required=True, + metavar="CSV") + +parser.add_argument("--part-db", + nargs="*", + required=True, + metavar="PART DB") + +parser.add_argument("--store", + default="us", + metavar="STORE CODE") + +parser.add_argument("--allow-incomplete", + action="store_true") + +args = parser.parse_args() +ee.tools.process_default_argparse_group(args) + +create_bom(Path(args.bom), [Path(p) for p in args.part_db], Path(args.out), args.store, args.allow_incomplete) diff --git a/src/ee/tools/ninja.py b/src/ee/tools/ninja.py index aa6952b..7d73d8f 100644 --- a/src/ee/tools/ninja.py +++ b/src/ee/tools/ninja.py @@ -21,6 +21,15 @@ class NinjaSupplier(object): def part_db(self): return "$public_dir/{}/parts.xml".format(self.key) + @property + def bom_input(self): + return "$public_dir/{}/bom.xml".format(self.key) + + @property + def bom_output(self): + # This needs to be configurable per supplier + return "$public_dir/{}/bom.csv".format(self.key) + def ninja_path_filter(s: Union[Path, str, List[str]]) -> str: if isinstance(s, Path): diff --git a/src/ee/tools/split_bom_by_supplier.py b/src/ee/tools/split_bom_by_supplier.py new file mode 100644 index 0000000..01a6ef3 --- /dev/null +++ b/src/ee/tools/split_bom_by_supplier.py @@ -0,0 +1,86 @@ +import argparse +from pathlib import Path +from typing import List, Tuple + +import ee.tools +from ee.db import ObjDb +from ee.logging import log +from ee.part import Part, load_db, save_db, PartDb +from ee.part.bom import load_bom, check_bom + + +class OrderPart(object): + def __init__(self, order_part: Part, part: Part): + self.order_part = order_part + self.part = part + + +def split_bom_by_supplier(bom_path: Path, part_files: List[Path], supplier_list: List[Tuple[str, str, Path]]): + # key, url, file + + bom_parts, supplier_parts = load_bom(bom_path, part_files) + check_bom(bom_parts, supplier_parts) + + uri_idx = supplier_parts.index("uri") + + for supplier, uri, path in supplier_list: + print(supplier, uri, path) + + supplier_info = {uri: path for _, uri, path in supplier_list} + part_dbs = {uri: PartDb() for _, uri, _ in supplier_list} + + for bom_part in bom_parts.values: + sch = bom_part.get_exactly_one_schematic_reference().referenceProp + part_reference = bom_part.get_only_part_reference() + + if not part_reference: + # log.warn("Missing part for {}, allowing for now".format(sch)) + continue + + # print("{}: {}".format(sch, part_reference.part_uriProp)) + part = uri_idx.get_single(part_reference.part_uriProp) + print("{}: {}".format(sch, part.uri)) + print("{}: {}".format(sch, part.supplier)) + + part_dbs[part.supplier].add_entry(bom_part, new=False) + + for uri, path in supplier_info.items(): + part_db = part_dbs[uri] + save_db(path, part_db) + + +parser = argparse.ArgumentParser() +ee.tools.add_default_argparse_group(parser) + +parser.add_argument("--bom", + required=True, + metavar="PART DB") + +parser.add_argument("--part-db", + nargs="*", + required=True, + metavar="PART DB") + +parser.add_argument("--supplier", + required=True, + nargs="*", + metavar="SUPPLIER") + +parser.add_argument("--order", + required=True, + nargs="*", + metavar="PART DB") + +args = parser.parse_args() +ee.tools.process_default_argparse_group(args) + +ss = [] + +order_files = {o.split("=", maxsplit=1)[0]: o.split("=", maxsplit=1)[1] for o in args.order} + +for s in args.supplier: + key, url = s.split("=", maxsplit=1) + file = order_files[key] + ss.append((key, url, Path(file))) + +split_bom_by_supplier(Path(args.bom), [Path(path) for path in args.part_db], ss) diff --git a/src/ee/tools/split_parts_by_supplier.py b/src/ee/tools/split_parts_by_supplier.py deleted file mode 100644 index 9202f0d..0000000 --- a/src/ee/tools/split_parts_by_supplier.py +++ /dev/null @@ -1,72 +0,0 @@ -import argparse -from pathlib import Path -from typing import List - -import ee.tools -from ee.db import ObjDb -from ee.part import Part, load_db, save_db, PartDb -from ee.project import Project - - -class OrderPart(object): - def __init__(self, order_part: Part, part: Part): - self.order_part = order_part - self.part = part - - -def uri_fn(part: Part): - return part.uri - - -def split_parts_by_supplier(project: Project, order_file: Path, part_dbs: List[Path], out_dir: Path): - parts: ObjDb[Part] = ObjDb[Part]() - part_by_uri = parts.add_unique_index("uri", uri_fn) - - for part_db in part_dbs: - for xml in load_db(part_db).iterparts(): - parts.add(Part(xml)) - - order_parts: ObjDb[OrderPart] = ObjDb() - supplier_idx = order_parts.add_index("supplier", lambda op: op.part.supplier) - for xml in load_db(order_file).iterparts(): - order_part = Part(xml) - part = part_by_uri.get_single(order_part.get_exactly_one_part_reference().part_uriProp) - order_parts.add(OrderPart(order_part, part)) - - for supplier, parts_for_supplier in supplier_idx.items(): - desc = project.get_supplier_by_uri(supplier) - - print("{}: {}".format(desc.name, len(parts))) - # supplier_db: ObjDb[Part] = ObjDb[Part]() - # supplier_db.add_unique_index("uri", uri_fn) - - supplier_descriptor = project.get_supplier_by_uri(supplier) - - db = PartDb() - for part_for_supplier in parts_for_supplier: - db.add_entry(part_for_supplier.part.underlying, False) - - save_db(out_dir / "{}.xml".format(supplier_descriptor.key), db, sort=True) - - -parser = argparse.ArgumentParser() -ee.tools.add_default_argparse_group(parser) - -parser.add_argument("--parts", - required=True, - metavar="PART DB") - -parser.add_argument("--part-db", - nargs="*", - required=True, - metavar="PART DB") - -parser.add_argument("--out-dir", - metavar="DIR FOR PART DBS") - -args = parser.parse_args() -ee.tools.process_default_argparse_group(args) - -part_db_dirs = [Path(part_db) for part_db in args.part_db] - -split_parts_by_supplier(Project.load(), Path(args.parts), part_db_dirs, Path(args.out_dir)) diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2 index 9fff519..1ab6102 100644 --- a/src/ee/tools/templates/build.ninja.j2 +++ b/src/ee/tools/templates/build.ninja.j2 @@ -56,8 +56,8 @@ rule element14-search-parts rule create-bom command = $ee create-bom {{ log }} --schematic $schematic --part-db $part_dbs --out $out $strategy -rule split-parts-by-supplier - command = $ee split-parts-by-supplier {{ log }} --parts $in --out $out $part_dbs +rule split-bom-by-supplier + command = $ee split-bom-by-supplier {{ log }} --bom $in $part_dbs $suppliers $orders rule import-parts-yaml description = import-parts-yaml $in @@ -151,6 +151,12 @@ build $public_dir/{{ s.key }}/souffle/out/fact.csv: souffle {{ alt if is_file(al build {{ s.part_db }}: part-apply-souffle-post $public_dir/{{ s.key }}/souffle/out/fact.csv in_sch = $public_dir/{{ s.key }}/downloaded.xml work = $public_dir/{{ s.key }}/souffle + +rule {{ s.key }}-create-bom + command = $ee {{ s.key }}-create-bom {{ log }} --bom $in --out $out $part_dbs --allow-incomplete + +build {{ s.bom_output }}: {{ s.key }}-create-bom {{ s.bom_input }} + part_dbs = --part-db {{ s.part_db }} {%- endfor %} {%- for f in parts_yaml_files %} @@ -167,11 +173,10 @@ build $public_dir/bom.xml | $report_dir/bom.rst: create-bom $public_dir/sch.xml {%- endif %} {%- set reports=reports+["$report_dir/bom.rst"] %} -{# -build $public_dir/orders/index.xml: split-parts-by-supplier $public_dir/bom.xml | {%- for s in suppliers %} {{ s.part_db }}{% endfor %} - part_dbs ={%- for s in suppliers %} {{ s.part_db }}{% endfor %} - suppliers = {% s for s in suppliers %} -#} +build {%- for s in suppliers %} {{ s.bom_input }}{% endfor %}: split-bom-by-supplier $public_dir/bom.xml | {%- for s in suppliers %} {{ s.part_db }}{% endfor %} + part_dbs = {%- for s in suppliers %} --part-db {{ s.part_db }}{% endfor %} + suppliers = {%- for s in suppliers %} --supplier {{ s.key }}={{ s.supplier.uri }}{% endfor %} + orders = {%- for s in suppliers %} --order {{ s.key }}={{ s.bom_input }}{% endfor %} rule seeed-download-opl description = seeed-download-opl $opl @@ -190,5 +195,6 @@ build $public_dir/seeed/opl/{{ opl }}.xml: seeed-download-opl # Reports build ee-reports: phony {{ " ".join(reports) }} -build part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %} -build ee-all: phony ee-reports part-dbs +build ee-part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %} +build ee-orders: phony {%- for s in suppliers %} {{ s.bom_output }}{% endfor %} +build ee-all: phony ee-reports ee-orders diff --git a/test/part/test_bom.py b/test/part/test_bom.py new file mode 100644 index 0000000..85eb932 --- /dev/null +++ b/test/part/test_bom.py @@ -0,0 +1,12 @@ +import pytest +from ee.part.bom import join_refs + + +@pytest.mark.parametrize("inputs, output", [ + [[], ""], + [["C1"], "C1"], + [["C1", "C2"], "C1,2"], + [["C1", "C2", "C3", "C5", "C6", "C9"], "C1-3,5,6,9"], +]) +def test_basic(inputs, output): + assert output == join_refs(inputs) diff --git a/test/test_bom.py b/test/test_bom.py deleted file mode 100644 index 50cae9f..0000000 --- a/test/test_bom.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest -import os.path -import sys - -from ee.kicad.bom import * -from ee.kicad.bom.io import read_bom - -basedir = os.path.dirname(os.path.abspath(__file__)) - - -@pytest.mark.parametrize("s, ref, val, rest", [ - ("C12", "C", 12, ""), - ("C12n", "C", 12, "n"), - ("C", "C", sys.maxsize, ""), - ("Foo", "Foo", sys.maxsize, ""), - ("+3.0VA1", "+3.0VA1", sys.maxsize, ""), -]) -def test_split_ref(s, ref, val, rest): - assert split_ref(s) == (ref, val, rest) - - -def test_read_bom_1(): - bom = read_bom(basedir + '/../demo/kicad/bom/A64-OlinuXino_Rev_C.xml') - assert len(bom.get_components()) == 425 - - -def test_read_bom_2(): - bom = read_bom(basedir + '/../demo/kicad/bom/gw.xml') - assert len(bom.get_components()) == 165 - - r5 = bom.get_component("R5") - assert r5.ref == "R5" - assert r5.value == "R0402_100R" - assert r5["value"] == "R0402_100R" - assert r5.footprint == "Resistors_SMD:R_0402" - assert r5.library.name == "gw-cache" - assert len(r5.fields) == 4 - assert r5.fields["Part Number"] == "CRCW0402100RFKED" - assert r5["Part Number"] == "CRCW0402100RFKED" - assert {'ref', 'value', 'Capacitance', 'Color', 'Description', 'Frequency', 'Impedance', 'Inductance', - 'Manufacturer', 'Part Number', 'Resistance'} == bom.all_field_names() - - assert "foo" not in r5 - with pytest.raises(KeyError): - r5["foo"] - - -def test_read_bom_2_pandas(): - bom = read_bom(basedir + '/../demo/kicad/bom/gw.xml').to_pandas() - assert len(bom) == 165 - - r5 = bom.loc["R5"] - # assert r5.index == "R5" - assert r5["ref"] == "R5" - assert r5["value"] == "R0402_100R" - assert r5["value"] == "R0402_100R" - # assert r5["footprint"] == "Resistors_SMD:R_0402" - # assert r5.library.name == "gw-cache" - # assert len(r5.fields) == 4 - # assert r5.fields["Part Number"] == "CRCW0402100RFKED" - assert r5["Part Number"] == "CRCW0402100RFKED" - # assert set(['ref', 'value', 'Capacitance', 'Color', 'Description', 'Frequency', 'Impedance', 'Inductance', 'Manufacturer', 'Part Number', 'Resistance']) == bom.all_field_names() - - assert "foo" not in r5 - with pytest.raises(KeyError): - r5["foo"] diff --git a/test/test_digikey.py b/test/test_digikey.py index f1b010b..4961a9d 100644 --- a/test/test_digikey.py +++ b/test/test_digikey.py @@ -83,7 +83,7 @@ def test_digikey_3(): p = next((p for p in res.products if p.part_number == "1655-1501-1-ND"), None) assert p.mpn == "RS1MTR" - assert p.url == "/product-detail/en/smc-diode-solutions/RS1MTR/1655-1501-1-ND/6022946" + assert p.url == "https://www.digikey.com/product-detail/en/smc-diode-solutions/RS1MTR/1655-1501-1-ND/6022946" @pytest.mark.digikey diff --git a/test/test_kicad_bom.py b/test/test_kicad_bom.py new file mode 100644 index 0000000..50cae9f --- /dev/null +++ b/test/test_kicad_bom.py @@ -0,0 +1,66 @@ +import pytest +import os.path +import sys + +from ee.kicad.bom import * +from ee.kicad.bom.io import read_bom + +basedir = os.path.dirname(os.path.abspath(__file__)) + + +@pytest.mark.parametrize("s, ref, val, rest", [ + ("C12", "C", 12, ""), + ("C12n", "C", 12, "n"), + ("C", "C", sys.maxsize, ""), + ("Foo", "Foo", sys.maxsize, ""), + ("+3.0VA1", "+3.0VA1", sys.maxsize, ""), +]) +def test_split_ref(s, ref, val, rest): + assert split_ref(s) == (ref, val, rest) + + +def test_read_bom_1(): + bom = read_bom(basedir + '/../demo/kicad/bom/A64-OlinuXino_Rev_C.xml') + assert len(bom.get_components()) == 425 + + +def test_read_bom_2(): + bom = read_bom(basedir + '/../demo/kicad/bom/gw.xml') + assert len(bom.get_components()) == 165 + + r5 = bom.get_component("R5") + assert r5.ref == "R5" + assert r5.value == "R0402_100R" + assert r5["value"] == "R0402_100R" + assert r5.footprint == "Resistors_SMD:R_0402" + assert r5.library.name == "gw-cache" + assert len(r5.fields) == 4 + assert r5.fields["Part Number"] == "CRCW0402100RFKED" + assert r5["Part Number"] == "CRCW0402100RFKED" + assert {'ref', 'value', 'Capacitance', 'Color', 'Description', 'Frequency', 'Impedance', 'Inductance', + 'Manufacturer', 'Part Number', 'Resistance'} == bom.all_field_names() + + assert "foo" not in r5 + with pytest.raises(KeyError): + r5["foo"] + + +def test_read_bom_2_pandas(): + bom = read_bom(basedir + '/../demo/kicad/bom/gw.xml').to_pandas() + assert len(bom) == 165 + + r5 = bom.loc["R5"] + # assert r5.index == "R5" + assert r5["ref"] == "R5" + assert r5["value"] == "R0402_100R" + assert r5["value"] == "R0402_100R" + # assert r5["footprint"] == "Resistors_SMD:R_0402" + # assert r5.library.name == "gw-cache" + # assert len(r5.fields) == 4 + # assert r5.fields["Part Number"] == "CRCW0402100RFKED" + assert r5["Part Number"] == "CRCW0402100RFKED" + # assert set(['ref', 'value', 'Capacitance', 'Color', 'Description', 'Frequency', 'Impedance', 'Inductance', 'Manufacturer', 'Part Number', 'Resistance']) == bom.all_field_names() + + assert "foo" not in r5 + with pytest.raises(KeyError): + r5["foo"] -- cgit v1.2.3