From 8d17fb5bc4b0dae0758e01a44d77d87acf2e686a Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Thu, 14 Mar 2019 06:27:16 +0100 Subject: o Adding module for searching on element14. o Starting on functionality create orders. Very WIP. o Adding a concept of an "ee project". Can load a gitconfig-like config file. o Adding a tool to import a yaml file into a part xml file. --- .gitignore | 3 +- README.md | 6 ++- src/ee/element14/__init__.py | 63 ++++++++++++++++++++++++ src/ee/element14/search_parts.py | 31 ++++++++++++ src/ee/order/__init__.py | 88 ++++++++++++++++++++++++++++++++++ src/ee/part/__init__.py | 5 +- src/ee/project/__init__.py | 12 +++++ src/ee/tools/create_order.py | 29 +++++++++++ src/ee/tools/element14_search_parts.py | 30 ++++++++++++ src/ee/tools/import_parts_yaml.py | 52 ++++++++++++++++++++ src/ee/tools/ninja.py | 50 ++++++++++++++----- src/ee/tools/templates/build.ninja.j2 | 51 ++++++++++++++++---- src/ee/xml/bom_file_utils.py | 4 +- xsd/ee-bom.xsd | 29 +++++++++++ 14 files changed, 425 insertions(+), 28 deletions(-) create mode 100644 src/ee/element14/__init__.py create mode 100644 src/ee/element14/search_parts.py create mode 100644 src/ee/order/__init__.py create mode 100644 src/ee/project/__init__.py create mode 100644 src/ee/tools/create_order.py create mode 100644 src/ee/tools/element14_search_parts.py create mode 100644 src/ee/tools/import_parts_yaml.py diff --git a/.gitignore b/.gitignore index 9636df4..972d5ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.tmp -digikey_cache +.ee +*.patch *.pyc env diff --git a/README.md b/README.md index d280377..06cbc0d 100644 --- a/README.md +++ b/README.md @@ -31,5 +31,9 @@ part. Some cases where this is not true: # TODOs -* Consistenly use `read` and `parse` in function names. +* Consistently use `read` and `parse` in function names. * Consider flattening the module tree. + +* Rename some stuff: + * Rename "distributor" to "supplier" + * Rename "distributor part number/DPN" to "SKU". diff --git a/src/ee/element14/__init__.py b/src/ee/element14/__init__.py new file mode 100644 index 0000000..17a8825 --- /dev/null +++ b/src/ee/element14/__init__.py @@ -0,0 +1,63 @@ +import json +from pathlib import Path +from typing import Optional +from urllib.parse import urlencode +from urllib.request import urlopen + +import ee._utils + +__all__ = [ + "Element14Config", + "Element14Client", +] + + +class Element14Config(object): + def __init__(self, store: Optional[str], api_key: Optional[str]): + self.store = store + self.api_key = api_key + + +class Element14Client(object): + def __init__(self, config: Element14Config, cache_dir: Path): + self.config = config + self.cache = ee._utils.maybe_cache(cache_dir) + + def search(self, term: Optional[str] = None): + url = "https://api.element14.com/catalog/products" + + kv = { + "callInfo.responseDataFormat": "XML", + } + + if self.config.api_key: + kv["callInfo.apiKey"] = self.config.api_key + + if self.config.store: + kv["storeInfo.id"] = self.config.store + + if term: + kv["term"] = term + + kv["resultsSettings.offset"] = "0" + kv["resultsSettings.numberOfResults"] = "100" + + print("params: {}".format(kv)) + + # &resultsSettings.refinements.filters={rohsCompliant,inStock} + # &resultsSettings.responseGroup={none,small,medium,large, Prices, Inventory} + + url = url + "?" + urlencode(kv) + + print("url={}".format(url)) + + data = "wat!!" + data = urlopen(url).read() + + print("-----------") + print(data) + print("-----------") + + search_response = json.loads(data) + + return diff --git a/src/ee/element14/search_parts.py b/src/ee/element14/search_parts.py new file mode 100644 index 0000000..724485c --- /dev/null +++ b/src/ee/element14/search_parts.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from ee.element14 import * +from ee.part import PartDb, load_db, save_db +from ee.xml import bom_file_utils, bomFile + +__all__ = ["search_parts"] + + +def search_parts(in_dir: Path, out_dir: Path, cache_dir: Path, config: Element14Config): + in_db = load_db(in_dir) + out_parts = PartDb() + + client = Element14Client(config, cache_dir) + + for part in in_db.iterparts(): + mpn = bom_file_utils.find_pn(part) + + query = mpn # TODO: suppor dpn + + out_id = query + + client.search(term="manuPartNum:" + query) + + out_part = bomFile.Part(id=out_id, + distributor_info=bomFile.DistributorInfo(), + part_numbers=part.part_numbersProp) + di = out_part.distributor_infoProp + + print("Saving {} work parts".format(out_parts.size())) + save_db(out_dir, out_parts) diff --git a/src/ee/order/__init__.py b/src/ee/order/__init__.py new file mode 100644 index 0000000..7c815b4 --- /dev/null +++ b/src/ee/order/__init__.py @@ -0,0 +1,88 @@ +from functools import total_ordering +from pathlib import Path +from typing import List, Tuple + +from ee.part import PartDb, load_db, save_db +from ee.xml import bomFile, bom_file_utils + +__all__ = ["create_order"] + + +@total_ordering +class PartInfo(object): + def __init__(self, part: bomFile.Part): + self.part = part + self.id = part.id + self.pn = bom_file_utils.find_pn(part) + self.available_from: List[Tuple[str, bomFile.Part]] = [] + + def __lt__(self, other: "PartInfo"): + return self.part.idProp == other.part.idProp + + +def create_order(schematic_dir: Path, out_dir: Path, part_db_dirs: List[Path], fail_on_missing_parts: bool): + sch_db = load_db(schematic_dir) + + dbs = [(path.parent.name, load_db(path)) for path in part_db_dirs] + + out_parts = PartDb() + + infos = [PartInfo(sch) for sch in sch_db.iterparts()] + + for info in infos: + print("Resolving {}".format(info.id)) + + for distributor, db in dbs: + for p in db.iterparts(sort=True): + if info.pn: + p_pn: str = bom_file_utils.find_pn(p) + + if info.pn == p_pn: + info.available_from.append((distributor, p)) + + for sch_pn_ in bom_file_utils.part_numbers(info.part): + sch_pn: bomFile.PartNumber = sch_pn_ + + for p_pn_ in bom_file_utils.part_numbers(p): + p_pn: bomFile.PartNumber = p_pn_ + + if sch_pn.distributorProp == p_pn.distributorProp and sch_pn.value == p_pn.value: + if p.idProp not in info.available_from: + info.available_from.append((distributor, p)) + + for info in infos: + print("Resolved {} to {}".format(info.id, info.available_from)) + + warnings = [] + has_missing_parts = False + for info in infos: + if len(info.available_from) == 0: + has_missing_parts = True + warnings.append("Could not find {} in any database, part numbers: {}". + format(info.id, ", ".join([p.value for p in bom_file_utils.part_numbers(info.part)]))) + + for w in sorted(warnings): + print(w) + + if has_missing_parts and fail_on_missing_parts: + print("has missing parts") + return False + + for info in infos: + part = bomFile.Part(id=info.part.id, + schema_reference=info.part.schema_reference, + part_numbers=bomFile.PartNumberList()) + + part_numbers = part.part_numbersProp.part_number + + if len(info.available_from) == 0: + continue + + distributor, distributor_part = info.available_from[0] + + part_numbers.append(bomFile.PartNumber(value=distributor_part.id, distributor=distributor)) + + out_parts.add_entry(part, True) + + print("Saving {} work parts".format(out_parts.size())) + save_db(out_dir, out_parts) diff --git a/src/ee/part/__init__.py b/src/ee/part/__init__.py index d4d99ac..26bc26c 100644 --- a/src/ee/part/__init__.py +++ b/src/ee/part/__init__.py @@ -40,8 +40,9 @@ class PartDb(object): if e.new: self.new_entries = self.new_entries + 1 - def iterparts(self) -> Iterator[bomFile.Part]: - return [e.part for e in self.parts] + def iterparts(self, sort=False) -> Iterator[bomFile.Part]: + it = (e.part for e in self.parts) + return sorted(it, key=lambda p: p.idProp) if sort else it def size(self) -> int: return len(self.parts) diff --git a/src/ee/project/__init__.py b/src/ee/project/__init__.py new file mode 100644 index 0000000..187fc3c --- /dev/null +++ b/src/ee/project/__init__.py @@ -0,0 +1,12 @@ +import configparser +from pathlib import Path + + +def load_config(project_dir: Path) -> configparser.ConfigParser: + config = configparser.ConfigParser() + + config_path = project_dir / ".ee" / "config" + with config_path.open("r") as f: + config.read_file(f, source=str(config_path)) + + return config diff --git a/src/ee/tools/create_order.py b/src/ee/tools/create_order.py new file mode 100644 index 0000000..e0bed58 --- /dev/null +++ b/src/ee/tools/create_order.py @@ -0,0 +1,29 @@ +import argparse +import sys +from pathlib import Path + +from ee.order import create_order + +parser = argparse.ArgumentParser() + +parser.add_argument("--schematic", + required=True, + metavar="DIR") + +parser.add_argument("--out", + required=True, + metavar="DIR") + +parser.add_argument("--part-db", + nargs="*", + required=True, + metavar="PART DB") + +args = parser.parse_args() + +part_db_dirs = [Path(part_db) for part_db in args.part_db] +fail_on_missing_parts = False + +ret = create_order(Path(args.schematic), Path(args.out), part_db_dirs, fail_on_missing_parts) + +sys.exit(1 if ret is False else 0) diff --git a/src/ee/tools/element14_search_parts.py b/src/ee/tools/element14_search_parts.py new file mode 100644 index 0000000..a4793c5 --- /dev/null +++ b/src/ee/tools/element14_search_parts.py @@ -0,0 +1,30 @@ +import argparse +from pathlib import Path + +from ee import project +from ee.element14 import Element14Config +from ee.element14.search_parts import search_parts + +parser = argparse.ArgumentParser() + +parser.add_argument("--in", + dest="in_", + required=True, + metavar="FILE") + +parser.add_argument("--out", + required=True, + metavar="FILE") + +args = parser.parse_args() + +cache_dir = ".ee/cache/element14" + +config = project.load_config(Path(".")) + +e14_config = Element14Config( + store=config.get("element14", "store", fallback=None), + api_key=config.get("element14", "api-key", fallback=None) +) + +search_parts(Path(args.in_), Path(args.out), Path(cache_dir), e14_config) diff --git a/src/ee/tools/import_parts_yaml.py b/src/ee/tools/import_parts_yaml.py new file mode 100644 index 0000000..81a23f8 --- /dev/null +++ b/src/ee/tools/import_parts_yaml.py @@ -0,0 +1,52 @@ +import argparse +import sys +import yaml +from pathlib import Path + +from ee.part import PartDb, save_db +from ee.xml import bomFile + + +def import_parts_yaml(in_file: Path, out_dir: Path): + with in_file.open("r") as f: + doc = yaml.safe_load(f) + + if not isinstance(doc, list): + print("Bad YAML document, the root must be a list", file=sys.stderr) + + parts = PartDb() + for item in doc: + if not isinstance(item, dict): + print("Bad YAML document, each part must be a dict", file=sys.stderr) + return + + part_number_list = bomFile.PartNumberList() + + mpn = item.get("mpn") + assert isinstance(mpn, str) + part_number_list.part_number.append(bomFile.PartNumber(value=mpn)) + + id_ = mpn + + part = bomFile.Part(id=id_) + part.part_numbersProp = part_number_list + parts.add_entry(part, True) + + print("Imported {} parts".format(parts.size())) + save_db(out_dir, parts) + + +parser = argparse.ArgumentParser() + +parser.add_argument("--in", + dest="in_", + required=True, + metavar="PARTS_YAML") + +parser.add_argument("--out", + required=True, + metavar="PART_ DB") + +args = parser.parse_args() + +import_parts_yaml(Path(args.in_), Path(args.out)) diff --git a/src/ee/tools/ninja.py b/src/ee/tools/ninja.py index cc090a0..ed8e91a 100644 --- a/src/ee/tools/ninja.py +++ b/src/ee/tools/ninja.py @@ -1,4 +1,5 @@ import argparse +import os.path import sys from pathlib import Path from typing import List, Union, Optional @@ -8,7 +9,10 @@ from jinja2 import Environment, PackageLoader, select_autoescape from ee.kicad import read_schematics -def ninja_path_filter(s: Union[str, List[str]]) -> str: +def ninja_path_filter(s: Union[Path, str, List[str]]) -> str: + if isinstance(s, Path): + s = str(s) + if isinstance(s, str): return s. \ replace("$", "$$"). \ @@ -23,8 +27,16 @@ def ninja_path_filter(s: Union[str, List[str]]) -> str: raise Exception("Unsupported argument type: {}".format(type(s))) -def parent_dir_filter(s: str) -> str: - return str(Path(s).parent) +def parent_dir_filter(s: str) -> Path: + return Path(s).parent + + +def basename_filter(s: Union[str, Path]) -> str: + return os.path.basename(str(s)) + + +def noext_filter(s: Union[str, Path]) -> str: + return os.path.splitext(os.path.basename(str(s)))[0] def generate(sch_path: Path, kicad_bom_strategy: Optional[str]): @@ -36,6 +48,8 @@ def generate(sch_path: Path, kicad_bom_strategy: Optional[str]): ) e.filters["ninja_path"] = ninja_path_filter e.filters["parent_dir"] = parent_dir_filter + e.filters["basename"] = basename_filter + e.filters["noext"] = noext_filter return e gerber_zip = "prod/gerber.zip" @@ -44,20 +58,30 @@ def generate(sch_path: Path, kicad_bom_strategy: Optional[str]): sch_files = sorted([s.path for s in sch.schematics]) - params = {} - import os.path - params["ee"] = "{} -m ee".format(os.path.relpath(sys.executable, Path("."))) - params["sch"] = sch_path - params["sch_files"] = sch_files - - params["kicad_bom_strategy"] = kicad_bom_strategy + part_dbs = [] + params = { + "ee": "{} -m ee".format(os.path.relpath(sys.executable, Path("."))), "sch": sch_path, + "sch_files": sch_files, "kicad_bom_strategy": kicad_bom_strategy, + "pcb": str(sch_path).replace(".sch", ".kicad_pcb"), + "part_dbs": part_dbs, + } - params["pcb"] = str(sch_path).replace(".sch", ".kicad_pcb") + # TODO: read from config + distributors = ["digikey"] + params["distributors"] = distributors if gerber_zip is not None: params["gerber_zip"] = gerber_zip - build_ninja = Path("build.ninja") + build_ninja = sch_path.parent / "build.ninja" + + ee_dir = sch_path.parent / "ee" + parts_yaml_files = [path for path in ee_dir.iterdir() if str(path).endswith("-parts.yaml")] + params["parts_yaml_files"] = parts_yaml_files + + # Local part databases first + part_dbs.extend([parent_dir_filter(p) / noext_filter(p) for p in parts_yaml_files]) + part_dbs.extend([Path("ee") / d / "normalized" for d in distributors]) with build_ninja.open("w") as f: env = _create_env() @@ -77,4 +101,4 @@ parser.add_argument("--kicad-bom-strategy", args = parser.parse_args() -generate(args.sch, args.kicad_bom_strategy) +generate(Path(args.sch), args.kicad_bom_strategy) diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2 index 79a2bc1..e91de6c 100644 --- a/src/ee/tools/templates/build.ninja.j2 +++ b/src/ee/tools/templates/build.ninja.j2 @@ -28,6 +28,22 @@ rule digikey-normalize-facts description = digikey-normalize-facts command = $ee digikey-normalize-facts --in $in_dir --out $out_dir +rule element14-search-parts + description = element14-search-parts + command = $ee element14-search-parts --in $in_dir --out $out_dir + +rule element14-normalize-facts + description = element14-normalize-facts + command = $ee element14-normalize-facts --in $in_dir --out $out_dir + +rule create-order + description = create-order + command = $ee create-order --schematic $sch_dir --part-db $part_dbs --out $out_dir + +rule import-parts-yaml + description = import-parts-yaml $in + command = $ee import-parts-yaml --in $in --out $out_dir + {% if gerber_zip is defined %} build gerbers: phony {{ gerber_zip }} build {{ gerber_zip }}: kicad-gerber $pcb @@ -38,16 +54,33 @@ build ee/sch/index.xml: kicad-make-bom $sch out_dir = ee/sch strategy = {{ "--strategy " + kicad_bom_strategy if kicad_bom_strategy else "" }} -build ee/digikey/search-list/index.xml: part-create-distributor-search-list ee/sch/index.xml +{%- for d in distributors %} +# Distributor {{ d }} +build ee/{{ d }}/search-list/index.xml: part-create-distributor-search-list ee/sch/index.xml in_dir = ee/sch - out_dir = ee/digikey/search-list + out_dir = ee/{{ d }}/search-list + +build ee/{{ d }}/downloaded/index.xml: {{ d }}-search-parts ee/{{ d }}/search-list/index.xml + in_dir = ee/{{ d }}/search-list + out_dir = ee/{{ d }}/downloaded + +build ee/{{ d }}/normalized/index.xml: {{ d }}-normalize-facts ee/{{ d }}/downloaded/index.xml + in_dir = ee/{{ d }}/downloaded + out_dir = ee/{{ d }}/normalized + +default ee/{{ d }}/normalized/index.xml +{%- endfor %} -build ee/digikey/downloaded/index.xml: digikey-search-parts ee/digikey/search-list/index.xml - in_dir = ee/digikey/search-list - out_dir = ee/digikey/downloaded +{%- for f in parts_yaml_files %} +{% set out=(f | parent_dir) / (f | noext) / "index.xml" %} +build {{ out }}: import-parts-yaml {{ f }} + out_dir = {{ f | parent_dir }}/{{ f | noext }} +# default {{ out }} +{% endfor %} -build ee/digikey/normalized/index.xml: digikey-normalize-facts ee/digikey/downloaded/index.xml - in_dir = ee/digikey/downloaded - out_dir = ee/digikey/normalized +build ee/order/index.xml: create-order ee/sch/index.xml {%- for p in part_dbs %} {{ p / "index.xml" }}{% endfor %} + sch_dir = ee/sch + out_dir = ee/order + part_dbs ={%- for p in part_dbs %} {{ p }}{% endfor %} -default ee/digikey/normalized/index.xml +default ee/order/index.xml diff --git a/src/ee/xml/bom_file_utils.py b/src/ee/xml/bom_file_utils.py index d1f8be9..063a7bd 100644 --- a/src/ee/xml/bom_file_utils.py +++ b/src/ee/xml/bom_file_utils.py @@ -28,13 +28,13 @@ def part_numbers(part: bomFile.Part) -> List[bomFile.PartNumber]: return pns.part_numberProp -def find_pn(part: bomFile.Part) -> str: +def find_pn(part: bomFile.Part) -> Optional[str]: for pn in part_numbers(part): if pn.distributor is None: return pn.value -def find_dpn(part: bomFile.Part, distributor: str) -> str: +def find_dpn(part: bomFile.Part, distributor: str) -> Optional[str]: for pn in part_numbers(part): if pn.distributor == distributor: return pn.value diff --git a/xsd/ee-bom.xsd b/xsd/ee-bom.xsd index dddfa8a..b88a87e 100644 --- a/xsd/ee-bom.xsd +++ b/xsd/ee-bom.xsd @@ -1,3 +1,31 @@ + + -- cgit v1.2.3