aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore3
-rw-r--r--README.md6
-rw-r--r--src/ee/element14/__init__.py63
-rw-r--r--src/ee/element14/search_parts.py31
-rw-r--r--src/ee/order/__init__.py88
-rw-r--r--src/ee/part/__init__.py5
-rw-r--r--src/ee/project/__init__.py12
-rw-r--r--src/ee/tools/create_order.py29
-rw-r--r--src/ee/tools/element14_search_parts.py30
-rw-r--r--src/ee/tools/import_parts_yaml.py52
-rw-r--r--src/ee/tools/ninja.py50
-rw-r--r--src/ee/tools/templates/build.ninja.j251
-rw-r--r--src/ee/xml/bom_file_utils.py4
-rw-r--r--xsd/ee-bom.xsd29
14 files changed, 425 insertions, 28 deletions
diff --git a/.gitignore b/.gitignore
index 9636df4..972d5ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
*.tmp
-digikey_cache
+.ee
+*.patch
*.pyc
env
diff --git a/README.md b/README.md
index d280377..06cbc0d 100644
--- a/README.md
+++ b/README.md
@@ -31,5 +31,9 @@ part. Some cases where this is not true:
# TODOs
-* Consistenly use `read` and `parse` in function names.
+* Consistently use `read` and `parse` in function names.
* Consider flattening the module tree.
+
+* Rename some stuff:
+ * Rename "distributor" to "supplier"
+ * Rename "distributor part number/DPN" to "SKU".
diff --git a/src/ee/element14/__init__.py b/src/ee/element14/__init__.py
new file mode 100644
index 0000000..17a8825
--- /dev/null
+++ b/src/ee/element14/__init__.py
@@ -0,0 +1,63 @@
+import json
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlencode
+from urllib.request import urlopen
+
+import ee._utils
+
+__all__ = [
+ "Element14Config",
+ "Element14Client",
+]
+
+
+class Element14Config(object):
+ def __init__(self, store: Optional[str], api_key: Optional[str]):
+ self.store = store
+ self.api_key = api_key
+
+
+class Element14Client(object):
+ def __init__(self, config: Element14Config, cache_dir: Path):
+ self.config = config
+ self.cache = ee._utils.maybe_cache(cache_dir)
+
+ def search(self, term: Optional[str] = None):
+ url = "https://api.element14.com/catalog/products"
+
+ kv = {
+ "callInfo.responseDataFormat": "XML",
+ }
+
+ if self.config.api_key:
+ kv["callInfo.apiKey"] = self.config.api_key
+
+ if self.config.store:
+ kv["storeInfo.id"] = self.config.store
+
+ if term:
+ kv["term"] = term
+
+ kv["resultsSettings.offset"] = "0"
+ kv["resultsSettings.numberOfResults"] = "100"
+
+ print("params: {}".format(kv))
+
+ # &resultsSettings.refinements.filters={rohsCompliant,inStock}
+ # &resultsSettings.responseGroup={none,small,medium,large, Prices, Inventory}
+
+ url = url + "?" + urlencode(kv)
+
+ print("url={}".format(url))
+
+ data = "wat!!"
+ data = urlopen(url).read()
+
+ print("-----------")
+ print(data)
+ print("-----------")
+
+ search_response = json.loads(data)
+
+ return
diff --git a/src/ee/element14/search_parts.py b/src/ee/element14/search_parts.py
new file mode 100644
index 0000000..724485c
--- /dev/null
+++ b/src/ee/element14/search_parts.py
@@ -0,0 +1,31 @@
+from pathlib import Path
+
+from ee.element14 import *
+from ee.part import PartDb, load_db, save_db
+from ee.xml import bom_file_utils, bomFile
+
+__all__ = ["search_parts"]
+
+
+def search_parts(in_dir: Path, out_dir: Path, cache_dir: Path, config: Element14Config):
+ in_db = load_db(in_dir)
+ out_parts = PartDb()
+
+ client = Element14Client(config, cache_dir)
+
+ for part in in_db.iterparts():
+ mpn = bom_file_utils.find_pn(part)
+
+ query = mpn # TODO: suppor dpn
+
+ out_id = query
+
+ client.search(term="manuPartNum:" + query)
+
+ out_part = bomFile.Part(id=out_id,
+ distributor_info=bomFile.DistributorInfo(),
+ part_numbers=part.part_numbersProp)
+ di = out_part.distributor_infoProp
+
+ print("Saving {} work parts".format(out_parts.size()))
+ save_db(out_dir, out_parts)
diff --git a/src/ee/order/__init__.py b/src/ee/order/__init__.py
new file mode 100644
index 0000000..7c815b4
--- /dev/null
+++ b/src/ee/order/__init__.py
@@ -0,0 +1,88 @@
+from functools import total_ordering
+from pathlib import Path
+from typing import List, Tuple
+
+from ee.part import PartDb, load_db, save_db
+from ee.xml import bomFile, bom_file_utils
+
+__all__ = ["create_order"]
+
+
+@total_ordering
+class PartInfo(object):
+ def __init__(self, part: bomFile.Part):
+ self.part = part
+ self.id = part.id
+ self.pn = bom_file_utils.find_pn(part)
+ self.available_from: List[Tuple[str, bomFile.Part]] = []
+
+ def __lt__(self, other: "PartInfo"):
+ return self.part.idProp == other.part.idProp
+
+
+def create_order(schematic_dir: Path, out_dir: Path, part_db_dirs: List[Path], fail_on_missing_parts: bool):
+ sch_db = load_db(schematic_dir)
+
+ dbs = [(path.parent.name, load_db(path)) for path in part_db_dirs]
+
+ out_parts = PartDb()
+
+ infos = [PartInfo(sch) for sch in sch_db.iterparts()]
+
+ for info in infos:
+ print("Resolving {}".format(info.id))
+
+ for distributor, db in dbs:
+ for p in db.iterparts(sort=True):
+ if info.pn:
+ p_pn: str = bom_file_utils.find_pn(p)
+
+ if info.pn == p_pn:
+ info.available_from.append((distributor, p))
+
+ for sch_pn_ in bom_file_utils.part_numbers(info.part):
+ sch_pn: bomFile.PartNumber = sch_pn_
+
+ for p_pn_ in bom_file_utils.part_numbers(p):
+ p_pn: bomFile.PartNumber = p_pn_
+
+ if sch_pn.distributorProp == p_pn.distributorProp and sch_pn.value == p_pn.value:
+ if p.idProp not in info.available_from:
+ info.available_from.append((distributor, p))
+
+ for info in infos:
+ print("Resolved {} to {}".format(info.id, info.available_from))
+
+ warnings = []
+ has_missing_parts = False
+ for info in infos:
+ if len(info.available_from) == 0:
+ has_missing_parts = True
+ warnings.append("Could not find {} in any database, part numbers: {}".
+ format(info.id, ", ".join([p.value for p in bom_file_utils.part_numbers(info.part)])))
+
+ for w in sorted(warnings):
+ print(w)
+
+ if has_missing_parts and fail_on_missing_parts:
+ print("has missing parts")
+ return False
+
+ for info in infos:
+ part = bomFile.Part(id=info.part.id,
+ schema_reference=info.part.schema_reference,
+ part_numbers=bomFile.PartNumberList())
+
+ part_numbers = part.part_numbersProp.part_number
+
+ if len(info.available_from) == 0:
+ continue
+
+ distributor, distributor_part = info.available_from[0]
+
+ part_numbers.append(bomFile.PartNumber(value=distributor_part.id, distributor=distributor))
+
+ out_parts.add_entry(part, True)
+
+ print("Saving {} work parts".format(out_parts.size()))
+ save_db(out_dir, out_parts)
diff --git a/src/ee/part/__init__.py b/src/ee/part/__init__.py
index d4d99ac..26bc26c 100644
--- a/src/ee/part/__init__.py
+++ b/src/ee/part/__init__.py
@@ -40,8 +40,9 @@ class PartDb(object):
if e.new:
self.new_entries = self.new_entries + 1
- def iterparts(self) -> Iterator[bomFile.Part]:
- return [e.part for e in self.parts]
+ def iterparts(self, sort=False) -> Iterator[bomFile.Part]:
+ it = (e.part for e in self.parts)
+ return sorted(it, key=lambda p: p.idProp) if sort else it
def size(self) -> int:
return len(self.parts)
diff --git a/src/ee/project/__init__.py b/src/ee/project/__init__.py
new file mode 100644
index 0000000..187fc3c
--- /dev/null
+++ b/src/ee/project/__init__.py
@@ -0,0 +1,12 @@
+import configparser
+from pathlib import Path
+
+
+def load_config(project_dir: Path) -> configparser.ConfigParser:
+ config = configparser.ConfigParser()
+
+ config_path = project_dir / ".ee" / "config"
+ with config_path.open("r") as f:
+ config.read_file(f, source=str(config_path))
+
+ return config
diff --git a/src/ee/tools/create_order.py b/src/ee/tools/create_order.py
new file mode 100644
index 0000000..e0bed58
--- /dev/null
+++ b/src/ee/tools/create_order.py
@@ -0,0 +1,29 @@
+import argparse
+import sys
+from pathlib import Path
+
+from ee.order import create_order
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--schematic",
+ required=True,
+ metavar="DIR")
+
+parser.add_argument("--out",
+ required=True,
+ metavar="DIR")
+
+parser.add_argument("--part-db",
+ nargs="*",
+ required=True,
+ metavar="PART DB")
+
+args = parser.parse_args()
+
+part_db_dirs = [Path(part_db) for part_db in args.part_db]
+fail_on_missing_parts = False
+
+ret = create_order(Path(args.schematic), Path(args.out), part_db_dirs, fail_on_missing_parts)
+
+sys.exit(1 if ret is False else 0)
diff --git a/src/ee/tools/element14_search_parts.py b/src/ee/tools/element14_search_parts.py
new file mode 100644
index 0000000..a4793c5
--- /dev/null
+++ b/src/ee/tools/element14_search_parts.py
@@ -0,0 +1,30 @@
+import argparse
+from pathlib import Path
+
+from ee import project
+from ee.element14 import Element14Config
+from ee.element14.search_parts import search_parts
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--in",
+ dest="in_",
+ required=True,
+ metavar="FILE")
+
+parser.add_argument("--out",
+ required=True,
+ metavar="FILE")
+
+args = parser.parse_args()
+
+cache_dir = ".ee/cache/element14"
+
+config = project.load_config(Path("."))
+
+e14_config = Element14Config(
+ store=config.get("element14", "store", fallback=None),
+ api_key=config.get("element14", "api-key", fallback=None)
+)
+
+search_parts(Path(args.in_), Path(args.out), Path(cache_dir), e14_config)
diff --git a/src/ee/tools/import_parts_yaml.py b/src/ee/tools/import_parts_yaml.py
new file mode 100644
index 0000000..81a23f8
--- /dev/null
+++ b/src/ee/tools/import_parts_yaml.py
@@ -0,0 +1,52 @@
+import argparse
+import sys
+import yaml
+from pathlib import Path
+
+from ee.part import PartDb, save_db
+from ee.xml import bomFile
+
+
+def import_parts_yaml(in_file: Path, out_dir: Path):
+ with in_file.open("r") as f:
+ doc = yaml.safe_load(f)
+
+ if not isinstance(doc, list):
+ print("Bad YAML document, the root must be a list", file=sys.stderr)
+
+ parts = PartDb()
+ for item in doc:
+ if not isinstance(item, dict):
+ print("Bad YAML document, each part must be a dict", file=sys.stderr)
+ return
+
+ part_number_list = bomFile.PartNumberList()
+
+ mpn = item.get("mpn")
+ assert isinstance(mpn, str)
+ part_number_list.part_number.append(bomFile.PartNumber(value=mpn))
+
+ id_ = mpn
+
+ part = bomFile.Part(id=id_)
+ part.part_numbersProp = part_number_list
+ parts.add_entry(part, True)
+
+ print("Imported {} parts".format(parts.size()))
+ save_db(out_dir, parts)
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--in",
+ dest="in_",
+ required=True,
+ metavar="PARTS_YAML")
+
+parser.add_argument("--out",
+ required=True,
+ metavar="PART_ DB")
+
+args = parser.parse_args()
+
+import_parts_yaml(Path(args.in_), Path(args.out))
diff --git a/src/ee/tools/ninja.py b/src/ee/tools/ninja.py
index cc090a0..ed8e91a 100644
--- a/src/ee/tools/ninja.py
+++ b/src/ee/tools/ninja.py
@@ -1,4 +1,5 @@
import argparse
+import os.path
import sys
from pathlib import Path
from typing import List, Union, Optional
@@ -8,7 +9,10 @@ from jinja2 import Environment, PackageLoader, select_autoescape
from ee.kicad import read_schematics
-def ninja_path_filter(s: Union[str, List[str]]) -> str:
+def ninja_path_filter(s: Union[Path, str, List[str]]) -> str:
+ if isinstance(s, Path):
+ s = str(s)
+
if isinstance(s, str):
return s. \
replace("$", "$$"). \
@@ -23,8 +27,16 @@ def ninja_path_filter(s: Union[str, List[str]]) -> str:
raise Exception("Unsupported argument type: {}".format(type(s)))
-def parent_dir_filter(s: str) -> str:
- return str(Path(s).parent)
+def parent_dir_filter(s: str) -> Path:
+ return Path(s).parent
+
+
+def basename_filter(s: Union[str, Path]) -> str:
+ return os.path.basename(str(s))
+
+
+def noext_filter(s: Union[str, Path]) -> str:
+ return os.path.splitext(os.path.basename(str(s)))[0]
def generate(sch_path: Path, kicad_bom_strategy: Optional[str]):
@@ -36,6 +48,8 @@ def generate(sch_path: Path, kicad_bom_strategy: Optional[str]):
)
e.filters["ninja_path"] = ninja_path_filter
e.filters["parent_dir"] = parent_dir_filter
+ e.filters["basename"] = basename_filter
+ e.filters["noext"] = noext_filter
return e
gerber_zip = "prod/gerber.zip"
@@ -44,20 +58,30 @@ def generate(sch_path: Path, kicad_bom_strategy: Optional[str]):
sch_files = sorted([s.path for s in sch.schematics])
- params = {}
- import os.path
- params["ee"] = "{} -m ee".format(os.path.relpath(sys.executable, Path(".")))
- params["sch"] = sch_path
- params["sch_files"] = sch_files
-
- params["kicad_bom_strategy"] = kicad_bom_strategy
+ part_dbs = []
+ params = {
+ "ee": "{} -m ee".format(os.path.relpath(sys.executable, Path("."))), "sch": sch_path,
+ "sch_files": sch_files, "kicad_bom_strategy": kicad_bom_strategy,
+ "pcb": str(sch_path).replace(".sch", ".kicad_pcb"),
+ "part_dbs": part_dbs,
+ }
- params["pcb"] = str(sch_path).replace(".sch", ".kicad_pcb")
+ # TODO: read from config
+ distributors = ["digikey"]
+ params["distributors"] = distributors
if gerber_zip is not None:
params["gerber_zip"] = gerber_zip
- build_ninja = Path("build.ninja")
+ build_ninja = sch_path.parent / "build.ninja"
+
+ ee_dir = sch_path.parent / "ee"
+ parts_yaml_files = [path for path in ee_dir.iterdir() if str(path).endswith("-parts.yaml")]
+ params["parts_yaml_files"] = parts_yaml_files
+
+ # Local part databases first
+ part_dbs.extend([parent_dir_filter(p) / noext_filter(p) for p in parts_yaml_files])
+ part_dbs.extend([Path("ee") / d / "normalized" for d in distributors])
with build_ninja.open("w") as f:
env = _create_env()
@@ -77,4 +101,4 @@ parser.add_argument("--kicad-bom-strategy",
args = parser.parse_args()
-generate(args.sch, args.kicad_bom_strategy)
+generate(Path(args.sch), args.kicad_bom_strategy)
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index 79a2bc1..e91de6c 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -28,6 +28,22 @@ rule digikey-normalize-facts
description = digikey-normalize-facts
command = $ee digikey-normalize-facts --in $in_dir --out $out_dir
+rule element14-search-parts
+ description = element14-search-parts
+ command = $ee element14-search-parts --in $in_dir --out $out_dir
+
+rule element14-normalize-facts
+ description = element14-normalize-facts
+ command = $ee element14-normalize-facts --in $in_dir --out $out_dir
+
+rule create-order
+ description = create-order
+ command = $ee create-order --schematic $sch_dir --part-db $part_dbs --out $out_dir
+
+rule import-parts-yaml
+ description = import-parts-yaml $in
+ command = $ee import-parts-yaml --in $in --out $out_dir
+
{% if gerber_zip is defined %}
build gerbers: phony {{ gerber_zip }}
build {{ gerber_zip }}: kicad-gerber $pcb
@@ -38,16 +54,33 @@ build ee/sch/index.xml: kicad-make-bom $sch
out_dir = ee/sch
strategy = {{ "--strategy " + kicad_bom_strategy if kicad_bom_strategy else "" }}
-build ee/digikey/search-list/index.xml: part-create-distributor-search-list ee/sch/index.xml
+{%- for d in distributors %}
+# Distributor {{ d }}
+build ee/{{ d }}/search-list/index.xml: part-create-distributor-search-list ee/sch/index.xml
in_dir = ee/sch
- out_dir = ee/digikey/search-list
+ out_dir = ee/{{ d }}/search-list
+
+build ee/{{ d }}/downloaded/index.xml: {{ d }}-search-parts ee/{{ d }}/search-list/index.xml
+ in_dir = ee/{{ d }}/search-list
+ out_dir = ee/{{ d }}/downloaded
+
+build ee/{{ d }}/normalized/index.xml: {{ d }}-normalize-facts ee/{{ d }}/downloaded/index.xml
+ in_dir = ee/{{ d }}/downloaded
+ out_dir = ee/{{ d }}/normalized
+
+default ee/{{ d }}/normalized/index.xml
+{%- endfor %}
-build ee/digikey/downloaded/index.xml: digikey-search-parts ee/digikey/search-list/index.xml
- in_dir = ee/digikey/search-list
- out_dir = ee/digikey/downloaded
+{%- for f in parts_yaml_files %}
+{% set out=(f | parent_dir) / (f | noext) / "index.xml" %}
+build {{ out }}: import-parts-yaml {{ f }}
+ out_dir = {{ f | parent_dir }}/{{ f | noext }}
+# default {{ out }}
+{% endfor %}
-build ee/digikey/normalized/index.xml: digikey-normalize-facts ee/digikey/downloaded/index.xml
- in_dir = ee/digikey/downloaded
- out_dir = ee/digikey/normalized
+build ee/order/index.xml: create-order ee/sch/index.xml {%- for p in part_dbs %} {{ p / "index.xml" }}{% endfor %}
+ sch_dir = ee/sch
+ out_dir = ee/order
+ part_dbs ={%- for p in part_dbs %} {{ p }}{% endfor %}
-default ee/digikey/normalized/index.xml
+default ee/order/index.xml
diff --git a/src/ee/xml/bom_file_utils.py b/src/ee/xml/bom_file_utils.py
index d1f8be9..063a7bd 100644
--- a/src/ee/xml/bom_file_utils.py
+++ b/src/ee/xml/bom_file_utils.py
@@ -28,13 +28,13 @@ def part_numbers(part: bomFile.Part) -> List[bomFile.PartNumber]:
return pns.part_numberProp
-def find_pn(part: bomFile.Part) -> str:
+def find_pn(part: bomFile.Part) -> Optional[str]:
for pn in part_numbers(part):
if pn.distributor is None:
return pn.value
-def find_dpn(part: bomFile.Part, distributor: str) -> str:
+def find_dpn(part: bomFile.Part, distributor: str) -> Optional[str]:
for pn in part_numbers(part):
if pn.distributor == distributor:
return pn.value
diff --git a/xsd/ee-bom.xsd b/xsd/ee-bom.xsd
index dddfa8a..b88a87e 100644
--- a/xsd/ee-bom.xsd
+++ b/xsd/ee-bom.xsd
@@ -1,3 +1,31 @@
+<!--
+Link:
+ - url
+ - relation
+ - media-type
+
+SupplierDatabase
+ Supplier
+ - url
+ - id (short name, used for UI)
+
+ExchangeRateDatabase
+ ExchangeRate
+ - from
+ - to
+ - rate
+
+PartListFile
+ PartList
+ Pricing (also used directly on a Part)
+
+PartListFile is used for
+* BOM (schema export)
+* downloading facts and prices from suppliers
+* Creating orders from sets of available part lists
+
+TODO: rename 'id' to 'url'.
+-->
<xs:schema
xmlns:xs="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://purl.org/ee/bom-file"
@@ -15,6 +43,7 @@
</xs:complexType>
<xs:complexType name="Part">
+ <!-- TODO: add links, references to product page,data sheets. -->
<xs:sequence>
<xs:element name="schema-reference" type="xs:string"/>
<xs:element name="part-type" type="xs:anyURI"/>
generated by cgit v1.2.3 (git 2.46.0) at 2024-12-22 22:49:08 +0000