o Adding module for searching on element14.

o Starting on functionality create orders. Very WIP.
o Adding a concept of an "ee project". Can load a gitconfig-like config
  file.
o Adding a tool to import a yaml file into a part xml file.

14 files changed, 425 insertions, 28 deletions

diff --git a/.gitignore b/.gitignore
index 9636df4..972d5ed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *.tmp
-digikey_cache
+.ee
+*.patch
 
 *.pyc
 env
diff --git a/README.md b/README.md
index d280377..06cbc0d 100644
--- a/README.md
+++ b/README.md
@@ -31,5 +31,9 @@ part. Some cases where this is not true:
 
 # TODOs
 
-* Consistenly use `read` and `parse` in function names.
+* Consistently use `read` and `parse` in function names.
 * Consider flattening the module tree.
+
+* Rename some stuff:
+  * Rename "distributor" to "supplier"
+  * Rename "distributor part number/DPN" to "SKU".
diff --git a/src/ee/element14/__init__.py b/src/ee/element14/__init__.py
new file mode 100644
index 0000000..17a8825
--- /dev/null
+++ b/src/ee/element14/__init__.py
@@ -0,0 +1,63 @@
+import json
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlencode
+from urllib.request import urlopen
+
+import ee._utils
+
+__all__ = [
+    "Element14Config",
+    "Element14Client",
+]
+
+
+class Element14Config(object):
+    def __init__(self, store: Optional[str], api_key: Optional[str]):
+        self.store = store
+        self.api_key = api_key
+
+
+class Element14Client(object):
+    def __init__(self, config: Element14Config, cache_dir: Path):
+        self.config = config
+        self.cache = ee._utils.maybe_cache(cache_dir)
+
+    def search(self, term: Optional[str] = None):
+        url = "https://api.element14.com/catalog/products"
+
+        kv = {
+            "callInfo.responseDataFormat": "XML",
+        }
+
+        if self.config.api_key:
+            kv["callInfo.apiKey"] = self.config.api_key
+
+        if self.config.store:
+            kv["storeInfo.id"] = self.config.store
+
+        if term:
+            kv["term"] = term
+
+        kv["resultsSettings.offset"] = "0"
+        kv["resultsSettings.numberOfResults"] = "100"
+
+        print("params: {}".format(kv))
+
+        # &resultsSettings.refinements.filters={rohsCompliant,inStock}
+        # &resultsSettings.responseGroup={none,small,medium,large, Prices, Inventory}
+
+        url = url + "?" + urlencode(kv)
+
+        print("url={}".format(url))
+
+        data = "wat!!"
+        data = urlopen(url).read()
+
+        print("-----------")
+        print(data)
+        print("-----------")
+
+        search_response = json.loads(data)
+
+        return
diff --git a/src/ee/element14/search_parts.py b/src/ee/element14/search_parts.py
new file mode 100644
index 0000000..724485c
--- /dev/null
+++ b/src/ee/element14/search_parts.py
@@ -0,0 +1,31 @@
+from pathlib import Path
+
+from ee.element14 import *
+from ee.part import PartDb, load_db, save_db
+from ee.xml import bom_file_utils, bomFile
+
+__all__ = ["search_parts"]
+
+
+def search_parts(in_dir: Path, out_dir: Path, cache_dir: Path, config: Element14Config):
+    in_db = load_db(in_dir)
+    out_parts = PartDb()
+
+    client = Element14Client(config, cache_dir)
+
+    for part in in_db.iterparts():
+        mpn = bom_file_utils.find_pn(part)
+
+        query = mpn  # TODO: suppor dpn
+
+        out_id = query
+
+        client.search(term="manuPartNum:" + query)
+
+        out_part = bomFile.Part(id=out_id,
+                                distributor_info=bomFile.DistributorInfo(),
+                                part_numbers=part.part_numbersProp)
+        di = out_part.distributor_infoProp
+
+    print("Saving {} work parts".format(out_parts.size()))
+    save_db(out_dir, out_parts)
diff --git a/src/ee/order/__init__.py b/src/ee/order/__init__.py
new file mode 100644
index 0000000..7c815b4
--- /dev/null
+++ b/src/ee/order/__init__.py
@@ -0,0 +1,88 @@
+from functools import total_ordering
+from pathlib import Path
+from typing import List, Tuple
+
+from ee.part import PartDb, load_db, save_db
+from ee.xml import bomFile, bom_file_utils
+
+__all__ = ["create_order"]
+
+
+@total_ordering
+class PartInfo(object):
+    def __init__(self, part: bomFile.Part):
+        self.part = part
+        self.id = part.id
+        self.pn = bom_file_utils.find_pn(part)
+        self.available_from: List[Tuple[str, bomFile.Part]] = []
+
+    def __lt__(self, other: "PartInfo"):
+        return self.part.idProp == other.part.idProp
+
+
+def create_order(schematic_dir: Path, out_dir: Path, part_db_dirs: List[Path], fail_on_missing_parts: bool):
+    sch_db = load_db(schematic_dir)
+
+    dbs = [(path.parent.name, load_db(path)) for path in part_db_dirs]
+
+    out_parts = PartDb()
+
+    infos = [PartInfo(sch) for sch in sch_db.iterparts()]
+
+    for info in infos:
+        print("Resolving {}".format(info.id))
+
+        for distributor, db in dbs:
+            for p in db.iterparts(sort=True):
+                if info.pn:
+                    p_pn: str = bom_file_utils.find_pn(p)
+
+                    if info.pn == p_pn:
+                        info.available_from.append((distributor, p))
+
+                for sch_pn_ in bom_file_utils.part_numbers(info.part):
+                    sch_pn: bomFile.PartNumber = sch_pn_
+
+                    for p_pn_ in bom_file_utils.part_numbers(p):
+                        p_pn: bomFile.PartNumber = p_pn_
+
+                        if sch_pn.distributorProp == p_pn.distributorProp and sch_pn.value == p_pn.value:
+                            if p.idProp not in info.available_from:
+                                info.available_from.append((distributor, p))
+
+    for info in infos:
+        print("Resolved {} to {}".format(info.id, info.available_from))
+
+    warnings = []
+    has_missing_parts = False
+    for info in infos:
+        if len(info.available_from) == 0:
+            has_missing_parts = True
+            warnings.append("Could not find {} in any database, part numbers: {}".
+                            format(info.id, ", ".join([p.value for p in bom_file_utils.part_numbers(info.part)])))
+
+    for w in sorted(warnings):
+        print(w)
+
+    if has_missing_parts and fail_on_missing_parts:
+        print("has missing parts")
+        return False
+
+    for info in infos:
+        part = bomFile.Part(id=info.part.id,
+                            schema_reference=info.part.schema_reference,
+                            part_numbers=bomFile.PartNumberList())
+
+        part_numbers = part.part_numbersProp.part_number
+
+        if len(info.available_from) == 0:
+            continue
+
+        distributor, distributor_part = info.available_from[0]
+
+        part_numbers.append(bomFile.PartNumber(value=distributor_part.id, distributor=distributor))
+
+        out_parts.add_entry(part, True)
+
+    print("Saving {} work parts".format(out_parts.size()))
+    save_db(out_dir, out_parts)
diff --git a/src/ee/part/__init__.py b/src/ee/part/__init__.py
index d4d99ac..26bc26c 100644
--- a/src/ee/part/__init__.py
+++ b/src/ee/part/__init__.py
@@ -40,8 +40,9 @@ class PartDb(object):
         if e.new:
             self.new_entries = self.new_entries + 1
 
-    def iterparts(self) -> Iterator[bomFile.Part]:
-        return [e.part for e in self.parts]
+    def iterparts(self, sort=False) -> Iterator[bomFile.Part]:
+        it = (e.part for e in self.parts)
+        return sorted(it, key=lambda p: p.idProp) if sort else it
 
     def size(self) -> int:
         return len(self.parts)
diff --git a/src/ee/project/__init__.py b/src/ee/project/__init__.py
new file mode 100644
index 0000000..187fc3c
--- /dev/null
+++ b/src/ee/project/__init__.py
@@ -0,0 +1,12 @@
+import configparser
+from pathlib import Path
+
+
+def load_config(project_dir: Path) -> configparser.ConfigParser:
+    config = configparser.ConfigParser()
+
+    config_path = project_dir / ".ee" / "config"
+    with config_path.open("r") as f:
+        config.read_file(f, source=str(config_path))
+
+    return config
diff --git a/src/ee/tools/create_order.py b/src/ee/tools/create_order.py
new file mode 100644
index 0000000..e0bed58
--- /dev/null
+++ b/src/ee/tools/create_order.py
@@ -0,0 +1,29 @@
+import argparse
+import sys
+from pathlib import Path
+
+from ee.order import create_order
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--schematic",
+                    required=True,
+                    metavar="DIR")
+
+parser.add_argument("--out",
+                    required=True,
+                    metavar="DIR")
+
+parser.add_argument("--part-db",
+                    nargs="*",
+                    required=True,
+                    metavar="PART DB")
+
+args = parser.parse_args()
+
+part_db_dirs = [Path(part_db) for part_db in args.part_db]
+fail_on_missing_parts = False
+
+ret = create_order(Path(args.schematic), Path(args.out), part_db_dirs, fail_on_missing_parts)
+
+sys.exit(1 if ret is False else 0)
diff --git a/src/ee/tools/element14_search_parts.py b/src/ee/tools/element14_search_parts.py
new file mode 100644
index 0000000..a4793c5
--- /dev/null
+++ b/src/ee/tools/element14_search_parts.py
@@ -0,0 +1,30 @@
+import argparse
+from pathlib import Path
+
+from ee import project
+from ee.element14 import Element14Config
+from ee.element14.search_parts import search_parts
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--in",
+                    dest="in_",
+                    required=True,
+                    metavar="FILE")
+
+parser.add_argument("--out",
+                    required=True,
+                    metavar="FILE")
+
+args = parser.parse_args()
+
+cache_dir = ".ee/cache/element14"
+
+config = project.load_config(Path("."))
+
+e14_config = Element14Config(
+    store=config.get("element14", "store", fallback=None),
+    api_key=config.get("element14", "api-key", fallback=None)
+)
+
+search_parts(Path(args.in_), Path(args.out), Path(cache_dir), e14_config)
diff --git a/src/ee/tools/import_parts_yaml.py b/src/ee/tools/import_parts_yaml.py
new file mode 100644
index 0000000..81a23f8
--- /dev/null
+++ b/src/ee/tools/import_parts_yaml.py
@@ -0,0 +1,52 @@
+import argparse
+import sys
+import yaml
+from pathlib import Path
+
+from ee.part import PartDb, save_db
+from ee.xml import bomFile
+
+
+def import_parts_yaml(in_file: Path, out_dir: Path):
+    with in_file.open("r") as f:
+        doc = yaml.safe_load(f)
+
+    if not isinstance(doc, list):
+        print("Bad YAML document, the root must be a list", file=sys.stderr)
+
+    parts = PartDb()
+    for item in doc:
+        if not isinstance(item, dict):
+            print("Bad YAML document, each part must be a dict", file=sys.stderr)
+            return
+
+        part_number_list = bomFile.PartNumberList()
+
+        mpn = item.get("mpn")
+        assert isinstance(mpn, str)
+        part_number_list.part_number.append(bomFile.PartNumber(value=mpn))
+
+        id_ = mpn
+
+        part = bomFile.Part(id=id_)
+        part.part_numbersProp = part_number_list
+        parts.add_entry(part, True)
+
+    print("Imported {} parts".format(parts.size()))
+    save_db(out_dir, parts)
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--in",
+                    dest="in_",
+                    required=True,
+                    metavar="PARTS_YAML")
+
+parser.add_argument("--out",
+                    required=True,
+                    metavar="PART_ DB")
+
+args = parser.parse_args()
+
+import_parts_yaml(Path(args.in_), Path(args.out))
diff --git a/src/ee/tools/ninja.py b/src/ee/tools/ninja.py
index cc090a0..ed8e91a 100644
--- a/src/ee/tools/ninja.py
+++ b/src/ee/tools/ninja.py
@@ -1,4 +1,5 @@
 import argparse
+import os.path
 import sys
 from pathlib import Path
 from typing import List, Union, Optional
@@ -8,7 +9,10 @@ from jinja2 import Environment, PackageLoader, select_autoescape
 from ee.kicad import read_schematics
 
 
-def ninja_path_filter(s: Union[str, List[str]]) -> str:
+def ninja_path_filter(s: Union[Path, str, List[str]]) -> str:
+    if isinstance(s, Path):
+        s = str(s)
+
     if isinstance(s, str):
         return s. \
             replace("$", "$$"). \
@@ -23,8 +27,16 @@ def ninja_path_filter(s: Union[str, List[str]]) -> str:
         raise Exception("Unsupported argument type: {}".format(type(s)))
 
 
-def parent_dir_filter(s: str) -> str:
-    return str(Path(s).parent)
+def parent_dir_filter(s: str) -> Path:
+    return Path(s).parent
+
+
+def basename_filter(s: Union[str, Path]) -> str:
+    return os.path.basename(str(s))
+
+
+def noext_filter(s: Union[str, Path]) -> str:
+    return os.path.splitext(os.path.basename(str(s)))[0]
 
 
 def generate(sch_path: Path, kicad_bom_strategy: Optional[str]):
@@ -36,6 +48,8 @@ def generate(sch_path: Path, kicad_bom_strategy: Optional[str]):
         )
         e.filters["ninja_path"] = ninja_path_filter
         e.filters["parent_dir"] = parent_dir_filter
+        e.filters["basename"] = basename_filter
+        e.filters["noext"] = noext_filter
         return e
 
     gerber_zip = "prod/gerber.zip"
@@ -44,20 +58,30 @@ def generate(sch_path: Path, kicad_bom_strategy: Optional[str]):
 
     sch_files = sorted([s.path for s in sch.schematics])
 
-    params = {}
-    import os.path
-    params["ee"] = "{} -m ee".format(os.path.relpath(sys.executable, Path(".")))
-    params["sch"] = sch_path
-    params["sch_files"] = sch_files
-
-    params["kicad_bom_strategy"] = kicad_bom_strategy
+    part_dbs = []
+    params = {
+        "ee": "{} -m ee".format(os.path.relpath(sys.executable, Path("."))), "sch": sch_path,
+        "sch_files": sch_files, "kicad_bom_strategy": kicad_bom_strategy,
+        "pcb": str(sch_path).replace(".sch", ".kicad_pcb"),
+        "part_dbs": part_dbs,
+    }
 
-    params["pcb"] = str(sch_path).replace(".sch", ".kicad_pcb")
+    # TODO: read from config
+    distributors = ["digikey"]
+    params["distributors"] = distributors
 
     if gerber_zip is not None:
         params["gerber_zip"] = gerber_zip
 
-    build_ninja = Path("build.ninja")
+    build_ninja = sch_path.parent / "build.ninja"
+
+    ee_dir = sch_path.parent / "ee"
+    parts_yaml_files = [path for path in ee_dir.iterdir() if str(path).endswith("-parts.yaml")]
+    params["parts_yaml_files"] = parts_yaml_files
+
+    # Local part databases first
+    part_dbs.extend([parent_dir_filter(p) / noext_filter(p) for p in parts_yaml_files])
+    part_dbs.extend([Path("ee") / d / "normalized" for d in distributors])
 
     with build_ninja.open("w") as f:
         env = _create_env()
@@ -77,4 +101,4 @@ parser.add_argument("--kicad-bom-strategy",
 
 args = parser.parse_args()
 
-generate(args.sch, args.kicad_bom_strategy)
+generate(Path(args.sch), args.kicad_bom_strategy)
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index 79a2bc1..e91de6c 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -28,6 +28,22 @@ rule digikey-normalize-facts
     description = digikey-normalize-facts
     command = $ee digikey-normalize-facts --in $in_dir --out $out_dir
 
+rule element14-search-parts
+    description = element14-search-parts
+    command = $ee element14-search-parts --in $in_dir --out $out_dir
+
+rule element14-normalize-facts
+    description = element14-normalize-facts
+    command = $ee element14-normalize-facts --in $in_dir --out $out_dir
+
+rule create-order
+    description = create-order
+    command = $ee create-order --schematic $sch_dir --part-db $part_dbs --out $out_dir
+
+rule import-parts-yaml
+    description = import-parts-yaml $in
+    command = $ee import-parts-yaml --in $in --out $out_dir
+
 {% if gerber_zip is defined %}
 build gerbers: phony {{ gerber_zip }}
 build {{ gerber_zip }}: kicad-gerber $pcb
@@ -38,16 +54,33 @@ build ee/sch/index.xml: kicad-make-bom $sch
     out_dir = ee/sch
     strategy = {{ "--strategy " + kicad_bom_strategy if kicad_bom_strategy else "" }}
 
-build ee/digikey/search-list/index.xml: part-create-distributor-search-list ee/sch/index.xml
+{%- for d in distributors %}
+# Distributor {{ d }}
+build ee/{{ d }}/search-list/index.xml: part-create-distributor-search-list ee/sch/index.xml
     in_dir = ee/sch
-    out_dir = ee/digikey/search-list
+    out_dir = ee/{{ d }}/search-list
+
+build ee/{{ d }}/downloaded/index.xml: {{ d }}-search-parts ee/{{ d }}/search-list/index.xml
+    in_dir = ee/{{ d }}/search-list
+    out_dir = ee/{{ d }}/downloaded
+
+build ee/{{ d }}/normalized/index.xml: {{ d }}-normalize-facts ee/{{ d }}/downloaded/index.xml
+    in_dir = ee/{{ d }}/downloaded
+    out_dir = ee/{{ d }}/normalized
+
+default ee/{{ d }}/normalized/index.xml
+{%- endfor %}
 
-build ee/digikey/downloaded/index.xml: digikey-search-parts ee/digikey/search-list/index.xml
-    in_dir = ee/digikey/search-list
-    out_dir = ee/digikey/downloaded
+{%- for f in parts_yaml_files %}
+{% set out=(f | parent_dir) / (f | noext) / "index.xml" %}
+build {{ out }}: import-parts-yaml {{ f }}
+    out_dir = {{ f | parent_dir }}/{{ f | noext }}
+# default {{ out }}
+{% endfor %}
 
-build ee/digikey/normalized/index.xml: digikey-normalize-facts ee/digikey/downloaded/index.xml
-    in_dir = ee/digikey/downloaded
-    out_dir = ee/digikey/normalized
+build ee/order/index.xml: create-order ee/sch/index.xml {%- for p in part_dbs %} {{ p / "index.xml" }}{% endfor %}
+    sch_dir = ee/sch
+    out_dir = ee/order
+    part_dbs ={%- for p in part_dbs %} {{ p }}{% endfor %}
 
-default ee/digikey/normalized/index.xml
+default ee/order/index.xml
diff --git a/src/ee/xml/bom_file_utils.py b/src/ee/xml/bom_file_utils.py
index d1f8be9..063a7bd 100644
--- a/src/ee/xml/bom_file_utils.py
+++ b/src/ee/xml/bom_file_utils.py
@@ -28,13 +28,13 @@ def part_numbers(part: bomFile.Part) -> List[bomFile.PartNumber]:
     return pns.part_numberProp
 
 
-def find_pn(part: bomFile.Part) -> str:
+def find_pn(part: bomFile.Part) -> Optional[str]:
     for pn in part_numbers(part):
         if pn.distributor is None:
             return pn.value
 
 
-def find_dpn(part: bomFile.Part, distributor: str) -> str:
+def find_dpn(part: bomFile.Part, distributor: str) -> Optional[str]:
     for pn in part_numbers(part):
         if pn.distributor == distributor:
             return pn.value
diff --git a/xsd/ee-bom.xsd b/xsd/ee-bom.xsd
index dddfa8a..b88a87e 100644
--- a/xsd/ee-bom.xsd
+++ b/xsd/ee-bom.xsd
@@ -1,3 +1,31 @@
+<!--
+Link:
+    - url
+    - relation
+    - media-type
+
+SupplierDatabase
+    Supplier
+        - url
+        - id (short name, used for UI)
+
+ExchangeRateDatabase
+    ExchangeRate
+        - from
+        - to
+        - rate
+
+PartListFile
+  PartList
+  Pricing (also used directly on a Part)
+
+PartListFile is used for
+* BOM (schema export)
+* downloading facts and prices from suppliers
+* Creating orders from sets of available part lists
+
+TODO: rename 'id' to 'url'.
+-->
 <xs:schema
         xmlns:xs="http://www.w3.org/2001/XMLSchema"
         targetNamespace="http://purl.org/ee/bom-file"
@@ -15,6 +43,7 @@
     </xs:complexType>
 
     <xs:complexType name="Part">
+        <!-- TODO: add links, references to product page,data sheets. -->
         <xs:sequence>
             <xs:element name="schema-reference" type="xs:string"/>
             <xs:element name="part-type" type="xs:anyURI"/>