seeed-download-opl

o Change to download one OPL at the time. Enables Ninja to run the
  downloads concurrently.
init/ninja:
o Learning about seeed-download-opl.

5 files changed, 118 insertions, 90 deletions

diff --git a/src/ee/part/create_distributor_search_list.py b/src/ee/part/create_distributor_search_list.py
index be362a8..fef1690 100644
--- a/src/ee/part/create_distributor_search_list.py
+++ b/src/ee/part/create_distributor_search_list.py
@@ -17,8 +17,8 @@ def create_distributor_search_list(in_path: Path, out_path: Path):
         pn_value = next((p.valueProp for p in part.get_mpns()), None)
 
         if pn_value is None:
-            # TODO: use schematic reference if found
-            print("Skipping part with no part number: uri={}".format(xml.uriProp))
+            refs = [ref.referenceProp for ref in part.get_schematic_references()]
+            print("Skipping part with no part number: schematic reference: {}".format(", ".join(refs)))
             continue
 
         entry = out_parts.find_by_pn(pn_value)
diff --git a/src/ee/supplier/seeed.py b/src/ee/supplier/seeed.py
index 45cff44..b4b17e9 100644
--- a/src/ee/supplier/seeed.py
+++ b/src/ee/supplier/seeed.py
@@ -58,92 +58,96 @@ def _get(obj, key):
     return value.strip()
 
 
-def download_opls(out_dir: Path, cache_dir: Path):
-    opls_types = ["SEEED", "HQCHIP"]
-
+def download_opl(out_path: Path, cache_dir: Path, opl_type: str):
     s = requests.Session()
 
     from ee.money import get_default_context
     money = get_default_context()
 
-    for opls_type in opls_types:
-        page_offset = 1
-        page_length = 30
-        count = 0
-
-        cd = cache_dir / opls_type if cache_dir else cache_dir
-        cache = ee._utils.maybe_cache(cd, ext="json")
-
-        db = PartDb()
-
-        while True:
-            cache_key = "{}-{}".format(opls_type, page_offset)
-            content = cache.lookup(cache_key)
-            if content is None:
-                form = {
-                    "page_offset": page_offset,
-                    "page_length": page_length,
-                    "keyword": "",
-                    "category": "",
-                    "type": opls_type,
-                }
-                query = {
-                    "guid": "A8D502008EB2E4FD3FE9CE5E0855E8E4",
-                    "appid": "en.pc.bazaar",
-                }
-                r = s.get("https://sapi.seeedstudio.com/fusion/opl/list", params=query, data=form)
-                content = r.text
-                cache.save(cache_key, content)
-
-            obj = json.loads(content)
-            supplier_uri = "http://purl.org/ee/supplier/seeed?{}".format(urlencode({"opl": opls_type}))
-
-            parts = obj["data"]["list"]
-
-            for p in parts:
-                # print(json.dumps(p, indent=4))
-                mpn = _get(p, "mpn")
-                sku = _get(p, "sku")
-                datasheet = _get(p, "datasheet")
-                desc = _get(p, "desc")
-                package = _get(p, "package")
-
-                if mpn is None and sku is None:
-                    continue
-
-                ladder_price = p["ladder_price"]
-
-                xml = types.Part(references=types.ReferencesList(), price_breaks=types.PriceBreakList())
-                xml.supplierProp = supplier_uri
-
-                xml.descriptionProp = desc
-
-                uri_params = {
-                    "opl": opls_type,
-                }
-                if sku:
-                    uri_params["sku"] = sku
-                if mpn:
-                    uri_params["mpn"] = mpn
-                xml.uriProp = "http://purl.org/ee/supplier/seeed?{}".format(urlencode(uri_params))
-
-                part = Part(xml)
-
-                if mpn:
-                    part.add_mpn(mpn)
-                if sku:
-                    part.add_spn(sku)
-
-                for item in ladder_price:
-                    price = money.parse(item["price"], currency="USD")
-                    part.add_price_break(item["qty"], price)
-
-                db.add_entry(part, True)
-
-            page_offset += 1
-            count += len(parts)
-            if len(parts) != page_length:
-                break
-
-        print("Imported {} parts from Seeed's {} library".format(count, opls_type))
-        save_db(out_dir / "{}.xml".format(opls_type), db, sort=True)
+    page_offset = 1
+    page_length = 30
+    count = 0
+
+    cd = cache_dir / opl_type if cache_dir else cache_dir
+    cache = ee._utils.maybe_cache(cd, ext="json")
+
+    db = PartDb()
+
+    while True:
+        cache_key = "{}-{}".format(opl_type, page_offset)
+        content = cache.lookup(cache_key)
+        if content is None:
+            form = {
+                "page_offset": page_offset,
+                "page_length": page_length,
+                "keyword": "",
+                "category": "",
+                "type": opl_type,
+            }
+            query = {
+                "guid": "A8D502008EB2E4FD3FE9CE5E0855E8E4",
+                "appid": "en.pc.bazaar",
+            }
+            r = s.get("https://sapi.seeedstudio.com/fusion/opl/list", params=query, data=form)
+            content = r.text
+            cache.save(cache_key, content)
+
+        obj = json.loads(content)
+        supplier_uri = "http://purl.org/ee/supplier/seeed?{}".format(urlencode({"opl": opl_type}))
+
+        parts = obj["data"]["list"]
+
+        for p in parts:
+            # print(json.dumps(p, indent=4))
+            mpn = _get(p, "mpn")
+            sku = _get(p, "sku")
+            datasheet = _get(p, "datasheet")
+            desc = _get(p, "desc")
+            package = _get(p, "package")
+
+            if mpn is None and sku is None:
+                continue
+
+            ladder_price = p["ladder_price"]
+
+            xml = types.Part(references=types.ReferencesList(), price_breaks=types.PriceBreakList())
+            xml.supplierProp = supplier_uri
+
+            xml.descriptionProp = desc
+
+            uri_params = {
+                "opl": opl_type,
+            }
+            if sku:
+                uri_params["sku"] = sku
+            if mpn:
+                uri_params["mpn"] = mpn
+            xml.uriProp = "http://purl.org/ee/supplier/seeed?{}".format(urlencode(uri_params))
+
+            part = Part(xml)
+
+            if mpn:
+                part.add_mpn(mpn)
+            if sku:
+                part.add_spn(sku)
+
+            for item in ladder_price:
+                price = money.parse(item["price"], currency="USD")
+                part.add_price_break(item["qty"], price)
+
+            db.add_entry(part, True)
+
+        page_offset += 1
+        count += len(parts)
+        if len(parts) != page_length:
+            break
+
+    print("Imported {} parts from Seeed's {} library".format(count, opl_type))
+    save_db(out_path, db, sort=True)
+
+
+def init_project(project):
+    opls = ["SEEED", "HQCHIP"]
+
+    project.cfg.add_section("seeed-opl")
+    project.cfg["seeed-opl"]["opls"] = ", ".join(opls)
diff --git a/src/ee/tools/init.py b/src/ee/tools/init.py
index 81bdda3..f87c958 100644
--- a/src/ee/tools/init.py
+++ b/src/ee/tools/init.py
@@ -7,7 +7,6 @@ import configparser
 
 
 def init_kicad_project(basedir: Path, cfg):
-    print("basedir={}".format(basedir))
     pro_files: List[Path] = [f for f in basedir.iterdir() if f.name.endswith(".pro")]
 
     if len(pro_files) == 0:
@@ -30,11 +29,17 @@ def init_kicad_project(basedir: Path, cfg):
         print("Found more than one kicad project file.")
 
 
+def init_seeed_opl(project: Project):
+    import ee.supplier.seeed
+    ee.supplier.seeed.init_project(project)
+
+
 def init(project_dir: Path, basedir: Path):
     cfg = configparser.ConfigParser()
     project = Project(project_dir, cfg)
 
     init_kicad_project(basedir, cfg)
+    init_seeed_opl(project)
 
     print("Saving project. Now run 'ee ninja' to generate Ninja build file")
     project.save()
diff --git a/src/ee/tools/seeed_download_opl.py b/src/ee/tools/seeed_download_opl.py
index e53361f..3f8b07b 100644
--- a/src/ee/tools/seeed_download_opl.py
+++ b/src/ee/tools/seeed_download_opl.py
@@ -5,12 +5,16 @@ from ee.supplier import seeed
 
 parser = argparse.ArgumentParser()
 
-parser.add_argument("--out-dir",
+parser.add_argument("--out",
                     required=True,
-                    metavar="DIR FOR PART DBS")
+                    metavar="PART DB")
+
+parser.add_argument("--opl",
+                    required=True,
+                    metavar="OPL")
 
 args = parser.parse_args()
 
 cache_dir = ".ee/cache/seeed"
 
-seeed.download_opls(Path(args.out_dir), Path(cache_dir))
+seeed.download_opl(Path(args.out), Path(cache_dir), args.opl)
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index 080ac65..227b68c 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -82,3 +82,18 @@ build ee/order.xml | $report_dir/order.rst: create-order ee/sch.xml {%- for p in
     part_dbs ={%- for p in part_dbs %} {{ p }}.xml{% endfor %}
 
 default ee/order.xml
+
+rule seeed-download-opl
+    description = seeed-download-opl $opl
+    command = $ee seeed-download-opl --out $out --opl $opl
+
+{% if project.cfg.has_section("seeed-opl") -%}
+{% set opls=project.cfg["seeed-opl"]["opls"].split(",") -%}
+build seeed-download-opls: phony{%- for opl in opls %} ee/seeed/opl/{{ opl.strip() }}.xml{% endfor %}
+{%- for opl in opls %}
+{% set opl=opl.strip() -%}
+
+build ee/seeed/opl/{{ opl }}.xml: seeed-download-opl
+    opl = {{ opl }}
+{% endfor -%}
+{% endif -%}