From d9daf4c1d0abf0d93702e56028e8f746b0f475bc Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Thu, 28 Mar 2019 17:42:05 +0100 Subject: seeed-download-opl o Change to download one OPL at the time. Enables Ninja to run the downloads concurrently. init/ninja: o Learning about seeed-download-opl. --- src/ee/supplier/seeed.py | 172 ++++++++++++++++++++++++----------------------- 1 file changed, 88 insertions(+), 84 deletions(-) (limited to 'src/ee/supplier') diff --git a/src/ee/supplier/seeed.py b/src/ee/supplier/seeed.py index 45cff44..b4b17e9 100644 --- a/src/ee/supplier/seeed.py +++ b/src/ee/supplier/seeed.py @@ -58,92 +58,96 @@ def _get(obj, key): return value.strip() -def download_opls(out_dir: Path, cache_dir: Path): - opls_types = ["SEEED", "HQCHIP"] - +def download_opl(out_path: Path, cache_dir: Path, opl_type: str): s = requests.Session() from ee.money import get_default_context money = get_default_context() - for opls_type in opls_types: - page_offset = 1 - page_length = 30 - count = 0 - - cd = cache_dir / opls_type if cache_dir else cache_dir - cache = ee._utils.maybe_cache(cd, ext="json") - - db = PartDb() - - while True: - cache_key = "{}-{}".format(opls_type, page_offset) - content = cache.lookup(cache_key) - if content is None: - form = { - "page_offset": page_offset, - "page_length": page_length, - "keyword": "", - "category": "", - "type": opls_type, - } - query = { - "guid": "A8D502008EB2E4FD3FE9CE5E0855E8E4", - "appid": "en.pc.bazaar", - } - r = s.get("https://sapi.seeedstudio.com/fusion/opl/list", params=query, data=form) - content = r.text - cache.save(cache_key, content) - - obj = json.loads(content) - supplier_uri = "http://purl.org/ee/supplier/seeed?{}".format(urlencode({"opl": opls_type})) - - parts = obj["data"]["list"] - - for p in parts: - # print(json.dumps(p, indent=4)) - mpn = _get(p, "mpn") - sku = _get(p, "sku") - datasheet = _get(p, "datasheet") - desc = _get(p, "desc") - package = _get(p, "package") - - if mpn is None and sku is None: - continue - - ladder_price = p["ladder_price"] - - xml = types.Part(references=types.ReferencesList(), price_breaks=types.PriceBreakList()) - xml.supplierProp = supplier_uri - - xml.descriptionProp = desc - - uri_params = { - "opl": opls_type, - } - if sku: - uri_params["sku"] = sku - if mpn: - uri_params["mpn"] = mpn - xml.uriProp = "http://purl.org/ee/supplier/seeed?{}".format(urlencode(uri_params)) - - part = Part(xml) - - if mpn: - part.add_mpn(mpn) - if sku: - part.add_spn(sku) - - for item in ladder_price: - price = money.parse(item["price"], currency="USD") - part.add_price_break(item["qty"], price) - - db.add_entry(part, True) - - page_offset += 1 - count += len(parts) - if len(parts) != page_length: - break - - print("Imported {} parts from Seeed's {} library".format(count, opls_type)) - save_db(out_dir / "{}.xml".format(opls_type), db, sort=True) + page_offset = 1 + page_length = 30 + count = 0 + + cd = cache_dir / opl_type if cache_dir else cache_dir + cache = ee._utils.maybe_cache(cd, ext="json") + + db = PartDb() + + while True: + cache_key = "{}-{}".format(opl_type, page_offset) + content = cache.lookup(cache_key) + if content is None: + form = { + "page_offset": page_offset, + "page_length": page_length, + "keyword": "", + "category": "", + "type": opl_type, + } + query = { + "guid": "A8D502008EB2E4FD3FE9CE5E0855E8E4", + "appid": "en.pc.bazaar", + } + r = s.get("https://sapi.seeedstudio.com/fusion/opl/list", params=query, data=form) + content = r.text + cache.save(cache_key, content) + + obj = json.loads(content) + supplier_uri = "http://purl.org/ee/supplier/seeed?{}".format(urlencode({"opl": opl_type})) + + parts = obj["data"]["list"] + + for p in parts: + # print(json.dumps(p, indent=4)) + mpn = _get(p, "mpn") + sku = _get(p, "sku") + datasheet = _get(p, "datasheet") + desc = _get(p, "desc") + package = _get(p, "package") + + if mpn is None and sku is None: + continue + + ladder_price = p["ladder_price"] + + xml = types.Part(references=types.ReferencesList(), price_breaks=types.PriceBreakList()) + xml.supplierProp = supplier_uri + + xml.descriptionProp = desc + + uri_params = { + "opl": opl_type, + } + if sku: + uri_params["sku"] = sku + if mpn: + uri_params["mpn"] = mpn + xml.uriProp = "http://purl.org/ee/supplier/seeed?{}".format(urlencode(uri_params)) + + part = Part(xml) + + if mpn: + part.add_mpn(mpn) + if sku: + part.add_spn(sku) + + for item in ladder_price: + price = money.parse(item["price"], currency="USD") + part.add_price_break(item["qty"], price) + + db.add_entry(part, True) + + page_offset += 1 + count += len(parts) + if len(parts) != page_length: + break + + print("Imported {} parts from Seeed's {} library".format(count, opl_type)) + save_db(out_path, db, sort=True) + + +def init_project(project): + opls = ["SEEED", "HQCHIP"] + + project.cfg.add_section("seeed-opl") + project.cfg["seeed-opl"]["opls"] = ", ".join(opls) -- cgit v1.2.3