diff options
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | src/ee/_utils.py | 9 | ||||
| -rw-r--r-- | src/ee/supplier/seeed.py | 151 | ||||
| -rw-r--r-- | src/ee/tools/seeed_download_opl.py | 16 | ||||
| -rw-r--r-- | src/ee/xml/types.py | 16 | ||||
| -rw-r--r-- | xsd/ee.xsd | 1 |
6 files changed, 189 insertions, 5 deletions
diff --git a/requirements.txt b/requirements.txt index ec2c2f6..a389f3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ tox==3.1.2 selenium==3.141.0 generateds==2.30.11 +requests==2.21.0 # for development jupyter==1.0.0 diff --git a/src/ee/_utils.py b/src/ee/_utils.py index f917847..08e75fa 100644 --- a/src/ee/_utils.py +++ b/src/ee/_utils.py @@ -44,8 +44,9 @@ def all(filters): class HttpCache(object): - def __init__(self, path: Path): + def __init__(self, path: Path, ext="html"): self.path = path + self.ext = ext def lookup(self, key): cache_path = self._make_path(key) @@ -62,7 +63,7 @@ class HttpCache(object): f.write(value) def _make_path(self, key) -> Path: - return self.path / "{}.html".format(key) + return self.path / "{}.{}".format(key, self.ext) class EmptyHttpCache(object): @@ -74,5 +75,5 @@ class EmptyHttpCache(object): pass -def maybe_cache(path: Optional[Path]) -> HttpCache: - return HttpCache(path) if path is not None else EmptyHttpCache() +def maybe_cache(path: Optional[Path], **kwargs) -> HttpCache: + return HttpCache(path, **kwargs) if path is not None else EmptyHttpCache() diff --git a/src/ee/supplier/seeed.py b/src/ee/supplier/seeed.py new file mode 100644 index 0000000..2df2068 --- /dev/null +++ b/src/ee/supplier/seeed.py @@ -0,0 +1,151 @@ +import binascii +import json +import re +from pathlib import Path +from typing import Optional +from urllib.parse import urlencode + +import requests +from selenium import webdriver + +import ee._utils +from ee.part import PartDb, save_db +from ee.xml import types + +_title_re = re.compile(r"^([^(]*)\( *[0-9]* *\) *") + +__all__ = [ + "SeeedClient", + "download_opls", +] + + +class SeeedClient(object): + def __init__(self, cache_dir: Path = None): + self.cache = ee._utils.maybe_cache(cache_dir) + self.driver: Optional[webdriver.Chrome] = None + + def get(self, url: str): + cache_key = abs(binascii.crc32(url.encode("utf-8"))) + cached = self.cache.lookup(cache_key) + if cached: + return cached + + if self.driver is None: + options = webdriver.ChromeOptions() + self.driver = webdriver.Chrome(chrome_options=options) + + self.driver.get(url) + + src = self.driver.page_source + self.cache.save(cache_key, src) + + return src + + +def _checksum_opl(tree): + for table in tree.xpath("//*[contains(@class, 'fusion-opl-list')]"): + rows = table.xpath(".//tr[contains(@class, 'f12')]") + for r in rows: + print(r) + break + + +def _get(obj, key): + value = obj.get(key, None) + if value is None: + return "" + return value.strip() + + +def download_opls(out_dir: Path, cache_dir: Path): + opls_types = ["SEEED", "HQCHIP"] + + s = requests.Session() + + from ee.money import get_default_context + money = get_default_context() + + for opls_type in opls_types: + page_offset = 1 + page_length = 30 + count = 0 + + cd = cache_dir / opls_type if cache_dir else cache_dir + cache = ee._utils.maybe_cache(cd, ext="json") + + db = PartDb() + + while True: + cache_key = "{}-{}".format(opls_type, page_offset) + content = cache.lookup(cache_key) + if content is None: + form = { + "page_offset": page_offset, + "page_length": page_length, + "keyword": "", + "category": "", + "type": opls_type, + } + query = { + "guid": "A8D502008EB2E4FD3FE9CE5E0855E8E4", + "appid": "en.pc.bazaar", + } + r = s.get("https://sapi.seeedstudio.com/fusion/opl/list", params=query, data=form) + content = r.text + cache.save(cache_key, content) + + obj = json.loads(content) + supplier_uri = "http://purl.org/ee/supplier/seeed?{}".format(urlencode({"opl": opls_type})) + + parts = obj["data"]["list"] + + for p in parts: + # print(json.dumps(p, indent=4)) + mpn = _get(p, "mpn") + sku = _get(p, "sku") + datasheet = _get(p, "datasheet") + desc = _get(p, "desc") + package = _get(p, "package") + + if mpn is None and sku is None: + continue + + ladder_price = p["ladder_price"] + + part = types.Part(references=types.ReferencesList(), price_breaks=types.PriceBreakList()) + part_numbers = part.referencesProp.part_numberProp + supplier_part_numbers = part.referencesProp.supplier_part_numberProp + pbs = part.price_breaksProp.price_breakProp + + if desc: + part.descriptionProp = desc + + uri_params = { + "opl": opls_type, + } + if sku: + uri_params["sku"] = sku + if mpn: + uri_params["mpn"] = mpn + part.uriProp = "http://purl.org/ee/supplier/seeed?{}".format(urlencode(uri_params)) + + if mpn: + part_numbers.append(types.PartNumber(mpn)) + if sku: + supplier_part_numbers.append(types.SupplierPartNumber(value=mpn, supplier=supplier_uri)) + + for item in ladder_price: + price = money.parse(item["price"], currency="USD") + amount = types.Amount(value=price.amount, currency=price.currency) + pbs.append(types.PriceBreak(quantity=item["qty"], amount=amount)) + + db.add_entry(part, True) + + page_offset += 1 + count += len(parts) + if len(parts) != page_length: + break + + print("Imported {} parts from Seeed's {} library".format(count, opls_type)) + save_db(out_dir / "{}.xml".format(opls_type), db, sort=True) diff --git a/src/ee/tools/seeed_download_opl.py b/src/ee/tools/seeed_download_opl.py new file mode 100644 index 0000000..e53361f --- /dev/null +++ b/src/ee/tools/seeed_download_opl.py @@ -0,0 +1,16 @@ +import argparse +from pathlib import Path + +from ee.supplier import seeed + +parser = argparse.ArgumentParser() + +parser.add_argument("--out-dir", + required=True, + metavar="DIR FOR PART DBS") + +args = parser.parse_args() + +cache_dir = ".ee/cache/seeed" + +seeed.download_opls(Path(args.out_dir), Path(cache_dir)) diff --git a/src/ee/xml/types.py b/src/ee/xml/types.py index 35cd664..a1ab7ec 100644 --- a/src/ee/xml/types.py +++ b/src/ee/xml/types.py @@ -807,11 +807,12 @@ class PartDb(GeneratedsSuper): class Part(GeneratedsSuper): subclass = None superclass = None - def __init__(self, uri=None, part_type=None, references=None, distributor_info=None, facts=None, price_breaks=None, **kwargs_): + def __init__(self, uri=None, part_type=None, description=None, references=None, distributor_info=None, facts=None, price_breaks=None, **kwargs_): self.original_tagname_ = None self.parent_object_ = kwargs_.get('parent_object_') self.uri = _cast(None, uri) self.part_type = part_type + self.description = description self.references = references self.distributor_info = distributor_info self.facts = facts @@ -832,6 +833,11 @@ class Part(GeneratedsSuper): def set_part_type(self, part_type): self.part_type = part_type part_typeProp = property(get_part_type, set_part_type) + def get_description(self): + return self.description + def set_description(self, description): + self.description = description + descriptionProp = property(get_description, set_description) def get_references(self): return self.references def set_references(self, references): @@ -860,6 +866,7 @@ class Part(GeneratedsSuper): def hasContent_(self): if ( self.part_type is not None or + self.description is not None or self.references is not None or self.distributor_info is not None or self.facts is not None or @@ -901,6 +908,9 @@ class Part(GeneratedsSuper): if self.part_type is not None: showIndent(outfile, level, pretty_print) outfile.write('<%spart-type>%s</%spart-type>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.part_type), input_name='part-type')), namespaceprefix_ , eol_)) + if self.description is not None: + showIndent(outfile, level, pretty_print) + outfile.write('<%sdescription>%s</%sdescription>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.description), input_name='description')), namespaceprefix_ , eol_)) if self.references is not None: self.references.export(outfile, level, namespaceprefix_, namespacedef_='', name_='references', pretty_print=pretty_print) if self.distributor_info is not None: @@ -926,6 +936,10 @@ class Part(GeneratedsSuper): part_type_ = child_.text part_type_ = self.gds_validate_string(part_type_, node, 'part_type') self.part_type = part_type_ + elif nodeName_ == 'description': + description_ = child_.text + description_ = self.gds_validate_string(description_, node, 'description') + self.description = description_ elif nodeName_ == 'references': obj_ = ReferencesList.factory(parent_object_=self) obj_.build(child_) @@ -46,6 +46,7 @@ TODO: rename 'id' to 'url'. <xs:sequence> <!-- TODO: this should be a fact --> <xs:element name="part-type" type="xs:anyURI"/> + <xs:element name="description" type="xs:string"/> <xs:element name="references" type="ReferencesList"/> <xs:element name="distributor-info" type="DistributorInfo"/> <xs:element name="facts" type="FactList"/> |