diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/ee/digikey/__init__.py | 59 | ||||
| -rw-r--r-- | src/ee/digikey/search_parts.py | 8 | ||||
| -rw-r--r-- | src/ee/money.py | 297 | ||||
| -rw-r--r-- | src/ee/xml/bomFile.py | 292 |
4 files changed, 645 insertions, 11 deletions
diff --git a/src/ee/digikey/__init__.py b/src/ee/digikey/__init__.py index 6baae84..f67e6b5 100644 --- a/src/ee/digikey/__init__.py +++ b/src/ee/digikey/__init__.py @@ -13,32 +13,35 @@ from lxml import html from selenium import webdriver import ee._utils +from ee.money import Money, get_default_context from ee.tools import mk_parents +money = get_default_context() -def normalize_filename(part): + +def normalize_filename(part) -> str: return part.replace('/', '_').replace(' ', '_') -def _clean(s): +def _clean(s) -> Optional[str]: if s is None: return None s = s.strip() return None if len(s) == 0 else s -def _to_string(e): +def _to_string(e) -> str: s = "" for t in e.itertext(): s += t return s.strip() -def _parse_int(s): +def _parse_int(s) -> int: return int(s.replace(',', '').replace('.', '')) -def _to_int(s): +def _to_int(s) -> Optional[int]: try: return _parse_int(s) except ValueError: @@ -69,16 +72,24 @@ class Digikey(object): return a +class PriceBreak(object): + def __init__(self, quantity: int, per_piece_price: Money, per_quantity_price: Money): + self.quantity = quantity + self.per_piece_price = per_piece_price + self.per_quantity_price = per_quantity_price + + @total_ordering class DigikeyProduct(object): def __init__(self, part_number, mpn, url, attributes: List["DigikeyAttributeValue"] = None, categories=None): self.part_number = _clean(part_number) self.mpn = _clean(mpn) self.url = url - self.attributes = attributes or [] # type: List["DigikeyAttributeValue"] + self.attributes = attributes or [] # type: List["DigikeyAttributeValue"] self.categories = categories or [] self.quantity_available = None self.description = None + self.price_breaks: List[PriceBreak] = [] assert self.part_number assert self.mpn @@ -214,7 +225,7 @@ class DigikeyClient(object): def __init__(self, cache_dir: Path = None, on_download=None): self.on_download = on_download or self.__nop self.cache = ee._utils.maybe_cache(cache_dir) - self.driver: webdriver.Chrome = None + self.driver: Optional[webdriver.Chrome] = None def search(self, query: str, page_size=10) -> str: return self.product_search(query, page_size) @@ -291,6 +302,7 @@ class DigikeyParser(object): if part_number and mpn: p = DigikeyProduct(part_number, mpn, url, attributes, categories) + p.price_breaks = self._parse_price_breaks(tree) for n in tree.xpath("//*[@itemprop='description']"): p.description = _to_string(n) return p @@ -298,6 +310,39 @@ class DigikeyParser(object): return None @staticmethod + def _find_currency(tree: html) -> Optional[str]: + for e in tree.xpath("//*[@id='cur-dropdown']"): + s = _clean(e.text) + if s: + return s + + def _parse_price_breaks(self, tree: html) -> List[PriceBreak]: + currency = self._find_currency(tree) + + price_breaks = [] + + ok = True + for row in tree.xpath("//table[@class='product-dollars']//tr"): + tds = list(row.xpath("./td")) + + if len(tds) != 3: + continue + + tds = ["".join(td.xpath("./descendant-or-self::*/text()")) for td in tds] + + quantity = _to_int(tds[0]) + price = money.try_parse(tds[1], currency=currency) + + if quantity is None or price is None: + ok = False + break + + price_breaks.append(PriceBreak(quantity=quantity, per_piece_price=price, + per_quantity_price=price * quantity)) + + return price_breaks if ok else [] + + @staticmethod def _handle_product_table(tree: html, res: DigikeySearchResponse): products = tree.xpath("//*[@itemtype='http://schema.org/Product']") diff --git a/src/ee/digikey/search_parts.py b/src/ee/digikey/search_parts.py index 9c8eb74..61c5c1b 100644 --- a/src/ee/digikey/search_parts.py +++ b/src/ee/digikey/search_parts.py @@ -27,6 +27,14 @@ def resolved(p: DigikeyProduct) -> bomFile.Part: key = make_digikey_fact_key(a.attribute_type.id) facts.append(bomFile.Fact(key=key, label=a.attribute_type.label, value=a.value)) + if len(p.price_breaks): + part.price_breaksProp = bomFile.PriceBreakList() + + price_breaks: List[bomFile.PriceBreak] = part.price_breaksProp.price_break + for pb in p.price_breaks: + amount = bomFile.Amount(value=str(pb.per_piece_price.amount), currency=pb.per_piece_price.currency) + price_breaks.append(bomFile.PriceBreak(pb.quantity, amount=amount)) + return part diff --git a/src/ee/money.py b/src/ee/money.py new file mode 100644 index 0000000..139d0d7 --- /dev/null +++ b/src/ee/money.py @@ -0,0 +1,297 @@ +from decimal import Decimal +from typing import Union, Optional, Mapping, Set + +from ee import EeException + +__all__ = [ + "Money", + "MoneyContext", + "get_default_context", + "set_default_context", +] + + +def _parse(amount: str) -> (str, str): + def read_space(): + nonlocal idx + while idx < end: + if amount[idx] != " ": + break + + idx += 1 + + def read_num(): + nonlocal idx + tmp = "" + while idx < end: + c_ = amount[idx] + if not c_.isnumeric(): + break + tmp += c_ + idx += 1 + + return tmp + + def read_word(): + nonlocal idx + tmp = "" + while idx < end: + c_ = amount[idx] + + if c_.isnumeric() or c_ in (" ", ",", "."): + break + + tmp += c_ + idx += 1 + + return tmp + + amount = amount.strip() + + idx = 0 + end = len(amount) + + if end == 0: + return None, None + + # extract leading currency + prefix = read_word() + + read_space() + + # If the number starts with '.' or ',' we assume it is a whole separator + # if idx < end: + # c = amount[idx] + # if c == "." or c == ",": + # s1 = c + # idx += 1 + + num = read_num() + # if len(num) > 3: + # return None, None + + # First and second separators + s1 = s2 = None + s0s = [] # stuff before first separator + s1s = [] + s2s = [] + + if num: + s0s.append(num) + + while idx < end: + c = amount[idx] + + is_separator = c in (",", ".") + + if not is_separator and c == " ": + next_c = amount[idx + 1] if idx + 1 < end else None + + if next_c is None or not next_c.isnumeric(): + idx += 1 + break + + is_separator = True + + if not is_separator: + return None, None + + if not s1: + s1 = c + elif s1 and c == s1: + pass + elif s1 and c != s1 and not s2: + s2 = c + else: + return None, None + idx += 1 + + num = read_num() + if not num: + if idx == end: + break + return None, None + + if s2: + s2s.append(num) + else: + s1s.append(num) + + del num + + if s1 == ' ': + s1 = s2 + s2 = None + s0s.extend(s1s) + s1s = s2s + s2s = [] + + if not s1 and not s2: + wholes = "".join(s0s) + parts = "0" + elif s1 and not s2: + wholes = "".join(s0s) + parts = "".join(s1s) + elif s1 and s2: + wholes = "".join(s0s) + "".join(s1s) + parts = "".join(s2s) + else: + return None, None + + read_space() + + postfix = read_word() + + # Check that there is no junk left + if amount[idx:].strip(): + return None, None + + # wholes = wholes if wholes else "0" + # parts = parts if parts else "0" + + currency = None + if len(prefix): + currency = prefix + + if len(postfix): + if currency: + return None, None + currency = postfix + + while parts.endswith("0"): + parts = parts[:-1] + + return wholes + "." + parts, currency + + +class Money(object): + def __init__(self, amount: Decimal, currency: Optional[str] = None): + self.amount: Decimal = amount + self.currency = currency + + def assert_same_currency(self, other): + if isinstance(other, int): + return + + if self.currency is None and other.currency is None: + return + if self.currency == other.currency: + return + + raise EeException("Can't relate to instances of Money with different currencies.") + + def __truediv__(self, other: Union["Money", int]): + self.assert_same_currency(other) + + if isinstance(other, int): + return Money(amount=self.amount.__truediv__(other), currency=self.currency) + + return Money(amount=self.amount.__truediv__(other), currency=self.currency) + + def __mul__(self, other: Union["Money", int]): + self.assert_same_currency(other) + + if isinstance(other, int): + return Money(amount=self.amount.__mul__(other), currency=self.currency) + + return Money(amount=self.amount.__mul__(other), currency=self.currency) + + def _amount_str(self): + amount = str(self.amount) + + (sign, _int, _exp) = self.amount.as_tuple() + if _exp >= 0: + return amount + + while amount.endswith("0"): + amount = amount[0:-1] + + if amount.endswith("."): + amount = amount[0:-1] + + return amount + + def __repr__(self): + return self._amount_str() + ((" " + self.currency) if self.currency else "") + + def __str__(self): + amount = self._amount_str() + + if self.currency is None: + return amount + + first, last = (self.currency, amount) if self.currency in _prefix_currencies else (amount, self.currency) + + return first + " " + last + + def __eq__(self, other: "Money"): + if not isinstance(other, Money): + return False + + return self.amount == other.amount and self.currency == other.currency + + +class MoneyContext(object): + def __init__(self, symbols_to_currency: Mapping[str, str], prefix_currencies: Set[str]): + self.symbols_to_currency = symbols_to_currency + self.prefix_currencies = prefix_currencies + + @staticmethod + def parse(amount: Union[str, int, float, Decimal], currency: Optional[str] = None) -> Money: + m = MoneyContext.try_parse(amount, currency) + + if m is None: + raise EeException("Could not parse money: '{}'".format(amount)) + + return m + + @staticmethod + def try_parse(amount: Union[str, int, float, Decimal], currency: Optional[str] = None) -> Optional[Money]: + if isinstance(amount, str): + (a, c) = _parse(amount) + + if a is None: + return None + + amount = Decimal(a) + + if c is not None: + if currency is not None and currency != c: + return None + + currency = c + elif isinstance(amount, int) or isinstance(amount, float): + amount = Decimal(amount) + elif isinstance(amount, Decimal): + pass + else: + raise EeException("Unsupported value type: {}".format(type(amount))) + + if currency is not None: + currency = _symbols_to_currency.get(currency, currency) + + return Money(amount, currency) + + +_symbols_to_currency = { + "$": "USD" +} + +_prefix_currencies = { + "USD", +} + +_default_context = None + + +def get_default_context(): + global _default_context + if _default_context is None: + _default_context = MoneyContext(_symbols_to_currency, _prefix_currencies) + + return _default_context + + +def set_default_context(ctx: MoneyContext): + global _default_context + _default_context = ctx diff --git a/src/ee/xml/bomFile.py b/src/ee/xml/bomFile.py index 7c14679..0557a6c 100644 --- a/src/ee/xml/bomFile.py +++ b/src/ee/xml/bomFile.py @@ -3,7 +3,7 @@ # # Generated by generateDS.py. -# Python 3.7.2+ (default, Feb 2 2019, 14:31:48) [GCC 8.2.0] +# Python 3.7.2+ (default, Feb 27 2019, 15:41:59) [GCC 8.2.0] # # Command line options: # ('-f', '') @@ -807,7 +807,7 @@ class BomFile(GeneratedsSuper): class Part(GeneratedsSuper): subclass = None superclass = None - def __init__(self, id=None, schema_reference=None, part_type=None, part_numbers=None, distributor_info=None, facts=None, **kwargs_): + def __init__(self, id=None, schema_reference=None, part_type=None, part_numbers=None, distributor_info=None, facts=None, price_breaks=None, **kwargs_): self.original_tagname_ = None self.parent_object_ = kwargs_.get('parent_object_') self.id = _cast(None, id) @@ -816,6 +816,7 @@ class Part(GeneratedsSuper): self.part_numbers = part_numbers self.distributor_info = distributor_info self.facts = facts + self.price_breaks = price_breaks def factory(*args_, **kwargs_): if CurrentSubclassModule_ is not None: subclass = getSubclassFromModule_( @@ -852,6 +853,11 @@ class Part(GeneratedsSuper): def set_facts(self, facts): self.facts = facts factsProp = property(get_facts, set_facts) + def get_price_breaks(self): + return self.price_breaks + def set_price_breaks(self, price_breaks): + self.price_breaks = price_breaks + price_breaksProp = property(get_price_breaks, set_price_breaks) def get_id(self): return self.id def set_id(self, id): @@ -863,7 +869,8 @@ class Part(GeneratedsSuper): self.part_type is not None or self.part_numbers is not None or self.distributor_info is not None or - self.facts is not None + self.facts is not None or + self.price_breaks is not None ): return True else: @@ -910,6 +917,8 @@ class Part(GeneratedsSuper): self.distributor_info.export(outfile, level, namespaceprefix_, namespacedef_='', name_='distributor-info', pretty_print=pretty_print) if self.facts is not None: self.facts.export(outfile, level, namespaceprefix_, namespacedef_='', name_='facts', pretty_print=pretty_print) + if self.price_breaks is not None: + self.price_breaks.export(outfile, level, namespaceprefix_, namespacedef_='', name_='price-breaks', pretty_print=pretty_print) def build(self, node): already_processed = set() self.buildAttributes(node, node.attrib, already_processed) @@ -946,6 +955,11 @@ class Part(GeneratedsSuper): obj_.build(child_) self.facts = obj_ obj_.original_tagname_ = 'facts' + elif nodeName_ == 'price-breaks': + obj_ = PriceBreakList.factory(parent_object_=self) + obj_.build(child_) + self.price_breaks = obj_ + obj_.original_tagname_ = 'price-breaks' # end class Part @@ -1492,6 +1506,273 @@ class DistributorInfo(GeneratedsSuper): # end class DistributorInfo +class Amount(GeneratedsSuper): + subclass = None + superclass = None + def __init__(self, value=None, currency=None, **kwargs_): + self.original_tagname_ = None + self.parent_object_ = kwargs_.get('parent_object_') + self.value = _cast(None, value) + self.currency = _cast(None, currency) + def factory(*args_, **kwargs_): + if CurrentSubclassModule_ is not None: + subclass = getSubclassFromModule_( + CurrentSubclassModule_, Amount) + if subclass is not None: + return subclass(*args_, **kwargs_) + if Amount.subclass: + return Amount.subclass(*args_, **kwargs_) + else: + return Amount(*args_, **kwargs_) + factory = staticmethod(factory) + def get_value(self): + return self.value + def set_value(self, value): + self.value = value + valueProp = property(get_value, set_value) + def get_currency(self): + return self.currency + def set_currency(self, currency): + self.currency = currency + currencyProp = property(get_currency, set_currency) + def hasContent_(self): + if ( + + ): + return True + else: + return False + def export(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='Amount', pretty_print=True): + imported_ns_def_ = GenerateDSNamespaceDefs_.get('Amount') + if imported_ns_def_ is not None: + namespacedef_ = imported_ns_def_ + if pretty_print: + eol_ = '\n' + else: + eol_ = '' + if self.original_tagname_ is not None: + name_ = self.original_tagname_ + showIndent(outfile, level, pretty_print) + outfile.write('<%s%s%s' % (namespaceprefix_, name_, namespacedef_ and ' ' + namespacedef_ or '', )) + already_processed = set() + self.exportAttributes(outfile, level, already_processed, namespaceprefix_, name_='Amount') + if self.hasContent_(): + outfile.write('>%s' % (eol_, )) + self.exportChildren(outfile, level + 1, namespaceprefix_, namespacedef_, name_='Amount', pretty_print=pretty_print) + outfile.write('</%s%s>%s' % (namespaceprefix_, name_, eol_)) + else: + outfile.write('/>%s' % (eol_, )) + def exportAttributes(self, outfile, level, already_processed, namespaceprefix_='', name_='Amount'): + if self.value is not None and 'value' not in already_processed: + already_processed.add('value') + outfile.write(' value=%s' % (self.gds_encode(self.gds_format_string(quote_attrib(self.value), input_name='value')), )) + if self.currency is not None and 'currency' not in already_processed: + already_processed.add('currency') + outfile.write(' currency=%s' % (self.gds_encode(self.gds_format_string(quote_attrib(self.currency), input_name='currency')), )) + def exportChildren(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='Amount', fromsubclass_=False, pretty_print=True): + pass + def build(self, node): + already_processed = set() + self.buildAttributes(node, node.attrib, already_processed) + for child in node: + nodeName_ = Tag_pattern_.match(child.tag).groups()[-1] + self.buildChildren(child, node, nodeName_) + return self + def buildAttributes(self, node, attrs, already_processed): + value = find_attr_value_('value', node) + if value is not None and 'value' not in already_processed: + already_processed.add('value') + self.value = value + value = find_attr_value_('currency', node) + if value is not None and 'currency' not in already_processed: + already_processed.add('currency') + self.currency = value + def buildChildren(self, child_, node, nodeName_, fromsubclass_=False): + pass +# end class Amount + + +class PriceBreak(GeneratedsSuper): + subclass = None + superclass = None + def __init__(self, quantity=None, amount=None, **kwargs_): + self.original_tagname_ = None + self.parent_object_ = kwargs_.get('parent_object_') + self.quantity = quantity + self.amount = amount + def factory(*args_, **kwargs_): + if CurrentSubclassModule_ is not None: + subclass = getSubclassFromModule_( + CurrentSubclassModule_, PriceBreak) + if subclass is not None: + return subclass(*args_, **kwargs_) + if PriceBreak.subclass: + return PriceBreak.subclass(*args_, **kwargs_) + else: + return PriceBreak(*args_, **kwargs_) + factory = staticmethod(factory) + def get_quantity(self): + return self.quantity + def set_quantity(self, quantity): + self.quantity = quantity + quantityProp = property(get_quantity, set_quantity) + def get_amount(self): + return self.amount + def set_amount(self, amount): + self.amount = amount + amountProp = property(get_amount, set_amount) + def hasContent_(self): + if ( + self.quantity is not None or + self.amount is not None + ): + return True + else: + return False + def export(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='PriceBreak', pretty_print=True): + imported_ns_def_ = GenerateDSNamespaceDefs_.get('PriceBreak') + if imported_ns_def_ is not None: + namespacedef_ = imported_ns_def_ + if pretty_print: + eol_ = '\n' + else: + eol_ = '' + if self.original_tagname_ is not None: + name_ = self.original_tagname_ + showIndent(outfile, level, pretty_print) + outfile.write('<%s%s%s' % (namespaceprefix_, name_, namespacedef_ and ' ' + namespacedef_ or '', )) + already_processed = set() + self.exportAttributes(outfile, level, already_processed, namespaceprefix_, name_='PriceBreak') + if self.hasContent_(): + outfile.write('>%s' % (eol_, )) + self.exportChildren(outfile, level + 1, namespaceprefix_, namespacedef_, name_='PriceBreak', pretty_print=pretty_print) + showIndent(outfile, level, pretty_print) + outfile.write('</%s%s>%s' % (namespaceprefix_, name_, eol_)) + else: + outfile.write('/>%s' % (eol_, )) + def exportAttributes(self, outfile, level, already_processed, namespaceprefix_='', name_='PriceBreak'): + pass + def exportChildren(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='PriceBreak', fromsubclass_=False, pretty_print=True): + if pretty_print: + eol_ = '\n' + else: + eol_ = '' + if self.quantity is not None: + showIndent(outfile, level, pretty_print) + outfile.write('<%squantity>%s</%squantity>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.quantity), input_name='quantity')), namespaceprefix_ , eol_)) + if self.amount is not None: + self.amount.export(outfile, level, namespaceprefix_, namespacedef_='', name_='amount', pretty_print=pretty_print) + def build(self, node): + already_processed = set() + self.buildAttributes(node, node.attrib, already_processed) + for child in node: + nodeName_ = Tag_pattern_.match(child.tag).groups()[-1] + self.buildChildren(child, node, nodeName_) + return self + def buildAttributes(self, node, attrs, already_processed): + pass + def buildChildren(self, child_, node, nodeName_, fromsubclass_=False): + if nodeName_ == 'quantity': + quantity_ = child_.text + quantity_ = self.gds_validate_string(quantity_, node, 'quantity') + self.quantity = quantity_ + elif nodeName_ == 'amount': + obj_ = Amount.factory(parent_object_=self) + obj_.build(child_) + self.amount = obj_ + obj_.original_tagname_ = 'amount' +# end class PriceBreak + + +class PriceBreakList(GeneratedsSuper): + subclass = None + superclass = None + def __init__(self, price_break=None, **kwargs_): + self.original_tagname_ = None + self.parent_object_ = kwargs_.get('parent_object_') + if price_break is None: + self.price_break = [] + else: + self.price_break = price_break + def factory(*args_, **kwargs_): + if CurrentSubclassModule_ is not None: + subclass = getSubclassFromModule_( + CurrentSubclassModule_, PriceBreakList) + if subclass is not None: + return subclass(*args_, **kwargs_) + if PriceBreakList.subclass: + return PriceBreakList.subclass(*args_, **kwargs_) + else: + return PriceBreakList(*args_, **kwargs_) + factory = staticmethod(factory) + def get_price_break(self): + return self.price_break + def set_price_break(self, price_break): + self.price_break = price_break + def add_price_break(self, value): + self.price_break.append(value) + def add_price_break(self, value): + self.price_break.append(value) + def insert_price_break_at(self, index, value): + self.price_break.insert(index, value) + def replace_price_break_at(self, index, value): + self.price_break[index] = value + price_breakProp = property(get_price_break, set_price_break) + def hasContent_(self): + if ( + self.price_break + ): + return True + else: + return False + def export(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='PriceBreakList', pretty_print=True): + imported_ns_def_ = GenerateDSNamespaceDefs_.get('PriceBreakList') + if imported_ns_def_ is not None: + namespacedef_ = imported_ns_def_ + if pretty_print: + eol_ = '\n' + else: + eol_ = '' + if self.original_tagname_ is not None: + name_ = self.original_tagname_ + showIndent(outfile, level, pretty_print) + outfile.write('<%s%s%s' % (namespaceprefix_, name_, namespacedef_ and ' ' + namespacedef_ or '', )) + already_processed = set() + self.exportAttributes(outfile, level, already_processed, namespaceprefix_, name_='PriceBreakList') + if self.hasContent_(): + outfile.write('>%s' % (eol_, )) + self.exportChildren(outfile, level + 1, namespaceprefix_, namespacedef_, name_='PriceBreakList', pretty_print=pretty_print) + showIndent(outfile, level, pretty_print) + outfile.write('</%s%s>%s' % (namespaceprefix_, name_, eol_)) + else: + outfile.write('/>%s' % (eol_, )) + def exportAttributes(self, outfile, level, already_processed, namespaceprefix_='', name_='PriceBreakList'): + pass + def exportChildren(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='PriceBreakList', fromsubclass_=False, pretty_print=True): + if pretty_print: + eol_ = '\n' + else: + eol_ = '' + for price_break_ in self.price_break: + price_break_.export(outfile, level, namespaceprefix_, namespacedef_='', name_='price-break', pretty_print=pretty_print) + def build(self, node): + already_processed = set() + self.buildAttributes(node, node.attrib, already_processed) + for child in node: + nodeName_ = Tag_pattern_.match(child.tag).groups()[-1] + self.buildChildren(child, node, nodeName_) + return self + def buildAttributes(self, node, attrs, already_processed): + pass + def buildChildren(self, child_, node, nodeName_, fromsubclass_=False): + if nodeName_ == 'price-break': + obj_ = PriceBreak.factory(parent_object_=self) + obj_.build(child_) + self.price_break.append(obj_) + obj_.original_tagname_ = 'price-break' +# end class PriceBreakList + + GDSClassesMapping = { 'bom-file': BomFile, 'part': Part, @@ -1622,6 +1903,7 @@ if __name__ == '__main__': __all__ = [ + "Amount", "BomFile", "DistributorInfo", "Fact", @@ -1629,5 +1911,7 @@ __all__ = [ "Part", "PartList", "PartNumber", - "PartNumberList" + "PartNumberList", + "PriceBreak", + "PriceBreakList" ] |