From a25bba9dd40b5dd6b0d85fd415634e2e768fe40b Mon Sep 17 00:00:00 2001 From: Trygve Laugstøl Date: Mon, 30 Jul 2018 21:21:59 +0200 Subject: o Changing resolve-schematic-components to create generic 'component-to-part-mapping' objects with seller=digikey o Implementing 'task_digikey_fetch_full_part_facts' that downloads info about each mapped parts with seller=digikey. --- demo/doit/dodo.py | 6 +- demo/doit/ee/digikey-100.csv | 6 +- demo/doit/ee/digikey-parts/data-set.ini | 3 + .../digikey-parts/digikey-part/493-4771-1-ND.ini | 27 +++++++ .../ee/digikey-parts/digikey-part/BC2AAPC-ND.ini | 18 +++++ .../digikey-part/RMCF1206JT10K0CT-ND.ini | 23 ++++++ .../digikey-BT1-BC2AAPC-ND.ini | 11 +++ .../digikey-C1-493-4771-1-ND.ini | 10 +++ .../digikey-R1-RMCF1206JT10K0CT-ND.ini | 10 +++ .../digikey-R2-RMCF1206JT10K0CT-ND.ini | 10 +++ .../digikey-part-stub/493-4771-1-ND.ini | 8 -- .../digikey-part-stub/BC2AAPC-ND.ini | 9 --- .../digikey-part-stub/RMCF1206JT10K0CT-ND.ini | 8 -- demo/doit/ee/digikey.csv | 6 +- src/ee/bom/doit.py | 71 ++++++++++++----- src/ee/digikey/doit.py | 91 ++++++++++++++++++---- src/ee/doit.py | 6 ++ src/ee/ds/__init__.py | 22 ++++-- src/ee/tools/digikey_download_facts.py | 1 - 19 files changed, 271 insertions(+), 75 deletions(-) create mode 100644 demo/doit/ee/digikey-parts/data-set.ini create mode 100644 demo/doit/ee/digikey-parts/digikey-part/493-4771-1-ND.ini create mode 100644 demo/doit/ee/digikey-parts/digikey-part/BC2AAPC-ND.ini create mode 100644 demo/doit/ee/digikey-parts/digikey-part/RMCF1206JT10K0CT-ND.ini create mode 100644 demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-BT1-BC2AAPC-ND.ini create mode 100644 demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-C1-493-4771-1-ND.ini create mode 100644 demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R1-RMCF1206JT10K0CT-ND.ini create mode 100644 demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R2-RMCF1206JT10K0CT-ND.ini delete mode 100644 demo/doit/ee/digikey-resolved-parts/digikey-part-stub/493-4771-1-ND.ini delete mode 100644 demo/doit/ee/digikey-resolved-parts/digikey-part-stub/BC2AAPC-ND.ini delete mode 100644 demo/doit/ee/digikey-resolved-parts/digikey-part-stub/RMCF1206JT10K0CT-ND.ini diff --git a/demo/doit/dodo.py b/demo/doit/dodo.py index 15b31c0..fe9b9ae 100644 --- a/demo/doit/dodo.py +++ b/demo/doit/dodo.py @@ -38,14 +38,18 @@ ee.digikey.doit.doit_config.configure(data_set_manager=dsm) def task_orders(): - data_sets = [ee.bom.doit.output_data_set_for_task(task_bom)] + data_sets = [ee.bom.doit.output_data_set_for_task(task_bom), + ee.digikey.doit.doit_config.out_data_set_for(task_digikey_resolve_schematic_components)] + yield create_task_order_csv( output_file="ee/order.csv", + out_data_set="order", data_sets=data_sets) for size in [1, 100]: yield create_task_order_csv( output_file="ee/digikey-{}.csv".format(size) if size != 1 else "ee/digikey.csv", style="digikey", + out_data_set="digikey-order-{}".format(size), data_sets=data_sets, count=size) diff --git a/demo/doit/ee/digikey-100.csv b/demo/doit/ee/digikey-100.csv index bad6173..4bf730c 100644 --- a/demo/doit/ee/digikey-100.csv +++ b/demo/doit/ee/digikey-100.csv @@ -1,4 +1,4 @@ Digi-Key Part Number,Customer Reference,Quantity,MPN -,BT1,100,BC2AAPC -,"R2,R1",200,RMCF1206JT10K0 -,C1,100,UCA2G100MPD1TD +BC2AAPC-ND,BT1,100,BC2AAPC +RMCF1206JT10K0CT-ND,"R2,R1",200,RMCF1206JT10K0 +493-4771-1-ND,C1,100,UCA2G100MPD1TD diff --git a/demo/doit/ee/digikey-parts/data-set.ini b/demo/doit/ee/digikey-parts/data-set.ini new file mode 100644 index 0000000..156b0ff --- /dev/null +++ b/demo/doit/ee/digikey-parts/data-set.ini @@ -0,0 +1,3 @@ +[data-set] +name = digikey-parts + diff --git a/demo/doit/ee/digikey-parts/digikey-part/493-4771-1-ND.ini b/demo/doit/ee/digikey-parts/digikey-part/493-4771-1-ND.ini new file mode 100644 index 0000000..17ab33d --- /dev/null +++ b/demo/doit/ee/digikey-parts/digikey-part/493-4771-1-ND.ini @@ -0,0 +1,27 @@ +[meta] +type = digikey-part + +[values] +part-number = 493-4771-1-ND +url = https://www.digikey.com/product-detail/en/nichicon/UCA2G100MPD1TD/493-4771-1-ND/3129112 +mpn = UCA2G100MPD1TD +7/packaging = Cut Tape (CT) +1989/part status = Active +2049/capacitance = 10µF +3/tolerance = ±20%% +2079/voltage - rated = 400V +724/esr (equivalent series resistance) = - +725/lifetime @ temp. = 10000 Hrs @ 105°C +252/operating temperature = -25°C ~ 105°C +52/polarization = - +707/ratings = - +405/applications = General Purpose +2253/ripple current @ low frequency = 140mA @ 120Hz +2260/ripple current @ high frequency = 280mA @ 100kHz +508/lead spacing = 0.197" (5.00mm) +46/size / dimension = 0.394" Dia (10.00mm) +1500/height - seated (max) = 0.846" (21.50mm) +884/surface mount land size = - +69/mounting type = Through Hole +16/package / case = Radial, Can + diff --git a/demo/doit/ee/digikey-parts/digikey-part/BC2AAPC-ND.ini b/demo/doit/ee/digikey-parts/digikey-part/BC2AAPC-ND.ini new file mode 100644 index 0000000..4e151d8 --- /dev/null +++ b/demo/doit/ee/digikey-parts/digikey-part/BC2AAPC-ND.ini @@ -0,0 +1,18 @@ +[meta] +type = digikey-part + +[values] +part-number = BC2AAPC-ND +url = https://www.digikey.com/product-detail/en/mpd-memory-protection-devices/BC2AAPC/BC2AAPC-ND/455777 +mpn = BC2AAPC +7/packaging = Active +1989/part status = Cylindrical, Holder +2049/capacitance = Holder (Open) +3/tolerance = AA +2079/voltage - rated = 2 +724/esr (equivalent series resistance) = - +725/lifetime @ temp. = PCB, Through Hole +252/operating temperature = PC Pin +52/polarization = 0.591" (15.01mm) +707/ratings = -10°C ~ 100°C + diff --git a/demo/doit/ee/digikey-parts/digikey-part/RMCF1206JT10K0CT-ND.ini b/demo/doit/ee/digikey-parts/digikey-part/RMCF1206JT10K0CT-ND.ini new file mode 100644 index 0000000..35eb796 --- /dev/null +++ b/demo/doit/ee/digikey-parts/digikey-part/RMCF1206JT10K0CT-ND.ini @@ -0,0 +1,23 @@ +[meta] +type = digikey-part + +[values] +part-number = RMCF1206JT10K0CT-ND +url = https://www.digikey.com/product-detail/en/stackpole-electronics-inc/RMCF1206JT10K0/RMCF1206JT10K0CT-ND/1942803 +mpn = RMCF1206JT10K0 +7/packaging = Cut Tape (CT) +1989/part status = Active +2049/capacitance = 10 kOhms +3/tolerance = ±5%% +2079/voltage - rated = 0.25W, 1/4W +724/esr (equivalent series resistance) = Thick Film +725/lifetime @ temp. = Automotive AEC-Q200 +252/operating temperature = ±200ppm/°C +52/polarization = -55°C ~ 155°C +707/ratings = 1206 (3216 Metric) +405/applications = 1206 +2253/ripple current @ low frequency = 0.126" L x 0.063" W (3.20mm x 1.60mm) +2260/ripple current @ high frequency = 0.028" (0.70mm) +508/lead spacing = 2 +46/size / dimension = - + diff --git a/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-BT1-BC2AAPC-ND.ini b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-BT1-BC2AAPC-ND.ini new file mode 100644 index 0000000..a8c30b4 --- /dev/null +++ b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-BT1-BC2AAPC-ND.ini @@ -0,0 +1,11 @@ +[meta] +type = component-to-part-mapping + +[values] +seller = digikey +ref = BT1 +part-number = BC2AAPC-ND +mpn = BC2AAPC +description = EMITTER IR 940NM 100MA RADIAL +url = https://www.digikey.com/product-detail/en/mpd-memory-protection-devices/BC2AAPC/BC2AAPC-ND/455777 + diff --git a/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-C1-493-4771-1-ND.ini b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-C1-493-4771-1-ND.ini new file mode 100644 index 0000000..7368272 --- /dev/null +++ b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-C1-493-4771-1-ND.ini @@ -0,0 +1,10 @@ +[meta] +type = component-to-part-mapping + +[values] +seller = digikey +ref = C1 +part-number = 493-4771-1-ND +mpn = UCA2G100MPD1TD +description = /product-detail/en/nichicon/UCA2G100MPD1TD/493-4771-1-ND/3129112 + diff --git a/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R1-RMCF1206JT10K0CT-ND.ini b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R1-RMCF1206JT10K0CT-ND.ini new file mode 100644 index 0000000..eca43a9 --- /dev/null +++ b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R1-RMCF1206JT10K0CT-ND.ini @@ -0,0 +1,10 @@ +[meta] +type = component-to-part-mapping + +[values] +seller = digikey +ref = R1 +part-number = RMCF1206JT10K0CT-ND +mpn = RMCF1206JT10K0 +description = /product-detail/en/stackpole-electronics-inc/RMCF1206JT10K0/RMCF1206JT10K0CT-ND/1942803 + diff --git a/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R2-RMCF1206JT10K0CT-ND.ini b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R2-RMCF1206JT10K0CT-ND.ini new file mode 100644 index 0000000..fae5b54 --- /dev/null +++ b/demo/doit/ee/digikey-resolved-parts/component-to-part-mapping/digikey-R2-RMCF1206JT10K0CT-ND.ini @@ -0,0 +1,10 @@ +[meta] +type = component-to-part-mapping + +[values] +seller = digikey +ref = R2 +part-number = RMCF1206JT10K0CT-ND +mpn = RMCF1206JT10K0 +description = /product-detail/en/stackpole-electronics-inc/RMCF1206JT10K0/RMCF1206JT10K0CT-ND/1942803 + diff --git a/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/493-4771-1-ND.ini b/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/493-4771-1-ND.ini deleted file mode 100644 index 3cfb7d9..0000000 --- a/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/493-4771-1-ND.ini +++ /dev/null @@ -1,8 +0,0 @@ -[meta] -type = digikey-part-stub - -[values] -part-number = 493-4771-1-ND -mpn = UCA2G100MPD1TD -url = /product-detail/en/nichicon/UCA2G100MPD1TD/493-4771-1-ND/3129112 - diff --git a/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/BC2AAPC-ND.ini b/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/BC2AAPC-ND.ini deleted file mode 100644 index 013dae3..0000000 --- a/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/BC2AAPC-ND.ini +++ /dev/null @@ -1,9 +0,0 @@ -[meta] -type = digikey-part-stub - -[values] -part-number = BC2AAPC-ND -mpn = BC2AAPC -url = https://www.digikey.com/product-detail/en/mpd-memory-protection-devices/BC2AAPC/BC2AAPC-ND/455777 -description = EMITTER IR 940NM 100MA RADIAL - diff --git a/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/RMCF1206JT10K0CT-ND.ini b/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/RMCF1206JT10K0CT-ND.ini deleted file mode 100644 index 86a5504..0000000 --- a/demo/doit/ee/digikey-resolved-parts/digikey-part-stub/RMCF1206JT10K0CT-ND.ini +++ /dev/null @@ -1,8 +0,0 @@ -[meta] -type = digikey-part-stub - -[values] -part-number = RMCF1206JT10K0CT-ND -mpn = RMCF1206JT10K0 -url = /product-detail/en/stackpole-electronics-inc/RMCF1206JT10K0/RMCF1206JT10K0CT-ND/1942803 - diff --git a/demo/doit/ee/digikey.csv b/demo/doit/ee/digikey.csv index a813638..ea71d42 100644 --- a/demo/doit/ee/digikey.csv +++ b/demo/doit/ee/digikey.csv @@ -1,4 +1,4 @@ Digi-Key Part Number,Customer Reference,Quantity,MPN -,BT1,1,BC2AAPC -,"R2,R1",2,RMCF1206JT10K0 -,C1,1,UCA2G100MPD1TD +BC2AAPC-ND,BT1,1,BC2AAPC +RMCF1206JT10K0CT-ND,"R2,R1",2,RMCF1206JT10K0 +493-4771-1-ND,C1,1,UCA2G100MPD1TD diff --git a/src/ee/bom/doit.py b/src/ee/bom/doit.py index bbb9241..521b8dd 100644 --- a/src/ee/bom/doit.py +++ b/src/ee/bom/doit.py @@ -5,7 +5,7 @@ from typing import Mapping, Union from configclass import Config from namedlist import namedlist -from ee.ds import DataSetManager, DataSet +from ee.ds import DataSetManager, DataSet, create_message logger = logging.getLogger(__name__) @@ -19,15 +19,20 @@ def change_data_sets_for_task(task, _callable): def output_data_set_for_task(task): - return _data_sets[task][0] + try: + return _data_sets[task][0] + except KeyError: + raise KeyError("No such task registered in this module: {}".format(task)) def input_data_sets_for_task(task): - return _data_sets[task][1] + try: + return _data_sets[task][1] + except KeyError: + raise KeyError("No such task registered in this module: {}".format(task)) -_config_template = Config({ -}) +_config_template = Config({}) _config = None # type: Mapping[str, str] @@ -79,9 +84,7 @@ def task_bom(): raise Exception("Missing ref") if not mpn: - output.create_object("message", "bom-{}".format(ref)). \ - set("level", "error"). \ - set("message", "Missing required field 'mpn'") + create_message(output, "Missing required field 'mpn'", "error") continue if ref in bom_components: @@ -101,7 +104,7 @@ def task_bom(): _data_sets[task_bom] = ["bom", ["components"]] -def order_csv(count: int, style: str, output_file: Path, data_sets): +def order_csv(count: int, style: str, output_file: Path, data_sets) -> DataSet: ds = _dsm.load_data_sets(data_sets) out = DataSet() @@ -114,26 +117,48 @@ def order_csv(count: int, style: str, output_file: Path, data_sets): for c in [o for o in ds.items() if o.object_type.name == "bom-component"]: ref = c.get("ref") mpn = c.get("mpn") - digikey_pn = c.get("digikey-pn") + + dk_part = None if style == "digikey": - dpn = digikey_pn.strip() if digikey_pn else "" + dk_parts = [o.get_req("part-number") for o in ds.items() if + o.object_type.name == "component-to-part-mapping" and + o.has("seller", "ref", "part-number") and + o.get("seller") == "digikey" and + o.get("ref") == ref] + + # The actual DK part should depend on the price breaks for the different packagings, but we don't have that + # info here. Luckily the DK store proposes cheaper packagings when ordering so that is something. + + dk_parts = set(dk_parts) + + if len(dk_parts) == 0: + create_message(out, "No part for component: ref={}, mpn={}".format(ref, mpn), "error") + continue + elif len(dk_parts) > 1: + create_message(out, "Multiple parts for component: ref={}, mpn={}. Don't know which one to select.". + format(ref, mpn), "error") + continue + else: + dk_part = next(iter(dk_parts)) - # TODO: implement part resolution - # if len(dpn) == 0: - # raise Exception("Missing digikey-pn for ref={}".format(ref)) + dk_part = dk_part.strip() + + if len(dk_part) == 0: + raise Exception("Missing digikey part number for ref={}".format(ref)) if mpn in parts: part = parts[mpn] - if part.digikey_pn != digikey_pn: - raise Exception("Bad data, inconsistent digikey-pn for mpn '{}'. Original digikey-pn='{}', new " - "digikey-pn='{}'".format(mpn, part.digikey_pn, digikey_pn)) + if dk_part: + if part.digikey_pn != dk_part: + raise Exception("Bad data, inconsistent digikey-pn for mpn '{}'. First digikey part number='{}', " + "new digikey part number='{}'".format(mpn, part.digikey_pn, dk_part)) part.cnt += 1 part.refs.append(ref) else: - parts[mpn] = Part(mpn=mpn, cnt=1, refs=[ref], digikey_pn=digikey_pn) + parts[mpn] = Part(mpn=mpn, cnt=1, refs=[ref], digikey_pn=dk_part) mpn_field = "MPN" count_field = "Count" @@ -163,10 +188,14 @@ def order_csv(count: int, style: str, output_file: Path, data_sets): include_extra_fields=include_extra_fields) -def create_task_order_csv(*, style: str = None, output_file: Union[str, Path], data_sets, count: int = 1): +def create_task_order_csv(*, style: str = None, output_file: Union[str, Path], out_data_set, data_sets, count: int = 1): + def action(): + with _dsm.create_rw(out_data_set, clean=True): + order_csv(count, style, Path(output_file), data_sets) + return { "name": "order-{}".format(count) if not style else "order-{}-{}".format(style, count), - "actions": [(order_csv, [count, style, Path(output_file), data_sets])], + "actions": [action], "file_dep": [_dsm.cookie_for_ds(ds) for ds in data_sets], - "targets": [output_file], + "targets": [_dsm.cookie_for_ds(out_data_set), output_file], } diff --git a/src/ee/digikey/doit.py b/src/ee/digikey/doit.py index 74259bb..2b30d9b 100644 --- a/src/ee/digikey/doit.py +++ b/src/ee/digikey/doit.py @@ -1,4 +1,6 @@ import logging +from itertools import groupby +from typing import List import ee.digikey as dk from ee.doit import DoItConfig @@ -11,38 +13,48 @@ doit_config = DoItConfig() def resolve_schematic_components(output: DataSet, in_ds: DataSet): def find(field, value): - return [o for o in output.items() if o.object_type.name == "digikey-part-stub" and o.get(field) == value] - - def save(p: dk.DigikeyProduct): - logger.info("Found part, dpn={}, mpn={}".format(p.part_number, p.mpn)) - - return output.create_object("digikey-part-stub", p.part_number, replace=True). \ - set("part-number", p.part_number). \ - set("mpn", p.mpn). \ - set("description", p.description). \ - set("quantity-available", p.quantity_available). \ - set("url", p.url) + return [o for o in output.items() if + o.object_type.name == "component-to-part-mapping" and o.get(field) == value] + + def save(refs: List[str], p: dk.DigikeyProduct): + logger.info("Found part, dpn={}, mpn={}, refs={}".format(p.part_number, p.mpn, ", ".join(refs))) + + for ref in refs: + # Make the key a bit long so we can have several parts that matches the MPN from a single seller + key = "digikey-{}-{}".format(ref, p.part_number) + output.create_object("component-to-part-mapping", key, replace=True). \ + set("seller", "digikey"). \ + set("ref", ref). \ + set("part-number", p.part_number). \ + set("mpn", p.mpn). \ + set("description", p.description). \ + set("quantity-available", p.quantity_available). \ + set("url", p.url) digikey = dk.Digikey() client = dk.DigikeyClient(digikey, on_download=logger.info) - components = [o for o in in_ds.items() if o.object_type.name == "component"] + components = [(o.get_req("mpn"), o.get_req("ref")) for o in in_ds.items() if + o.object_type.name == "component" and o.has("mpn", "ref")] + components = sorted(components, key=lambda c: c[0]) + + for mpn, components in groupby(components, key=lambda c: c[0]): + references = [c[1] for c in components] - for mpn in sorted({c.get("mpn") for c in components if c.get("mpn")}): # TODO: support searching by value and digikey part number directly. Priority should be "digikey", "mpn" and # "value", first field present should be used. dk_components = find("mpn", mpn) if len(dk_components): - logger.info("Already resolved {} to {}".format(mpn, ", ".join([c.get("mpn") for c in dk_components]))) + logger.info("Already resolved {} to {}".format(mpn, ", ".join(sorted(set([c.get("part-number") for c in dk_components]))))) continue logger.info("Looking up {}".format(mpn)) response = client.search(mpn) if response.response_type == dk.SearchResponseTypes.SINGLE: - save(response.products[0]) + save(references, response.products[0]) elif response.response_type == dk.SearchResponseTypes.MANY: # A search for "FOO" might return products "FOO" and "FOOZ" so we pick out the ones with the matching mpn # This will often be more than one product, but digikey shows the same part in different packagings. @@ -60,7 +72,7 @@ def resolve_schematic_components(output: DataSet, in_ds: DataSet): logger.info("Got multiple hits ({})".format(len(viable_products))) - save(part) + save(references, part) elif response.response_type == dk.SearchResponseTypes.TOO_MANY: logger.warning("to many matches") elif response.response_type == dk.SearchResponseTypes.NO_MATCHES: @@ -85,3 +97,50 @@ def task_digikey_resolve_schematic_components(): doit_config.set_data_sets_for(task_digikey_resolve_schematic_components, "digikey-resolved-parts", "components") + + +def download_part_facts(output: DataSet, in_ds: DataSet): + digikey = dk.Digikey() + client = dk.DigikeyClient(digikey, on_download=logger.info) + + parts = [o for o in in_ds.items() + if o.object_type.name == "component-to-part-mapping" and o.get("seller") == "digikey"] + + for pn in sorted({part.get("part-number") for part in parts}): + logger.info("Downloading facts for {}".format(pn)) + + response = client.search(pn) + + if response.response_type == dk.SearchResponseTypes.SINGLE: + product = response.products[0] + + o = output.create_object("digikey-part", pn, replace=True). \ + set("part-number", product.part_number). \ + set("url", product.url). \ + set("mpn", product.mpn) + + for a in product.attributes: + key = "{}/{}".format(a.attribute_type.id, a.attribute_type.label) + key = key.replace("%", "_") + value = a.value.replace("%", "%%") + o.set(key, value) + + +def task_digikey_fetch_full_part_facts(): + out_data_set, in_data_sets = doit_config.data_sets_for(task_digikey_fetch_full_part_facts) + + def action(): + in_ds = doit_config.dsm.load_data_sets(in_data_sets) + + with doit_config.dsm.create_rw(out_data_set, clean=False) as output: + download_part_facts(output, in_ds) + + return dict( + file_dep=[doit_config.dsm.cookie_for_ds(ds) for ds in in_data_sets], + actions=[action], + targets=[doit_config.dsm.cookie_for_ds(out_data_set)], + ) + + +doit_config.set_data_sets_for(task_digikey_fetch_full_part_facts, + "digikey-parts", "digikey-resolved-parts") diff --git a/src/ee/doit.py b/src/ee/doit.py index 014af05..dc89fae 100644 --- a/src/ee/doit.py +++ b/src/ee/doit.py @@ -35,5 +35,11 @@ class DoItConfig(object): def data_sets_for(self, task): return self._data_sets[task] + def out_data_set_for(self, task): + return self.data_sets_for(task)[0] + + def input_data_sets_for(self, task): + return self.data_sets_for(task)[1] + def set_data_sets_for(self, task, out_dataset: str, *in_datasets: str): self._data_sets[task] = [out_dataset, in_datasets] diff --git a/src/ee/ds/__init__.py b/src/ee/ds/__init__.py index 14bbb91..e48cab6 100644 --- a/src/ee/ds/__init__.py +++ b/src/ee/ds/__init__.py @@ -49,6 +49,9 @@ class ObjectType(object): self._fields.append(field) return len(self._fields) - 1 + def has(self, *keys: str): + return all([key in self._fields for key in keys]) + class Object(object): def __init__(self, ds: "DataSet", ot: ObjectType, key: str): @@ -77,6 +80,9 @@ class Object(object): for k in other._ot.fields: self.set(k, other.get(k)) + def has(self, *keys: str): + return self.object_type.has(*keys) + def get(self, key: str) -> Optional[str]: idx = self._ot.index_of(key) return self._data[idx] if idx is not None and idx < len(self._data) else None @@ -293,7 +299,7 @@ class DataSetManager(object): ds_dir = Path(_path) if Path(_path).is_absolute() else self._basedir / _path ds_dir = ds_dir if ds_dir.is_dir() else ds_dir.parent - logger.debug("Loading DS from '{}'".format(ds_dir)) + # logger.debug("Loading DS from '{}'".format(ds_dir)) self._load_ini(ds_dir / "data-set.ini") @@ -304,12 +310,12 @@ class DataSetManager(object): continue ot = ot_path.name - logger.debug(" Loading type '{}'".format(ot)) + # logger.debug(" Loading type '{}'".format(ot)) for o_path in ot_path.glob("*.ini"): count += 1 key = o_path.name[:-4] - logger.debug(" Loading key '{}'".format(key)) + # logger.debug(" Loading key '{}'".format(key)) ini = self._load_ini(o_path) o = ds.create_object(ot, key) for k, v in ini.items("values"): @@ -318,13 +324,13 @@ class DataSetManager(object): if freeze: ds.freeze() - logger.debug("Loaded {} items".format(count)) + # logger.debug("Loaded {} items".format(count)) return ds def store(self, ds: DataSet, ds_name: str): ds_dir = self._basedir / ds_name items = list(ds.items()) - logger.info("Storing DS '{}' with {} objects to {}".format(ds_name, len(items), ds_dir)) + # logger.info("Storing DS '{}' with {} objects to {}".format(ds_name, len(items), ds_dir)) os.makedirs(ds_dir, exist_ok=True) ini = self._blank_ini() @@ -428,3 +434,9 @@ class LazyRwDataSet(object): def __exit__(self, *args): self._dsm.store(self._ds, self._name) return False + + +def create_message(data_set: DataSet, message: str, level: str): + return data_set.create_object("message", "message-{}".format(str(hash(message)))). \ + set("message", message). \ + set("level", level) diff --git a/src/ee/tools/digikey_download_facts.py b/src/ee/tools/digikey_download_facts.py index 923b996..ee4cf20 100644 --- a/src/ee/tools/digikey_download_facts.py +++ b/src/ee/tools/digikey_download_facts.py @@ -1,5 +1,4 @@ import argparse -from itertools import * from functools import total_ordering import ee.digikey as dk -- cgit v1.2.3