diff options
Diffstat (limited to 'src/ee/tools')
| -rw-r--r-- | src/ee/tools/part_download_datasheets.py | 117 | ||||
| -rw-r--r-- | src/ee/tools/templates/build.ninja.j2 | 11 |
2 files changed, 127 insertions, 1 deletions
diff --git a/src/ee/tools/part_download_datasheets.py b/src/ee/tools/part_download_datasheets.py new file mode 100644 index 0000000..1689502 --- /dev/null +++ b/src/ee/tools/part_download_datasheets.py @@ -0,0 +1,117 @@ +import argparse +import time +from pathlib import Path + +import requests +from selenium import webdriver +from selenium.webdriver.support.wait import WebDriverWait + +from ee.part import Part, load_db +from ee.xml import uris + + +class Job(object): + def __init__(self, url, title): + self.url = url + self.title = title + + self.path = Path(url).name + + +def work(in_path: Path, out_dir: Path, index_path: Path): + in_parts = load_db(in_path) + + ds_rel = uris.make_link_rel("datasheet") + + jobs = {} + for xml in in_parts.iterparts(): + part = Part(xml) + + for link in part.get_links(): + rels = link.relationProp.split(" ") + if ds_rel not in rels: + continue + + jobs[link.urlProp] = (Job(link.urlProp, link.titleProp)) + + options = webdriver.ChromeOptions() + print("out_dir={}".format(out_dir)) + options.add_experimental_option("prefs", { + "download.default_directory": str(out_dir), + }) + driver = webdriver.Chrome(options=options) + + # FILE_SAVER_MIN_JS_URL = "https://raw.githubusercontent.com/eligrey/FileSaver.js/1.2.1/FileSaver.js" + FILE_SAVER_MIN_JS_URL = "https://raw.githubusercontent.com/eligrey/FileSaver.js/v2.0.2/dist/FileSaver.js" + file_saver_min_js = requests.get(FILE_SAVER_MIN_JS_URL).text + + # print("---------------------- file_saver_min_js ----------------------") + # print(file_saver_min_js) + # print("---------------------- file_saver_min_js ----------------------") + driver.execute_script(file_saver_min_js) + + for j in jobs.values(): + print("Fetching {}".format(j.url)) + + driver.execute_script(''' + window.done = false; + var args = arguments; + function work() { + console.log("WORK"); + return fetch(args[0], { + "credentials": "same-origin", + "referrerPolicy": "no-referrer-when-downgrade", + "body": null, + "method": "HEAD", + "mode": "no-cors" + }).then(resp => { + window.resp = resp; + return resp; + }); + } + setTimeout(work, 2000); + + return fetch(arguments[0], { + "credentials": "same-origin", + "referrerPolicy": "no-referrer-when-downgrade", + "body": null, + "method": "GET", + "mode": "no-cors" + }).then(resp => { + return resp.blob(); + }).then(blob => { + saveAs(blob, arguments[1]); + window.done = true; + }); + ''', j.url, j.path) + + timeout = 5 * 1000 + done = WebDriverWait(driver, timeout).until(lambda d: d.execute_script('return window.done')) + print("done={}".format(done)) + + time.sleep(1000) + + break + + with index_path.open("w") as f: + print("<root/>", file=f) + + +parser = argparse.ArgumentParser() + +parser.add_argument("--in", + dest="in_path", + required=True, + metavar="PART DB") + +parser.add_argument("--destination", + required=True, + metavar="DIRECTORY") + +parser.add_argument("--index", + required=True, + metavar="INDEX") + +args = parser.parse_args() + +work(Path(args.in_path), Path(args.destination), Path(args.index)) diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2 index c68f505..492363d 100644 --- a/src/ee/tools/templates/build.ninja.j2 +++ b/src/ee/tools/templates/build.ninja.j2 @@ -1,4 +1,5 @@ {% set reports=[] -%} +{% set datasheets=[] -%} ee = {{ ee }} uuid = {{ project.uuid }} public_dir = {{ project.public_dir }} @@ -49,6 +50,9 @@ rule part-apply-souffle-post rule part-find-requirements command = $ee part-find-requirements {{ log }} --in $in --out $out $report +rule part-download-datasheets + command = $ee part-download-datasheets --in $in --destination $dir --index $out + rule part-validate-parts command = $ee part-validate-parts {{ log }} --bom $bom --sch $sch --report $out --part-db $part_dbs @@ -135,6 +139,10 @@ build $public_dir/{{ s.key }}/pn-part-search-list.xml: pn-part-search-list $publ build $public_dir/{{ s.key }}/downloaded.xml | $public_dir/{{ s.key }}/downloaded.rst: {{ s.key }}-search-parts $public_dir/{{ s.key }}/pn-part-search-list.xml {%- set reports=reports+["$report_dir/" + s.key + "/downloaded.rst"] %} +build $public_dir/{{ s.key }}/datasheets.xml: part-download-datasheets $public_dir/{{ s.key }}/downloaded.xml + dir = $public_dir/{{ s.key }}/datasheets +{%- set _=datasheets.append("$public_dir/" + s.key + "/datasheets.xml") %} + {# build $public_dir/{{ s.key }}/parts.xml: part-apply-function $public_dir/{{ s.key }}/downloaded.xml execution = {{ s.key }} @@ -206,6 +214,7 @@ default ee.ninja # Reports build ee-reports: phony {{ " ".join(reports) }} +build ee-download-datasheets: phony {{ " ".join(datasheets) }} build ee-part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %} build ee-orders: phony {%- for s in suppliers %} {{ s.bom_output }}{% endfor %} -build ee-all: phony ee-reports ee-orders +build ee-all: phony ee-reports ee-download-datasheets ee-part-dbs ee-orders |