aboutsummaryrefslogtreecommitdiff
path: root/src/ee/tools
diff options
context:
space:
mode:
Diffstat (limited to 'src/ee/tools')
-rw-r--r--src/ee/tools/part_download_datasheets.py117
-rw-r--r--src/ee/tools/templates/build.ninja.j211
2 files changed, 127 insertions, 1 deletions
diff --git a/src/ee/tools/part_download_datasheets.py b/src/ee/tools/part_download_datasheets.py
new file mode 100644
index 0000000..1689502
--- /dev/null
+++ b/src/ee/tools/part_download_datasheets.py
@@ -0,0 +1,117 @@
+import argparse
+import time
+from pathlib import Path
+
+import requests
+from selenium import webdriver
+from selenium.webdriver.support.wait import WebDriverWait
+
+from ee.part import Part, load_db
+from ee.xml import uris
+
+
+class Job(object):
+ def __init__(self, url, title):
+ self.url = url
+ self.title = title
+
+ self.path = Path(url).name
+
+
+def work(in_path: Path, out_dir: Path, index_path: Path):
+ in_parts = load_db(in_path)
+
+ ds_rel = uris.make_link_rel("datasheet")
+
+ jobs = {}
+ for xml in in_parts.iterparts():
+ part = Part(xml)
+
+ for link in part.get_links():
+ rels = link.relationProp.split(" ")
+ if ds_rel not in rels:
+ continue
+
+ jobs[link.urlProp] = (Job(link.urlProp, link.titleProp))
+
+ options = webdriver.ChromeOptions()
+ print("out_dir={}".format(out_dir))
+ options.add_experimental_option("prefs", {
+ "download.default_directory": str(out_dir),
+ })
+ driver = webdriver.Chrome(options=options)
+
+ # FILE_SAVER_MIN_JS_URL = "https://raw.githubusercontent.com/eligrey/FileSaver.js/1.2.1/FileSaver.js"
+ FILE_SAVER_MIN_JS_URL = "https://raw.githubusercontent.com/eligrey/FileSaver.js/v2.0.2/dist/FileSaver.js"
+ file_saver_min_js = requests.get(FILE_SAVER_MIN_JS_URL).text
+
+ # print("---------------------- file_saver_min_js ----------------------")
+ # print(file_saver_min_js)
+ # print("---------------------- file_saver_min_js ----------------------")
+ driver.execute_script(file_saver_min_js)
+
+ for j in jobs.values():
+ print("Fetching {}".format(j.url))
+
+ driver.execute_script('''
+ window.done = false;
+ var args = arguments;
+ function work() {
+ console.log("WORK");
+ return fetch(args[0], {
+ "credentials": "same-origin",
+ "referrerPolicy": "no-referrer-when-downgrade",
+ "body": null,
+ "method": "HEAD",
+ "mode": "no-cors"
+ }).then(resp => {
+ window.resp = resp;
+ return resp;
+ });
+ }
+ setTimeout(work, 2000);
+
+ return fetch(arguments[0], {
+ "credentials": "same-origin",
+ "referrerPolicy": "no-referrer-when-downgrade",
+ "body": null,
+ "method": "GET",
+ "mode": "no-cors"
+ }).then(resp => {
+ return resp.blob();
+ }).then(blob => {
+ saveAs(blob, arguments[1]);
+ window.done = true;
+ });
+ ''', j.url, j.path)
+
+ timeout = 5 * 1000
+ done = WebDriverWait(driver, timeout).until(lambda d: d.execute_script('return window.done'))
+ print("done={}".format(done))
+
+ time.sleep(1000)
+
+ break
+
+ with index_path.open("w") as f:
+ print("<root/>", file=f)
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--in",
+ dest="in_path",
+ required=True,
+ metavar="PART DB")
+
+parser.add_argument("--destination",
+ required=True,
+ metavar="DIRECTORY")
+
+parser.add_argument("--index",
+ required=True,
+ metavar="INDEX")
+
+args = parser.parse_args()
+
+work(Path(args.in_path), Path(args.destination), Path(args.index))
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index c68f505..492363d 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -1,4 +1,5 @@
{% set reports=[] -%}
+{% set datasheets=[] -%}
ee = {{ ee }}
uuid = {{ project.uuid }}
public_dir = {{ project.public_dir }}
@@ -49,6 +50,9 @@ rule part-apply-souffle-post
rule part-find-requirements
command = $ee part-find-requirements {{ log }} --in $in --out $out $report
+rule part-download-datasheets
+ command = $ee part-download-datasheets --in $in --destination $dir --index $out
+
rule part-validate-parts
command = $ee part-validate-parts {{ log }} --bom $bom --sch $sch --report $out --part-db $part_dbs
@@ -135,6 +139,10 @@ build $public_dir/{{ s.key }}/pn-part-search-list.xml: pn-part-search-list $publ
build $public_dir/{{ s.key }}/downloaded.xml | $public_dir/{{ s.key }}/downloaded.rst: {{ s.key }}-search-parts $public_dir/{{ s.key }}/pn-part-search-list.xml
{%- set reports=reports+["$report_dir/" + s.key + "/downloaded.rst"] %}
+build $public_dir/{{ s.key }}/datasheets.xml: part-download-datasheets $public_dir/{{ s.key }}/downloaded.xml
+ dir = $public_dir/{{ s.key }}/datasheets
+{%- set _=datasheets.append("$public_dir/" + s.key + "/datasheets.xml") %}
+
{#
build $public_dir/{{ s.key }}/parts.xml: part-apply-function $public_dir/{{ s.key }}/downloaded.xml
execution = {{ s.key }}
@@ -206,6 +214,7 @@ default ee.ninja
# Reports
build ee-reports: phony {{ " ".join(reports) }}
+build ee-download-datasheets: phony {{ " ".join(datasheets) }}
build ee-part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %}
build ee-orders: phony {%- for s in suppliers %} {{ s.bom_output }}{% endfor %}
-build ee-all: phony ee-reports ee-orders
+build ee-all: phony ee-reports ee-download-datasheets ee-part-dbs ee-orders