datasheet: wip

2 files changed, 127 insertions, 1 deletions

diff --git a/src/ee/tools/part_download_datasheets.py b/src/ee/tools/part_download_datasheets.py
new file mode 100644
index 0000000..1689502
--- /dev/null
+++ b/src/ee/tools/part_download_datasheets.py
@@ -0,0 +1,117 @@
+import argparse
+import time
+from pathlib import Path
+
+import requests
+from selenium import webdriver
+from selenium.webdriver.support.wait import WebDriverWait
+
+from ee.part import Part, load_db
+from ee.xml import uris
+
+
+class Job(object):
+    def __init__(self, url, title):
+        self.url = url
+        self.title = title
+
+        self.path = Path(url).name
+
+
+def work(in_path: Path, out_dir: Path, index_path: Path):
+    in_parts = load_db(in_path)
+
+    ds_rel = uris.make_link_rel("datasheet")
+
+    jobs = {}
+    for xml in in_parts.iterparts():
+        part = Part(xml)
+
+        for link in part.get_links():
+            rels = link.relationProp.split(" ")
+            if ds_rel not in rels:
+                continue
+
+            jobs[link.urlProp] = (Job(link.urlProp, link.titleProp))
+
+    options = webdriver.ChromeOptions()
+    print("out_dir={}".format(out_dir))
+    options.add_experimental_option("prefs", {
+        "download.default_directory": str(out_dir),
+    })
+    driver = webdriver.Chrome(options=options)
+
+    # FILE_SAVER_MIN_JS_URL = "https://raw.githubusercontent.com/eligrey/FileSaver.js/1.2.1/FileSaver.js"
+    FILE_SAVER_MIN_JS_URL = "https://raw.githubusercontent.com/eligrey/FileSaver.js/v2.0.2/dist/FileSaver.js"
+    file_saver_min_js = requests.get(FILE_SAVER_MIN_JS_URL).text
+
+    # print("---------------------- file_saver_min_js ----------------------")
+    # print(file_saver_min_js)
+    # print("---------------------- file_saver_min_js ----------------------")
+    driver.execute_script(file_saver_min_js)
+
+    for j in jobs.values():
+        print("Fetching {}".format(j.url))
+
+        driver.execute_script('''
+            window.done = false;
+            var args = arguments;
+            function work() {
+                console.log("WORK");
+                return fetch(args[0], {
+                    "credentials": "same-origin",
+                    "referrerPolicy": "no-referrer-when-downgrade",
+                    "body": null,
+                    "method": "HEAD",
+                    "mode": "no-cors"
+                }).then(resp => {
+                    window.resp = resp;
+                    return resp;
+                });
+            }
+            setTimeout(work, 2000); 
+
+            return fetch(arguments[0], {
+                "credentials": "same-origin",
+                "referrerPolicy": "no-referrer-when-downgrade",
+                "body": null,
+                "method": "GET",
+                "mode": "no-cors"
+            }).then(resp => {
+                return resp.blob();
+            }).then(blob => {
+                saveAs(blob, arguments[1]);
+                window.done = true;
+            });
+            ''', j.url, j.path)
+
+        timeout = 5 * 1000
+        done = WebDriverWait(driver, timeout).until(lambda d: d.execute_script('return window.done'))
+        print("done={}".format(done))
+
+        time.sleep(1000)
+
+        break
+
+    with index_path.open("w") as f:
+        print("<root/>", file=f)
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--in",
+                    dest="in_path",
+                    required=True,
+                    metavar="PART DB")
+
+parser.add_argument("--destination",
+                    required=True,
+                    metavar="DIRECTORY")
+
+parser.add_argument("--index",
+                    required=True,
+                    metavar="INDEX")
+
+args = parser.parse_args()
+
+work(Path(args.in_path), Path(args.destination), Path(args.index))
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index c68f505..492363d 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -1,4 +1,5 @@
 {% set reports=[] -%}
+{% set datasheets=[] -%}
 ee = {{ ee }}
 uuid = {{ project.uuid }}
 public_dir = {{ project.public_dir }}
@@ -49,6 +50,9 @@ rule part-apply-souffle-post
 rule part-find-requirements
     command = $ee part-find-requirements {{ log }} --in $in --out $out $report
 
+rule part-download-datasheets
+    command = $ee part-download-datasheets --in $in --destination $dir --index $out
+
 rule part-validate-parts
     command = $ee part-validate-parts {{ log }} --bom $bom --sch $sch --report $out --part-db $part_dbs
 
@@ -135,6 +139,10 @@ build $public_dir/{{ s.key }}/pn-part-search-list.xml: pn-part-search-list $publ
 build $public_dir/{{ s.key }}/downloaded.xml | $public_dir/{{ s.key }}/downloaded.rst: {{ s.key }}-search-parts $public_dir/{{ s.key }}/pn-part-search-list.xml
 {%- set reports=reports+["$report_dir/" + s.key + "/downloaded.rst"] %}
 
+build $public_dir/{{ s.key }}/datasheets.xml: part-download-datasheets $public_dir/{{ s.key }}/downloaded.xml
+    dir = $public_dir/{{ s.key }}/datasheets
+{%- set _=datasheets.append("$public_dir/" + s.key + "/datasheets.xml") %}
+
 {#
 build $public_dir/{{ s.key }}/parts.xml: part-apply-function $public_dir/{{ s.key }}/downloaded.xml
     execution = {{ s.key }}
@@ -206,6 +214,7 @@ default ee.ninja
 
 # Reports
 build ee-reports: phony {{ " ".join(reports) }}
+build ee-download-datasheets: phony {{ " ".join(datasheets) }}
 build ee-part-dbs: phony {%- for s in suppliers %} {{ s.part_db }}{% endfor %}
 build ee-orders: phony {%- for s in suppliers %} {{ s.bom_output }}{% endfor %}
-build ee-all: phony ee-reports ee-orders
+build ee-all: phony ee-reports ee-download-datasheets ee-part-dbs ee-orders