aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/ee/db.py4
-rw-r--r--src/ee/digikey/__init__.py50
-rw-r--r--src/ee/digikey/search_parts.py23
-rw-r--r--src/ee/project/__init__.py15
-rw-r--r--src/ee/tools/digikey_download_facts.py8
-rw-r--r--src/ee/tools/digikey_search_parts.py8
-rw-r--r--src/ee/tools/split_parts_by_supplier.py69
-rw-r--r--src/ee/tools/templates/build.ninja.j28
-rw-r--r--src/ee/xml/uris.py4
-rw-r--r--test/test_digikey.py26
10 files changed, 184 insertions, 31 deletions
diff --git a/src/ee/db.py b/src/ee/db.py
index 57533ef..79d48ee 100644
--- a/src/ee/db.py
+++ b/src/ee/db.py
@@ -1,4 +1,4 @@
-from typing import TypeVar, Generic, Callable, MutableMapping, List, Iterable, Union, Any, Mapping
+from typing import TypeVar, Generic, Callable, MutableMapping, List, Iterable, Union, Any, Mapping, Tuple, AbstractSet
K = TypeVar('K')
V = TypeVar('V')
@@ -69,7 +69,7 @@ class ListIndex(Index[K, V]):
def __iter__(self):
return self.idx.__iter__()
- def items(self):
+ def items(self) -> AbstractSet[Tuple[K, V]]:
return self.idx.items()
def values(self):
diff --git a/src/ee/digikey/__init__.py b/src/ee/digikey/__init__.py
index 9815a8f..31d5c41 100644
--- a/src/ee/digikey/__init__.py
+++ b/src/ee/digikey/__init__.py
@@ -8,6 +8,7 @@ import urllib.parse
from functools import total_ordering
from pathlib import Path
from typing import List, Optional
+from urllib.parse import urlparse, parse_qs
from lxml import html
from selenium import webdriver
@@ -59,6 +60,48 @@ def _first(collection, default=None):
return next(iter(collection), default)
+class DigikeyStore(object):
+ BASEURL = "http://purl.org/ee/supplier/digikey"
+
+ def __init__(self, url, store, products_url):
+ self.url = url
+ self.store = store
+ self.products_url = products_url
+
+ @staticmethod
+ def from_store_code(store_code):
+ url = "{}?store={}".format(DigikeyStore.BASEURL, store_code)
+
+ products_url = "https://www.digikey.com/products/en" if store_code == "us" else \
+ "https://www.digikey.{}/products/en".format(store_code)
+
+ return DigikeyStore(url, store_code, products_url)
+
+ @staticmethod
+ def from_url(store_url) -> Optional["DigikeyStore"]:
+ base = urlparse(DigikeyStore.BASEURL)
+ url = urlparse(store_url)
+
+ if base.scheme != url.scheme or \
+ base.netloc != url.netloc or \
+ base.path != url.path:
+ return None
+
+ q = parse_qs((url.query or "").strip())
+
+ store = q.get("store")
+
+ if not store:
+ return None
+
+ del q["store"]
+
+ if len(q):
+ return None
+
+ return DigikeyStore.from_store_code(store[0])
+
+
class Digikey(object):
def __init__(self):
self.attribute_types = {}
@@ -230,7 +273,8 @@ class DigikeyClient(object):
def __nop(self, message):
pass
- def __init__(self, cache_dir: Path = None, on_download=None):
+ def __init__(self, baseurl, cache_dir: Path = None, on_download=None):
+ self.baseurl = baseurl
self.on_download = on_download or self.__nop
self.cache = ee._utils.maybe_cache(cache_dir)
self.driver: Optional[webdriver.Chrome] = None
@@ -241,13 +285,11 @@ class DigikeyClient(object):
def product_search(self, query: str, page_size=10) -> str:
params = {'lang': 'en', 'site': 'us', 'keywords': query, 'pageSize': str(page_size)}
cache_key = urllib.parse.quote(query)
- page = self._req("https://www.digikey.com/products/en", cache_key=cache_key, params=params)
+ page = self._req(self.baseurl, cache_key=cache_key, params=params)
return page
def _req(self, url, cache_key, params=None):
- if not url.startswith("http"):
- url = "https://www.digikey.com" + url
url = url + ("" if not params else "?" + urllib.parse.urlencode(params))
cached = self.cache.lookup(cache_key)
diff --git a/src/ee/digikey/search_parts.py b/src/ee/digikey/search_parts.py
index fec5615..c7f981e 100644
--- a/src/ee/digikey/search_parts.py
+++ b/src/ee/digikey/search_parts.py
@@ -1,19 +1,18 @@
from pathlib import Path
from typing import List
-from ee.digikey import Digikey, DigikeyParser, DigikeyClient, SearchResponseTypes, DigikeyProduct
+from ee.digikey import Digikey, DigikeyParser, DigikeyClient, SearchResponseTypes, DigikeyProduct, DigikeyStore
from ee.part import PartDb, load_db, save_db, Part
-from ee.project import Project
from ee.xml import types
-from ee.xml.uris import DIGIKEY_URI, make_digikey_fact_key
+from ee.xml.uris import make_digikey_fact_key
__all__ = ["search_parts"]
-def resolved(p: DigikeyProduct) -> Part:
+def resolved(supplier, p: DigikeyProduct) -> Part:
# TODO: fix uri
xml = types.Part(uri="https://digikey.com/pn#{}".format(p.part_number),
- supplier=DIGIKEY_URI,
+ supplier=supplier,
description=p.description,
distributor_info=types.DistributorInfo(state="resolved"),
links=types.LinkList(),
@@ -50,15 +49,17 @@ def resolved(p: DigikeyProduct) -> Part:
return part
-def search_parts(in_path: Path, out_path: Path, cache_dir: Path):
+def search_parts(in_path: Path, out_path: Path, cache_dir: Path, store_code):
in_db = load_db(in_path)
out_parts = PartDb()
+ store = DigikeyStore.from_store_code(store_code)
+
parser = DigikeyParser(Digikey())
- client = DigikeyClient(cache_dir)
+ client = DigikeyClient(store.products_url, cache_dir)
for xml in in_db.iterparts():
- if xml.supplierProp is not None and xml.supplierProp != DIGIKEY_URI:
+ if xml.supplierProp is not None and xml.supplierProp != store.url:
continue
part = Part(xml)
@@ -83,7 +84,7 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path):
out_id = query
out_part = types.Part(uri=out_id,
distributor_info=types.DistributorInfo(),
- supplier=DIGIKEY_URI,
+ supplier=store.url,
references=xml.referencesProp)
di = out_part.distributor_infoProp
@@ -91,7 +92,7 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path):
response = parser.parse_string(text)
if response.response_type == SearchResponseTypes.SINGLE:
- out_part = resolved(response.products[0])
+ out_part = resolved(store.url, response.products[0])
elif response.response_type == SearchResponseTypes.MANY:
# find those with an exact match. Digikey uses a prefix search so a query for "FOO" will return "FOO"
@@ -108,7 +109,7 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path):
response = parser.parse_string(client.search(dpn))
if response.response_type == SearchResponseTypes.SINGLE:
- out_part = resolved(response.products[0])
+ out_part = resolved(store.url, response.products[0])
else:
di.stateProp = "many"
diff --git a/src/ee/project/__init__.py b/src/ee/project/__init__.py
index 24635ee..0857e7c 100644
--- a/src/ee/project/__init__.py
+++ b/src/ee/project/__init__.py
@@ -2,8 +2,8 @@ import configparser
from pathlib import Path
from ee import EeException
+from ee.digikey import DigikeyStore
from ee.tools import mk_parents
-from ee.xml.uris import DIGIKEY_URI
def load_config(project_dir: Path) -> configparser.ConfigParser:
@@ -20,9 +20,10 @@ def load_config(project_dir: Path) -> configparser.ConfigParser:
class SupplierDescriptor(object):
- def __init__(self, key: str, uri: str):
+ def __init__(self, key: str, uri: str, name: str):
self.key = key
self.uri = uri
+ self.name = name
class Project(object):
@@ -34,7 +35,9 @@ class Project(object):
self._cfg = cfg
# TODO: read from config
- self._suppliers = [SupplierDescriptor("digikey", DIGIKEY_URI)]
+ self._suppliers = []
+ digikey_store = DigikeyStore.from_store_code("us")
+ self._suppliers.append(SupplierDescriptor("digikey", digikey_store.url, "Digikey"))
def get_supplier_by_key(self, key) -> SupplierDescriptor:
sd = next((s for s in self._suppliers if s.key == key), None)
@@ -42,6 +45,12 @@ class Project(object):
return sd
raise EeException("No such supplier configured: {}".format(key))
+ def get_supplier_by_uri(self, uri) -> SupplierDescriptor:
+ sd = next((s for s in self._suppliers if s.uri == uri), None)
+ if sd:
+ return sd
+ raise EeException("No such supplier configured: {}".format(uri))
+
@property
def cfg(self):
return self._cfg
diff --git a/src/ee/tools/digikey_download_facts.py b/src/ee/tools/digikey_download_facts.py
index 3ab8551..5cfdba7 100644
--- a/src/ee/tools/digikey_download_facts.py
+++ b/src/ee/tools/digikey_download_facts.py
@@ -5,6 +5,9 @@ import ee.digikey as dk
from ee.digikey import SearchResponseTypes, DigikeyProduct
from ee.tools import log
+if True:
+ raise Exception("This module is deprecated.")
+
@total_ordering
class Query(object):
@@ -45,7 +48,7 @@ parser.add_argument("--force",
args = parser.parse_args()
digikey = dk.Digikey()
-client = dk.DigikeyClient(on_download=log.debug)
+client = dk.DigikeyClient("https://www.digikey.com/products/en", on_download=log.debug)
parser = dk.DigikeyParser(digikey)
repo = dk.DigikeyRepository(digikey, args.out)
@@ -99,7 +102,8 @@ for q in queries:
# Pick the first one, should be as good as any
part_number = sorted(viable_products, key=lambda p: p.part_number)[0].part_number
- log.info("Got many hits for term '{}', will use {} for downloading attributes.".format(q.query, part_number))
+ log.info(
+ "Got many hits for term '{}', will use {} for downloading attributes.".format(q.query, part_number))
todos.append(part_number)
else:
log.warn("Got many results: {}".format(", ".join([p.part_number for p in response.products])))
diff --git a/src/ee/tools/digikey_search_parts.py b/src/ee/tools/digikey_search_parts.py
index 6cf104d..28ccbb9 100644
--- a/src/ee/tools/digikey_search_parts.py
+++ b/src/ee/tools/digikey_search_parts.py
@@ -15,9 +15,15 @@ parser.add_argument("--out",
required=True,
metavar="PART DB")
+parser.add_argument("--store",
+ default="us",
+ metavar="STORE CODE")
+
args = parser.parse_args()
project = Project.load()
cache_dir = project.cache_dir / "digikey"
-search_parts(Path(args.in_path), Path(args.out), cache_dir)
+store_code = args.store
+
+search_parts(Path(args.in_path), Path(args.out), cache_dir, store_code)
diff --git a/src/ee/tools/split_parts_by_supplier.py b/src/ee/tools/split_parts_by_supplier.py
new file mode 100644
index 0000000..1ebf094
--- /dev/null
+++ b/src/ee/tools/split_parts_by_supplier.py
@@ -0,0 +1,69 @@
+import argparse
+from pathlib import Path
+from typing import List
+
+from ee.db import ObjDb
+from ee.part import Part, load_db, save_db, PartDb
+from ee.project import Project
+
+
+class OrderPart(object):
+ def __init__(self, order_part: Part, part: Part):
+ self.order_part = order_part
+ self.part = part
+
+
+def uri_fn(part: Part):
+ return part.uri
+
+
+def split_parts_by_supplier(project: Project, order_file: Path, part_dbs: List[Path], out_dir: Path):
+ parts: ObjDb[Part] = ObjDb[Part]()
+ part_by_uri = parts.add_unique_index("uri", uri_fn)
+
+ for part_db in part_dbs:
+ for xml in load_db(part_db).iterparts():
+ parts.add(Part(xml))
+
+ order_parts: ObjDb[OrderPart] = ObjDb()
+ supplier_idx = order_parts.add_index("supplier", lambda op: op.part.supplier)
+ for xml in load_db(order_file).iterparts():
+ order_part = Part(xml)
+ part = part_by_uri.get_single(order_part.get_exactly_one_part_reference().part_uriProp)
+ order_parts.add(OrderPart(order_part, part))
+
+ for supplier, parts_for_supplier in supplier_idx.items():
+ desc = project.get_supplier_by_uri(supplier)
+
+ print("{}: {}".format(desc.name, len(parts)))
+ # supplier_db: ObjDb[Part] = ObjDb[Part]()
+ # supplier_db.add_unique_index("uri", uri_fn)
+
+ supplier_descriptor = project.get_supplier_by_uri(supplier)
+
+ db = PartDb()
+ for part_for_supplier in parts_for_supplier:
+ db.add_entry(part_for_supplier.part.underlying, False)
+
+ save_db(out_dir / "{}.xml".format(supplier_descriptor.key), db, sort=True)
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--parts",
+ required=True,
+ metavar="PART DB")
+
+parser.add_argument("--part-db",
+ nargs="*",
+ required=True,
+ metavar="PART DB")
+
+parser.add_argument("--out-dir",
+ metavar="DIR FOR PART DBS")
+
+args = parser.parse_args()
+
+part_db_dirs = [Path(part_db) for part_db in args.part_db]
+
+split_parts_by_supplier(Project.load(), Path(args.parts), part_db_dirs, Path(args.out_dir))
diff --git a/src/ee/tools/templates/build.ninja.j2 b/src/ee/tools/templates/build.ninja.j2
index ea979da..d62f350 100644
--- a/src/ee/tools/templates/build.ninja.j2
+++ b/src/ee/tools/templates/build.ninja.j2
@@ -45,9 +45,9 @@ rule create-bom
description = create-bom
command = $ee create-bom --schematic $schematic --part-db $part_dbs --out $out $strategy
-rule export-order
- description = export-order
- command = $ee export-order --order $order $part_dbs --out-dir $out_dir
+rule split-parts-by-supplier
+ description = split-parts-by-supplier
+ command = $ee split-parts-by-supplier --parts $order $part_dbs --out-dir $out_dir
rule import-parts-yaml
description = import-parts-yaml $in
@@ -90,7 +90,7 @@ build ee/bom.xml | $report_dir/bom.rst: create-bom ee/sch.xml {%- for p in part_
strategy = --strategy {{ project.cfg["create-bom"]["strategy"] }}
{%- endif %}
-build ee/orders/index.xml: export-order ee/bom.xml {%- for p in part_dbs %} {{ p }}.xml{% endfor %}
+build ee/orders/index.xml: split-parts-by-supplier ee/bom.xml {%- for p in part_dbs %} {{ p }}.xml{% endfor %}
order = ee/bom.xml
part_dbs ={%- for p in part_dbs %} --part-db {{ p }}.xml{% endfor %}
out_dir = ee/orders
diff --git a/src/ee/xml/uris.py b/src/ee/xml/uris.py
index f0a1022..c716103 100644
--- a/src/ee/xml/uris.py
+++ b/src/ee/xml/uris.py
@@ -6,9 +6,7 @@ DIODE = "http://purl.org/ee/part-type#diode"
INDUCTOR = "http://purl.org/ee/part-type#inductor"
CRYSTAL = "http://purl.org/ee/part-type#inductor"
-DIGIKEY_URI = "https://digikey.com"
-
-_DIGIKEY_FACT_KEY_PREFIX = "http://purl.org/trygvis/ee/digikey-fact-key#"
+_DIGIKEY_FACT_KEY_PREFIX = "http://purl.org/ee/digikey-fact-key#"
def make_digikey_fact_key(key: int) -> str:
diff --git a/test/test_digikey.py b/test/test_digikey.py
index 2e9b1f1..568360e 100644
--- a/test/test_digikey.py
+++ b/test/test_digikey.py
@@ -12,7 +12,8 @@ static_copies = basedir / "digikey" / "static-copies" # type: Path
digikey = dk.Digikey()
parser = dk.DigikeyParser(digikey)
-client = dk.DigikeyClient(cache_dir=basedir / "digikey" / "static-copies", on_download=print)
+client = dk.DigikeyClient("https://www.digikey.com/products/en", cache_dir=basedir / "digikey" / "static-copies",
+ on_download=print)
force_refresh = False # Set to True to always fetch the updated html files
@@ -83,3 +84,26 @@ def test_digikey_3():
p = next((p for p in res.products if p.part_number == "1655-1501-1-ND"), None)
assert p.mpn == "RS1MTR"
assert p.url == "/product-detail/en/smc-diode-solutions/RS1MTR/1655-1501-1-ND/6022946"
+
+
+@pytest.mark.digikey
+def test_store():
+ store = dk.DigikeyStore.from_store_code("us")
+ assert store.products_url == "https://www.digikey.com/products/en"
+ assert store.url == "http://purl.org/ee/supplier/digikey?store=us"
+ assert store.store == "us"
+
+ store = dk.DigikeyStore.from_url(store.url)
+ assert store.products_url == "https://www.digikey.com/products/en"
+ assert store.url == "http://purl.org/ee/supplier/digikey?store=us"
+ assert store.store == "us"
+
+ store = dk.DigikeyStore.from_store_code("no")
+ assert store.products_url == "https://www.digikey.no/products/en"
+ assert store.url == "http://purl.org/ee/supplier/digikey?store=no"
+ assert store.store == "no"
+
+ store = dk.DigikeyStore.from_url(store.url)
+ assert store.products_url == "https://www.digikey.no/products/en"
+ assert store.url == "http://purl.org/ee/supplier/digikey?store=no"
+ assert store.store == "no"
generated by cgit v1.2.3 (git 2.46.0) at 2024-12-22 21:36:54 +0000