aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTrygve Laugstøl <trygvis@inamo.no>2017-09-30 23:05:00 +0200
committerTrygve Laugstøl <trygvis@inamo.no>2017-09-30 23:05:00 +0200
commit122850d7a90428b6d7b92fe6100a1f2a6df2a1eb (patch)
tree5408d8509b42fc6ee53b608674da3dcb8e0578f7
parent88e878626eac92d255b763b1ba5da5b393bfdc2f (diff)
downloadee-python-122850d7a90428b6d7b92fe6100a1f2a6df2a1eb.tar.gz
ee-python-122850d7a90428b6d7b92fe6100a1f2a6df2a1eb.tar.bz2
ee-python-122850d7a90428b6d7b92fe6100a1f2a6df2a1eb.tar.xz
ee-python-122850d7a90428b6d7b92fe6100a1f2a6df2a1eb.zip
o Switching from YAML to INI files for downloaded facts.
o Improved fact downloader.
-rw-r--r--requirements.txt1
-rw-r--r--src/ee/digikey/__init__.py80
-rw-r--r--src/ee/tools/digikey_download_facts.py66
-rw-r--r--test/test_digikey.py40
4 files changed, 144 insertions, 43 deletions
diff --git a/requirements.txt b/requirements.txt
index 4278b1f..05d8a6b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,6 @@ pandas==0.20.3
parsec==3.3
Pillow==4.2.1
pytest==3.2.0
-pyyaml==3.12
requests==2.18.4
sympy==1.0
typing==3.6.2; python_version < '3.0'
diff --git a/src/ee/digikey/__init__.py b/src/ee/digikey/__init__.py
index af5977b..1315f96 100644
--- a/src/ee/digikey/__init__.py
+++ b/src/ee/digikey/__init__.py
@@ -6,7 +6,8 @@ import re
import requests
import os
import os.path
-import yaml
+import configparser
+import glob
from cachecontrol import CacheControl
from cachecontrol.caches.file_cache import FileCache
from cachecontrol.heuristics import ExpiresAfter
@@ -90,13 +91,32 @@ class DigikeyProduct(object):
def __hash__(self):
return self.part_number.__hash__()
- def to_yaml(self):
- yaml = {"part_number": self.part_number}
+ def to_ini(self):
+ c = configparser.ConfigParser()
+ c["overview"] = {}; overview = c["overview"]
+ overview["part_number"] = self.part_number
if self.mpn:
- yaml["mpn"] = self.mpn
- yaml["attributes"] = [{"type": {"id": a.attribute_type.id, "label": a.attribute_type.label}, "value": a.value}
- for a in self.attributes]
- return yaml
+ overview["mpn"] = self.mpn
+ c["attributes"] = {}; attributes = c["attributes"]
+ for a in self.attributes:
+ key = "{}/{}".format(a.attribute_type.id, a.attribute_type.label)
+ key = key.replace("%", "_")
+ value = a.value.replace("%", "%%")
+ attributes[key] = value
+ return c
+
+ from_ini_r = re.compile("([^/]*)/(.*)")
+
+ @staticmethod
+ def from_ini(digikey, c):
+ overview = c["overview"]
+ attributes = []
+ for k, value in c.items("attributes"):
+# print("k={}".format(k))
+ (type_id, label) = DigikeyProduct.from_ini_r.match(k).groups()
+ a_type = digikey.get_attribute_type(type_id, label)
+ attributes.append(DigikeyAttributeValue(value, a_type))
+ return DigikeyProduct(overview["part_number"], overview["mpn"], attributes)
class DigikeyAttributeType(object):
@@ -228,8 +248,8 @@ class DigikeyClient(object):
products = tree.xpath("//*[@itemtype='http://schema.org/Product']")
for product in products:
- part_number = _first(product.xpath("//*[@itemprop='productid' and @content]"))
- mpn = _first(product.xpath("//*[@itemprop='name']"))
+ part_number = _first(product.xpath(".//*[@itemprop='productid' and @content]"))
+ mpn = _first(product.xpath(".//*[@itemprop='name']"))
if part_number is not None and mpn is not None:
res.append(DigikeyProduct(
@@ -272,19 +292,45 @@ class DigikeyClient(object):
return DigikeySearchResponse(1, SearchResponseTypes.NO_MATCHES)
class DigikeyRepository(object):
- def __init__(self, path):
+ def __init__(self, digikey, path):
+ self._digikey = digikey
self._path = path
+ self._products = {}
def mpn_to_path(self, mpn):
- return "{}/{}.yaml".format(self._path, mpn)
-
- def has_product(self, mpn):
- filename = self.mpn_to_path(mpn)
- return os.path.isfile(filename)
+ mpn = mpn.replace("/", "_").replace(" ", "_")
+ return "{}/{}.ini".format(self._path, mpn)
+
+ def has_product(self, mpn=None, dpn=None):
+ if mpn is not None:
+ filename = self.mpn_to_path(mpn)
+ return os.path.isfile(filename)
+ if dpn is not None:
+ for p in self.products:
+ if p.part_number == dpn:
+ return p
def save(self, product: DigikeyProduct):
- y = product.to_yaml()
+ y = product.to_ini()
filename = self.mpn_to_path(product.mpn)
mk_parents(filename)
with open(filename, "w") as f:
- yaml.dump(y, f, encoding="utf-8", allow_unicode=True)
+ y.write(f)
+
+ def load_all(self):
+ [self._load(path) for path in glob.glob(self._path + "/*.ini")]
+
+ def _load(self, path):
+ c = configparser.ConfigParser()
+ c.read(path)
+ p = DigikeyProduct.from_ini(self._digikey, c)
+ self._products[p.mpn] = p
+ return p
+
+ @property
+ def products(self):
+ self.load_all()
+ return self._products.values()
+
+ def find_by_mpn(self, mpn):
+ return [p for p in self.products if p.mpn == mpn]
diff --git a/src/ee/tools/digikey_download_facts.py b/src/ee/tools/digikey_download_facts.py
index 0550c5f..950623f 100644
--- a/src/ee/tools/digikey_download_facts.py
+++ b/src/ee/tools/digikey_download_facts.py
@@ -1,5 +1,6 @@
import argparse
from itertools import *
+from functools import total_ordering
from colors import color
@@ -7,6 +8,22 @@ import ee.digikey as dk
from ee.digikey import SearchResponseTypes, DigikeyProduct
from ee.tools import mk_parents
+
+@total_ordering
+class Query(object):
+ def __init__(self, is_mpn, query):
+ self.is_mpn = is_mpn
+ self.query = query
+
+ def __eq__(self, other):
+ return (self.is_mpn, self.query) == (other.is_mpn, other.query)
+
+ def __lt__(self, other):
+ return (self.is_mpn, self.query) < (other.is_mpn, other.query)
+
+ def __hash__(self):
+ return hash((self.is_mpn, self.query))
+
parser = argparse.ArgumentParser(description="Download facts about parts from Digi-Key")
parser.add_argument("--out",
@@ -31,7 +48,7 @@ args = parser.parse_args()
digikey = dk.Digikey()
client = dk.DigikeyClient(digikey, on_download=lambda s: print(color(s, 'grey')))
-repo = dk.DigikeyRepository(args.out)
+repo = dk.DigikeyRepository(digikey, args.out)
def on_product(product: DigikeyProduct):
@@ -41,7 +58,7 @@ def on_product(product: DigikeyProduct):
parts = []
if args.part:
- parts.append(args.part)
+ [parts.append(Query(True, p)) for p in args.part]
if args.sch:
from ee.kicad import read_schematic, to_bom
@@ -49,39 +66,52 @@ if args.sch:
for c in to_bom(sch):
digikey = c.get_field("digikey")
if digikey:
- parts.append(digikey.value)
+ parts.append(Query(False, digikey.value))
mpn = c.get_field("mpn")
if mpn:
- parts.append(mpn.value)
+ parts.append(Query(True, mpn.value))
-for p in parts:
- print(color("Searching for {}".format(p), "white"))
+parts = sorted(set(parts))
- if repo.has_product(p) and not args.force:
+for q in parts:
+ p = q.query
+ if repo.has_product(mpn = p if q.is_mpn else None, dpn = p if not q.is_mpn else None) and not args.force:
+ print(color("Already have facts for {}".format(p), "white"))
continue
+ print(color("Searching for {}".format(p), "white"))
response = client.search(p)
+ todos = []
+
if response.response_type == SearchResponseTypes.SINGLE:
p = response.products[0]
- print(color("Direct match {}".format(p.mpn), "white"))
+ print(color("Direct match mpn/dpn: {}/{}".format(p.mpn, p.part_number), "white"))
on_product(p)
elif response.response_type == SearchResponseTypes.MANY:
- hits = list(groupby(sorted(response.products), lambda p: p.mpn))
+ hits = [(mpn, list(products)) for mpn, products in groupby(sorted(response.products, key=lambda p: p.mpn), lambda p: p.mpn)]
if len(hits) == 1:
(mpn, products) = hits[0]
- products = list(products)
-
- if len(products) == 1:
- print(color("Got many results, but they all point to the same part: {}".format(mpn), "white"))
- on_product(products[0])
- continue
- for k, g in hits:
- print(color("Got many results with many parts: {}: {}".format(k, list(g)), "white"))
- on_product(list(g)[0])
+ part_number = products[0].part_number
+ print(color("Got many results, but they all point to the same part: {}. Will use {} for downloading attributes.".format(mpn, ", ".join([p.part_number for p in products])), "white"))
+ todos.append(part_number)
+ else:
+ for k, g in hits:
+ print(color("Got many results with many parts: {}: {}".format(k, list(g)), "white"))
+ on_product(list(g)[0])
elif response.response_type == SearchResponseTypes.TOO_MANY:
print(color("Too many results ({}), select a category first".format(response.count), 'red'))
elif response.response_type == SearchResponseTypes.NO_MATCHES:
print(color("Part not found", "orange"))
+
+ for part_number in todos:
+ response = client.search(part_number)
+
+ if response.response_type == SearchResponseTypes.SINGLE:
+ p = response.products[0]
+ print(color("Downloaded {}".format(p.mpn), "white"))
+ on_product(p)
+ else:
+ print("Got response of type {} when really expecting a single product.".format(response.response_type))
diff --git a/test/test_digikey.py b/test/test_digikey.py
index 7509e2b..04b4642 100644
--- a/test/test_digikey.py
+++ b/test/test_digikey.py
@@ -1,7 +1,8 @@
import ee.digikey as dk
import os.path
-import yaml
import sys
+import io
+from itertools import groupby
basedir = os.path.dirname(os.path.abspath(__file__))
@@ -9,20 +10,45 @@ digikey = dk.Digikey()
client = dk.DigikeyClient(digikey, on_download=print)
-def test_digikey_1():
+def test_digikey_1(tmpdir):
res = client.search("TCR2LF18LM(CTTR-ND")
assert res.response_type == dk.SearchResponseTypes.SINGLE
p = res.products[0]
assert p.part_number == "TCR2LF18LM(CTTR-ND"
assert p.mpn == "TCR2LF18,LM(CT"
assert len(p.attributes) > 5
- x = p.to_yaml()
- print(str(x))
- yaml.dump(x, sys.stdout)
+ x = io.StringIO()
+ p.to_ini().write(x)
+ print(x.getvalue())
+ repo = dk.DigikeyRepository(digikey, str(tmpdir))
+ repo.save(p)
+ repo = dk.DigikeyRepository(digikey, str(tmpdir))
+ repo.load_all()
+ ps = repo.find_by_mpn(p.mpn)
+ assert len(ps) == 1
+ y = io.StringIO()
+ p.to_ini().write(y)
+ assert x.getvalue() == y.getvalue()
-def test_digikey_2():
+
+def test_digikey_2(tmpdir):
res = client.search("TCR2LF", page_size=500)
assert res.response_type == dk.SearchResponseTypes.MANY
- [print(p.part_number) for p in res.products]
+ [print("dpn={}, mpn={}".format(p.part_number, p.mpn)) for p in res.products]
assert len(res.products) == 28
+
+ print("path={}".format(tmpdir))
+ repo = dk.DigikeyRepository(digikey, tmpdir)
+
+ for mpn, parts in groupby(sorted(res.products, key=lambda p: p.mpn), lambda p: p.mpn):
+ parts = list(parts)
+ print("mpn={}, parts={}".format(mpn, [p.part_number for p in parts]))
+
+ dpn = parts[0].part_number
+ print("Downloading {} as {}".format(mpn, dpn))
+ res = client.search(dpn)
+
+ assert res.response_type == dk.SearchResponseTypes.SINGLE
+ p = res.products[0]
+ repo.save(p)