ee.xsd:

o Removing distributor info, wasn't useful.
o Removing part type, using a fact instead.
part-search-list:
o Putting in some smart rules about values for parts. Might be too smart
  for its own good.
o Removing duplication checking, that is up to the searcher to decide.

8 files changed, 134 insertions, 168 deletions

diff --git a/src/ee/digikey/search_parts.py b/src/ee/digikey/search_parts.py
index c248831..5d6a17a 100644
--- a/src/ee/digikey/search_parts.py
+++ b/src/ee/digikey/search_parts.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 from typing import List
 
+from ee.db import ObjDb
 from ee.digikey import Digikey, DigikeyParser, DigikeyClient, SearchResponseTypes, DigikeyProduct, DigikeyStore
 from ee.part import PartDb, load_db, save_db, Part
 from ee.xml import types
@@ -14,7 +15,6 @@ def resolved(supplier, p: DigikeyProduct) -> Part:
     xml = types.Part(uri="https://digikey.com/pn#{}".format(p.part_number),
                      supplier=supplier,
                      description=p.description,
-                     distributor_info=types.DistributorInfo(state="resolved"),
                      links=types.LinkList(),
                      facts=types.FactList(),
                      references=types.ReferenceList())
@@ -49,15 +49,18 @@ def resolved(supplier, p: DigikeyProduct) -> Part:
     return part
 
 
-def search_parts(in_path: Path, out_path: Path, cache_dir: Path, store_code):
+def search_parts(in_path: Path, out_path: Path, cache_dir: Path, store_code, print_result=True):
     in_db = load_db(in_path)
-    out_parts = PartDb()
 
     store = DigikeyStore.from_store_code(store_code)
 
     parser = DigikeyParser(Digikey())
     client = DigikeyClient(store.products_url, cache_dir)
 
+    out_parts: ObjDb[Part] = ObjDb[Part]()
+    uri_idx = out_parts.add_unique_index("uri", lambda p: p.uri)
+    pn_idx = out_parts.add_index("pn", lambda p: [pn.value for pn in p.get_part_references()], multiple=True)
+    spn_idx = out_parts.add_index("spn", lambda p: [pn.value for pn in p.get_spns()], multiple=True)
     for xml in in_db.iterparts():
         if xml.supplierProp is not None and xml.supplierProp != store.url:
             continue
@@ -81,12 +84,8 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path, store_code):
             print("could not find pn or dpn: part.uri={}".format(xml.uriProp))
             continue
 
-        out_id = query
-        out_part = types.Part(uri=out_id,
-                              distributor_info=types.DistributorInfo(),
-                              supplier=store.url,
-                              references=xml.referencesProp)
-        di = out_part.distributor_infoProp
+        out_part = None
+        result = None
 
         text = client.search(query)
         response = parser.parse_string(text)
@@ -103,7 +102,7 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path, store_code):
             filtered_products = [p for p in response.products if get_field(p) == query]
 
             if len(filtered_products) == 0:
-                di.stateProp = "not-found"
+                result = "not-found"
             else:
                 dpn = sorted(filtered_products, key=lambda p: p.part_number)[0].part_number
 
@@ -111,14 +110,21 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path, store_code):
                 if response.response_type == SearchResponseTypes.SINGLE:
                     out_part = resolved(store.url, response.products[0])
                 else:
-                    di.stateProp = "many"
+                    result = "many"
 
         elif response.response_type == SearchResponseTypes.TOO_MANY:
-            di.stateProp = "too-many"
+            result = "too-many"
         elif response.response_type == SearchResponseTypes.NO_MATCHES:
-            di.stateProp = "not-found"
+            result = "not-found"
 
-        out_parts.add_entry(out_part, True)
+        if out_part:
+            if out_part.uri not in uri_idx:
+                out_parts.add(out_part)
 
-    print("Saving {} work parts".format(out_parts.size()))
-    save_db(out_path, out_parts, sort=True)
+        if print_result and result:
+            print("{}: {}".format(query, result))
+
+    part_db = PartDb()
+    for part in out_parts:
+        part_db.add_entry(part, True)
+    save_db(out_path, part_db, sort=True)
diff --git a/src/ee/element14/search_parts.py b/src/ee/element14/search_parts.py
index cfa1d34..4a8b0a9 100644
--- a/src/ee/element14/search_parts.py
+++ b/src/ee/element14/search_parts.py
@@ -23,9 +23,7 @@ def search_parts(in_path: Path, out_path: Path, cache_dir: Path, config: Element
         client.search(term="manuPartNum:" + query)
 
         out_part = types.Part(id=out_id,
-                              distributor_info=types.DistributorInfo(),
                               references=part.referencesProp)
-        di = out_part.distributor_infoProp
 
     print("Saving {} work parts".format(out_parts.size()))
     save_db(out_path, out_parts, sort=True)
diff --git a/src/ee/kicad/make_bom.py b/src/ee/kicad/make_bom.py
index bec42de..7c8d8f0 100644
--- a/src/ee/kicad/make_bom.py
+++ b/src/ee/kicad/make_bom.py
@@ -1,3 +1,4 @@
+import re
 import pydoc
 from pathlib import Path
 from typing import Optional, List, Callable, Mapping
@@ -37,24 +38,65 @@ def apply_strategies(c: Component, part: Part, strategies: List[StrategyCallable
     return part
 
 
-def part_type_strategy(component: Component, part: Part) -> Part:
-    fp = component.footprint
-    if fp is None:
-        return part
-
+def part_type_from_footprint(fp):
     lib, part_name = fp.split(":")
 
-    xml = part.underlying
     if lib == "Capacitor_SMD":
-        xml.part_typeProp = uris.CAPACITOR
+        return uris.CAPACITOR
     elif lib == "Resistor_SMD":
-        xml.part_typeProp = uris.RESISTOR
+        return uris.RESISTOR
     elif lib == "Diode_SMD":
-        xml.part_typeProp = uris.DIODE
+        return uris.DIODE
     elif lib == "Inductor_SMD":
-        xml.part_typeProp = uris.INDUCTOR
+        return uris.INDUCTOR
     elif lib == "Crystal":
-        xml.part_typeProp = uris.CRYSTAL
+        return uris.CRYSTAL
+
+
+def part_type_from_ref_type(ref_type):
+    if ref_type == "C":
+        return uris.CAPACITOR
+    elif ref_type == "R":
+        return uris.RESISTOR
+    elif ref_type == "D":
+        return uris.DIODE
+    elif ref_type == "L":
+        return uris.INDUCTOR
+    elif ref_type == "X":
+        return uris.CRYSTAL
+    elif ref_type == "Q":
+        return uris.TRANSISTOR
+
+
+def part_type_strategy(component: Component, part: Part) -> Part:
+    pt = None
+
+    fp = component.footprint
+    if fp is not None:
+        pt = part_type_from_footprint(fp)
+
+    if pt is None and component.has_ref_num:
+        pt = part_type_from_ref_type(component.ref_type)
+
+    if pt is not None:
+        part.get_facts().append(types.Fact(uris.make_fact_key("type"), value=pt))
+
+    return part
+
+
+def fix_value_strategy(component: Component, part: Part) -> Part:
+    if not component.has_ref_num:
+        return part
+
+    rt = component.ref_type
+    v = component.value
+
+    print("component.symbol.name={}".format(component.symbol.name))
+    if rt in ("D", "R", "L", "C") and v == component.symbol.name:
+        part.remove_fact(uris.make_fact_key("value"))
+
+    if rt == "Q" and v == component.symbol.name and re.match("^Q_[NP]MOS_[DSG]{3}$", component.symbol.name):
+        part.remove_fact(uris.make_fact_key("value"))
 
     return part
 
@@ -85,6 +127,7 @@ class MakeBomStrategy():
     def __init__(self):
         self.dpn_mappings = {}
         self.default_strategies = [
+            fix_value_strategy,
             mpn_strategy,
             part_type_strategy,
             dpn_strategy_factory(self.dpn_mappings),
diff --git a/src/ee/part/__init__.py b/src/ee/part/__init__.py
index 996eeff..43e4cfa 100644
--- a/src/ee/part/__init__.py
+++ b/src/ee/part/__init__.py
@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List, MutableMapping, Optional, Iterator, Union
+from typing import List, Optional, Iterator, Union
 
 from ee import EeException
 from ee.money import Money
@@ -43,6 +43,12 @@ class PartNumber(Reference):
     def to_xml(self):
         return types.PartNumber(value=self.value)
 
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __hash__(self):
+        return hash(self.value)
+
 
 class SupplierPartNumber(Reference):
     def __init__(self, value: str):
@@ -51,6 +57,12 @@ class SupplierPartNumber(Reference):
     def to_xml(self):
         return types.SupplierPartNumber(value=self.value)
 
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __hash__(self):
+        return hash(self.value)
+
 
 class ReferenceList(object):
     def __init__(self, part_uri):
@@ -296,6 +308,9 @@ class Part(object):
     def find_fact(self, key: str) -> Optional[types.Fact]:
         return next((f for f in self.get_facts() if f.keyProp == key), None)
 
+    def remove_fact(self, key):
+        self.xml.factsProp.fact = [f for f in self.xml.factsProp.fact if f.keyProp != key]
+
 
 class Facts(object):
     def __init__(self, part):
@@ -331,7 +346,6 @@ class Assembly(object):
 class PartDb(object):
     def __init__(self):
         self.parts: List[Entry] = []
-        self.pn_index: MutableMapping[str, Entry] = {}
         self.new_entries = 0
         self._assembly: Optional[Assembly] = None
 
@@ -342,9 +356,6 @@ class PartDb(object):
         e = Entry(new, part)
         self.parts.append(e)
 
-        if e.pn:
-            self.pn_index[e.pn] = e
-
         if e.new:
             self.new_entries = self.new_entries + 1
 
@@ -355,10 +366,6 @@ class PartDb(object):
     def size(self) -> int:
         return len(self.parts)
 
-    def find_by_pn(self, pn: str) -> Optional[types.Part]:
-        entry = self.pn_index.get(pn, None)
-        return entry.part if entry else None
-
     @property
     def has_assembly(self):
         return self._assembly is not None
diff --git a/src/ee/part/pn_part_search_list.py b/src/ee/part/pn_part_search_list.py
index 049d0b8..343048e 100644
--- a/src/ee/part/pn_part_search_list.py
+++ b/src/ee/part/pn_part_search_list.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 
 from ee.part import PartDb, load_db, save_db, Part, fact_keys
-from ee.xml import types
+from ee.xml import types, uris
 
 __all__ = ["pn_part_search_list"]
 
@@ -10,35 +10,61 @@ ignored_part_classes = [
 ]
 
 
+def valid_pns(part: Part):
+    """Check if the part has any MPNs or SPNs"""
+    return len(part.get_mpns()) > 0 or len(part.get_spns()) > 0
+
+
+def get_value(part: Part):
+    """Check if the part has a value and it is not a resistor, capacitor or inductor. Their value is not useful for a
+    part number-based lookup"""
+    value = part.find_fact(uris.make_fact_key("value"))
+
+    if value is None:
+        return
+
+    typ = part.find_fact(uris.make_fact_key("type"))
+    if typ is None:
+        return
+
+    # if type is Zener, it's value could be a voltage
+
+    return value.valueProp if typ.valueProp not in (uris.RESISTOR, uris.CAPACITOR, uris.INDUCTOR) else None
+
+
 def pn_part_search_list(in_path: Path, out_path: Path, supplier: str):
     in_parts = load_db(in_path)
     out_parts = PartDb()
 
-    # print("loaded {} existing parts".format(in_parts.size()))
-
+    count = 0
+    skipped = list()
     for xml in in_parts.iterparts():
+        count += 1
         part = Part(xml)
-        pn_value = next((p.valueProp for p in part.get_mpns()), None)
+
+        refs = [ref.referenceProp for ref in part.get_schematic_references()]
 
         part_class = part.find_fact(fact_keys.part_class)
         if part_class:
             if part_class.valueProp in ignored_part_classes:
+                skipped.append(refs)
                 continue
 
-        if pn_value is None:
-            refs = [ref.referenceProp for ref in part.get_schematic_references()]
-            print("Skipping part with no part number: schematic reference: {}".format(", ".join(refs)))
-            continue
-
-        entry = out_parts.find_by_pn(pn_value)
+        if not valid_pns(part):
+            value = get_value(part)
+            if not value:
+                skipped.append(refs)
+                continue
 
-        if entry is not None:
-            continue
+            # Use the value of the component as a part number to search for
+            xml.referencesProp.part_number.append(types.PartNumber(value=value))
 
         new_part = types.Part()
         new_part.referencesProp = xml.referencesProp
 
         out_parts.add_entry(new_part, True)
 
-    # print("Saving {} work parts".format(out_parts.size()))
+    if len(skipped) > 0:
+        print("Skipped {} of {} parts".format(len(skipped), count))
+
     save_db(out_path, out_parts)
diff --git a/src/ee/xml/types.py b/src/ee/xml/types.py
index a6ef46b..bfda4fa 100644
--- a/src/ee/xml/types.py
+++ b/src/ee/xml/types.py
@@ -821,16 +821,14 @@ class PartDb(GeneratedsSuper):
 class Part(GeneratedsSuper):
     subclass = None
     superclass = None
-    def __init__(self, uri=None, supplier=None, part_type=None, description=None, links=None, references=None, distributor_info=None, facts=None, price_breaks=None, **kwargs_):
+    def __init__(self, uri=None, supplier=None, description=None, links=None, references=None, facts=None, price_breaks=None, **kwargs_):
         self.original_tagname_ = None
         self.parent_object_ = kwargs_.get('parent_object_')
         self.uri = _cast(None, uri)
         self.supplier = supplier
-        self.part_type = part_type
         self.description = description
         self.links = links
         self.references = references
-        self.distributor_info = distributor_info
         self.facts = facts
         self.price_breaks = price_breaks
     def factory(*args_, **kwargs_):
@@ -849,11 +847,6 @@ class Part(GeneratedsSuper):
     def set_supplier(self, supplier):
         self.supplier = supplier
     supplierProp = property(get_supplier, set_supplier)
-    def get_part_type(self):
-        return self.part_type
-    def set_part_type(self, part_type):
-        self.part_type = part_type
-    part_typeProp = property(get_part_type, set_part_type)
     def get_description(self):
         return self.description
     def set_description(self, description):
@@ -869,11 +862,6 @@ class Part(GeneratedsSuper):
     def set_references(self, references):
         self.references = references
     referencesProp = property(get_references, set_references)
-    def get_distributor_info(self):
-        return self.distributor_info
-    def set_distributor_info(self, distributor_info):
-        self.distributor_info = distributor_info
-    distributor_infoProp = property(get_distributor_info, set_distributor_info)
     def get_facts(self):
         return self.facts
     def set_facts(self, facts):
@@ -892,11 +880,9 @@ class Part(GeneratedsSuper):
     def hasContent_(self):
         if (
             self.supplier is not None or
-            self.part_type is not None or
             self.description is not None or
             self.links is not None or
             self.references is not None or
-            self.distributor_info is not None or
             self.facts is not None or
             self.price_breaks is not None
         ):
@@ -936,9 +922,6 @@ class Part(GeneratedsSuper):
         if self.supplier is not None:
             showIndent(outfile, level, pretty_print)
             outfile.write('<%ssupplier>%s</%ssupplier>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.supplier), input_name='supplier')), namespaceprefix_ , eol_))
-        if self.part_type is not None:
-            showIndent(outfile, level, pretty_print)
-            outfile.write('<%spart-type>%s</%spart-type>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.part_type), input_name='part-type')), namespaceprefix_ , eol_))
         if self.description is not None:
             showIndent(outfile, level, pretty_print)
             outfile.write('<%sdescription>%s</%sdescription>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.description), input_name='description')), namespaceprefix_ , eol_))
@@ -946,8 +929,6 @@ class Part(GeneratedsSuper):
             self.links.export(outfile, level, namespaceprefix_, namespacedef_='', name_='links', pretty_print=pretty_print)
         if self.references is not None:
             self.references.export(outfile, level, namespaceprefix_, namespacedef_='', name_='references', pretty_print=pretty_print)
-        if self.distributor_info is not None:
-            self.distributor_info.export(outfile, level, namespaceprefix_, namespacedef_='', name_='distributor-info', pretty_print=pretty_print)
         if self.facts is not None:
             self.facts.export(outfile, level, namespaceprefix_, namespacedef_='', name_='facts', pretty_print=pretty_print)
         if self.price_breaks is not None:
@@ -969,10 +950,6 @@ class Part(GeneratedsSuper):
             supplier_ = child_.text
             supplier_ = self.gds_validate_string(supplier_, node, 'supplier')
             self.supplier = supplier_
-        elif nodeName_ == 'part-type':
-            part_type_ = child_.text
-            part_type_ = self.gds_validate_string(part_type_, node, 'part_type')
-            self.part_type = part_type_
         elif nodeName_ == 'description':
             description_ = child_.text
             description_ = self.gds_validate_string(description_, node, 'description')
@@ -987,11 +964,6 @@ class Part(GeneratedsSuper):
             obj_.build(child_)
             self.references = obj_
             obj_.original_tagname_ = 'references'
-        elif nodeName_ == 'distributor-info':
-            obj_ = DistributorInfo.factory(parent_object_=self)
-            obj_.build(child_)
-            self.distributor_info = obj_
-            obj_.original_tagname_ = 'distributor-info'
         elif nodeName_ == 'facts':
             obj_ = FactList.factory(parent_object_=self)
             obj_.build(child_)
@@ -1790,84 +1762,6 @@ class FactList(GeneratedsSuper):
 # end class FactList
 
 
-class DistributorInfo(GeneratedsSuper):
-    subclass = None
-    superclass = None
-    def __init__(self, state=None, **kwargs_):
-        self.original_tagname_ = None
-        self.parent_object_ = kwargs_.get('parent_object_')
-        self.state = state
-    def factory(*args_, **kwargs_):
-        if CurrentSubclassModule_ is not None:
-            subclass = getSubclassFromModule_(
-                CurrentSubclassModule_, DistributorInfo)
-            if subclass is not None:
-                return subclass(*args_, **kwargs_)
-        if DistributorInfo.subclass:
-            return DistributorInfo.subclass(*args_, **kwargs_)
-        else:
-            return DistributorInfo(*args_, **kwargs_)
-    factory = staticmethod(factory)
-    def get_state(self):
-        return self.state
-    def set_state(self, state):
-        self.state = state
-    stateProp = property(get_state, set_state)
-    def hasContent_(self):
-        if (
-            self.state is not None
-        ):
-            return True
-        else:
-            return False
-    def export(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='DistributorInfo', pretty_print=True):
-        imported_ns_def_ = GenerateDSNamespaceDefs_.get('DistributorInfo')
-        if imported_ns_def_ is not None:
-            namespacedef_ = imported_ns_def_
-        if pretty_print:
-            eol_ = '\n'
-        else:
-            eol_ = ''
-        if self.original_tagname_ is not None:
-            name_ = self.original_tagname_
-        showIndent(outfile, level, pretty_print)
-        outfile.write('<%s%s%s' % (namespaceprefix_, name_, namespacedef_ and ' ' + namespacedef_ or '', ))
-        already_processed = set()
-        self.exportAttributes(outfile, level, already_processed, namespaceprefix_, name_='DistributorInfo')
-        if self.hasContent_():
-            outfile.write('>%s' % (eol_, ))
-            self.exportChildren(outfile, level + 1, namespaceprefix_, namespacedef_, name_='DistributorInfo', pretty_print=pretty_print)
-            showIndent(outfile, level, pretty_print)
-            outfile.write('</%s%s>%s' % (namespaceprefix_, name_, eol_))
-        else:
-            outfile.write('/>%s' % (eol_, ))
-    def exportAttributes(self, outfile, level, already_processed, namespaceprefix_='', name_='DistributorInfo'):
-        pass
-    def exportChildren(self, outfile, level, namespaceprefix_='', namespacedef_='', name_='DistributorInfo', fromsubclass_=False, pretty_print=True):
-        if pretty_print:
-            eol_ = '\n'
-        else:
-            eol_ = ''
-        if self.state is not None:
-            showIndent(outfile, level, pretty_print)
-            outfile.write('<%sstate>%s</%sstate>%s' % (namespaceprefix_ , self.gds_encode(self.gds_format_string(quote_xml(self.state), input_name='state')), namespaceprefix_ , eol_))
-    def build(self, node):
-        already_processed = set()
-        self.buildAttributes(node, node.attrib, already_processed)
-        for child in node:
-            nodeName_ = Tag_pattern_.match(child.tag).groups()[-1]
-            self.buildChildren(child, node, nodeName_)
-        return self
-    def buildAttributes(self, node, attrs, already_processed):
-        pass
-    def buildChildren(self, child_, node, nodeName_, fromsubclass_=False):
-        if nodeName_ == 'state':
-            state_ = child_.text
-            state_ = self.gds_validate_string(state_, node, 'state')
-            self.state = state_
-# end class DistributorInfo
-
-
 class Amount(GeneratedsSuper):
     subclass = None
     superclass = None
@@ -2747,7 +2641,6 @@ __all__ = [
     "Assembly",
     "AssemblyPart",
     "AssemblyPartList",
-    "DistributorInfo",
     "Fact",
     "FactList",
     "Link",
diff --git a/src/ee/xml/uris.py b/src/ee/xml/uris.py
index c716103..f168c59 100644
--- a/src/ee/xml/uris.py
+++ b/src/ee/xml/uris.py
@@ -1,10 +1,12 @@
 from typing import Optional
 
+# Values for `..#type` facts
 CAPACITOR = "http://purl.org/ee/part-type#capacitor"
 RESISTOR = "http://purl.org/ee/part-type#resistor"
 DIODE = "http://purl.org/ee/part-type#diode"
 INDUCTOR = "http://purl.org/ee/part-type#inductor"
 CRYSTAL = "http://purl.org/ee/part-type#inductor"
+TRANSISTOR = "http://purl.org/ee/part-type#transistor"
 
 _DIGIKEY_FACT_KEY_PREFIX = "http://purl.org/ee/digikey-fact-key#"
 
diff --git a/xsd/ee.xsd b/xsd/ee.xsd
index 32c97b3..6b7260b 100644
--- a/xsd/ee.xsd
+++ b/xsd/ee.xsd
@@ -46,12 +46,9 @@ TODO: rename 'id' to 'url'.
     <xs:complexType name="Part">
         <xs:sequence>
             <xs:element name="supplier" type="xs:anyURI" minOccurs="0"/>
-            <!-- TODO: this should be a fact -->
-            <xs:element name="part-type" type="xs:anyURI"/>
             <xs:element name="description" type="xs:string"/>
             <xs:element name="links" type="LinkList"/>
             <xs:element name="references" type="ReferenceList"/>
-            <xs:element name="distributor-info" type="DistributorInfo"/>
             <xs:element name="facts" type="FactList"/>
             <xs:element name="price-breaks" type="PriceBreakList"/>
         </xs:sequence>
@@ -114,12 +111,6 @@ TODO: rename 'id' to 'url'.
         </xs:sequence>
     </xs:complexType>
 
-    <xs:complexType name="DistributorInfo">
-        <xs:sequence>
-            <xs:element name="state" type="xs:string"/>
-        </xs:sequence>
-    </xs:complexType>
-
     <xs:complexType name="Amount">
         <xs:attribute name="value" use="required"/>
         <xs:attribute name="currency"/>