wip.

o Support for loading CSV files.

6 files changed, 248 insertions, 53 deletions

diff --git a/demo/doit/dodo.py b/demo/doit/dodo.py
index 27ceb67..858e673 100644
--- a/demo/doit/dodo.py
+++ b/demo/doit/dodo.py
@@ -1,4 +1,6 @@
 import ee.kicad.doit
+from ee.kicad.doit import *
+from ee.fact import DataSetManager
 
 prj = "demo"
 sch = "{}.sch".format(prj)
@@ -6,12 +8,16 @@ kicad_pcb = "{}.kicad_pcb".format(prj)
 
 DOIT_CONFIG = {'check_file_uptodate': 'timestamp'}
 
+dsm = DataSetManager("ee")
+
+kicad_footprint = "kicad-footprint"
+dsm.add_ds("csv", kicad_footprint, "kicad-footprint-mapping", path="ee/kicad-footprint.csv")
+
+ee.kicad.doit.change_data_sets_for_task(task_kicad_create_component_data_set, lambda ds: ds + [kicad_footprint])
+
 ee.kicad.doit.init(
+    data_set_manager=dsm,
     sch=sch,
     kicad_pcb=kicad_pcb,
     gerber_dir="gerber",
-    data_set_dir="ee",
 )
-
-# noinspection PyUnresolvedReferences
-from ee.kicad.doit import *
diff --git a/demo/doit/ee/kicad-footprint.csv b/demo/doit/ee/kicad-footprint.csv
new file mode 100644
index 0000000..a9ccfd7
--- /dev/null
+++ b/demo/doit/ee/kicad-footprint.csv
@@ -0,0 +1,2 @@
+kicad,common
+Resistor_SMD:R_1206_3216Metric,SMD_3216
diff --git a/src/ee/fact/__init__.py b/src/ee/fact/__init__.py
index 02e6b2a..c8c21dd 100644
--- a/src/ee/fact/__init__.py
+++ b/src/ee/fact/__init__.py
@@ -1,9 +1,10 @@
 import configparser
 import logging
 import os
+import csv
 from functools import total_ordering
 from pathlib import Path
-from typing import MutableMapping, Optional, List, Tuple
+from typing import MutableMapping, Optional, List, Tuple, Union, Iterator
 
 logger = logging.getLogger(__name__)
 
@@ -37,12 +38,12 @@ class ObjectType(object):
     def fields(self):
         return self._fields
 
-    def index_of(self, field: str, create: bool = False) -> int:
+    def index_of(self, field: str, create: bool = False) -> Optional[int]:
         try:
             return self._fields.index(field)
         except ValueError as e:
             if not create:
-                raise e
+                return None
 
             self._fields.append(field)
             return len(self._fields) - 1
@@ -75,7 +76,16 @@ class Object(object):
 
     def get(self, key: str) -> Optional[str]:
         idx = self._ot.index_of(key)
-        return self._data[idx] if idx < len(self._data) else None
+        return self._data[idx] if idx is not None and idx < len(self._data) else None
+
+    def get_all(self, *keys: str) -> Optional[List[str]]:
+        values = []
+        for key in keys:
+            idx = self._ot.index_of(key)
+            if not idx or idx >= len(self._data):
+                return None
+            values.append(self._data[idx])
+        return values
 
 
 class DataSet(object):
@@ -89,6 +99,9 @@ class DataSet(object):
     def name(self):
         return self._name
 
+    def __len__(self):
+        return sum((len(objects) for objects in self._objects_by_type.values()))
+
     def freeze(self):
         self._frozen = True
 
@@ -168,9 +181,10 @@ class DataSet(object):
 
         return self._check_object(object_type, key, True)
 
-    def items(self):
-        from itertools import chain
-        return list(chain.from_iterable([objects.values() for objects in self._objects_by_type.values()]))
+    def items(self) -> Iterator[Object]:
+        for objects in self._objects_by_type.values():
+            for o in objects.values():
+                yield o
 
     def merge(self, other: "DataSet") -> "DataSet":
         ds = DataSet(self._name)
@@ -186,11 +200,15 @@ class DataSet(object):
 
 
 class DataSetManager(object):
-    def __init__(self, basedir: Path):
+    def __init__(self, basedir: Union[Path, str]):
         self._basedir = Path(basedir)
+        self._csv = {}  # type: MutableMapping[str, Tuple[str, Path]]
 
-    def metafile_for_ds(self, ds_name) -> Path:
-        return self._basedir / ds_name / "data-set.ini"
+    def cookie_for_ds(self, ds_name) -> Path:
+        try:
+            return self._csv[ds_name][1]
+        except KeyError:
+            return self._basedir / ds_name / "data-set.ini"
 
     def create_rw(self, name, inputs: List[str] = None) -> "LazyDataSet":
         return LazyDataSet(self, False, name, inputs if inputs else [])
@@ -198,7 +216,52 @@ class DataSetManager(object):
     def create_ro(self, inputs: List[str]) -> "LazyDataSet":
         return LazyDataSet(self, True, None, inputs)
 
+    def add_ds(self, ds_type: str, name: str, object_type: str, path: str = None):
+        if ds_type == "csv":
+            if name in self._csv:
+                raise Exception("Data source already exists: {}".format(name))
+
+            self._csv[name] = object_type, Path(path),
+        else:
+            raise Exception("Unknown data source type: {}".format(ds_type))
+
+    def ds_type(self, name: str):
+        return "csv" if name in self._csv else "ini-dir"
+
     def load(self, name, freeze=False) -> DataSet:
+        try:
+            object_type, path = self._csv[name]
+
+            if not freeze:
+                raise Exception("CSV data sources must be frozen")
+
+            return DataSetManager._load_csv(name, object_type, path, freeze)
+        except KeyError:
+            return self._load_ini_dir(name, freeze)
+
+    @staticmethod
+    def _load_csv(name: str, object_type: str, path: Path, freeze: bool) -> DataSet:
+        logger.info("Loading CSV file {}".format(path))
+        ds = DataSet(name)
+
+        with open(str(path), newline='') as f:
+            r = csv.reader(f)
+
+            header = next(r, None)
+            for row in r:
+                if len(row) == 0:
+                    continue
+
+                key = row[0]
+
+                o = ds.create_object(object_type, key)
+                for idx, value in zip(range(0, min(len(row), len(header))), row):
+                    o.set(header[idx], value)
+
+        logger.debug("Loaded {} objects".format(len(ds)))
+        return ds
+
+    def _load_ini_dir(self, name: str, freeze: bool) -> DataSet:
         ds_dir = Path(name) if Path(name).is_absolute() else self._basedir / name
         ds_dir = ds_dir if ds_dir.is_dir() else ds_dir.parent
 
@@ -233,7 +296,8 @@ class DataSetManager(object):
 
     def store(self, ds: DataSet):
         ds_dir = self._basedir / ds.name
-        logger.info("Storing DS '{}' with {} objects to {}".format(ds.name, len(ds.items()), ds_dir))
+        items = list(ds.items())
+        logger.info("Storing DS '{}' with {} objects to {}".format(ds.name, len(items), ds_dir))
 
         os.makedirs(ds_dir, exist_ok=True)
         ini = self._blank_ini()
@@ -241,7 +305,7 @@ class DataSetManager(object):
         ini.set("data-set", "name", ds.name)
         self._store_ini(ini, ds_dir / "data-set.ini")
 
-        for o in ds.items():
+        for o in items:
             ot = o.object_type
             key = o.key
 
diff --git a/src/ee/kicad/doit.py b/src/ee/kicad/doit.py
index 7273076..41e3155 100644
--- a/src/ee/kicad/doit.py
+++ b/src/ee/kicad/doit.py
@@ -1,5 +1,7 @@
 import logging
 import os.path
+from pathlib import Path
+from typing import Mapping
 
 from configclass import Config
 
@@ -14,27 +16,33 @@ _config_template = Config({
     "kicad_pcb": None,
     "gerber_dir": None,
     "gerber_zip": None,
-    "data_set_dir": None,
 })
 
-_config = None
+_config = None  # type: Mapping[str, str]
 
 _dsm = None  # type: DataSetManager
 
+_data_sets = {}
 
-def init(**kwargs):
+
+def change_data_sets_for_task(task, callable):
+    _data_sets[task] = callable(_data_sets[task])
+
+
+def init(data_set_manager: DataSetManager, **kwargs):
     global _config
     _config = _config_template.make(kwargs)
 
+    ee_logger = logging.getLogger("ee")
     formatter = logging.Formatter("%(levelname)s: %(message)s")
     ch = logging.StreamHandler()
     ch.setFormatter(formatter)
-    logger.addHandler(ch)
+    ee_logger.addHandler(ch)
 
-    logger.setLevel(logging.DEBUG)
+    ee_logger.setLevel(logging.DEBUG)
 
     global _dsm
-    _dsm = DataSetManager(_config["data_set_dir"])
+    _dsm = data_set_manager
 
 
 def task_kicad_gerber():
@@ -69,7 +77,6 @@ def task_kicad_gerber():
                 z.write(p, arcname=p.relative_to(gerber_dir))
 
     return {
-        "basename": "kicad-gerber",
         "targets": [gerber_zip],
         "actions": [mkdir, export_gerber, make_zip],
         "file_dep": [kicad_pcb],
@@ -78,14 +85,14 @@ def task_kicad_gerber():
 
 def task_kicad_sch_to_data_set():
     out_data_set = "kicad-sch"
-    in_data_sets = []
+    in_data_sets = _data_sets[task_kicad_sch_to_data_set]
 
     sch = _config["sch"]
 
     def action():
         from ee.kicad.model import ComponentField
 
-        with _dsm.create_rw(out_data_set, inputs=in_data_sets) as ds:
+        with _dsm.create_rw(out_data_set) as ds:
             schematics = ee.kicad.read_schematics(sch)
             for c in [c for c in schematics.components]:
                 o = ds.create_object("kicad-schematic-component", c.timestamp)
@@ -102,18 +109,20 @@ def task_kicad_sch_to_data_set():
                         o.set(f.name, str(f.value))
 
     return {
-        "basename": "kicad-sch-to-data-set",
-        "file_dep": [sch] + [_dsm.metafile_for_ds(ds) for ds in in_data_sets],
+        "file_dep": [Path(sch)] + [_dsm.cookie_for_ds(ds) for ds in in_data_sets],
         "actions": [action],
-        "targets": [_dsm.metafile_for_ds(out_data_set)],
+        "targets": [_dsm.cookie_for_ds(out_data_set)],
     }
 
 
+_data_sets[task_kicad_sch_to_data_set] = []
+
+
 def task_kicad_pcb_to_data_set():
     kicad_pcb = _config["kicad_pcb"]
 
     out_data_set = "kicad-pcb"
-    in_data_sets = []
+    in_data_sets = _data_sets[task_kicad_pcb_to_data_set]
 
     def action():
         from ee.kicad.pcb import KicadPcb, Module
@@ -130,7 +139,7 @@ def task_kicad_pcb_to_data_set():
                 o = ds.create_object("kicad-pcb-component", m.tstamp)
 
                 ref_text = next((t for t in m.fp_texts if t.kind == "reference"), None)
-                o.set("reference", ref_text.value)
+                o.set("ref", ref_text.value)
 
                 x, y, rot = m.at
                 o.set("placement-x", x)
@@ -139,31 +148,91 @@ def task_kicad_pcb_to_data_set():
                 o.set("layer", m.layer)
 
     return {
-        "basename": "kicad-pcb-to-data-set",
-        "file_dep": [kicad_pcb] + [_dsm.metafile_for_ds(ds) for ds in in_data_sets],
+        "file_dep": [Path(kicad_pcb)] + [_dsm.cookie_for_ds(ds) for ds in in_data_sets],
         "actions": [action],
-        "targets": [_dsm.metafile_for_ds(out_data_set)],
+        "targets": [_dsm.cookie_for_ds(out_data_set)],
     }
 
 
+_data_sets[task_kicad_pcb_to_data_set] = []
+
+
 def task_kicad_create_component_data_set():
     out_data_set = "components"
-    in_data_sets = ["kicad-sch", "kicad-pcb"]
+    in_data_sets = _data_sets[task_kicad_create_component_data_set]
+
+    def action():
+        logger.info("in_data_sets={}, out_data_set={}".format(in_data_sets, out_data_set))
+
+        with _dsm.create_ro(in_data_sets) as in_ds:
+            # for o in in_ds.items():
+            #     logger.info("item: {}/{}".format(o.object_type.name, o.key))
+
+            def map_footprint(footprint):
+                for o in in_ds.items():
+                    if not o.object_type.name == "kicad-footprint-mapping":
+                        continue
+
+                    common = o.get("common")
+                    if common:
+                        return common
 
-    def action(targets, *args, **kwargs):
-        logger.info("targets={}, args={}, kwargs={}".format(targets, args, kwargs))
+                return footprint
 
-        with _dsm.create_ro(in_data_sets) as input:
             with _dsm.create_rw(out_data_set) as output:
-                items = input.items()
-                logger.info("Got {} objects".format(len(items)))
-                for o in items:
-                    ot = o.object_type
-                    logger.info("processing {}:{}".format(ot.name, o.key))
+                kicad_sch = [o for o in in_ds.items() if o.object_type.name == "kicad-schematic-component"]
+
+                logger.info("processing {} kicad-sch".format(len(kicad_sch)))
+
+                for sch in kicad_sch:
+                    ref = sch.get("ref")
+                    ref_num = sch.get("ref-num")
+                    if not ref or not ref_num:
+                        logger.debug("Missing ref or ref-num")
+                        continue
+
+                    ref_type = sch.get("ref-type")
+                    if not ref_type or ref_type == "#PWR":
+                        logger.debug("Missing ref-type or bad ref-type: ref={}, ref-type={}".format(ref, ref_type))
+                        continue
+
+                    c = output.create_object("component", ref)
+                    c.set("ref", ref)
+                    c.set("ref-num", ref_num)
+                    c.set("ref-type", ref_type)
+
+                    fp = sch.get("footprint")
+                    if fp:
+                        fp = map_footprint(fp)
+                        c.set("footprint", fp)
+
+                    def pcb_match(o):
+                        return o.object_type.name == "kicad-pcb-component" and \
+                               o.get("ref") == ref
+
+                    pcb = [o for o in in_ds.items() if pcb_match(o)]
+
+                    if not pcb:
+                        logger.info("Could not find PCB component for {}".format(ref))
+
+                        # TODO: check that the SCH and PCB footprint are the same
+                        # c.set("footprint", pcb.)
 
     return {
-        "basename": "kicad-create-component-data-set",
-        "file_dep": [_dsm.metafile_for_ds(ds) for ds in in_data_sets],
+        "file_dep": [_dsm.cookie_for_ds(ds) for ds in in_data_sets],
         "actions": [action],
-        "targets": [_dsm.metafile_for_ds(out_data_set)],
+        "targets": [_dsm.cookie_for_ds(out_data_set)],
     }
+
+
+_data_sets[task_kicad_create_component_data_set] = ["kicad-sch", "kicad-pcb"]
+
+__all__ = [
+    init.__name__,
+    change_data_sets_for_task.__name__,
+
+    task_kicad_create_component_data_set.__name__,
+    task_kicad_gerber.__name__,
+    task_kicad_pcb_to_data_set.__name__,
+    task_kicad_sch_to_data_set.__name__,
+]
diff --git a/test/doit/schematics/ee/kicad-footprint.csv b/test/doit/schematics/ee/kicad-footprint.csv
new file mode 100644
index 0000000..fa5cd5f
--- /dev/null
+++ b/test/doit/schematics/ee/kicad-footprint.csv
@@ -0,0 +1,45 @@
+kicad,common
+Capacitors_SMD:C_0402,0402
+Capacitors_SMD:C_0402_NoSilk,0402
+Capacitors_SMD:C_0603,0603
+Capacitors_SMD:C_0603_NoSilk,0603
+Capacitors_SMD:C_0805,0805
+Capacitors_SMD:C_0805_NoSilk,0805
+Diodes_SMD:D_SMA_Handsoldering,SMA
+Diodes_SMD:D_SMA,SMA
+Resistors_SMD:R_0402,0402
+Resistors_SMD:R_0402_NoSilk,0402
+Resistors_SMD:R_0805,0805
+TO_SOT_Packages_SMD:SOT-223,TO-261-4
+TO_SOT_Packages_SMD:SOT-23-6_Handsoldering,SOT-23-6
+TO_SOT_Packages_SMD:SOT-23,SOT-23
+Package_TO_SOT_SMD:SOT-23,SOT-23
+Bitraf:SOT-563,SOT-563
+
+Diode_SMD:D_SOD-323,SOD-323
+
+IPC7351-Nominal:CAPC1005X55,0402
+IPC7351-Nominal:CAPC1608X55,0603
+IPC7351-Nominal:CAPC2012X70,0805
+IPC7351-Nominal:CAPC3216X70,1206
+IPC7351-Nominal:CAPC3225X88,1210
+IPC7351-Nominal:CAPC4532X102,1812
+IPC7351-Nominal:CAPC4564X203,1825
+
+IPC7351-Nominal:LEDC1005X110,0402
+IPC7351-Nominal:LEDC1310X110,0504
+IPC7351-Nominal:LEDC1608X110,0603
+IPC7351-Nominal:LEDC2012X110,0805
+IPC7351-Nominal:LEDC3216X110,1206
+IPC7351-Nominal:LEDC3225X110,1210
+IPC7351-Nominal:LEDC4532X110,1812
+IPC7351-Nominal:LEDC4564X110,1825
+
+IPC7351-Nominal:RESC1005X38,0402
+IPC7351-Nominal:RESC1310X51,0504
+IPC7351-Nominal:RESC1608X63,0603
+IPC7351-Nominal:RESC2012X50,0805
+IPC7351-Nominal:RESC3216X60,1206
+IPC7351-Nominal:RESC3225X60,1210
+IPC7351-Nominal:RESC4532X70,1812
+IPC7351-Nominal:RESC4564X110,1825
diff --git a/test/doit/test_doit.py b/test/doit/test_doit.py
index 4076af8..8c98345 100644
--- a/test/doit/test_doit.py
+++ b/test/doit/test_doit.py
@@ -1,16 +1,18 @@
 import inspect
-import logging
 import os
 import os.path
+import logging
+from pathlib import Path
+from ee.fact import DataSetManager
 from inspect import Parameter
-
 import ee.kicad.doit
-from ee.kicad.doit import task_kicad_sch_to_data_set, task_kicad_pcb_to_data_set, task_kicad_create_component_data_set
+
+from ee.kicad.doit import *
 
 logger = logging.getLogger(__name__)
 
-file_dir = os.path.dirname(os.path.abspath(__file__))
-schematics_dir = os.path.join(file_dir, "schematics")
+file_dir = Path(__file__).parent
+schematics_dir = file_dir / "schematics"
 
 
 def exec_task(task):
@@ -31,9 +33,16 @@ def exec_task(task):
 
 # noinspection SpellCheckingInspection
 def test_doit(tmpdir, caplog):
-    ee.kicad.doit.init(sch=os.path.join(schematics_dir, "schematic-1.sch"),
-                       kicad_pcb=os.path.join(schematics_dir, "schematic-1.kicad_pcb"),
-                       data_set_dir=os.path.join(tmpdir, "ee"), )
+    dsm = DataSetManager(os.path.join(tmpdir, "ee"))
+
+    dsm.add_ds("csv", "kicad-footprint", "kicad-footprint-mapping",
+               path=schematics_dir / "ee" / "kicad-footprint.csv")
+
+    ee.kicad.doit.change_data_sets_for_task(task_kicad_create_component_data_set, lambda ds: ds + ["kicad-footprint"])
+
+    ee.kicad.doit.init(data_set_manager=dsm,
+                       sch=schematics_dir / "schematic-1.sch",
+                       kicad_pcb=schematics_dir / "schematic-1.kicad_pcb",)
 
     exec_task(task_kicad_sch_to_data_set())
     exec_task(task_kicad_pcb_to_data_set())