aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorTrygve Laugstøl <trygvis@inamo.no>2018-07-23 22:53:17 +0200
committerTrygve Laugstøl <trygvis@inamo.no>2018-07-23 22:53:17 +0200
commit894ff92c770320e264961bba55bbf0fba16efe56 (patch)
tree1cd723415aaaf5ad45a7aa32ac77b4e8e5645db9 /src
parent3e5819caef56c449606e27ec80f9d563d519a907 (diff)
downloadee-python-894ff92c770320e264961bba55bbf0fba16efe56.tar.gz
ee-python-894ff92c770320e264961bba55bbf0fba16efe56.tar.bz2
ee-python-894ff92c770320e264961bba55bbf0fba16efe56.tar.xz
ee-python-894ff92c770320e264961bba55bbf0fba16efe56.zip
wip.
o Support for loading CSV files.
Diffstat (limited to 'src')
-rw-r--r--src/ee/fact/__init__.py88
-rw-r--r--src/ee/kicad/doit.py127
2 files changed, 174 insertions, 41 deletions
diff --git a/src/ee/fact/__init__.py b/src/ee/fact/__init__.py
index 02e6b2a..c8c21dd 100644
--- a/src/ee/fact/__init__.py
+++ b/src/ee/fact/__init__.py
@@ -1,9 +1,10 @@
import configparser
import logging
import os
+import csv
from functools import total_ordering
from pathlib import Path
-from typing import MutableMapping, Optional, List, Tuple
+from typing import MutableMapping, Optional, List, Tuple, Union, Iterator
logger = logging.getLogger(__name__)
@@ -37,12 +38,12 @@ class ObjectType(object):
def fields(self):
return self._fields
- def index_of(self, field: str, create: bool = False) -> int:
+ def index_of(self, field: str, create: bool = False) -> Optional[int]:
try:
return self._fields.index(field)
except ValueError as e:
if not create:
- raise e
+ return None
self._fields.append(field)
return len(self._fields) - 1
@@ -75,7 +76,16 @@ class Object(object):
def get(self, key: str) -> Optional[str]:
idx = self._ot.index_of(key)
- return self._data[idx] if idx < len(self._data) else None
+ return self._data[idx] if idx is not None and idx < len(self._data) else None
+
+ def get_all(self, *keys: str) -> Optional[List[str]]:
+ values = []
+ for key in keys:
+ idx = self._ot.index_of(key)
+ if not idx or idx >= len(self._data):
+ return None
+ values.append(self._data[idx])
+ return values
class DataSet(object):
@@ -89,6 +99,9 @@ class DataSet(object):
def name(self):
return self._name
+ def __len__(self):
+ return sum((len(objects) for objects in self._objects_by_type.values()))
+
def freeze(self):
self._frozen = True
@@ -168,9 +181,10 @@ class DataSet(object):
return self._check_object(object_type, key, True)
- def items(self):
- from itertools import chain
- return list(chain.from_iterable([objects.values() for objects in self._objects_by_type.values()]))
+ def items(self) -> Iterator[Object]:
+ for objects in self._objects_by_type.values():
+ for o in objects.values():
+ yield o
def merge(self, other: "DataSet") -> "DataSet":
ds = DataSet(self._name)
@@ -186,11 +200,15 @@ class DataSet(object):
class DataSetManager(object):
- def __init__(self, basedir: Path):
+ def __init__(self, basedir: Union[Path, str]):
self._basedir = Path(basedir)
+ self._csv = {} # type: MutableMapping[str, Tuple[str, Path]]
- def metafile_for_ds(self, ds_name) -> Path:
- return self._basedir / ds_name / "data-set.ini"
+ def cookie_for_ds(self, ds_name) -> Path:
+ try:
+ return self._csv[ds_name][1]
+ except KeyError:
+ return self._basedir / ds_name / "data-set.ini"
def create_rw(self, name, inputs: List[str] = None) -> "LazyDataSet":
return LazyDataSet(self, False, name, inputs if inputs else [])
@@ -198,7 +216,52 @@ class DataSetManager(object):
def create_ro(self, inputs: List[str]) -> "LazyDataSet":
return LazyDataSet(self, True, None, inputs)
+ def add_ds(self, ds_type: str, name: str, object_type: str, path: str = None):
+ if ds_type == "csv":
+ if name in self._csv:
+ raise Exception("Data source already exists: {}".format(name))
+
+ self._csv[name] = object_type, Path(path),
+ else:
+ raise Exception("Unknown data source type: {}".format(ds_type))
+
+ def ds_type(self, name: str):
+ return "csv" if name in self._csv else "ini-dir"
+
def load(self, name, freeze=False) -> DataSet:
+ try:
+ object_type, path = self._csv[name]
+
+ if not freeze:
+ raise Exception("CSV data sources must be frozen")
+
+ return DataSetManager._load_csv(name, object_type, path, freeze)
+ except KeyError:
+ return self._load_ini_dir(name, freeze)
+
+ @staticmethod
+ def _load_csv(name: str, object_type: str, path: Path, freeze: bool) -> DataSet:
+ logger.info("Loading CSV file {}".format(path))
+ ds = DataSet(name)
+
+ with open(str(path), newline='') as f:
+ r = csv.reader(f)
+
+ header = next(r, None)
+ for row in r:
+ if len(row) == 0:
+ continue
+
+ key = row[0]
+
+ o = ds.create_object(object_type, key)
+ for idx, value in zip(range(0, min(len(row), len(header))), row):
+ o.set(header[idx], value)
+
+ logger.debug("Loaded {} objects".format(len(ds)))
+ return ds
+
+ def _load_ini_dir(self, name: str, freeze: bool) -> DataSet:
ds_dir = Path(name) if Path(name).is_absolute() else self._basedir / name
ds_dir = ds_dir if ds_dir.is_dir() else ds_dir.parent
@@ -233,7 +296,8 @@ class DataSetManager(object):
def store(self, ds: DataSet):
ds_dir = self._basedir / ds.name
- logger.info("Storing DS '{}' with {} objects to {}".format(ds.name, len(ds.items()), ds_dir))
+ items = list(ds.items())
+ logger.info("Storing DS '{}' with {} objects to {}".format(ds.name, len(items), ds_dir))
os.makedirs(ds_dir, exist_ok=True)
ini = self._blank_ini()
@@ -241,7 +305,7 @@ class DataSetManager(object):
ini.set("data-set", "name", ds.name)
self._store_ini(ini, ds_dir / "data-set.ini")
- for o in ds.items():
+ for o in items:
ot = o.object_type
key = o.key
diff --git a/src/ee/kicad/doit.py b/src/ee/kicad/doit.py
index 7273076..41e3155 100644
--- a/src/ee/kicad/doit.py
+++ b/src/ee/kicad/doit.py
@@ -1,5 +1,7 @@
import logging
import os.path
+from pathlib import Path
+from typing import Mapping
from configclass import Config
@@ -14,27 +16,33 @@ _config_template = Config({
"kicad_pcb": None,
"gerber_dir": None,
"gerber_zip": None,
- "data_set_dir": None,
})
-_config = None
+_config = None # type: Mapping[str, str]
_dsm = None # type: DataSetManager
+_data_sets = {}
-def init(**kwargs):
+
+def change_data_sets_for_task(task, callable):
+ _data_sets[task] = callable(_data_sets[task])
+
+
+def init(data_set_manager: DataSetManager, **kwargs):
global _config
_config = _config_template.make(kwargs)
+ ee_logger = logging.getLogger("ee")
formatter = logging.Formatter("%(levelname)s: %(message)s")
ch = logging.StreamHandler()
ch.setFormatter(formatter)
- logger.addHandler(ch)
+ ee_logger.addHandler(ch)
- logger.setLevel(logging.DEBUG)
+ ee_logger.setLevel(logging.DEBUG)
global _dsm
- _dsm = DataSetManager(_config["data_set_dir"])
+ _dsm = data_set_manager
def task_kicad_gerber():
@@ -69,7 +77,6 @@ def task_kicad_gerber():
z.write(p, arcname=p.relative_to(gerber_dir))
return {
- "basename": "kicad-gerber",
"targets": [gerber_zip],
"actions": [mkdir, export_gerber, make_zip],
"file_dep": [kicad_pcb],
@@ -78,14 +85,14 @@ def task_kicad_gerber():
def task_kicad_sch_to_data_set():
out_data_set = "kicad-sch"
- in_data_sets = []
+ in_data_sets = _data_sets[task_kicad_sch_to_data_set]
sch = _config["sch"]
def action():
from ee.kicad.model import ComponentField
- with _dsm.create_rw(out_data_set, inputs=in_data_sets) as ds:
+ with _dsm.create_rw(out_data_set) as ds:
schematics = ee.kicad.read_schematics(sch)
for c in [c for c in schematics.components]:
o = ds.create_object("kicad-schematic-component", c.timestamp)
@@ -102,18 +109,20 @@ def task_kicad_sch_to_data_set():
o.set(f.name, str(f.value))
return {
- "basename": "kicad-sch-to-data-set",
- "file_dep": [sch] + [_dsm.metafile_for_ds(ds) for ds in in_data_sets],
+ "file_dep": [Path(sch)] + [_dsm.cookie_for_ds(ds) for ds in in_data_sets],
"actions": [action],
- "targets": [_dsm.metafile_for_ds(out_data_set)],
+ "targets": [_dsm.cookie_for_ds(out_data_set)],
}
+_data_sets[task_kicad_sch_to_data_set] = []
+
+
def task_kicad_pcb_to_data_set():
kicad_pcb = _config["kicad_pcb"]
out_data_set = "kicad-pcb"
- in_data_sets = []
+ in_data_sets = _data_sets[task_kicad_pcb_to_data_set]
def action():
from ee.kicad.pcb import KicadPcb, Module
@@ -130,7 +139,7 @@ def task_kicad_pcb_to_data_set():
o = ds.create_object("kicad-pcb-component", m.tstamp)
ref_text = next((t for t in m.fp_texts if t.kind == "reference"), None)
- o.set("reference", ref_text.value)
+ o.set("ref", ref_text.value)
x, y, rot = m.at
o.set("placement-x", x)
@@ -139,31 +148,91 @@ def task_kicad_pcb_to_data_set():
o.set("layer", m.layer)
return {
- "basename": "kicad-pcb-to-data-set",
- "file_dep": [kicad_pcb] + [_dsm.metafile_for_ds(ds) for ds in in_data_sets],
+ "file_dep": [Path(kicad_pcb)] + [_dsm.cookie_for_ds(ds) for ds in in_data_sets],
"actions": [action],
- "targets": [_dsm.metafile_for_ds(out_data_set)],
+ "targets": [_dsm.cookie_for_ds(out_data_set)],
}
+_data_sets[task_kicad_pcb_to_data_set] = []
+
+
def task_kicad_create_component_data_set():
out_data_set = "components"
- in_data_sets = ["kicad-sch", "kicad-pcb"]
+ in_data_sets = _data_sets[task_kicad_create_component_data_set]
+
+ def action():
+ logger.info("in_data_sets={}, out_data_set={}".format(in_data_sets, out_data_set))
+
+ with _dsm.create_ro(in_data_sets) as in_ds:
+ # for o in in_ds.items():
+ # logger.info("item: {}/{}".format(o.object_type.name, o.key))
+
+ def map_footprint(footprint):
+ for o in in_ds.items():
+ if not o.object_type.name == "kicad-footprint-mapping":
+ continue
+
+ common = o.get("common")
+ if common:
+ return common
- def action(targets, *args, **kwargs):
- logger.info("targets={}, args={}, kwargs={}".format(targets, args, kwargs))
+ return footprint
- with _dsm.create_ro(in_data_sets) as input:
with _dsm.create_rw(out_data_set) as output:
- items = input.items()
- logger.info("Got {} objects".format(len(items)))
- for o in items:
- ot = o.object_type
- logger.info("processing {}:{}".format(ot.name, o.key))
+ kicad_sch = [o for o in in_ds.items() if o.object_type.name == "kicad-schematic-component"]
+
+ logger.info("processing {} kicad-sch".format(len(kicad_sch)))
+
+ for sch in kicad_sch:
+ ref = sch.get("ref")
+ ref_num = sch.get("ref-num")
+ if not ref or not ref_num:
+ logger.debug("Missing ref or ref-num")
+ continue
+
+ ref_type = sch.get("ref-type")
+ if not ref_type or ref_type == "#PWR":
+ logger.debug("Missing ref-type or bad ref-type: ref={}, ref-type={}".format(ref, ref_type))
+ continue
+
+ c = output.create_object("component", ref)
+ c.set("ref", ref)
+ c.set("ref-num", ref_num)
+ c.set("ref-type", ref_type)
+
+ fp = sch.get("footprint")
+ if fp:
+ fp = map_footprint(fp)
+ c.set("footprint", fp)
+
+ def pcb_match(o):
+ return o.object_type.name == "kicad-pcb-component" and \
+ o.get("ref") == ref
+
+ pcb = [o for o in in_ds.items() if pcb_match(o)]
+
+ if not pcb:
+ logger.info("Could not find PCB component for {}".format(ref))
+
+ # TODO: check that the SCH and PCB footprint are the same
+ # c.set("footprint", pcb.)
return {
- "basename": "kicad-create-component-data-set",
- "file_dep": [_dsm.metafile_for_ds(ds) for ds in in_data_sets],
+ "file_dep": [_dsm.cookie_for_ds(ds) for ds in in_data_sets],
"actions": [action],
- "targets": [_dsm.metafile_for_ds(out_data_set)],
+ "targets": [_dsm.cookie_for_ds(out_data_set)],
}
+
+
+_data_sets[task_kicad_create_component_data_set] = ["kicad-sch", "kicad-pcb"]
+
+__all__ = [
+ init.__name__,
+ change_data_sets_for_task.__name__,
+
+ task_kicad_create_component_data_set.__name__,
+ task_kicad_gerber.__name__,
+ task_kicad_pcb_to_data_set.__name__,
+ task_kicad_sch_to_data_set.__name__,
+]