diff options
| -rw-r--r-- | demo/doit/dodo.py | 14 | ||||
| -rw-r--r-- | demo/doit/ee/kicad-footprint.csv | 2 | ||||
| -rw-r--r-- | src/ee/fact/__init__.py | 88 | ||||
| -rw-r--r-- | src/ee/kicad/doit.py | 127 | ||||
| -rw-r--r-- | test/doit/schematics/ee/kicad-footprint.csv | 45 | ||||
| -rw-r--r-- | test/doit/test_doit.py | 25 |
6 files changed, 248 insertions, 53 deletions
diff --git a/demo/doit/dodo.py b/demo/doit/dodo.py index 27ceb67..858e673 100644 --- a/demo/doit/dodo.py +++ b/demo/doit/dodo.py @@ -1,4 +1,6 @@ import ee.kicad.doit +from ee.kicad.doit import * +from ee.fact import DataSetManager prj = "demo" sch = "{}.sch".format(prj) @@ -6,12 +8,16 @@ kicad_pcb = "{}.kicad_pcb".format(prj) DOIT_CONFIG = {'check_file_uptodate': 'timestamp'} +dsm = DataSetManager("ee") + +kicad_footprint = "kicad-footprint" +dsm.add_ds("csv", kicad_footprint, "kicad-footprint-mapping", path="ee/kicad-footprint.csv") + +ee.kicad.doit.change_data_sets_for_task(task_kicad_create_component_data_set, lambda ds: ds + [kicad_footprint]) + ee.kicad.doit.init( + data_set_manager=dsm, sch=sch, kicad_pcb=kicad_pcb, gerber_dir="gerber", - data_set_dir="ee", ) - -# noinspection PyUnresolvedReferences -from ee.kicad.doit import * diff --git a/demo/doit/ee/kicad-footprint.csv b/demo/doit/ee/kicad-footprint.csv new file mode 100644 index 0000000..a9ccfd7 --- /dev/null +++ b/demo/doit/ee/kicad-footprint.csv @@ -0,0 +1,2 @@ +kicad,common +Resistor_SMD:R_1206_3216Metric,SMD_3216 diff --git a/src/ee/fact/__init__.py b/src/ee/fact/__init__.py index 02e6b2a..c8c21dd 100644 --- a/src/ee/fact/__init__.py +++ b/src/ee/fact/__init__.py @@ -1,9 +1,10 @@ import configparser import logging import os +import csv from functools import total_ordering from pathlib import Path -from typing import MutableMapping, Optional, List, Tuple +from typing import MutableMapping, Optional, List, Tuple, Union, Iterator logger = logging.getLogger(__name__) @@ -37,12 +38,12 @@ class ObjectType(object): def fields(self): return self._fields - def index_of(self, field: str, create: bool = False) -> int: + def index_of(self, field: str, create: bool = False) -> Optional[int]: try: return self._fields.index(field) except ValueError as e: if not create: - raise e + return None self._fields.append(field) return len(self._fields) - 1 @@ -75,7 +76,16 @@ class Object(object): def get(self, key: str) -> Optional[str]: idx = self._ot.index_of(key) - return self._data[idx] if idx < len(self._data) else None + return self._data[idx] if idx is not None and idx < len(self._data) else None + + def get_all(self, *keys: str) -> Optional[List[str]]: + values = [] + for key in keys: + idx = self._ot.index_of(key) + if not idx or idx >= len(self._data): + return None + values.append(self._data[idx]) + return values class DataSet(object): @@ -89,6 +99,9 @@ class DataSet(object): def name(self): return self._name + def __len__(self): + return sum((len(objects) for objects in self._objects_by_type.values())) + def freeze(self): self._frozen = True @@ -168,9 +181,10 @@ class DataSet(object): return self._check_object(object_type, key, True) - def items(self): - from itertools import chain - return list(chain.from_iterable([objects.values() for objects in self._objects_by_type.values()])) + def items(self) -> Iterator[Object]: + for objects in self._objects_by_type.values(): + for o in objects.values(): + yield o def merge(self, other: "DataSet") -> "DataSet": ds = DataSet(self._name) @@ -186,11 +200,15 @@ class DataSet(object): class DataSetManager(object): - def __init__(self, basedir: Path): + def __init__(self, basedir: Union[Path, str]): self._basedir = Path(basedir) + self._csv = {} # type: MutableMapping[str, Tuple[str, Path]] - def metafile_for_ds(self, ds_name) -> Path: - return self._basedir / ds_name / "data-set.ini" + def cookie_for_ds(self, ds_name) -> Path: + try: + return self._csv[ds_name][1] + except KeyError: + return self._basedir / ds_name / "data-set.ini" def create_rw(self, name, inputs: List[str] = None) -> "LazyDataSet": return LazyDataSet(self, False, name, inputs if inputs else []) @@ -198,7 +216,52 @@ class DataSetManager(object): def create_ro(self, inputs: List[str]) -> "LazyDataSet": return LazyDataSet(self, True, None, inputs) + def add_ds(self, ds_type: str, name: str, object_type: str, path: str = None): + if ds_type == "csv": + if name in self._csv: + raise Exception("Data source already exists: {}".format(name)) + + self._csv[name] = object_type, Path(path), + else: + raise Exception("Unknown data source type: {}".format(ds_type)) + + def ds_type(self, name: str): + return "csv" if name in self._csv else "ini-dir" + def load(self, name, freeze=False) -> DataSet: + try: + object_type, path = self._csv[name] + + if not freeze: + raise Exception("CSV data sources must be frozen") + + return DataSetManager._load_csv(name, object_type, path, freeze) + except KeyError: + return self._load_ini_dir(name, freeze) + + @staticmethod + def _load_csv(name: str, object_type: str, path: Path, freeze: bool) -> DataSet: + logger.info("Loading CSV file {}".format(path)) + ds = DataSet(name) + + with open(str(path), newline='') as f: + r = csv.reader(f) + + header = next(r, None) + for row in r: + if len(row) == 0: + continue + + key = row[0] + + o = ds.create_object(object_type, key) + for idx, value in zip(range(0, min(len(row), len(header))), row): + o.set(header[idx], value) + + logger.debug("Loaded {} objects".format(len(ds))) + return ds + + def _load_ini_dir(self, name: str, freeze: bool) -> DataSet: ds_dir = Path(name) if Path(name).is_absolute() else self._basedir / name ds_dir = ds_dir if ds_dir.is_dir() else ds_dir.parent @@ -233,7 +296,8 @@ class DataSetManager(object): def store(self, ds: DataSet): ds_dir = self._basedir / ds.name - logger.info("Storing DS '{}' with {} objects to {}".format(ds.name, len(ds.items()), ds_dir)) + items = list(ds.items()) + logger.info("Storing DS '{}' with {} objects to {}".format(ds.name, len(items), ds_dir)) os.makedirs(ds_dir, exist_ok=True) ini = self._blank_ini() @@ -241,7 +305,7 @@ class DataSetManager(object): ini.set("data-set", "name", ds.name) self._store_ini(ini, ds_dir / "data-set.ini") - for o in ds.items(): + for o in items: ot = o.object_type key = o.key diff --git a/src/ee/kicad/doit.py b/src/ee/kicad/doit.py index 7273076..41e3155 100644 --- a/src/ee/kicad/doit.py +++ b/src/ee/kicad/doit.py @@ -1,5 +1,7 @@ import logging import os.path +from pathlib import Path +from typing import Mapping from configclass import Config @@ -14,27 +16,33 @@ _config_template = Config({ "kicad_pcb": None, "gerber_dir": None, "gerber_zip": None, - "data_set_dir": None, }) -_config = None +_config = None # type: Mapping[str, str] _dsm = None # type: DataSetManager +_data_sets = {} -def init(**kwargs): + +def change_data_sets_for_task(task, callable): + _data_sets[task] = callable(_data_sets[task]) + + +def init(data_set_manager: DataSetManager, **kwargs): global _config _config = _config_template.make(kwargs) + ee_logger = logging.getLogger("ee") formatter = logging.Formatter("%(levelname)s: %(message)s") ch = logging.StreamHandler() ch.setFormatter(formatter) - logger.addHandler(ch) + ee_logger.addHandler(ch) - logger.setLevel(logging.DEBUG) + ee_logger.setLevel(logging.DEBUG) global _dsm - _dsm = DataSetManager(_config["data_set_dir"]) + _dsm = data_set_manager def task_kicad_gerber(): @@ -69,7 +77,6 @@ def task_kicad_gerber(): z.write(p, arcname=p.relative_to(gerber_dir)) return { - "basename": "kicad-gerber", "targets": [gerber_zip], "actions": [mkdir, export_gerber, make_zip], "file_dep": [kicad_pcb], @@ -78,14 +85,14 @@ def task_kicad_gerber(): def task_kicad_sch_to_data_set(): out_data_set = "kicad-sch" - in_data_sets = [] + in_data_sets = _data_sets[task_kicad_sch_to_data_set] sch = _config["sch"] def action(): from ee.kicad.model import ComponentField - with _dsm.create_rw(out_data_set, inputs=in_data_sets) as ds: + with _dsm.create_rw(out_data_set) as ds: schematics = ee.kicad.read_schematics(sch) for c in [c for c in schematics.components]: o = ds.create_object("kicad-schematic-component", c.timestamp) @@ -102,18 +109,20 @@ def task_kicad_sch_to_data_set(): o.set(f.name, str(f.value)) return { - "basename": "kicad-sch-to-data-set", - "file_dep": [sch] + [_dsm.metafile_for_ds(ds) for ds in in_data_sets], + "file_dep": [Path(sch)] + [_dsm.cookie_for_ds(ds) for ds in in_data_sets], "actions": [action], - "targets": [_dsm.metafile_for_ds(out_data_set)], + "targets": [_dsm.cookie_for_ds(out_data_set)], } +_data_sets[task_kicad_sch_to_data_set] = [] + + def task_kicad_pcb_to_data_set(): kicad_pcb = _config["kicad_pcb"] out_data_set = "kicad-pcb" - in_data_sets = [] + in_data_sets = _data_sets[task_kicad_pcb_to_data_set] def action(): from ee.kicad.pcb import KicadPcb, Module @@ -130,7 +139,7 @@ def task_kicad_pcb_to_data_set(): o = ds.create_object("kicad-pcb-component", m.tstamp) ref_text = next((t for t in m.fp_texts if t.kind == "reference"), None) - o.set("reference", ref_text.value) + o.set("ref", ref_text.value) x, y, rot = m.at o.set("placement-x", x) @@ -139,31 +148,91 @@ def task_kicad_pcb_to_data_set(): o.set("layer", m.layer) return { - "basename": "kicad-pcb-to-data-set", - "file_dep": [kicad_pcb] + [_dsm.metafile_for_ds(ds) for ds in in_data_sets], + "file_dep": [Path(kicad_pcb)] + [_dsm.cookie_for_ds(ds) for ds in in_data_sets], "actions": [action], - "targets": [_dsm.metafile_for_ds(out_data_set)], + "targets": [_dsm.cookie_for_ds(out_data_set)], } +_data_sets[task_kicad_pcb_to_data_set] = [] + + def task_kicad_create_component_data_set(): out_data_set = "components" - in_data_sets = ["kicad-sch", "kicad-pcb"] + in_data_sets = _data_sets[task_kicad_create_component_data_set] + + def action(): + logger.info("in_data_sets={}, out_data_set={}".format(in_data_sets, out_data_set)) + + with _dsm.create_ro(in_data_sets) as in_ds: + # for o in in_ds.items(): + # logger.info("item: {}/{}".format(o.object_type.name, o.key)) + + def map_footprint(footprint): + for o in in_ds.items(): + if not o.object_type.name == "kicad-footprint-mapping": + continue + + common = o.get("common") + if common: + return common - def action(targets, *args, **kwargs): - logger.info("targets={}, args={}, kwargs={}".format(targets, args, kwargs)) + return footprint - with _dsm.create_ro(in_data_sets) as input: with _dsm.create_rw(out_data_set) as output: - items = input.items() - logger.info("Got {} objects".format(len(items))) - for o in items: - ot = o.object_type - logger.info("processing {}:{}".format(ot.name, o.key)) + kicad_sch = [o for o in in_ds.items() if o.object_type.name == "kicad-schematic-component"] + + logger.info("processing {} kicad-sch".format(len(kicad_sch))) + + for sch in kicad_sch: + ref = sch.get("ref") + ref_num = sch.get("ref-num") + if not ref or not ref_num: + logger.debug("Missing ref or ref-num") + continue + + ref_type = sch.get("ref-type") + if not ref_type or ref_type == "#PWR": + logger.debug("Missing ref-type or bad ref-type: ref={}, ref-type={}".format(ref, ref_type)) + continue + + c = output.create_object("component", ref) + c.set("ref", ref) + c.set("ref-num", ref_num) + c.set("ref-type", ref_type) + + fp = sch.get("footprint") + if fp: + fp = map_footprint(fp) + c.set("footprint", fp) + + def pcb_match(o): + return o.object_type.name == "kicad-pcb-component" and \ + o.get("ref") == ref + + pcb = [o for o in in_ds.items() if pcb_match(o)] + + if not pcb: + logger.info("Could not find PCB component for {}".format(ref)) + + # TODO: check that the SCH and PCB footprint are the same + # c.set("footprint", pcb.) return { - "basename": "kicad-create-component-data-set", - "file_dep": [_dsm.metafile_for_ds(ds) for ds in in_data_sets], + "file_dep": [_dsm.cookie_for_ds(ds) for ds in in_data_sets], "actions": [action], - "targets": [_dsm.metafile_for_ds(out_data_set)], + "targets": [_dsm.cookie_for_ds(out_data_set)], } + + +_data_sets[task_kicad_create_component_data_set] = ["kicad-sch", "kicad-pcb"] + +__all__ = [ + init.__name__, + change_data_sets_for_task.__name__, + + task_kicad_create_component_data_set.__name__, + task_kicad_gerber.__name__, + task_kicad_pcb_to_data_set.__name__, + task_kicad_sch_to_data_set.__name__, +] diff --git a/test/doit/schematics/ee/kicad-footprint.csv b/test/doit/schematics/ee/kicad-footprint.csv new file mode 100644 index 0000000..fa5cd5f --- /dev/null +++ b/test/doit/schematics/ee/kicad-footprint.csv @@ -0,0 +1,45 @@ +kicad,common +Capacitors_SMD:C_0402,0402 +Capacitors_SMD:C_0402_NoSilk,0402 +Capacitors_SMD:C_0603,0603 +Capacitors_SMD:C_0603_NoSilk,0603 +Capacitors_SMD:C_0805,0805 +Capacitors_SMD:C_0805_NoSilk,0805 +Diodes_SMD:D_SMA_Handsoldering,SMA +Diodes_SMD:D_SMA,SMA +Resistors_SMD:R_0402,0402 +Resistors_SMD:R_0402_NoSilk,0402 +Resistors_SMD:R_0805,0805 +TO_SOT_Packages_SMD:SOT-223,TO-261-4 +TO_SOT_Packages_SMD:SOT-23-6_Handsoldering,SOT-23-6 +TO_SOT_Packages_SMD:SOT-23,SOT-23 +Package_TO_SOT_SMD:SOT-23,SOT-23 +Bitraf:SOT-563,SOT-563 + +Diode_SMD:D_SOD-323,SOD-323 + +IPC7351-Nominal:CAPC1005X55,0402 +IPC7351-Nominal:CAPC1608X55,0603 +IPC7351-Nominal:CAPC2012X70,0805 +IPC7351-Nominal:CAPC3216X70,1206 +IPC7351-Nominal:CAPC3225X88,1210 +IPC7351-Nominal:CAPC4532X102,1812 +IPC7351-Nominal:CAPC4564X203,1825 + +IPC7351-Nominal:LEDC1005X110,0402 +IPC7351-Nominal:LEDC1310X110,0504 +IPC7351-Nominal:LEDC1608X110,0603 +IPC7351-Nominal:LEDC2012X110,0805 +IPC7351-Nominal:LEDC3216X110,1206 +IPC7351-Nominal:LEDC3225X110,1210 +IPC7351-Nominal:LEDC4532X110,1812 +IPC7351-Nominal:LEDC4564X110,1825 + +IPC7351-Nominal:RESC1005X38,0402 +IPC7351-Nominal:RESC1310X51,0504 +IPC7351-Nominal:RESC1608X63,0603 +IPC7351-Nominal:RESC2012X50,0805 +IPC7351-Nominal:RESC3216X60,1206 +IPC7351-Nominal:RESC3225X60,1210 +IPC7351-Nominal:RESC4532X70,1812 +IPC7351-Nominal:RESC4564X110,1825 diff --git a/test/doit/test_doit.py b/test/doit/test_doit.py index 4076af8..8c98345 100644 --- a/test/doit/test_doit.py +++ b/test/doit/test_doit.py @@ -1,16 +1,18 @@ import inspect -import logging import os import os.path +import logging +from pathlib import Path +from ee.fact import DataSetManager from inspect import Parameter - import ee.kicad.doit -from ee.kicad.doit import task_kicad_sch_to_data_set, task_kicad_pcb_to_data_set, task_kicad_create_component_data_set + +from ee.kicad.doit import * logger = logging.getLogger(__name__) -file_dir = os.path.dirname(os.path.abspath(__file__)) -schematics_dir = os.path.join(file_dir, "schematics") +file_dir = Path(__file__).parent +schematics_dir = file_dir / "schematics" def exec_task(task): @@ -31,9 +33,16 @@ def exec_task(task): # noinspection SpellCheckingInspection def test_doit(tmpdir, caplog): - ee.kicad.doit.init(sch=os.path.join(schematics_dir, "schematic-1.sch"), - kicad_pcb=os.path.join(schematics_dir, "schematic-1.kicad_pcb"), - data_set_dir=os.path.join(tmpdir, "ee"), ) + dsm = DataSetManager(os.path.join(tmpdir, "ee")) + + dsm.add_ds("csv", "kicad-footprint", "kicad-footprint-mapping", + path=schematics_dir / "ee" / "kicad-footprint.csv") + + ee.kicad.doit.change_data_sets_for_task(task_kicad_create_component_data_set, lambda ds: ds + ["kicad-footprint"]) + + ee.kicad.doit.init(data_set_manager=dsm, + sch=schematics_dir / "schematic-1.sch", + kicad_pcb=schematics_dir / "schematic-1.kicad_pcb",) exec_task(task_kicad_sch_to_data_set()) exec_task(task_kicad_pcb_to_data_set()) |