Support for Eagle / Fusion 360 Electronics (#216)

* initial commit - added EagleParser

This adds a third parser for JSON files produced from Eagle or Fusion 360 Electronics using the ULP available at https://github.com/Funkenjaeger/brd2json

* Stylistic fixes

* Refactored EagleParser to GenericJsonParser, added 'spec version' logic and more robust error handling

* _type and _spec_version in top level object only

* #216 code review updates

* Fix format string syntax

* Initial implementation of JSON validation with schema in GenericJsonParser

* Updated to implement the rest of the pcbdata struct as defined in DATAFORMAT.md

* Fix ExtraData (array vs. object)

* Initial cut at support for extra_fields in generic JSON schema & parser

* More schema updates based on code review

* Removed parser-specific code from the core code in ibom.py

Pushed all parsing of extra_fields data down into the respective parsers.  This includes some experimental code associated with GenericJsonParser, as well as the existing netlist-based (kicad-specific) code that was in ibom.py

* #216 code review updates

* Revert to returning extra_fields only within components

* Extra field data embedded in components for kicad parser also

* Override board bounding box from generic JSON based on edges

* Restore warning for outdated netlist/xml in kicad parser

* Fix clerical issues noted in code review

* Fix improper access of footprint ref in kicad.py

Author: Funkenjaeger

InteractiveHtmlBom/core/ibom.py

@@ -50,8 +50,8 @@ def warn(self, msg):
 log = None  # type: Logger or None
 
 
-def skip_component(m, config, extra_data):
-    # type: (Component, Config, dict) -> bool
+def skip_component(m, config):
+    # type: (Component, Config) -> bool
     # skip blacklisted components
     ref_prefix = re.findall('^[A-Z]*', m.ref)[0]
     if m.ref in config.component_blacklist:
@@ -67,29 +67,27 @@ def skip_component(m, config, extra_data):
         return True
 
     # skip components with dnp field not empty
-    if config.dnp_field and m.ref in extra_data \
-            and config.dnp_field in extra_data[m.ref] \
-            and extra_data[m.ref][config.dnp_field]:
+    if config.dnp_field \
+            and config.dnp_field in m.extra_fields \
+            and m.extra_fields[config.dnp_field]:
         return True
 
     # skip components with wrong variant field
     if config.board_variant_field and config.board_variant_whitelist:
-        if m.ref in extra_data:
-            ref_variant = extra_data[m.ref].get(config.board_variant_field, '')
-            if ref_variant not in config.board_variant_whitelist:
-                return True
+        ref_variant = m.extra_fields.get(config.board_variant_field, '')
+        if ref_variant not in config.board_variant_whitelist:
+            return True
 
     if config.board_variant_field and config.board_variant_blacklist:
-        if m.ref in extra_data:
-            ref_variant = extra_data[m.ref].get(config.board_variant_field, '')
-            if ref_variant and ref_variant in config.board_variant_blacklist:
-                return True
+        ref_variant = m.extra_fields.get(config.board_variant_field, '')
+        if ref_variant and ref_variant in config.board_variant_blacklist:
+            return True
 
     return False
 
 
-def generate_bom(pcb_footprints, config, extra_data):
-    # type: (list, Config, dict) -> dict
+def generate_bom(pcb_footprints, config):
+    # type: (list, Config) -> dict
     """
     Generate BOM from pcb layout.
     :param pcb_footprints: list of footprints on the pcb
@@ -113,11 +111,10 @@ def natural_sort(l):
         return sorted(l, key=lambda r: (alphanum_key(r[0]), r[1]))
 
     # build grouped part list
-    warning_shown = False
     skipped_components = []
     part_groups = {}
     for i, f in enumerate(pcb_footprints):
-        if skip_component(f, config, extra_data):
+        if skip_component(f, config):
             skipped_components.append(i)
             continue
 
@@ -126,23 +123,13 @@ def natural_sort(l):
 
         extras = []
         if config.extra_fields:
-            if f.ref in extra_data:
-                extras = [extra_data[f.ref].get(ef, '')
-                          for ef in config.extra_fields]
-            else:
-                # Some components are on pcb but not in schematic data.
-                # Show a warning about possibly outdated netlist/xml file.
-                log.warn(
-                        'Component %s is missing from schematic data.' % f.ref)
-                warning_shown = True
-                extras = [''] * len(config.extra_fields)
+            extras = [f.extra_fields.get(ef, '')
+                      for ef in config.extra_fields]
 
         group_key = (norm_value, tuple(extras), f.footprint, f.attr)
         valrefs = part_groups.setdefault(group_key, [f.val, []])
         valrefs[1].append((f.ref, i))
 
-    if warning_shown:
-        log.warn('Netlist/xml file is likely out of date.')
     # build bom table, sort refs
     bom_table = []
     for (norm_value, extras, footprint, attr), valrefs in part_groups.items():
@@ -230,7 +217,6 @@ def get_file_content(file_name):
         with io.open(path, 'r', encoding='utf-8') as f:
             return f.read()
 
-
     if os.path.isabs(config.bom_dest_dir):
         bom_file_dir = config.bom_dest_dir
     else:
@@ -274,36 +260,11 @@ def main(parser, config, logger):
     pcb_file_name = os.path.basename(parser.file_name)
     pcb_file_dir = os.path.dirname(parser.file_name)
 
-    # Get extra field data
-    extra_fields = None
-    if config.netlist_file and os.path.isfile(config.netlist_file):
-        extra_fields = parser.extra_data_func(
-                config.netlist_file, config.normalize_field_case)
-
-    need_extra_fields = (config.extra_fields or
-                         config.board_variant_whitelist or
-                         config.board_variant_blacklist or
-                         config.dnp_field)
-
-    if not config.netlist_file and need_extra_fields:
-        logger.warn('Ignoring extra fields related config parameters '
-                    'since no netlist/xml file was specified.')
-        config.extra_fields = []
-        config.board_variant_whitelist = []
-        config.board_variant_blacklist = []
-        config.dnp_field = ''
-        need_extra_fields = False
-
-    if extra_fields is None and need_extra_fields:
-        raise ParsingException('Failed parsing %s' % config.netlist_file)
-
-    extra_fields = extra_fields[1] if extra_fields else None
-
     pcbdata, components = parser.parse()
     if not pcbdata and not components:
         raise ParsingException('Parsing failed.')
 
-    pcbdata["bom"] = generate_bom(components, config, extra_fields)
+    pcbdata["bom"] = generate_bom(components, config)
     pcbdata["ibom_version"] = config.version
 
     # build BOM

InteractiveHtmlBom/ecad/__init__.py

@@ -6,7 +6,15 @@ def get_parser_by_extension(file_name, config, logger):
     if ext == '.kicad_pcb':
         return get_kicad_parser(file_name, config, logger)
     elif ext == '.json':
-        return get_easyeda_parser(file_name, config, logger)
+        """.json file may be from EasyEDA or a generic json format"""
+        import io
+        import json
+        with io.open(file_name, 'r') as f:
+            obj = json.load(f)
+        if 'pcbdata' in obj:
+            return get_generic_json_parser(file_name, config, logger)
+        else:
+            return get_easyeda_parser(file_name, config, logger)
     else:
         return None
 
@@ -19,3 +27,8 @@ def get_kicad_parser(file_name, config, logger, board=None):
 def get_easyeda_parser(file_name, config, logger):
     from .easyeda import EasyEdaParser
     return EasyEdaParser(file_name, config, logger)
+
+
+def get_generic_json_parser(file_name, config, logger):
+    from .genericjson import GenericJsonParser
+    return GenericJsonParser(file_name, config, logger)

InteractiveHtmlBom/ecad/common.py

@@ -19,7 +19,8 @@ def __init__(self, file_name, config, logger):
     def parse(self):
         """
         Abstract method that should be overridden in implementations.
-        Performs all the parsing and returns a tuple of (pcbdata, components)
+        Performs all the parsing and returns a tuple of
+        (pcbdata, components)
         pcbdata is described in DATAFORMAT.md
         components is list of Component objects
         :return:
@@ -35,16 +36,49 @@ def latest_extra_data(self, extra_dirs=None):
         """
         return None
 
+    def add_drawing_bounding_box(self, drawing, bbox):
+        # type: (dict, BoundingBox) -> None
+
+        def add_segment():
+            bbox.add_segment(drawing['start'][0], drawing['start'][1],
+                             drawing['end'][0], drawing['end'][1],
+                             drawing['width'] / 2)
+
+        def add_circle():
+            bbox.add_circle(drawing['start'][0], drawing['start'][1],
+                            drawing['radius'] + drawing['width'] / 2)
+
+        def add_svgpath():
+            width = drawing.get('width', 0)
+            bbox.add_svgpath(drawing['svgpath'], width, self.logger)
+
+        def add_polygon():
+            if 'polygons' not in drawing:
+                add_svgpath()
+                return
+            polygon = drawing['polygons'][0]
+            for point in polygon:
+                bbox.add_point(point[0], point[1])
+
+        {
+            'segment': add_segment,
+            'circle': add_circle,
+            'arc': add_svgpath,
+            'polygon': add_polygon,
+            'text': lambda: None,  # text is not really needed for bounding box
+        }.get(drawing['type'])()
+
 
 class Component(object):
     """Simple data object to store component data needed for bom table."""
 
-    def __init__(self, ref, val, footprint, layer, attr=None):
+    def __init__(self, ref, val, footprint, layer, attr=None, extra_fields={}):
         self.ref = ref
         self.val = val
         self.footprint = footprint
         self.layer = layer
         self.attr = attr
+        self.extra_fields = extra_fields
 
 
 class BoundingBox(object):

InteractiveHtmlBom/ecad/easyeda.py

@@ -273,38 +273,6 @@ def add_custom():
             'custom': add_custom,
         }.get(pad['shape'])()
 
-    def add_drawing_bounding_box(self, drawing, bbox):
-        # type: (dict, BoundingBox) -> None
-
-        def add_segment():
-            bbox.add_segment(drawing['start'][0], drawing['start'][1],
-                             drawing['end'][0], drawing['end'][1],
-                             drawing['width'] / 2)
-
-        def add_circle():
-            bbox.add_circle(drawing['start'][0], drawing['start'][1],
-                            drawing['radius'] + drawing['width'] / 2)
-
-        def add_svgpath():
-            width = drawing.get('width', 0)
-            bbox.add_svgpath(drawing['svgpath'], width, self.logger)
-
-        def add_polygon():
-            if 'polygons' not in drawing:
-                add_svgpath()
-                return
-            polygon = drawing['polygons'][0]
-            for point in polygon:
-                bbox.add_point(point[0], point[1])
-
-        {
-            'segment': add_segment,
-            'circle': add_circle,
-            'arc': add_svgpath,
-            'polygon': add_polygon,
-            'text': lambda: None,  # text is not really needed for bounding box
-        }.get(drawing['type'])()
-
     def parse_lib(self, shape):
         parts = self.sharp_split(shape)
         head = self.tilda_split(parts[0])

InteractiveHtmlBom/ecad/genericjson.py

@@ -0,0 +1,80 @@
+import io
+import json
+from jsonschema import validate, ValidationError
+
+from .common import EcadParser, Component, BoundingBox
+
+
+class GenericJsonParser(EcadParser):
+    COMPATIBLE_SPEC_VERSIONS = [1]
+
+    def get_generic_json_pcb(self):
+        from os import path
+        with io.open(self.file_name, 'r') as f:
+            pcb = json.load(f)
+
+        if 'spec_version' not in pcb:
+            raise ValidationError("'spec_version' is a required property")
+
+        if pcb['spec_version'] not in self.COMPATIBLE_SPEC_VERSIONS:
+            raise ValidationError("Unsupported spec_version ({})"
+                                  .format(pcb['spec_version']))
+
+        schema_dir = path.join(path.dirname(__file__), 'schema')
+        schema_file_name = path.join(schema_dir,
+                                     'genericjsonpcbdata_v{}.schema'
+                                     .format(pcb['spec_version']))
+
+        with io.open(schema_file_name, 'r') as f:
+            schema = json.load(f)
+
+        validate(instance=pcb, schema=schema)
+
+        return pcb
+
+    def _verify(self, pcb):
+
+        """Spot check the pcb object."""
+
+        if len(pcb['pcbdata']['footprints']) != len(pcb['components']):
+            self.logger.error("Length of components list doesn't match"
+                              " length of footprints list.")
+            return False
+
+        return True
+
+    def parse(self):
+        try:
+            pcb = self.get_generic_json_pcb()
+        except ValidationError as e:
+            self.logger.error('File {f} does not comply with json schema. {m}'
+                              .format(f=self.file_name, m=e.message))
+            return None, None
+
+        if not self._verify(pcb):
+            self.logger.error('File {} does not appear to be valid generic'
+                              ' InteractiveHtmlBom json file.'
+                              .format(self.file_name))
+            return None, None
+
+        self.logger.info('Successfully parsed {}'.format(self.file_name))
+
+        pcbdata = pcb['pcbdata']
+        components = [Component(**c) for c in pcb['components']]
+
+        # override board bounding box based on edges
+        board_outline_bbox = BoundingBox()
+        for drawing in pcbdata['edges']:
+            self.add_drawing_bounding_box(drawing, board_outline_bbox)
+        if board_outline_bbox.initialized():
+            pcbdata['edges_bbox'] = board_outline_bbox.to_dict()
+
+        if self.config.extra_fields:
+            for c in components:
+                extra_field_data = {}
+                for f in self.config.extra_fields:
+                    fv = ("" if f not in c.extra_fields else c.extra_fields[f])
+                    extra_field_data[f] = fv
+                c.extra_fields = extra_field_data
+
+        return pcbdata, components