Make standard fields optional, grouping configurable

Rename extra fields GUI tab to fields
Add ability to include fields without affecting grouping
Add flags --show-fields, --group-fields which default to
"Footprint,Value"
--extra-fields is now a shortcut to show and group additional
fields (along with standard Footprint, Value)

Fixes #167

Author: qu1ck

DATAFORMAT.md

@@ -101,6 +101,13 @@ pcbdata = {
     "B":  [bomrow1, bomrow2, ...],
     // numeric IDs of DNP components that are not in BOM
     "skipped": [id1, id2, ...]
+    // Fields map is keyed on component ID with values being field data.
+    // It's order corresponds to order of fields data in config struct.
+    "fields" {
+      id1: [field1, field2, ...],
+      id2: [field1, field2, ...],
+      ...
+    }
   },
   // Contains parsed stroke data from newstroke font for
   // characters used on the pcb.
@@ -333,17 +340,7 @@ Footprints are a collection of pads, drawings and some metadata.
 
 # bom row struct
 
-Bom row is a 5-tuple (array in js)
-
-```js
-[
-  group_component_count,
-  value,
-  footprint_name,
-  reference_set,
-  extra_data
-]
-```
+Bom row is a list of reference sets
 
 Reference set is array of tuples of (ref, id) where id is just
 a unique numeric identifier for each footprint that helps avoid
@@ -356,13 +353,6 @@ collisions when references are duplicated.
 ]
 ```
 
-Extra data is array of extra field data. It's order corresponds
-to order of extra field data in config struct.
-
-```js
-[field1_value, field2_value, ...]
-```
-
 # config struct
 
 ```js

InteractiveHtmlBom/core/config.py

@@ -38,8 +38,9 @@ class Config:
     html_config_fields = [
         'dark_mode', 'show_pads', 'show_fabrication', 'show_silkscreen',
         'highlight_pin1', 'redraw_on_drag', 'board_rotation', 'checkboxes',
-        'bom_view', 'layer_view', 'extra_fields'
+        'bom_view', 'layer_view'
     ]
+    default_show_group_fields = ["Value", "Footprint"]
 
     # Defaults
 
@@ -70,7 +71,8 @@ class Config:
     # Extra fields section
     extra_data_file = None
     netlist_initial_directory = ''  # This is relative to pcb file directory
-    extra_fields = []
+    show_fields = default_show_group_fields
+    group_fields = default_show_group_fields
     normalize_field_case = False
     board_variant_field = ''
     board_variant_whitelist = []
@@ -79,7 +81,7 @@ class Config:
 
     @staticmethod
     def _split(s):
-        """Splits string by ',' and drops empty strings from resulting array."""
+        """Splits string by ',' and drops empty strings from resulting array"""
         return [a.replace('\\,', ',') for a in re.split(r'(?<!\\),', s) if a]
 
     @staticmethod
@@ -99,9 +101,9 @@ def load_from_ini(self):
         self.dark_mode = f.ReadBool('dark_mode', self.dark_mode)
         self.show_pads = f.ReadBool('show_pads', self.show_pads)
         self.show_fabrication = f.ReadBool(
-                'show_fabrication', self.show_fabrication)
+            'show_fabrication', self.show_fabrication)
         self.show_silkscreen = f.ReadBool(
-                'show_silkscreen', self.show_silkscreen)
+            'show_silkscreen', self.show_silkscreen)
         self.highlight_pin1 = f.ReadBool('highlight_pin1', self.highlight_pin1)
         self.redraw_on_drag = f.ReadBool('redraw_on_drag', self.redraw_on_drag)
         self.board_rotation = f.ReadInt('board_rotation', self.board_rotation)
@@ -115,32 +117,33 @@ def load_from_ini(self):
         self.bom_dest_dir = f.Read('bom_dest_dir', self.bom_dest_dir)
         self.bom_name_format = f.Read('bom_name_format', self.bom_name_format)
         self.component_sort_order = self._split(f.Read(
-                'component_sort_order',
-                ','.join(self.component_sort_order)))
+            'component_sort_order',
+            ','.join(self.component_sort_order)))
         self.component_blacklist = self._split(f.Read(
-                'component_blacklist',
-                ','.join(self.component_blacklist)))
+            'component_blacklist',
+            ','.join(self.component_blacklist)))
         self.blacklist_virtual = f.ReadBool(
-                'blacklist_virtual', self.blacklist_virtual)
+            'blacklist_virtual', self.blacklist_virtual)
         self.blacklist_empty_val = f.ReadBool(
-                'blacklist_empty_val', self.blacklist_empty_val)
+            'blacklist_empty_val', self.blacklist_empty_val)
         self.include_tracks = f.ReadBool('include_tracks', self.include_tracks)
         self.include_nets = f.ReadBool('include_nets', self.include_nets)
 
-        f.SetPath('/extra_fields')
-        self.extra_fields = self._split(f.Read(
-                'extra_fields',
-                self._join(self.extra_fields)))
+        f.SetPath('/fields')
+        self.show_fields = self._split(f.Read(
+            'show_fields', self._join(self.show_fields)))
+        self.group_fields = self._split(f.Read(
+            'group_fields', self._join(self.group_fields)))
         self.normalize_field_case = f.ReadBool(
-                'normalize_field_case', self.normalize_field_case)
+            'normalize_field_case', self.normalize_field_case)
         self.board_variant_field = f.Read(
-                'board_variant_field', self.board_variant_field)
+            'board_variant_field', self.board_variant_field)
         self.board_variant_whitelist = self._split(f.Read(
-                'board_variant_whitelist',
-                ','.join(self.board_variant_whitelist)))
+            'board_variant_whitelist',
+            self._join(self.board_variant_whitelist)))
         self.board_variant_blacklist = self._split(f.Read(
-                'board_variant_blacklist',
-                ','.join(self.board_variant_blacklist)))
+            'board_variant_blacklist',
+            self._join(self.board_variant_blacklist)))
         self.dnp_field = f.Read('dnp_field', self.dnp_field)
 
     def save(self):
@@ -164,7 +167,7 @@ def save(self):
         bom_dest_dir = self.bom_dest_dir
         if bom_dest_dir.startswith(self.netlist_initial_directory):
             bom_dest_dir = os.path.relpath(
-                    bom_dest_dir, self.netlist_initial_directory)
+                bom_dest_dir, self.netlist_initial_directory)
         f.Write('bom_dest_dir', bom_dest_dir)
         f.Write('bom_name_format', self.bom_name_format)
         f.Write('component_sort_order',
@@ -176,14 +179,15 @@ def save(self):
         f.WriteBool('include_tracks', self.include_tracks)
         f.WriteBool('include_nets', self.include_nets)
 
-        f.SetPath('/extra_fields')
-        f.Write('extra_fields', self._join(self.extra_fields))
+        f.SetPath('/fields')
+        f.Write('show_fields', self._join(self.show_fields))
+        f.Write('group_fields', self._join(self.group_fields))
         f.WriteBool('normalize_field_case', self.normalize_field_case)
         f.Write('board_variant_field', self.board_variant_field)
         f.Write('board_variant_whitelist',
-                ','.join(self.board_variant_whitelist))
+                self._join(self.board_variant_whitelist))
         f.Write('board_variant_blacklist',
-                ','.join(self.board_variant_blacklist))
+                self._join(self.board_variant_blacklist))
         f.Write('dnp_field', self.dnp_field)
         f.Flush()
 
@@ -216,19 +220,20 @@ def set_from_dialog(self, dlg):
         self.include_tracks = dlg.general.includeTracksCheckbox.IsChecked()
         self.include_nets = dlg.general.includeNetsCheckbox.IsChecked()
 
-        # Extra fields
-        self.extra_data_file = dlg.extra.extraDataFilePicker.Path
-        self.extra_fields = list(dlg.extra.extraFieldsList.GetCheckedStrings())
-        self.normalize_field_case = dlg.extra.normalizeCaseCheckbox.Value
-        self.board_variant_field = dlg.extra.boardVariantFieldBox.Value
-        if self.board_variant_field == dlg.extra.NONE_STRING:
+        # Fields
+        self.extra_data_file = dlg.fields.extraDataFilePicker.Path
+        self.show_fields = dlg.fields.GetShowFields()
+        self.group_fields = dlg.fields.GetGroupFields()
+        self.normalize_field_case = dlg.fields.normalizeCaseCheckbox.Value
+        self.board_variant_field = dlg.fields.boardVariantFieldBox.Value
+        if self.board_variant_field == dlg.fields.NONE_STRING:
             self.board_variant_field = ''
         self.board_variant_whitelist = list(
-                dlg.extra.boardVariantWhitelist.GetCheckedStrings())
+            dlg.fields.boardVariantWhitelist.GetCheckedStrings())
         self.board_variant_blacklist = list(
-                dlg.extra.boardVariantBlacklist.GetCheckedStrings())
-        self.dnp_field = dlg.extra.dnpFieldBox.Value
-        if self.dnp_field == dlg.extra.NONE_STRING:
+            dlg.fields.boardVariantBlacklist.GetCheckedStrings())
+        self.dnp_field = dlg.fields.dnpFieldBox.Value
+        if self.dnp_field == dlg.fields.NONE_STRING:
             self.dnp_field = ''
 
     def transfer_to_dialog(self, dlg):
@@ -243,9 +248,9 @@ def transfer_to_dialog(self, dlg):
         dlg.html.boardRotationSlider.Value = self.board_rotation
         dlg.html.bomCheckboxesCtrl.Value = self.checkboxes
         dlg.html.bomDefaultView.Selection = self.bom_view_choices.index(
-                self.bom_view)
+            self.bom_view)
         dlg.html.layerDefaultView.Selection = self.layer_view_choices.index(
-                self.layer_view)
+            self.layer_view)
         dlg.html.compressionCheckbox.Value = self.compression
         dlg.html.openBrowserCheckbox.Value = self.open_browser
 
@@ -255,7 +260,7 @@ def transfer_to_dialog(self, dlg):
             dlg.general.bomDirPicker.Path = self.bom_dest_dir
         else:
             dlg.general.bomDirPicker.Path = os.path.join(
-                    self.netlist_initial_directory, self.bom_dest_dir)
+                self.netlist_initial_directory, self.bom_dest_dir)
         dlg.general.fileNameFormatTextControl.Value = self.bom_name_format
         dlg.general.componentSortOrderBox.SetItems(self.component_sort_order)
         dlg.general.blacklistBox.SetItems(self.component_blacklist)
@@ -264,28 +269,28 @@ def transfer_to_dialog(self, dlg):
         dlg.general.includeTracksCheckbox.Value = self.include_tracks
         dlg.general.includeNetsCheckbox.Value = self.include_nets
 
-        # Extra fields
-        dlg.extra.extraDataFilePicker.SetInitialDirectory(
-                self.netlist_initial_directory)
+        # Fields
+        dlg.fields.extraDataFilePicker.SetInitialDirectory(
+            self.netlist_initial_directory)
 
         def safe_set_checked_strings(clb, strings):
-            safe_strings = list(clb.GetStrings())
-            if safe_strings:
-                present_strings = [s for s in strings if s in safe_strings]
-                not_present_strings = [s for s in safe_strings if s not in strings]
+            current = list(clb.GetStrings())
+            if current:
+                present_strings = [s for s in strings if s in current]
+                not_present_strings = [s for s in current if s not in strings]
                 clb.Clear()
                 clb.InsertItems(present_strings + not_present_strings, 0)
                 clb.SetCheckedStrings(present_strings)
 
-        safe_set_checked_strings(dlg.extra.extraFieldsList, self.extra_fields)
-        dlg.extra.normalizeCaseCheckbox.Value = self.normalize_field_case
-        dlg.extra.boardVariantFieldBox.Value = self.board_variant_field
-        dlg.extra.OnBoardVariantFieldChange(None)
-        safe_set_checked_strings(dlg.extra.boardVariantWhitelist,
+        dlg.fields.SetCheckedFields(self.show_fields, self.group_fields)
+        dlg.fields.normalizeCaseCheckbox.Value = self.normalize_field_case
+        dlg.fields.boardVariantFieldBox.Value = self.board_variant_field
+        dlg.fields.OnBoardVariantFieldChange(None)
+        safe_set_checked_strings(dlg.fields.boardVariantWhitelist,
                                  self.board_variant_whitelist)
-        safe_set_checked_strings(dlg.extra.boardVariantBlacklist,
+        safe_set_checked_strings(dlg.fields.boardVariantBlacklist,
                                  self.board_variant_blacklist)
-        dlg.extra.dnpFieldBox.Value = self.dnp_field
+        dlg.fields.dnpFieldBox.Value = self.dnp_field
 
         dlg.finish_init()
 
@@ -350,24 +355,31 @@ def add_options(self, parser, file_name_format_hint):
         parser.add_argument('--blacklist',
                             default=','.join(self.component_blacklist),
                             help='List of comma separated blacklisted '
-                                 'components or prefixes with *. E.g. "X1,MH*"')
+                                 'components or prefixes with *. '
+                                 'E.g. "X1,MH*"')
         parser.add_argument('--no-blacklist-virtual', action='store_true',
                             help='Do not blacklist virtual components.')
         parser.add_argument('--blacklist-empty-val', action='store_true',
                             help='Blacklist components with empty value.')
 
-        # Extra fields section
+        # Fields section
         parser.add_argument('--netlist-file',
                             help='(Deprecated) Path to netlist or xml file.')
         parser.add_argument('--extra-data-file',
                             help='Path to netlist or xml file.')
         parser.add_argument('--extra-fields',
-                            default=self._join(self.extra_fields),
-                            help='Comma separated list of extra fields to '
-                                 'pull from netlist or xml file.')
+                            help='Passing --extra-fields "X,Y" is a shortcut '
+                                 'for --show-fields and --group-fields '
+                                 'with values "Value,Footprint,X,Y"')
+        parser.add_argument('--show-fields',
+                            default=self._join(self.show_fields),
+                            help='List of fields to show in the BOM.')
+        parser.add_argument('--group-fields',
+                            default=self._join(self.group_fields),
+                            help='Fields that components will be grouped by.')
         parser.add_argument('--normalize-field-case',
                             help='Normalize extra field name case. E.g. "MPN" '
-                                 'and "mpn" will be considered the same field.',
+                                 ', "mpn" will be considered the same field.',
                             action='store_true')
         parser.add_argument('--variant-field',
                             help='Name of the extra field that stores board '
@@ -380,8 +392,8 @@ def add_options(self, parser, file_name_format_hint):
                                  'exclude from the BOM.')
         parser.add_argument('--dnp-field', default=self.dnp_field,
                             help='Name of the extra field that indicates '
-                                 'do not populate status. Components with this '
-                                 'field not empty will be blacklisted.')
+                                 'do not populate status. Components with '
+                                 'this field not empty will be excluded.')
 
     def set_from_args(self, args):
         # type: (argparse.Namespace) -> None
@@ -411,9 +423,15 @@ def set_from_args(self, args):
         self.include_tracks = args.include_tracks
         self.include_nets = args.include_nets
 
-        # Extra
+        # Fields
         self.extra_data_file = args.extra_data_file or args.netlist_file
-        self.extra_fields = self._split(args.extra_fields)
+        if args.extra_fields is not None:
+            self.show_fields = self.default_show_group_fields + \
+                self._split(args.extra_fields)
+            self.group_fields = self.show_fields
+        else:
+            self.show_fields = self._split(args.show_fields)
+            self.group_fields = self._split(args.group_fields)
         self.normalize_field_case = args.normalize_field_case
         self.board_variant_field = args.variant_field
         self.board_variant_whitelist = self._split(args.variants_whitelist)
@@ -423,7 +441,5 @@ def set_from_args(self, args):
     def get_html_config(self):
         import json
         d = {f: getattr(self, f) for f in self.html_config_fields}
-        # Temporary until currently hardcoded columns are made configurable
-        d["fields"] = ["References"] + self.extra_fields + \
-            ["Value", "Footprint", "Quantity"]
+        d["fields"] = self.show_fields
         return json.dumps(d)