Compositional generators (#250)

Resolves #215. One of those rare PRs with more lines removed than added!

Changes generators to be compositional: instead of a monolithic single
`self.generator(fn, ...args...)`, this now invokes `self.generate()`
(structurally like `self.contents()`) which can `self.get(...)` on any
parameter. Parameters must be declared with `self.generator_param(...)`
ahead of time (in `__init__` and `contents`), but this can be called
multiple times.

Refactors libraries to use mixins and have better separation of
concerns. Major changes:
- Parts table based selectors now have individual select functionality
in each mixin class, e.g. the footprint class handles filtering by
footprint spec, and the smd-standard-package class handles filtering by
the minimum-smd-package
- Microcontrollers (application circuits) no longer duplicate code (in
each microcontroller class) for generating oscillators or stuffing
additional pins onto SWD ports, these have been refactored into common
mixins enabled by compositional generators.
- Deduplicate microcontroller pin mapping / pin allocation code with the
new `BaseIoControllerPinmapGenerator` base class.
- (Syntactic) refactoring of libraries to use the new generators tyle

This provides no functionality changes (aside from the generator API,
though a deprecation is kept), this is an implementation change only.
Future features will build on this base infrastructure.

Minor changes:
- Capacitors: unifies the single and multiple-parallel code
- The microcontroller refactor changes the connection and component
ordering, which changes the netlists. However, the netlists should still
be structurally the same (isomorphic).
- Add standard pinning to the (abstract) LED base class
- Refactor FT232 to have port arrays on ACBUS and ADBUS
- Add types to requirements.txt

---

Notes:
- Another generator API could be defining the generator params as
objects using `param = self.GeneratorParam(...wrapped...)` and
supporting `param.get()`, akin to `self.ArgParameter(...)`. This was
actually attempted first, but in practice this creates a duplicate
object with the underlying parameter so it's not clear what should be
used where. However, it is a bit more type-safe (since current we can
`self.get(...)` anything, whereas if you have a GeneratorParam object,
`.get()` is guaranted as long as it's within the generate phase).
- This causes a 10-15% performance penalty in the unit test suite.
Likely, the parts table selectors are a lot less efficient, because of
`super()` call overhead, and possibly other additional work done because
while the new structure enables code deduplication, it constrains
optimizations. In particular, the unified single and multiple-parallel
capacitors does more work in the single case now since parallel caps are
being computed for all cases, across the entire parts table.
- Some parts are have potentially bad performance
  - Resistor arrays repeat redundant calculations per-row.
- This supports internal-facing mixins (how blocks can share
implementations) per #104, while external mixins (eg, how to specify "I
want an abstract voltage regulator with a EN pin") would be a future
feature.
- Closes #166, since this uses `self.connect(...)` to create a node in a
way where it can't have type information. In the bigger picture, it
probably makes sense to separate the capabilities of a port and the
capabilities of a connection.

Author: ducky64

edg_core/Array.py

@@ -125,6 +125,10 @@ def __getitem__(self, item: str) -> VectorType:
     assert self._elts is not None, "no elts defined"
     return self._elts[item]
 
+  def items(self) -> ItemsView[str, VectorType]:
+    assert self._elts is not None, "no elts defined"
+    return self._elts.items()
+
   # unlike most other LibraryElement types, the names are stored in _elts and _allocates
   def _name_of_child(self, subelt: Any) -> str:
     from .HierarchyBlock import Block

edg_core/Generator.py

@@ -47,5 +47,6 @@ class BlockDefinitionError(EdslUserError):
   def __init__(self, block, exc: str, resolution: str = ''):
     super().__init__(f"invalid block definition for {type(block)}: {exc}", resolution)
 
+
 class ChainError(BlockDefinitionError):
   """Base error for bad elements in a chain connect"""

edg_core/HdlUserExceptions.py

@@ -17,10 +17,11 @@ class TestBlockSinkElasticArray(GeneratorBlock):
   def __init__(self) -> None:
     super().__init__()
     self.sinks = self.Port(Vector(TestPortSink()))
-    self.generator(self.generate, self.sinks.requested())
+    self.generator_param(self.sinks.requested())
 
-  def generate(self, requests: List[str]):
-    for request in requests:
+  def generate(self):
+    super().generate()
+    for request in self.get(self.sinks.requested()):
       self.sinks.append_elt(TestPortSink(), request)
 
 

edg_core/test_connect_array.py

@@ -15,9 +15,8 @@ def __init__(self) -> None:
     super().__init__()
     # Because this doesn't have dependency parameters, this is the top-level design
     self.float_param = self.Parameter(FloatExpr())
-    self.generator(self.float_gen)
 
-  def float_gen(self) -> None:
+  def generate(self) -> None:
     self.assign(self.float_param, 2.0)
 
 
@@ -32,10 +31,11 @@ class GeneratorDependency(GeneratorBlock):
   def __init__(self, float_preset: FloatLike) -> None:
     super().__init__()
     self.float_param = self.Parameter(FloatExpr())
-    self.generator(self.float_gen, float_preset)
+    self.float_preset = self.ArgParameter(float_preset)
+    self.generator_param(self.float_preset)
 
-  def float_gen(self, float_preset: float) -> None:
-    self.assign(self.float_param, float_preset * 2)
+  def generate(self) -> None:
+    self.assign(self.float_param, self.get(self.float_preset) * 2)
 
 
 class TestGeneratorMultiParameter(Block):
@@ -50,11 +50,13 @@ def __init__(self, float_preset1: FloatLike, float_preset2: FloatLike) -> None:
     super().__init__()
     self.float_param1 = self.Parameter(FloatExpr())
     self.float_param2 = self.Parameter(FloatExpr())
-    self.generator(self.float_gen, float_preset1, float_preset2)
+    self.float_preset1 = self.ArgParameter(float_preset1)
+    self.float_preset2 = self.ArgParameter(float_preset2)
+    self.generator_param(self.float_preset1, self.float_preset2)
 
-  def float_gen(self, float_preset1: float, float_preset2: float) -> None:
-    self.assign1 = self.assign(self.float_param1, float_preset1 * 3)
-    self.assign2 = self.assign(self.float_param2, float_preset2 + 7)
+  def generate(self) -> None:
+    self.assign1 = self.assign(self.float_param1, self.get(self.float_preset1) * 3)
+    self.assign2 = self.assign(self.float_param2, self.get(self.float_preset2) + 7)
 
 
 class TestGenerator(unittest.TestCase):
@@ -118,11 +120,12 @@ class GeneratorIsConnected(GeneratorBlock):
   def __init__(self) -> None:
     super().__init__()
     self.port = self.Port(TestPortSource(2.0), optional=True)
-    self.generator(self.generate_assign, self.port.is_connected())
+
+    self.generator_param(self.port.is_connected())
     self.connected = self.Parameter(BoolExpr())
 
-  def generate_assign(self, connected: bool) -> None:
-    if connected:
+  def generate(self) -> None:
+    if self.get(self.port.is_connected()):
       self.assign(self.connected, True)
     else:
       self.assign(self.connected, False)
@@ -146,7 +149,6 @@ class GeneratorInnerConnect(GeneratorBlock):
   def __init__(self) -> None:
     super().__init__()
     self.port = self.Port(TestPortSource(), optional=True)
-    self.generator(self.generate)
 
   def generate(self) -> None:
     self.inner = self.Block(TestBlockSource(4.5))
@@ -194,9 +196,8 @@ def __init__(self) -> None:
 class GeneratorFailure(GeneratorBlock):
   def __init__(self) -> None:
     super().__init__()
-    self.generator(self.errorfn)
 
-  def errorfn(self) -> None:
+  def generate(self) -> None:
     def helperfn() -> None:
       raise TestGeneratorException("test text")
     helperfn()

edg_core/test_generator.py

@@ -1,45 +1,29 @@
 import unittest
 
 from . import *
-from .ScalaCompilerInterface import ScalaCompiler
+from .HdlUserExceptions import BlockDefinitionError
 
 
 class BadGeneratorTestCase(unittest.TestCase):
   # These are internal classes to avoid this error case being auto-discovered in a library
 
   class InvalidMissingGeneratorBlock(GeneratorBlock):
-    def __init__(self) -> None:
-      super().__init__()
-      # this is missing a self.generator statement
+    """This doesn't implement generator()"""
 
   def test_missing_generator(self) -> None:
-    with self.assertRaises(AssertionError):
+    with self.assertRaises(BlockDefinitionError):
       self.InvalidMissingGeneratorBlock()._elaborated_def_to_proto()
 
 
-  class InvalidMultiGeneratorBlock(GeneratorBlock):
-    def __init__(self) -> None:
-      super().__init__()
-      self.generator(self.dummy_gen)
-      self.generator(self.dummy_gen)
-
-    def dummy_gen(self) -> None:
-      pass
-
-  def test_multi_generator(self) -> None:
-    with self.assertRaises(AssertionError):
-      self.InvalidMultiGeneratorBlock()._elaborated_def_to_proto()
-
-
   class InvalidNonArgGeneratorBlock(GeneratorBlock):
     def __init__(self) -> None:
       super().__init__()
       self.param = self.Parameter(FloatExpr())
-      self.generator(self.dummy_gen, self.param)
+      self.generator_param(self.param)
 
-    def dummy_gen(self, x: float) -> None:
-      pass
+    def generate(self) -> None:
+      super().generate()
 
   def test_non_arg_generator(self) -> None:
-    with self.assertRaises(AssertionError):
+    with self.assertRaises(BlockDefinitionError):
       self.InvalidNonArgGeneratorBlock()._elaborated_def_to_proto()

edg_core/test_generator_error.py

@@ -11,10 +11,10 @@ class GeneratorInnerBlock(GeneratorBlock):
   def __init__(self) -> None:
     super().__init__()
     self.ports = self.Port(Vector(TestPortSink()))
-    self.generator(self.generate, self.ports.requested())
+    self.generator_param(self.ports.requested())
 
-  def generate(self, elements: List[str]) -> None:
-    assert elements == ['0', 'named', '1'], f"bad elements {elements}"
+  def generate(self) -> None:
+    assert self.get(self.ports.requested()) == ['0', 'named', '1']
     self.ports.append_elt(TestPortSink((-1, 1)))
     self.ports.append_elt(TestPortSink((-5, 5)), 'named')
     self.ports.append_elt(TestPortSink((-2, 2)))
@@ -48,29 +48,26 @@ def test_initializer(self):
     self.assertEqual(pb.constraints[3].value, edgir.AssignLit(['ports', '1', 'range_param'], Range(-2, 2)))
 
 
-class GeneratorInnerBlockInvalid(GeneratorBlock):
+class InnerBlockInvalid(Block):
   def __init__(self) -> None:
     super().__init__()
     self.ports = self.Port(Vector(TestPortSink()))
-    self.generator(self.generate, self.ports.requested())
-
-  def generate(self, elements: List[str]) -> None:
     self.ports.append_elt(TestPortSink(), 'haha')
 
 
-class TestGeneratorElementsInvalid(Block):
+class TestElementsInvalid(Block):
   def __init__(self) -> None:
     super().__init__()
-    self.block = self.Block(GeneratorInnerBlockInvalid())
+    self.block = self.Block(InnerBlockInvalid())
 
     self.source0 = self.Block(TestBlockSource(1.0))
     self.connect(self.source0.port, self.block.ports.request('nope'))
 
 
-class TestGeneratorPortVectorInvalid(unittest.TestCase):
+class TestPortVectorInvalid(unittest.TestCase):
   def test_generator_error(self):
     with self.assertRaises(CompilerCheckError):
-      ScalaCompiler.compile(TestGeneratorElementsInvalid)
+      ScalaCompiler.compile(TestElementsInvalid)
 
 
 class GeneratorWrapperBlock(Block):
@@ -119,10 +116,11 @@ class GeneratorArrayParam(GeneratorBlock):
   def __init__(self, param: ArrayRangeLike) -> None:
     super().__init__()
     self.ports = self.Port(Vector(TestPortSink()))
-    self.generator(self.generate, param)
+    self.param = self.ArgParameter(param)
+    self.generator_param(self.param)
 
-  def generate(self, elements: List[Range]) -> None:
-    for elt in elements:
+  def generate(self) -> None:
+    for elt in self.get(self.param):
       created_port = self.ports.append_elt(TestPortSink(elt))  # any port
     self.require(created_port.link().sinks_range == Range(-2, 1))
 

edg_core/test_generator_portvector.py

@@ -40,11 +40,15 @@ class AnalogSwitchTree(AnalogSwitch, GeneratorBlock):
   @init_in_parent
   def __init__(self, switch_size: IntLike = 0):
     super().__init__()
-    self.generator(self.generate, self.inputs.requested(), switch_size)
+    self.switch_size = self.ArgParameter(switch_size)
+    self.generator_param(self.switch_size, self.inputs.requested())
 
-  def generate(self, elts: List[str], switch_size: int):
+  def generate(self):
     import math
+    super().generate()
 
+    switch_size = self.get(self.switch_size)
+    elts = self.get(self.inputs.requested())
     assert switch_size > 1, f"switch size {switch_size} must be greater than 1"
     assert len(elts) > 1, "passthrough AnalogSwitchTree not (yet?) supported"
     self.sw = ElementDict[AnalogSwitch]()
@@ -121,11 +125,12 @@ def __init__(self) -> None:
       impedance=self.device.analog_on_resistance + self.inputs.hull(lambda x: x.link().source_impedance)
     )))
 
-    self.generator(self.generate, self.inputs.requested())
+    self.generator_param(self.inputs.requested())
 
-  def generate(self, elts: List[str]):
+  def generate(self):
+    super().generate()
     self.inputs.defined()
-    for elt in elts:
+    for elt in self.get(self.inputs.requested()):
       self.connect(
         self.inputs.append_elt(AnalogSink().empty(), elt),
         self.device.inputs.request(elt).adapt_to(AnalogSink(
@@ -161,11 +166,12 @@ def __init__(self) -> None:
       impedance=self.device.analog_on_resistance + self.outputs.hull(lambda x: x.link().sink_impedance)
     )))
 
-    self.generator(self.generate, self.outputs.requested())
+    self.generator_param(self.outputs.requested())
 
-  def generate(self, elts: List[str]):
+  def generate(self):
+    super().generate()
     self.outputs.defined()
-    for elt in elts:
+    for elt in self.get(self.outputs.requested()):
       self.connect(
         self.outputs.append_elt(AnalogSource().empty(), elt),
         self.device.inputs.request(elt).adapt_to(AnalogSource(