Restructure into subpackages (#352)

Instead of a bunch of top-level packages, moves things like `edg_core`
into `edg.core`. Also renames some subpackages for simplicity:
`electronics_abstract_parts` -> `abstract_parts`, `electronics_lib` ->
`parts`. Should not break compatibility for those using `from edg import
*`, but will break for those importing internals.

This structure should allow this to be used as a submodule and is
overall cleaner. The compiler will detect if PolymorphicBlocks exists in
the current path, and if so, uses the qualified path.

`examples` is still independent and is not part of the public API or
package structure.

Author: ducky64

compiler/src/main/scala/edg/compiler/Compiler.scala

@@ -116,7 +116,7 @@ class AssignNamer() {
 }
 
 object Compiler {
-  final val kExpectedProtoVersion = 4
+  final val kExpectedProtoVersion = 5
 }
 
 /** Compiler for a particular design, with an associated library to elaborate references from.

compiler/src/main/scala/edg/compiler/CompilerServerMain.scala

@@ -13,8 +13,8 @@ import java.io.{File, PrintWriter, StringWriter}
 
 // a PythonInterface that uses the on-event hooks to forward stderr and stdout
 // without this, the compiler can freeze on large stdout/stderr data, possibly because of queue sizing
-class ForwardingPythonInterface(serverFile: Option[File], pythonPaths: Seq[String])
-    extends PythonInterface(serverFile, pythonPaths) {
+class ForwardingPythonInterface(pythonPaths: Seq[String] = Seq())
+    extends PythonInterface(pythonPaths = pythonPaths) {
   def forwardProcessOutput(): Unit = {
     StreamUtils.forAvailable(processOutputStream) { data =>
       System.out.print(new String(data))
@@ -94,9 +94,7 @@ object CompilerServerMain {
   }
 
   def main(args: Array[String]): Unit = {
-    val hdlServerOption = PythonInterface.serverFileOption(None) // local relative path
-    hdlServerOption.foreach { serverFile => println(s"Using local $serverFile") }
-    val pyIf = new ForwardingPythonInterface(hdlServerOption, Seq(new File(".").getAbsolutePath))
+    val pyIf = new ForwardingPythonInterface()
     (pyIf.getProtoVersion() match {
       case Errorable.Success(pyVersion) if pyVersion == Compiler.kExpectedProtoVersion => None
       case Errorable.Success(pyMismatchVersion) => Some(pyMismatchVersion.toString)

compiler/src/main/scala/edg/compiler/PythonInterface.scala

@@ -20,8 +20,8 @@ object ProtobufStdioSubprocess {
 
 class ProtobufStdioSubprocess[RequestType <: scalapb.GeneratedMessage, ResponseType <: scalapb.GeneratedMessage](
     responseType: scalapb.GeneratedMessageCompanion[ResponseType],
-    pythonPaths: Seq[String],
-    args: Seq[String]
+    pythonPaths: Seq[String] = Seq(),
+    args: Seq[String] = Seq()
 ) {
   protected val process: Either[Process, Throwable] =
     try {
@@ -128,32 +128,22 @@ class ProtobufStdioSubprocess[RequestType <: scalapb.GeneratedMessage, ResponseT
   }
 }
 
-object PythonInterface {
-  private val kHdlServerFilePath = "edg_hdl_server/__main__.py"
-  // returns the HDL server Python script if it exists locally, otherwise returns None.
-  def serverFileOption(root: Option[File] = None): Option[File] = {
-    val hdlServerFile = root match {
-      case Some(root) => new File(root, kHdlServerFilePath)
-      case None => new File(kHdlServerFilePath)
-    }
-    if (hdlServerFile.exists()) {
-      Some(hdlServerFile)
-    } else {
-      None
-    }
-  }
-}
-
 /** An interface to the Python HDL elaborator, which reads in Python HDL code and (partially) compiles them down to IR.
   * The underlying Python HDL should not change while this is open. This will not reload updated Python HDL files.
   *
-  * If the serverFile is specified, run that; otherwise use "python -m edg_hdl_server" for the global package.
+  * This invokes "python -m edg.hdl_server", using either the local or global (pip) module as available.
   */
-class PythonInterface(serverFile: Option[File], pythonPaths: Seq[String], pythonInterpreter: String = "python") {
-  val command = serverFile match { // -u for unbuffered mode
-    case Some(serverFile) => Seq(pythonInterpreter, "-u", serverFile.getAbsolutePath)
-    case None => Seq(pythonInterpreter, "-u", "-m", "edg_hdl_server")
+class PythonInterface(interpreter: String = "python", pythonPaths: Seq[String] = Seq()) {
+  val submoduleSearchPaths = if (pythonPaths.nonEmpty) pythonPaths else Seq(".")
+  val isSubmoduled = submoduleSearchPaths.map { searchPath => // check if submoduled, if so prepend the submodule name
+    new File(new File(searchPath), "PolymorphicBlocks/edg/hdl_server/__init__.py").exists()
+  }.exists(identity)
+  private val packageName = if (isSubmoduled) {
+    "PolymorphicBlocks.edg.hdl_server"
+  } else {
+    "edg.hdl_server"
   }
+  private val command = Seq(interpreter, "-u", "-m", packageName)
   protected val process = new ProtobufStdioSubprocess[edgrpc.HdlRequest, edgrpc.HdlResponse](
     edgrpc.HdlResponse,
     pythonPaths,

compiler/src/main/scala/edg/wir/LibraryConnectivityAnalysis.scala

@@ -7,11 +7,11 @@ import edgir.elem.elem
 object LibraryConnectivityAnalysis {
   // Shared library path to the base PortBridge class
   val portBridge =
-    ref.LibraryPath(target = Some(ref.LocalStep(step = ref.LocalStep.Step.Name("edg_core.PortBlocks.PortBridge"))))
+    ref.LibraryPath(target = Some(ref.LocalStep(step = ref.LocalStep.Step.Name("edg.core.PortBlocks.PortBridge"))))
   val portBridges = Set( // TODO this currently is a hack to avoid proper (multiple-level) subclass resolution
     portBridge,
     ref.LibraryPath(target =
-      Some(ref.LocalStep(step = ref.LocalStep.Step.Name("electronics_model.CircuitBlock.CircuitPortBridge")))
+      Some(ref.LocalStep(step = ref.LocalStep.Step.Name("edg.electronics_model.CircuitBlock.CircuitPortBridge")))
     )
   )
   val portBridgeOuterPort = "outer_port"

compiler/src/test/scala/edg/compiler/PythonInterfaceTest.scala

@@ -12,8 +12,8 @@ class PythonInterfaceTest extends AnyFlatSpec {
     val compiledDir = new File(getClass.getResource("").getPath)
     // above returns compiler/target/scala-2.xx/test-classes/edg/compiler, get the root repo dir
     val repoDir = compiledDir.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile.getParentFile
-    val pyIf = new PythonInterface(Some(new File(repoDir, "edg_hdl_server/__main__.py")), Seq(repoDir.getAbsolutePath))
-    pyIf.indexModule("edg_core").getClass should equal(classOf[Errorable.Success[Seq[LibraryPath]]])
+    val pyIf = new PythonInterface(pythonPaths = Seq(repoDir.getAbsolutePath))
+    pyIf.indexModule("edg.core").getClass should equal(classOf[Errorable.Success[Seq[LibraryPath]]])
     pyIf.shutdown() should equal(0)
   }
 }

edg/BoardCompiler.py

@@ -3,10 +3,11 @@
 from contextlib import suppress
 from typing import Type, Optional, Tuple
 
-from edg_core import Block, ScalaCompiler, CompiledDesign
-from electronics_model import NetlistBackend, SvgPcbBackend
-from electronics_model.RefdesRefinementPass import RefdesRefinementPass
-from electronics_model.BomBackend import GenerateBom
+from .core import Block, ScalaCompiler, CompiledDesign
+from .electronics_model.NetlistBackend import NetlistBackend  # imported separately b/c mypy confuses with the modules
+from .electronics_model.SvgPcbBackend import SvgPcbBackend
+from .electronics_model.RefdesRefinementPass import RefdesRefinementPass
+from .electronics_model.BomBackend import GenerateBom
 
 
 def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]]) -> CompiledDesign:
@@ -41,8 +42,8 @@ def compile_board(design: Type[Block], target_dir_name: Optional[Tuple[str, str]
       raw_file.write(compiled.design.SerializeToString())
 
   if compiled.errors:
-    import edg_core
-    raise edg_core.ScalaCompilerInterface.CompilerCheckError(f"error during compilation:\n{compiled.errors_str()}")
+    from . import core
+    raise core.ScalaCompilerInterface.CompilerCheckError(f"error during compilation:\n{compiled.errors_str()}")
 
   netlist_all = NetlistBackend().run(compiled)
   netlist_refdes = NetlistBackend().run(compiled, {'RefdesMode': 'refdes'})

edg/BoardTop.py

@@ -1,4 +1,4 @@
-from electronics_lib import *
+from .parts import *
 
 
 class BaseBoardTop(DesignTop):

edg/__init__.py

@@ -1,9 +1,9 @@
 # A metapackage for all the packages needed for electronics design with EDG
 
-from edg_core import *
-from electronics_model import *
-from electronics_abstract_parts import *
-from electronics_lib import *
+from .core import *
+from .electronics_model import *
+from .abstract_parts import *
+from .parts import *
 
 from .BoardTop import BoardTop, SimpleBoardTop, JlcBoardTop
 

…s_abstract_parts/AbstractAnalogSwitch.py …g/abstract_parts/AbstractAnalogSwitch.pyelectronics_abstract_parts/AbstractAnalogSwitch.py renamed to edg/abstract_parts/AbstractAnalogSwitch.py electronics_abstract_parts/AbstractAnalogSwitch.py renamed to edg/abstract_parts/AbstractAnalogSwitch.py

@@ -1,6 +1,6 @@
 from typing import List, cast, Optional, Dict
 
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import Interface
 
 

…ronics_abstract_parts/AbstractAntenna.py edg/abstract_parts/AbstractAntenna.pyelectronics_abstract_parts/AbstractAntenna.py renamed to edg/abstract_parts/AbstractAntenna.py electronics_abstract_parts/AbstractAntenna.py renamed to edg/abstract_parts/AbstractAntenna.py

@@ -1,4 +1,4 @@
-from electronics_model import *
+from ..electronics_model import *
 from .Categories import *
 from .PartsTable import PartsTableColumn, PartsTableRow
 from .PartsTablePart import PartsTableSelector