Centralize code for creating new code generation backends

Makefile

@@ -86,16 +86,16 @@ TEST_CMD=-m unittest discover -s test/ -p *_test.py -v
 GEN_FILES=test/*codegen*.py
 
 test_symengine:
-	$(TEST_ENV) SYMFORCE_BACKEND=symengine $(PYTHON) $(TEST_CMD)
+	$(TEST_ENV) SYMFORCE_SYMBOLIC_API=symengine $(PYTHON) $(TEST_CMD)
 
 test_sympy:
-	$(TEST_ENV) SYMFORCE_BACKEND=sympy $(PYTHON) $(TEST_CMD)
+	$(TEST_ENV) SYMFORCE_SYMBOLIC_API=sympy $(PYTHON) $(TEST_CMD)
 
 test: test_symengine test_sympy
 
 # Generic target to run a SymEngine codegen test with --update
 update_%:
-	$(TEST_ENV) SYMFORCE_BACKEND=symengine $(PYTHON) test/$*.py --update
+	$(TEST_ENV) SYMFORCE_SYMBOLIC_API=symengine $(PYTHON) test/$*.py --update
 
 # All SymForce codegen tests, formatted as update_my_codegen_test targets
 GEN_FILES_UPDATE_TARGETS=$(shell \
@@ -106,7 +106,7 @@ test_update: $(GEN_FILES_UPDATE_TARGETS)
 
 # Generic target to run a SymPy codegen test with --update --run_slow_tests
 sympy_update_%:
-	$(TEST_ENV) SYMFORCE_BACKEND=sympy $(PYTHON) test/$*.py --update --run_slow_tests
+	$(TEST_ENV) SYMFORCE_SYMBOLIC_API=sympy $(PYTHON) test/$*.py --update --run_slow_tests
 
 # All SymForce codegen tests, formatted as sympy_update_my_codegen_test targets
 GEN_FILES_SYMPY_UPDATE_TARGETS=$(shell \

docs/development.rst

@@ -95,11 +95,11 @@ Much of the core functionality of SymForce is in generating code using the `Jinj
 For example template files, see ``symforce/codegen/cpp_templates``.
 
 *************************************************
-Symbolic Backends
+Symbolic API
 *************************************************
-SymForce uses the `SymPy <https://www.sympy.org/en/index.html>`_ API, but supports two backend implementations of it. The SymPy backend is pure Python, whereas the `SymEngine <https://github.com/symengine/symengine>`_ backend is wrapped C++. It can be 100-200 times faster for many operations, but is less fully featured and requires a C++ build.
+SymForce uses the `SymPy <https://www.sympy.org/en/index.html>`_ API, but supports two implementations of it. The SymPy implementation is pure Python, whereas the `SymEngine <https://github.com/symengine/symengine>`_ implementation is wrapped C++. It can be 100-200 times faster for many operations, but is less fully featured and requires a C++ build.
 
-To set the backend, you can either use :func:`symforce.set_backend()` before any other imports, or use the ``SYMFORCE_BACKEND`` environment variable with the options ``sympy`` or ``symengine``. By default SymEngine will be used if found, otherwise SymPy.
+To set the symbolic API, you can either use :func:`symforce.set_symbolic_api()` before any other imports, or use the ``SYMFORCE_SYMBOLIC_API`` environment variable with the options ``sympy`` or ``symengine``. By default SymEngine will be used if found, otherwise SymPy.
 
 *************************************************
 Building wheels

notebooks/storage_D_tangent.ipynb

@@ -8,7 +8,7 @@
    "source": [
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce import geo\n",

notebooks/symbolic_computation_speedups.ipynb

@@ -23,7 +23,7 @@
     "# Setup\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "\n",
     "from symforce import geo\n",
     "from symforce import sympy as sm\n",

notebooks/tangent_D_storage.ipynb

@@ -8,7 +8,7 @@
    "source": [
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce import geo\n",

notebooks/tutorials/cameras_tutorial.ipynb

@@ -23,7 +23,7 @@
     "# Setup\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce.notebook_util import display\n",

notebooks/tutorials/codegen_tutorial.ipynb

@@ -36,7 +36,7 @@
     "import os\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"symengine\")\n",
+    "symforce.set_symbolic_api(\"symengine\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce import codegen\n",

notebooks/tutorials/geometry_tutorial.ipynb

@@ -25,7 +25,7 @@
     "# Setup\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce.notebook_util import display\n",

notebooks/tutorials/ops_tutorial.ipynb

@@ -27,7 +27,7 @@
     "# Setup\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce.notebook_util import display\n",

notebooks/tutorials/sympy_tutorial.ipynb

@@ -24,7 +24,7 @@
     "# Configuration (optional)\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "from symforce.notebook_util import display, print_expression_tree"
    ]
@@ -33,7 +33,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Always import the SymPy API through SymForce, because symforce can switch out the backend implementation of the API and adds a few minor but important augmentations. Let's define some algebraic symbols:"
+    "Always import the SymPy API through SymForce, because symforce can switch out the symbolic implementation of the API and adds a few minor but important augmentations. Let's define some algebraic symbols:"
    ]
   },
   {

notebooks/tutorials/values_tutorial.ipynb

@@ -25,7 +25,7 @@
     "# Setup\n",
     "import symforce\n",
     "\n",
-    "symforce.set_backend(\"sympy\")\n",
+    "symforce.set_symbolic_api(\"sympy\")\n",
     "symforce.set_log_level(\"warning\")\n",
     "\n",
     "from symforce import geo\n",

symforce/__init__.py

@@ -61,18 +61,18 @@ def set_log_level(log_level: str) -> None:
 set_log_level(os.environ.get("SYMFORCE_LOGLEVEL", "WARNING"))
 
 # -------------------------------------------------------------------------------------------------
-# Symbolic backend configuration
+# Symbolic API configuration
 # -------------------------------------------------------------------------------------------------
 sympy: T.Any = None
 
 from . import initialization
 
 
-def _set_backend(sympy_module: ModuleType) -> None:
+def _set_symbolic_api(sympy_module: ModuleType) -> None:
     # Make symforce-specific modifications to the sympy API
     initialization.modify_symbolic_api(sympy_module)
 
-    # Set this as the default backend
+    # Set this as the default symbolic API
     global sympy  # pylint: disable=global-statement
     sympy = sympy_module
 
@@ -143,13 +143,13 @@ def _use_symengine() -> None:
         logger.critical("Commanded to use symengine, but failed to import.")
         raise
 
-    _set_backend(symengine)
+    _set_symbolic_api(symengine)
 
 
 def _use_sympy() -> None:
     import sympy as sympy_py
 
-    _set_backend(sympy_py)
+    _set_symbolic_api(sympy_py)
     sympy_py.init_printing()
 
 
@@ -171,54 +171,54 @@ def set_symengine_eval_on_sympify(eval_on_sympy: bool = True) -> None:
         logger.debug("set_symengine_fast_sympify has no effect when not using symengine")
 
 
-def set_backend(backend: str) -> None:
+def set_symbolic_api(name: str) -> None:
     """
-    Set the symbolic backend for symforce. The sympy backend is the default and pure python,
-    whereas the symengine backend is C++ and requires building the symengine library. It can
+    Set the symbolic API for symforce. The sympy API is the default and pure python,
+    whereas the symengine API is C++ and requires building the symengine library. It can
     be 100-200 times faster for many operations, but is less fully featured.
 
     The default is symengine if available else sympy, but can be set by one of:
 
-        1) The SYMFORCE_BACKEND environment variable
+        1) The SYMFORCE_SYMBOLIC_API environment variable
         2) Calling this function before any other symforce imports
 
     Args:
-        backend (str): {sympy, symengine}
+        name (str): {sympy, symengine}
     """
     # TODO(hayk): Could do a better job of checking what's imported and raising an error
     # if this isn't the first thing imported/called from symforce.
 
-    if sympy and backend == sympy.__package__:
-        logger.debug(f'already on backend "{backend}"')
+    if sympy and name == sympy.__package__:
+        logger.debug(f'already on symbolic API "{name}"')
         return
     else:
-        logger.debug(f'backend: "{backend}"')
+        logger.debug(f'symbolic API: "{name}"')
 
-    if backend == "sympy":
+    if name == "sympy":
         _use_sympy()
-    elif backend == "symengine":
+    elif name == "symengine":
         _use_symengine()
     else:
-        raise NotImplementedError(f'Unknown backend: "{backend}"')
+        raise NotImplementedError(f'Unknown symbolic API: "{name}"')
 
 
 # Set default to symengine if available, else sympy
-if "SYMFORCE_BACKEND" in os.environ:
-    set_backend(os.environ["SYMFORCE_BACKEND"])
+if "SYMFORCE_SYMBOLIC_API" in os.environ:
+    set_symbolic_api(os.environ["SYMFORCE_SYMBOLIC_API"])
 else:
     try:
         symengine = _import_symengine_from_build()
 
-        logger.debug("No SYMFORCE_BACKEND set, found and using symengine.")
-        set_backend("symengine")
+        logger.debug("No SYMFORCE_SYMBOLIC_API set, found and using symengine.")
+        set_symbolic_api("symengine")
     except ImportError:
-        logger.debug("No SYMFORCE_BACKEND set, no symengine found, using sympy.")
-        set_backend("sympy")
+        logger.debug("No SYMFORCE_SYMBOLIC_API set, no symengine found, using sympy.")
+        set_symbolic_api("sympy")
 
 
-def get_backend() -> str:
+def get_symbolic_api() -> str:
     """
-    Return the current backend as a string.
+    Return the current symbolic API as a string.
 
     Returns:
         str:

symforce/codegen/__init__.py

@@ -7,5 +7,8 @@
 Package for executable code generation from symbolic expressions.
 """
 
-from .codegen_config import CodegenConfig, CppConfig, PythonConfig
 from .codegen import Codegen, LinearizationMode, GeneratedPaths
+from .codegen_config import CodegenConfig
+
+from .backends.cpp.cpp_config import CppConfig
+from .backends.python.python_config import PythonConfig

symforce/codegen/backends/README.md

@@ -0,0 +1,14 @@
+# Code Generation Backends
+
+SymForce takes symbolic functions and generates runtime functions for multiple target backends. It is quite straightforward to add new backends. Before you do this, you should be familiar with [SymPy printing](https://docs.sympy.org/latest/modules/printing.html) for getting code from symbolic expressions, and with [Jinja templating](https://realpython.com/primer-on-jinja-templating/) for rendering output files.
+
+The minimal steps to support a new backend are:
+
+ 1. Choose a name for your backend (for example 'julia') and create a corresponding package in `symforce/codegen/backends`.
+ 2. Implement a subclass of `sympy.CodePrinter` that emits backend math code while traversing symbolic expressions. Sometimes SymPy already contains the backend and the best pattern is to inherit from it and customize as needed. The best way to do this is by looking at existing backends as examples.
+ 3. Implement a subclass of `symforce.codegen.codegen_config.CodegenConfig`. This is the spec that users pass to the `Codegen` object to use your backend. Again, see existing examples. Optionally import your config in `symforce/codegen/__init__.py`.
+ 4. Create a `templates` directory containing jinja templates that are used to generate the actual output files. They specify the high level structure and APIs around the math code. Your codegen config has a `templates_to_render` method that should match your templates. A typical start is just one function template.
+ 5. Add your backend's extensions to `FileType` in `symforce/codegen/template_util.py`, filling out relevant methods there.
+ 6. Add tests to `test/symforce_codegen_test.py`.
+
+This will result in being able to generate functions for your backend that deal with scalars and arrays, but the `sym` geometry and camera classes. To implement those, follow the examples in `geo_package_codegen` and `cam_package_codegen`.

symforce/codegen/printers/cpp_code_printer.py → symforce/codegen/backends/cpp/cpp_code_printer.py

@@ -3,14 +3,12 @@
 # This source code is under the Apache 2.0 license found in the LICENSE file.
 # ----------------------------------------------------------------------------
 
+import sympy
 from sympy.printing.c import get_math_macros
 from sympy.printing.cxx import CXX11CodePrinter
 
 from symforce import typing as T
 
-# Everything in this file is SymPy, not SymEngine (even when SymForce is on the SymEngine backend)
-import sympy
-
 
 class CppCodePrinter(CXX11CodePrinter):
     """

symforce/codegen/backends/cpp/cpp_config.py

@@ -0,0 +1,74 @@
+# ----------------------------------------------------------------------------
+# SymForce - Copyright 2022, Skydio, Inc.
+# This source code is under the Apache 2.0 license found in the LICENSE file.
+# ----------------------------------------------------------------------------
+from dataclasses import dataclass
+from pathlib import Path
+from sympy.printing.codeprinter import CodePrinter
+
+from symforce import typing as T
+from symforce.codegen.codegen_config import CodegenConfig
+
+CURRENT_DIR = Path(__file__).parent
+
+
+@dataclass
+class CppConfig(CodegenConfig):
+    """
+    Code generation config for the C++ backend.
+
+    Args:
+        doc_comment_line_prefix: Prefix applied to each line in a docstring
+        line_length: Maximum allowed line length in docstrings; used for formatting docstrings.
+        use_eigen_types: Use eigen_lcm types for vectors instead of lists
+        autoformat: Run a code formatter on the generated code
+        cse_optimizations: Optimizations argument to pass to sm.cse
+        support_complex: Generate code that can work with std::complex or with regular float types
+        force_no_inline: Mark generated functions as `__attribute__((noinline))`
+        zero_initialization_sparsity_threshold: Threshold between 0 and 1 for the sparsity below
+                                                which we'll initialize an output matrix to 0, so we
+                                                don't have to generate a line to set each zero
+                                                element to 0 individually
+        explicit_template_instantiation_types: Explicity instantiates templated functions in a `.cc`
+            file for each given type. This allows the generated function to be compiled in its own
+            translation unit. Useful for large functions which take a long time to compile.
+    """
+
+    doc_comment_line_prefix: str = " * "
+    line_length: int = 100
+    use_eigen_types: bool = True
+    support_complex: bool = False
+    force_no_inline: bool = False
+    zero_initialization_sparsity_threshold: float = 0.5
+    explicit_template_instantiation_types: T.Optional[T.Sequence[str]] = None
+
+    @classmethod
+    def backend_name(cls) -> str:
+        return "cpp"
+
+    @classmethod
+    def template_dir(cls) -> Path:
+        return CURRENT_DIR / "templates"
+
+    def templates_to_render(self, generated_file_name: str) -> T.List[T.Tuple[str, str]]:
+        # Generate code into a header (since the code is templated)
+        templates = [("function/FUNCTION.h.jinja", f"{generated_file_name}.h")]
+
+        # Generate a cc file only if we need explicit instantiation.
+        if self.explicit_template_instantiation_types is not None:
+            templates.append(("function/FUNCTION.cc.jinja", f"{generated_file_name}.cc"))
+
+        return templates
+
+    def printer(self) -> CodePrinter:
+        # NOTE(hayk): Is there any benefit to this being lazy?
+        from symforce.codegen.backends.cpp import cpp_code_printer
+
+        if self.support_complex:
+            return cpp_code_printer.ComplexCppCodePrinter()
+        else:
+            return cpp_code_printer.CppCodePrinter()
+
+    @staticmethod
+    def format_data_accessor(prefix: str, index: int) -> str:
+        return f"{prefix}.Data()[{index}]"

symforce/codegen/printers/python_code_printer.py → symforce/codegen/backends/python/python_code_printer.py

@@ -3,10 +3,8 @@
 # This source code is under the Apache 2.0 license found in the LICENSE file.
 # ----------------------------------------------------------------------------
 
-from sympy.printing.pycode import PythonCodePrinter as _PythonCodePrinter
-
-# Everything in this file is SymPy, not SymEngine (even when SymForce is on the SymEngine backend)
 import sympy
+from sympy.printing.pycode import PythonCodePrinter as _PythonCodePrinter
 
 
 class PythonCodePrinter(_PythonCodePrinter):