add FaustProcessor compileFromBitcode for pickling

DBraun · DBraun · commit b891902bae3e · 2026-02-06T22:37:29.000-05:00
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -8,23 +8,22 @@ DawDreamer is a Digital Audio Workstation (DAW) framework for Python enabling pr
 
 ---
 
-## Quick Install (Library Already Built)
+## Quick Install
 
-If `dawdreamer/dawdreamer.so` exists:
+`setup.py` handles the full flow: building C++ (if needed), copying the `.so`, and installing. It detects source changes and skips recompilation when up-to-date.
 
 ```bash
-# Minimal output (recommended for LLMs)
-python3 setup.py develop --quiet && echo "✓ Installed" || echo "✗ Failed"
+pip install -e .
 
 # Verify
-python3 -c "import dawdreamer; dawdreamer.RenderEngine(44100, 512)" && echo "✓ Working"
+python3 -c "import dawdreamer; dawdreamer.RenderEngine(44100, 512)"
 ```
 
-**WSL2 Note**: Takes 1-2 minutes (normal - processing Faust libraries).
+**WSL2 Note**: First install takes several minutes (C++ build + processing Faust libraries). Subsequent installs skip the C++ build if sources haven't changed.
 
 ---
 
-## Quick Build (Library Doesn't Exist)
+## First-Time Setup (Prerequisites)
 
 ### Linux
 
@@ -34,22 +33,12 @@ sudo apt-get install -yq build-essential clang cmake git python3-dev \
   libboost-all-dev libfreetype6-dev libncurses-dev \
   libx11-dev libxrandr-dev libasound2-dev libxcomposite-dev
 
-# Setup and build
+# Submodules and Faust
 git submodule update --init --recursive
 cd thirdparty/libfaust && python3 download_libfaust.py && cd ../..
 
-export PYTHONLIBPATH=/usr/lib/python3.12
-export PYTHONINCLUDEPATH=/usr/include/python3.12
-
-cd thirdparty/libsamplerate && \
-  cmake -DCMAKE_BUILD_TYPE=Release -Bbuild_release && \
-  cmake --build build_release && cd ../..
-
-cd Builds/LinuxMakefile && \
-  make CONFIG=Release CXXFLAGS="-I$PYTHONINCLUDEPATH" LDFLAGS="-L$PYTHONLIBPATH" && \
-  cd ../..
-
-python3 setup.py develop --quiet && echo "✓ Done"
+# Build and install (builds C++, copies .so, installs package)
+pip install -e .
 ```
 
 ### macOS
@@ -74,7 +63,8 @@ python setup.py develop
 
 | Issue | Solution |
 |-------|----------|
-| Slow install (WSL2) | Normal - wait 1-2 min |
+| Slow first install (WSL2) | Normal - C++ build + Faust libs |
+| C++ changes not detected (WSL2) | `rm -rf Builds/LinuxMakefile/build` then reinstall |
 | Missing libtinfo (Linux) | `sudo apt-get install libncurses-dev` |
 | Import fails | Check Python 3.11-3.14: `python3 --version` |
 | Arch mismatch (macOS) | Set `ARCHS=arm64` or `x86_64` |
diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -166,19 +166,27 @@ cmake --build build_release --config Release
 cd ../..
 ```
 
-#### Build DawDreamer:
+#### Build, Copy, and Install:
+
+`setup.py` handles the full flow: building C++ via `make`, copying the `.so`, and installing the Python package. If the C++ sources haven't changed since the last build, it skips recompilation.
+
+```bash
+pip install -e .
+```
+
+To build manually (e.g., for debugging):
 ```bash
 cd Builds/LinuxMakefile
 make CONFIG=Release CXXFLAGS="-I$PYTHONINCLUDEPATH" LDFLAGS="-L$PYTHONLIBPATH"
 cd ../..
 ```
 
-Output: `dawdreamer/dawdreamer.so`
+Build output: `Builds/LinuxMakefile/build/libdawdreamer.so`
+`setup.py` copies this to `dawdreamer/dawdreamer.so` automatically.
 
-#### Install Python Package:
+**WSL2 Note**: The `make` build system may not detect source changes across the Windows/Linux filesystem boundary due to timestamp caching. If changes aren't picked up, delete the build directory first:
 ```bash
-python3 setup.py develop
-# Or for wheel: python3 -m build --wheel
+rm -rf Builds/LinuxMakefile/build
 ```
 
 ### macOS
@@ -262,11 +270,12 @@ See [Issue #82](https://github.com/DBraun/DawDreamer/issues/82#issuecomment-1097
 ### Modifying C++ Source
 
 1. Edit source files in `Source/`
-2. Open `DawDreamer.jucer` with [JUCE Projucer](https://juce.com/get-juce)
+2. If adding/removing files, open `DawDreamer.jucer` with [JUCE Projucer](https://juce.com/get-juce)
    - Projucer regenerates platform-specific build files
    - **DO NOT** edit .xcodeproj/.sln files directly
-3. Rebuild via platform-specific build system
-4. Reinstall: `python3 setup.py develop` (fast, just updates the .so)
+3. Rebuild and reinstall: `pip install -e .`
+   - Detects source changes, rebuilds C++, copies `.so`, installs package
+   - On WSL2, if changes aren't detected: `rm -rf Builds/LinuxMakefile/build` first
 
 **Adding a New Processor:**
 1. Create `Source/MyProcessor.h` (header-only or with .cpp)
@@ -414,6 +423,10 @@ python -m pip install dist/dawdreamer-*.whl
 - Normal on WSL2 due to cross-filesystem I/O (1-2 minutes)
 - Processing ~200+ Faust library files
 
+**C++ changes not detected (WSL2)**
+- WSL2 cross-filesystem timestamps may be stale, causing `make` to skip rebuilds
+- Fix: `rm -rf Builds/LinuxMakefile/build` then run `pip install -e .` again
+
 ### Runtime Issues
 
 **ImportError: cannot import dawdreamer**
diff --git a/Source/FaustProcessor.cpp b/Source/FaustProcessor.cpp
@@ -543,62 +543,76 @@ bool FaustProcessor::compile()
                                  pathToFaustLibraries);
     }
 
-    //// print where faustlib is looking for stdfaust.lib and the other lib files.
-    // auto pathnames = m_factory->getIncludePathnames();
-    // std::cout << "pathnames:\n" << std::endl;
-    // for (auto name : pathnames) {
-    //    std::cout << name << "\n" << std::endl;
-    //}
-    // std::cout << "library list:\n" << std::endl;
-    // auto librarylist = m_factory->getLibraryList();
-    // for (auto name : librarylist) {
-    //    std::cout << name << "\n" << std::endl;
-    //}
+    return initFromFactory();
+}
+
+bool FaustProcessor::compileFromBitcode(const std::string& bitcode)
+{
+    m_compileState = kNotCompiled;
+    clear();
+
+    std::string error_msg;
+    m_factory = readDSPFactoryFromBitcode(bitcode, getTarget(), error_msg, m_llvmOptLevel);
+
+    if (!m_factory)
+    {
+        throw std::runtime_error("FaustProcessor::compileFromBitcode(): Failed to restore factory "
+                                 "from bitcode: " +
+                                 error_msg);
+    }
+
+    return initFromFactory();
+}
+
+bool FaustProcessor::initFromFactory()
+{
+    // Create DSP instance from the factory (either mono or poly).
+    // Called by compile() after creating the factory from source code,
+    // and by setPickleState() after restoring the factory from bitcode.
+    bool is_polyphonic = m_nvoices > 0;
 
     try
     {
         if (is_polyphonic)
         {
-            // (false, true) works
             m_dsp_poly =
                 m_poly_factory->createPolyDSPInstance(m_nvoices, m_dynamicVoices, m_groupVoices);
             if (!m_dsp_poly)
             {
                 clear();
                 throw std::runtime_error(
-                    "FaustProcessor::compile(): Cannot create Poly DSP instance.");
+                    "FaustProcessor::initFromFactory(): Cannot create Poly DSP instance.");
             }
-            // m_dsp_poly->setReleaseLength(m_releaseLengthSec); // Removed in Faust 2.81.10
         }
         else
         {
-            // create DSP instance
             m_dsp = m_factory->createDSPInstance();
             if (!m_dsp)
             {
                 clear();
-                throw std::runtime_error("FaustProcessor::compile(): Cannot create DSP instance.");
+                throw std::runtime_error(
+                    "FaustProcessor::initFromFactory(): Cannot create DSP instance.");
             }
         }
     }
     catch (const std::exception& e)
     {
         clear();
         throw std::runtime_error(
-            std::string("FaustProcessor::compile(): Exception creating DSP instance: ") + e.what());
+            std::string("FaustProcessor::initFromFactory(): Exception creating DSP instance: ") +
+            e.what());
     }
     catch (...)
     {
         clear();
         throw std::runtime_error(
-            "FaustProcessor::compile(): Unknown exception creating DSP instance");
+            "FaustProcessor::initFromFactory(): Unknown exception creating DSP instance");
     }
 
     dsp* theDsp = is_polyphonic ? m_dsp_poly : m_dsp;
 
     try
     {
-        // get channels
         int inputs = theDsp->getNumInputs();
         int outputs = theDsp->getNumOutputs();
 
@@ -607,7 +621,6 @@ bool FaustProcessor::compile()
 
         setMainBusInputsAndOutputs(inputs, outputs);
 
-        // make new UI
         if (is_polyphonic)
         {
             m_midi_handler = rt_midi("my_midi");
@@ -625,7 +638,6 @@ bool FaustProcessor::compile()
         m_soundUI = new MySoundUI(&m_SoundfileMap, m_faustAssetsPaths, sr);
         theDsp->buildUserInterface(m_soundUI);
 
-        // init
         theDsp->init(sr);
 
         createParameterLayout();
@@ -636,14 +648,15 @@ bool FaustProcessor::compile()
     {
         clear();
         throw std::runtime_error(
-            std::string("FaustProcessor::compile(): Exception during DSP initialization: ") +
+            std::string(
+                "FaustProcessor::initFromFactory(): Exception during DSP initialization: ") +
             e.what());
     }
     catch (...)
     {
         clear();
         throw std::runtime_error(
-            "FaustProcessor::compile(): Unknown exception during DSP initialization");
+            "FaustProcessor::initFromFactory(): Unknown exception during DSP initialization");
     }
 
     return true;
diff --git a/Source/FaustProcessor.h b/Source/FaustProcessor.h
@@ -198,6 +198,8 @@ class FaustProcessor : public ProcessorBase
     // faust stuff
     void clear();
     bool compile();
+    bool compileFromBitcode(const std::string& bitcode);
+    bool initFromFactory();
     bool setDSPString(const std::string& code);
     bool setDSPFile(const std::string& path);
     bool setParamWithIndex(const int index, float p);
@@ -309,6 +311,14 @@ class FaustProcessor : public ProcessorBase
         state["midi_qn"] = MidiSerialization::serializeMidiBuffer(myMidiBufferQN);
         state["midi_sec"] = MidiSerialization::serializeMidiBuffer(myMidiBufferSec);
 
+        // Serialize compiled LLVM bitcode to avoid recompilation on unpickle.
+        // Only supported for mono (non-polyphonic) factories.
+        // Polyphonic factories fall back to recompilation from source.
+        if (m_factory)
+        {
+            state["bitcode"] = writeDSPFactoryToBitcode(m_factory);
+        }
+
         return state;
     }
 
@@ -368,8 +378,23 @@ class FaustProcessor : public ProcessorBase
         m_llvmOptLevel = nb::cast<int>(state["llvm_opt_level"]);
         m_releaseLengthSec = nb::cast<double>(state["release_length"]);
 
-        // Compile the Faust code
-        if (!m_code.empty())
+        // Restore from compiled LLVM bitcode if available (fast path),
+        // otherwise fall back to full recompilation from source (slow path).
+        if (state.contains("bitcode"))
+        {
+            std::string bitcode = nb::cast<std::string>(state["bitcode"]);
+            std::string error_msg;
+            m_factory = readDSPFactoryFromBitcode(bitcode, getTarget(), error_msg, m_llvmOptLevel);
+            if (!m_factory)
+            {
+                throw std::runtime_error(
+                    "FaustProcessor::setPickleState(): Failed to restore factory "
+                    "from bitcode: " +
+                    error_msg);
+            }
+            initFromFactory();
+        }
+        else if (!m_code.empty())
         {
             compile();
         }
@@ -583,6 +608,10 @@ inline void create_bindings_for_faust_processor(nb::module_& m)
         .def("compile", &FaustProcessor::compile,
              "Compile the FAUST object. You must have already set a dsp file "
              "path or dsp string.")
+        .def("compile_from_bitcode", &FaustProcessor::compileFromBitcode, arg("bitcode"),
+             "Restore a compiled FAUST object from LLVM bitcode, avoiding "
+             "recompilation from source. Bitcode can be obtained from the "
+             "processor's pickle state dict.")
         .def_prop_rw("auto_import", &FaustProcessor::getAutoImport, &FaustProcessor::setAutoImport,
                      "The auto import string. Default is `import(\"stdfaust.lib\");`")
         .def("get_parameters_description", &FaustProcessor::getPluginParametersDescription,
diff --git a/Source/RenderEngine.h b/Source/RenderEngine.h
@@ -122,10 +122,16 @@ class RenderEngine : public AudioPlayHead
             state["bpm_automation"] = nb::ndarray<nb::numpy, float>(array_data, 1, shape, capsule);
         }
 
-        // Save processors with their types and states
+        // Save all registered processors (not just connected ones)
         nb::list processors_list;
-        for (auto* proc : m_connectedProcessors)
+        for (const auto& [proc_name, nodeID] : m_UniqueNameToNodeID)
         {
+            auto node = m_mainProcessorGraph->getNodeForId(nodeID);
+            if (!node)
+                continue;
+            auto* proc = dynamic_cast<ProcessorBase*>(node->getProcessor());
+            if (!proc)
+                continue;
             nb::dict proc_dict;
 
             // Helper lambda to save automation for any processor
@@ -400,15 +406,25 @@ class RenderEngine : public AudioPlayHead
                     faust_proc->setLLVMOpt(nb::cast<int>(proc_state["llvm_opt_level"]));
                     faust_proc->setReleaseLength(nb::cast<double>(proc_state["release_length"]));
 
-                    // Compile the Faust code
-                    faust_proc->compile();
-
-                    // Restore soundfiles
+                    // Restore soundfiles BEFORE compilation because
+                    // setSoundfiles resets m_compileState.
                     if (proc_state.contains("soundfiles"))
                     {
                         faust_proc->setSoundfiles(nb::cast<nb::dict>(proc_state["soundfiles"]));
                     }
 
+                    // Restore from LLVM bitcode if available (fast path),
+                    // otherwise fall back to full recompilation from source.
+                    if (proc_state.contains("bitcode"))
+                    {
+                        faust_proc->compileFromBitcode(
+                            nb::cast<std::string>(proc_state["bitcode"]));
+                    }
+                    else
+                    {
+                        faust_proc->compile();
+                    }
+
                     // Save parameters for later restoration (after graph is loaded)
                     if (proc_state.contains("parameters"))
                     {
diff --git a/setup.py b/setup.py
