[lldb][docs] Use sphinx instead of epydoc to generate LLDB's Python reference

Currently LLDB uses epydoc to generate the Python API reference for the website.
epydoc however is unmaintained since more than a decade and no longer works with
Python 3. Also whatever setup we had once for generating the documentation on
the website server no longer seems to work, so the current website documentation
has been stale since more than a year.

This patch replaces epydoc with sphinx and its automodapi plugin that can
generate Python API references. LLVM already uses sphinx for the rest of the
documentation, so this way we are more consistent with the rest of LLVM. The
only new dependency is the automodapi plugin for sphinx.

This patch effectively does the following things:
* Remove the epydoc code.
* Make a new dummy Python API page in our website that just calls the Sphinx
  command for generated the API documentation.
* Add a mock _lldb module that is only used when generating the Python API.
 This way we don't have to build all of LLDB to generate the API reference.

Some notes:
* The long list of skips is necessary due to boilerplate functions that SWIG
  is generating. Sadly automodapi is not really scriptable from what I can see,
  so we have to blacklist this stuff manually.
* The .gitignore change because automodapi wants a subfolder of our
  documentation directory to place generated documentation files there. The path
  is also what is used on the website, so we can't really workaround this
  (without copying the whole `docs` dir somewhere else when we build).
* We have to use environment variables to pass our build path to our sphinx
  configuration. Sphinx doesn't support passing variables onto that script.

Reviewed By: JDevlieghere

Differential Revision: https://reviews.llvm.org/D94489

Author: Teemperor

.gitignore

@@ -63,3 +63,5 @@ pythonenv*
 /clang/utils/analyzer/projects/*/PatchedSource
 /clang/utils/analyzer/projects/*/ScanBuildResults
 /clang/utils/analyzer/projects/*/RefScanBuildResults
+# automodapi puts generated documentation files here.
+/lldb/docs/python_api/

lldb/docs/CMakeLists.txt

@@ -15,60 +15,39 @@ if(DOXYGEN_FOUND)
   )
 endif()
 
-if (LLDB_ENABLE_PYTHON)
-  find_program(EPYDOC_EXECUTABLE NAMES epydoc epydoc.py)
-  if(EPYDOC_EXECUTABLE)
-    message(STATUS "Found epydoc - ${EPYDOC_EXECUTABLE}")
-
-    find_program(DOT_EXECUTABLE dot)
-    if(DOT_EXECUTABLE)
-      set(EPYDOC_OPTIONS ${EPYDOC_OPTIONS} --graph all --dotpath ${DOT_EXECUTABLE})
-      message(STATUS "Found dot - ${DOT_EXECUTABLE}")
-    endif()
+if (LLVM_ENABLE_SPHINX)
+  include(AddSphinxTarget)
+endif()
 
-    # Pretend to make a python package so that we can generate the reference.
-    # Because we don't build liblldb, epydoc will complain that the import of
-    # _lldb.so failed, but that doesn't prevent it from generating the docs.
+if (LLDB_ENABLE_PYTHON AND SPHINX_FOUND)
+  if (${SPHINX_OUTPUT_HTML})
+    # Pretend that the SWIG generated API is a Python package.
     file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lldb)
     get_target_property(lldb_bindings_dir swig_wrapper_python BINARY_DIR)
     add_custom_target(lldb-python-doc-package
       COMMAND "${CMAKE_COMMAND}" -E copy "${lldb_bindings_dir}/lldb.py" "${CMAKE_CURRENT_BINARY_DIR}/lldb/__init__.py"
-      COMMENT "Copying lldb.py to pretend package.")
+      COMMENT "Copying lldb.py to pretend its a Python package.")
     add_dependencies(lldb-python-doc-package swig_wrapper_python)
 
-    set(DOC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/doc")
-    file(MAKE_DIRECTORY "${DOC_DIR}")
-    add_custom_target(lldb-python-doc
-      ${EPYDOC_EXECUTABLE}
-      --html
-      lldb
-      -o ${CMAKE_CURRENT_BINARY_DIR}/python_reference
-      --name "LLDB python API"
-      --url "http://lldb.llvm.org"
-      ${EPYDOC_OPTIONS}
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-      COMMENT "Generating LLDB Python API reference with epydoc" VERBATIM
-    )
-    add_dependencies(lldb-python-doc swig_wrapper_python lldb-python-doc-package)
-  else()
-    message(STATUS "Could NOT find epydoc")
+    # FIXME: Don't treat Sphinx warnings as errors. The files generated by
+    # automodapi are full of warnings (partly caused by SWIG, our documentation
+    # and probably also automodapi itself), so those warnings need to be fixed
+    # first before we can turn this on.
+    set(SPHINX_WARNINGS_AS_ERRORS Off)
+
+    # The sphinx config needs to know where the generated LLDB Python module is.
+    # There is no way to pass a variable into our sphinx config, so just pass
+    # the path to the module via the LLDB_SWIG_MODULE environment variable.
+    add_sphinx_target(html lldb ENV_VARS "LLDB_SWIG_MODULE=${CMAKE_CURRENT_BINARY_DIR}")
+    # Sphinx does not reliably update the custom CSS files, so force
+    # a clean rebuild of the documentation every time.
+    add_custom_target(clean-lldb-html COMMAND "${CMAKE_COMMAND}" -E
+      remove_directory ${CMAKE_CURRENT_BINARY_DIR}/html)
+    add_dependencies(docs-lldb-html swig_wrapper_python
+                     lldb-python-doc-package clean-lldb-html)
   endif()
-endif ()
-
-if (LLVM_ENABLE_SPHINX)
-  include(AddSphinxTarget)
-  if (SPHINX_FOUND)
-    if (${SPHINX_OUTPUT_HTML})
-      add_sphinx_target(html lldb)
-      # Sphinx does not reliably update the custom CSS files, so force
-      # a clean rebuild of the documentation every time.
-      add_custom_target(clean-lldb-html COMMAND "${CMAKE_COMMAND}" -E
-        remove_directory ${CMAKE_CURRENT_BINARY_DIR}/html)
-      add_dependencies(docs-lldb-html clean-lldb-html)
-    endif()
 
-    if (${SPHINX_OUTPUT_MAN})
-      add_sphinx_target(man lldb)
-    endif()
+  if (${SPHINX_OUTPUT_MAN})
+    add_sphinx_target(man lldb)
   endif()
 endif()

lldb/docs/_lldb/__init__.py

@@ -0,0 +1,9 @@
+from unittest.mock import Mock
+import sys
+import types
+
+# This package acts as a mock implementation of the native _lldb module so
+# that generating the LLDB documentation doesn't actually require building all
+# of LLDB.
+module_name = '_lldb'
+sys.modules[module_name] = Mock()

lldb/docs/conf.py

@@ -18,7 +18,17 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+
+# Add the current directory that contains the mock _lldb native module which
+# is imported by the `lldb` module.
+sys.path.insert(0, os.path.abspath("."))
+# Add the build directory that contains the `lldb` module. LLDB_SWIG_MODULE is
+# set by CMake.
+sys.path.insert(0, os.getenv("LLDB_SWIG_MODULE"))
+
+# Put the generated Python API documentation in the 'python_api' folder. This
+# also defines the URL these files will have in the generated website.
+automodapi_toctreedirnm = 'python_api'
 
 # -- General configuration -----------------------------------------------------
 
@@ -27,7 +37,8 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.intersphinx']
+extensions = ['sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.intersphinx',
+              'sphinx_automodapi.automodapi']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

lldb/docs/index.rst

@@ -164,7 +164,7 @@ interesting areas to contribute to lldb.
    :maxdepth: 1
    :caption: Reference
 
-   Public Python API <https://lldb.llvm.org/python_reference/index.html>
+   Public Python API <python_api>
    Public C++ API <https://lldb.llvm.org/cpp_reference/namespacelldb.html>
    Private C++ API <https://lldb.llvm.org/cpp_reference/index.html>
    Man Page <man/lldb>

lldb/docs/python_api.rst

@@ -0,0 +1,98 @@
+LLDB Python API
+================================
+
+..
+  The long list of "skip" filters out several global functions that are
+  generated by SWIG (but which are not useful as they are only the
+  backend for their respective static functions in the classes).
+  Without this list
+.. automodapi:: lldb
+  :no-inheritance-diagram:
+  :skip: SBBreakpoint_EventIsBreakpointEvent
+  :skip: SBBreakpoint_GetBreakpointEventTypeFromEvent
+  :skip: SBBreakpoint_GetBreakpointFromEvent
+  :skip: SBBreakpoint_GetBreakpointLocationAtIndexFromEvent
+  :skip: SBBreakpoint_GetNumBreakpointLocationsFromEvent
+  :skip: SBCommandInterpreter_EventIsCommandInterpreterEvent
+  :skip: SBCommandInterpreter_GetArgumentDescriptionAsCString
+  :skip: SBCommandInterpreter_GetArgumentTypeAsCString
+  :skip: SBCommandInterpreter_GetBroadcasterClass
+  :skip: SBCommunication_GetBroadcasterClass
+  :skip: SBData_CreateDataFromCString
+  :skip: SBData_CreateDataFromDoubleArray
+  :skip: SBData_CreateDataFromSInt32Array
+  :skip: SBData_CreateDataFromSInt64Array
+  :skip: SBData_CreateDataFromUInt32Array
+  :skip: SBData_CreateDataFromUInt64Array
+  :skip: SBDebugger_Create
+  :skip: SBDebugger_Create
+  :skip: SBDebugger_Destroy
+  :skip: SBDebugger_FindDebuggerWithID
+  :skip: SBDebugger_GetBuildConfiguration
+  :skip: SBDebugger_GetDefaultArchitecture
+  :skip: SBDebugger_GetInternalVariableValue
+  :skip: SBDebugger_GetVersionString
+  :skip: SBDebugger_Initialize
+  :skip: SBDebugger_InitializeWithErrorHandling
+  :skip: SBDebugger_MemoryPressureDetected
+  :skip: SBDebugger_SetDefaultArchitecture
+  :skip: SBDebugger_SetInternalVariable
+  :skip: SBDebugger_StateAsCString
+  :skip: SBDebugger_StateIsRunningState
+  :skip: SBDebugger_StateIsStoppedState
+  :skip: SBDebugger_Terminate
+  :skip: SBEvent_GetCStringFromEvent
+  :skip: SBFileSpec_ResolvePath
+  :skip: SBFile_MakeBorrowed
+  :skip: SBFile_MakeBorrowedForcingIOMethods
+  :skip: SBFile_MakeForcingIOMethods
+  :skip: SBHostOS_GetLLDBPath
+  :skip: SBHostOS_GetLLDBPythonPath
+  :skip: SBHostOS_GetProgramFileSpec
+  :skip: SBHostOS_GetUserHomeDirectory
+  :skip: SBHostOS_ThreadCancel
+  :skip: SBHostOS_ThreadCreate
+  :skip: SBHostOS_ThreadCreated
+  :skip: SBHostOS_ThreadDetach
+  :skip: SBHostOS_ThreadJoin
+  :skip: SBLanguageRuntime_GetLanguageTypeFromString
+  :skip: SBLanguageRuntime_GetNameForLanguageType
+  :skip: SBModuleSpecList_GetModuleSpecifications
+  :skip: SBModule_GarbageCollectAllocatedModules
+  :skip: SBModule_GetNumberAllocatedModules
+  :skip: SBPlatform_GetHostPlatform
+  :skip: SBProcess_EventIsProcessEvent
+  :skip: SBProcess_EventIsStructuredDataEvent
+  :skip: SBProcess_GetBroadcasterClassName
+  :skip: SBProcess_GetInterruptedFromEvent
+  :skip: SBProcess_GetNumRestartedReasonsFromEvent
+  :skip: SBProcess_GetProcessFromEvent
+  :skip: SBProcess_GetRestartedFromEvent
+  :skip: SBProcess_GetRestartedReasonAtIndexFromEvent
+  :skip: SBProcess_GetStateFromEvent
+  :skip: SBProcess_GetStructuredDataFromEvent
+  :skip: SBReproducer_Capture
+  :skip: SBReproducer_PassiveReplay
+  :skip: SBReproducer_SetAutoGenerate
+  :skip: SBReproducer_SetWorkingDirectory
+  :skip: SBTarget_EventIsTargetEvent
+  :skip: SBTarget_GetBroadcasterClassName
+  :skip: SBTarget_GetModuleAtIndexFromEvent
+  :skip: SBTarget_GetNumModulesFromEvent
+  :skip: SBTarget_GetTargetFromEvent
+  :skip: SBThread_EventIsThreadEvent
+  :skip: SBThread_GetBroadcasterClassName
+  :skip: SBThread_GetStackFrameFromEvent
+  :skip: SBThread_GetThreadFromEvent
+  :skip: SBTypeSummary_CreateWithFunctionName
+  :skip: SBTypeSummary_CreateWithScriptCode
+  :skip: SBTypeSummary_CreateWithSummaryString
+  :skip: SBTypeSynthetic_CreateWithClassName
+  :skip: SBTypeSynthetic_CreateWithScriptCode
+  :skip: SBWatchpoint_EventIsWatchpointEvent
+  :skip: SBWatchpoint_GetWatchpointEventTypeFromEvent
+  :skip: SBWatchpoint_GetWatchpointFromEvent
+  :skip: command
+  :skip: in_range
+  :skip: is_numeric_type
+  :skip: lldb_iter

llvm/cmake/modules/AddSphinxTarget.cmake

@@ -17,8 +17,13 @@ endif()
 # the sphinx-build command.
 #
 # ``project`` should be the project name
+#
+# Named arguments:
+# ``ENV_VARS`` should be a list of environment variables that should be set when
+#              running Sphinx. Each environment variable should be a string with
+#              the form KEY=VALUE.
 function (add_sphinx_target builder project)
-  cmake_parse_arguments(ARG "" "SOURCE_DIR" "" ${ARGN})
+  cmake_parse_arguments(ARG "" "SOURCE_DIR" "ENV_VARS" ${ARGN})
   set(SPHINX_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/${builder}")
   set(SPHINX_DOC_TREE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_doctrees-${project}-${builder}")
   set(SPHINX_TARGET_NAME docs-${project}-${builder})
@@ -34,7 +39,8 @@ function (add_sphinx_target builder project)
   endif()
 
   add_custom_target(${SPHINX_TARGET_NAME}
-                    COMMAND ${SPHINX_EXECUTABLE}
+                    COMMAND ${CMAKE_COMMAND} -E env ${ARG_ENV_VARS}
+                            ${SPHINX_EXECUTABLE}
                             -b ${builder}
                             -d "${SPHINX_DOC_TREE_DIR}"
                             -q # Quiet: no output other than errors and warnings.