docs: speed up incremental builds

On each documentation build (‘make html’), doxygen regenerates XML
files. In addition to that, gen-dxd.py regenerates API reference
files under _build/inc/. This results in Sphinx flagging about half
of the input files as modified, and incremental builds taking long
time.

With this change, XML files generated by Doxygen are copied into
docs/xml_in directory only when they are changed. Breathe is pointed
to docs/xml_in directory instead of docs/xml. In addition to that,
gen-dxd.py is modified to only write to the output file when contents
change.

Overall, incremental build time (with no source files changed) is
reduced from ~7 minutes to ~8 seconds (on a particular OS X
computer).

Due to the way Breathe includes Doxygen XML files, there is still
going to be a massive rebuild every time functions, enums, macros,
structures are added or removed from the header files scanned
by Doxygen, but at least individual .rst files can be edited
at a much faster pace.

Author: igrr

.gitignore

@@ -32,7 +32,9 @@ docs/doxygen-warning-log.txt
 docs/sphinx-warning-log.txt
 docs/sphinx-warning-log-sanitized.txt
 docs/xml/
+docs/xml_in/
 docs/man/
+docs/doxygen_sqlite3.db
 
 # Unit test app files
 tools/unit-test-app/sdkconfig

docs/conf.py

@@ -22,15 +22,26 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('.'))
 
-from repo_util import run_cmd_get_output
+from local_util import run_cmd_get_output, copy_if_modified
+
+builddir = '_build'
+if 'BUILDDIR' in os.environ:
+    builddir = os.environ['BUILDDIR']
 
 # Call Doxygen to get XML files from the header files
 print "Calling Doxygen to generate latest XML files"
 call('doxygen')
+# Doxygen has generated XML files in 'xml' directory.
+# Copy them to 'xml_in', only touching the files which have changed.
+copy_if_modified('xml/', 'xml_in/')
+
 # Generate 'api_name.inc' files using the XML files by Doxygen
-os.system("python gen-dxd.py")
+os.system('python gen-dxd.py')
+
 # Generate 'kconfig.inc' file from components' Kconfig files
-os.system("python gen-kconfig-doc.py > _build/inc/kconfig.inc")
+kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir)
+os.system('python gen-kconfig-doc.py > ' + kconfig_inc_path + '.in')
+copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
 
 # http://stackoverflow.com/questions/12772927/specifying-an-online-image-in-sphinx-restructuredtext-format
 # 
@@ -63,7 +74,11 @@
 packetdiag_fontpath = '_static/DejaVuSans.ttf'
 
 # Breathe extension variables
-breathe_projects = { "esp32-idf": "xml/" }
+
+# Doxygen regenerates files in 'xml/' directory every time,
+# but we copy files to 'xml_in/' only when they change, to speed up
+# incremental builds.
+breathe_projects = { "esp32-idf": "xml_in/" }
 breathe_default_project = "esp32-idf"
 
 # Add any paths that contain templates here, relative to this directory.

docs/gen-dxd.py

@@ -10,6 +10,11 @@
 import os
 import re
 
+# Determime build directory
+builddir = '_build'
+if 'BUILDDIR' in os.environ:
+    builddir = os.environ['BUILDDIR']
+
 # Script configuration
 header_file_path_prefix = "../components/"
 """string: path prefix for header files.
@@ -20,7 +25,7 @@
 xml_directory_path = "xml"
 """string: path to directory with XML files by Doxygen.
 """
-inc_directory_path = "_build/inc"
+inc_directory_path = os.path.join(builddir, 'inc')
 """string: path prefix for header files.
 """
 all_kinds = [
@@ -263,9 +268,15 @@ def generate_api_inc_files():
         api_name = get_api_name(header_file_path)
         inc_file_path = inc_directory_path + "/" + api_name + ".inc"
         rst_output = generate_directives(header_file_path)
-        inc_file = open(inc_file_path, "w")
-        inc_file.write(rst_output)
-        inc_file.close()
+
+        previous_rst_output = ''
+        if os.path.isfile(inc_file_path):
+            with open(inc_file_path, "r") as inc_file_old:
+                previous_rst_output = inc_file_old.read()
+
+        if previous_rst_output != rst_output:
+            with open(inc_file_path, "w") as inc_file:
+                inc_file.write(rst_output)
 
 
 if __name__ == "__main__":

docs/link-roles.py

@@ -2,7 +2,7 @@
 
 import re
 from docutils import nodes
-from repo_util import run_cmd_get_output
+from local_util import run_cmd_get_output
 
 def get_github_rev():
     path = run_cmd_get_output('git rev-parse --short HEAD')

docs/local_util.py

@@ -0,0 +1,53 @@
+# Utility functions used in conf.py
+#
+# Copyright 2017 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import os
+import shutil
+
+def run_cmd_get_output(cmd):
+    return os.popen(cmd).read().strip()
+
+def files_equal(path_1, path_2):
+    if not os.path.exists(path_1) or not os.path.exists(path_2):
+        return False
+    file_1_contents = ''
+    with open(path_1, "r") as f_1:
+        file_1_contents = f_1.read()
+    file_2_contents = ''
+    with open(path_2, "r") as f_2:
+        file_2_contents = f_2.read()
+    return file_1_contents == file_2_contents
+
+def copy_file_if_modified(src_file_path, dst_file_path):
+    if not files_equal(src_file_path, dst_file_path):
+        dst_dir_name = os.path.dirname(dst_file_path)
+        if not os.path.isdir(dst_dir_name):
+            os.makedirs(dst_dir_name)
+        shutil.copy(src_file_path, dst_file_path)
+
+def copy_if_modified(src_path, dst_path):
+    if os.path.isfile(src_path):
+        copy_file_if_modified(src_path, dst_path)
+        return
+
+    src_path_len = len(src_path)
+    for root, dirs, files in os.walk(src_path):
+        for src_file_name in files:
+            src_file_path = os.path.join(root, src_file_name)
+            dst_file_path = os.path.join(dst_path + root[src_path_len:], src_file_name)
+            copy_file_if_modified(src_file_path, dst_file_path)
+