Merge branch 'bugfix/docs_incremental_build' into 'master'

docs: speed up incremental builds

See merge request idf/esp-idf!2012

Author: igrr

.gitignore

@@ -32,7 +32,9 @@ docs/doxygen-warning-log.txt
 docs/sphinx-warning-log.txt
 docs/sphinx-warning-log-sanitized.txt
 docs/xml/
+docs/xml_in/
 docs/man/
+docs/doxygen_sqlite3.db
 
 # Unit test app files
 tools/unit-test-app/sdkconfig

docs/conf.py

@@ -22,15 +22,26 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('.'))
 
-from repo_util import run_cmd_get_output
+from local_util import run_cmd_get_output, copy_if_modified
+
+builddir = '_build'
+if 'BUILDDIR' in os.environ:
+    builddir = os.environ['BUILDDIR']
 
 # Call Doxygen to get XML files from the header files
 print "Calling Doxygen to generate latest XML files"
 call('doxygen')
+# Doxygen has generated XML files in 'xml' directory.
+# Copy them to 'xml_in', only touching the files which have changed.
+copy_if_modified('xml/', 'xml_in/')
+
 # Generate 'api_name.inc' files using the XML files by Doxygen
-os.system("python gen-dxd.py")
+os.system('python gen-dxd.py')
+
 # Generate 'kconfig.inc' file from components' Kconfig files
-os.system("python gen-kconfig-doc.py > _build/inc/kconfig.inc")
+kconfig_inc_path = '{}/inc/kconfig.inc'.format(builddir)
+os.system('python gen-kconfig-doc.py > ' + kconfig_inc_path + '.in')
+copy_if_modified(kconfig_inc_path + '.in', kconfig_inc_path)
 
 # http://stackoverflow.com/questions/12772927/specifying-an-online-image-in-sphinx-restructuredtext-format
 # 
@@ -63,7 +74,11 @@
 packetdiag_fontpath = '_static/DejaVuSans.ttf'
 
 # Breathe extension variables
-breathe_projects = { "esp32-idf": "xml/" }
+
+# Doxygen regenerates files in 'xml/' directory every time,
+# but we copy files to 'xml_in/' only when they change, to speed up
+# incremental builds.
+breathe_projects = { "esp32-idf": "xml_in/" }
 breathe_default_project = "esp32-idf"
 
 # Add any paths that contain templates here, relative to this directory.

docs/gen-dxd.py

@@ -10,6 +10,11 @@
 import os
 import re
 
+# Determime build directory
+builddir = '_build'
+if 'BUILDDIR' in os.environ:
+    builddir = os.environ['BUILDDIR']
+
 # Script configuration
 header_file_path_prefix = "../components/"
 """string: path prefix for header files.
@@ -20,7 +25,7 @@
 xml_directory_path = "xml"
 """string: path to directory with XML files by Doxygen.
 """
-inc_directory_path = "_build/inc"
+inc_directory_path = os.path.join(builddir, 'inc')
 """string: path prefix for header files.
 """
 all_kinds = [
@@ -263,9 +268,15 @@ def generate_api_inc_files():
         api_name = get_api_name(header_file_path)
         inc_file_path = inc_directory_path + "/" + api_name + ".inc"
         rst_output = generate_directives(header_file_path)
-        inc_file = open(inc_file_path, "w")
-        inc_file.write(rst_output)
-        inc_file.close()
+
+        previous_rst_output = ''
+        if os.path.isfile(inc_file_path):
+            with open(inc_file_path, "r") as inc_file_old:
+                previous_rst_output = inc_file_old.read()
+
+        if previous_rst_output != rst_output:
+            with open(inc_file_path, "w") as inc_file:
+                inc_file.write(rst_output)
 
 
 if __name__ == "__main__":

docs/link-roles.py

@@ -2,7 +2,7 @@
 
 import re
 from docutils import nodes
-from repo_util import run_cmd_get_output
+from local_util import run_cmd_get_output
 
 def get_github_rev():
     path = run_cmd_get_output('git rev-parse --short HEAD')

docs/local_util.py

@@ -0,0 +1,53 @@
+# Utility functions used in conf.py
+#
+# Copyright 2017 Espressif Systems (Shanghai) PTE LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import os
+import shutil
+
+def run_cmd_get_output(cmd):
+    return os.popen(cmd).read().strip()
+
+def files_equal(path_1, path_2):
+    if not os.path.exists(path_1) or not os.path.exists(path_2):
+        return False
+    file_1_contents = ''
+    with open(path_1, "r") as f_1:
+        file_1_contents = f_1.read()
+    file_2_contents = ''
+    with open(path_2, "r") as f_2:
+        file_2_contents = f_2.read()
+    return file_1_contents == file_2_contents
+
+def copy_file_if_modified(src_file_path, dst_file_path):
+    if not files_equal(src_file_path, dst_file_path):
+        dst_dir_name = os.path.dirname(dst_file_path)
+        if not os.path.isdir(dst_dir_name):
+            os.makedirs(dst_dir_name)
+        shutil.copy(src_file_path, dst_file_path)
+
+def copy_if_modified(src_path, dst_path):
+    if os.path.isfile(src_path):
+        copy_file_if_modified(src_path, dst_path)
+        return
+
+    src_path_len = len(src_path)
+    for root, dirs, files in os.walk(src_path):
+        for src_file_name in files:
+            src_file_path = os.path.join(root, src_file_name)
+            dst_file_path = os.path.join(dst_path + root[src_path_len:], src_file_name)
+            copy_file_if_modified(src_file_path, dst_file_path)
+