docs: Fix 'mkdocs serve' endless build loop (#3662)

* docs: Fix 'mkdocs serve' endless build loop

When developing and previewing docs using `mkdocs serve`, the site gets
stuck into an endless build loop when any content is changed. This is
caused by two hooks that copy content into the site-src/ directory on
site build, mkdocs-copy-geps.py and mkdocs-generate-conformance.py,
triggering another build due to the changed content.

The solution is to use MkDocs Files API to programmatically add the files
into the site instead of copying them into the site directory.

See mkdocs/mkdocs#2544 for more information.

* Fix failing test by upgrading to Python 3.12

The Path.walk() function was added in Python 3.12.

* Ensure that reportedImplementationsPath is sorted

Author: blake

.gitignore

@@ -56,4 +56,3 @@ go.work.sum
 /cache
 .venv/
 release/
-site-src/geps

hack/mkdocs-copy-geps.py

@@ -15,11 +15,50 @@
 import shutil
 import logging
 from mkdocs import plugins
+from mkdocs.structure.files import File
+from pathlib import Path
 
-log = logging.getLogger('mkdocs')
+log = logging.getLogger(f'mkdocs.plugins.{__name__}')
 
 
 @plugins.event_priority(100)
-def on_pre_build(config, **kwargs):
-    log.info("copying geps")
-    shutil.copytree("geps", "site-src/geps", dirs_exist_ok=True)
+def on_files(files, config, **kwargs):
+    log.info("adding geps")
+
+    # Check if site-src/geps exists (files copied out-of-band from MkDocs)
+    site_src_geps = Path('site-src/geps')
+    if site_src_geps.exists() and site_src_geps.is_dir():
+      log.info("Found site-src/gep/ directory. Deleting...")
+
+      # Iterate over the list of files in this directory and remove them from
+      # MkDocs
+      for root_dir, _, gep_files  in Path(site_src_geps).walk():
+        for filename in gep_files:
+          # Exclude the leading 'site-src/' to get the relative path as it
+          # exists on the site. (i.e., geps/overview.md)
+          path = '/'.join(root_dir.parts[1:])
+
+          existing_file = files.get_file_from_path(f'{path}/{filename}')
+          if existing_file:
+            files.remove(existing_file)
+
+      # Delete the 'site-src/geps' directory
+      shutil.rmtree(site_src_geps)
+
+    for root_dir, _, gep_files in Path('geps').walk():
+
+      # Iterate over the all the files in the GEP folder and add them to the site
+      for filename in gep_files:
+        file_path = str(root_dir / filename)
+
+        if files.get_file_from_path(file_path) is None:
+          new_file = File(
+            path=file_path,
+            src_dir='./',
+            dest_dir=config['site_dir'],
+            use_directory_urls=config['use_directory_urls']
+          )
+
+          files.append(new_file)
+
+    return files

hack/mkdocs-generate-conformance.py

@@ -13,26 +13,41 @@
 # limitations under the License.
 
 import logging
+from io import StringIO
 from mkdocs import plugins
+from mkdocs.structure.files import File
 import yaml
 import pandas
 from fnmatch import fnmatch
 import glob
 import os
 
-log = logging.getLogger('mkdocs')
+log = logging.getLogger(f'mkdocs.plugins.{__name__}')
 
 
 @plugins.event_priority(100)
-def on_pre_build(config, **kwargs):
+def on_files(files, config, **kwargs):
     log.info("generating conformance")
 
     vers = getConformancePaths()
+    # Iterate over the list of versions. Exclude the pre 1.0 versions.
     for v in vers[3:]:
 
         confYamls = getYaml(v)
         releaseVersion = v.split(os.sep)[-2]
-        generate_conformance_tables(confYamls, releaseVersion)
+        file = generate_conformance_tables(confYamls, releaseVersion, config)
+
+        if file:
+          existing_file = files.get_file_from_path(file.src_uri)
+          if existing_file:
+              # Remove the existing file that is likely present in the
+              # repository
+              files.remove(existing_file)
+
+          # Add the generated file to the site
+          files.append(file)
+
+    return files
 
 
 desc = """
@@ -51,7 +66,10 @@ def on_pre_build(config, **kwargs):
 
 
 
-def generate_conformance_tables(reports, currVersion):
+def generate_conformance_tables(reports, currVersion, mkdocsConfig):
+
+    # Enable Pandas copy-on-write
+    pandas.options.mode.copy_on_write = True
 
     gateway_tls_table = pandas.DataFrame()
     gateway_grpc_table = pandas.DataFrame()
@@ -79,7 +97,8 @@ def generate_conformance_tables(reports, currVersion):
     if entries.Project < 3:
         return
 
-    with open('site-src/implementations/'+versionFile+'.md', 'w') as f:
+    try:
+        f = StringIO()
 
         f.write(desc)
         f.write("\n\n")
@@ -90,7 +109,7 @@ def generate_conformance_tables(reports, currVersion):
         f.write("## Gateway Profile\n\n")
         f.write("### HTTPRoute\n\n")
         f.write(gateway_http_table.to_markdown()+'\n\n')
-        if currVersion != 'v1.0.0': 
+        if currVersion != 'v1.0.0':
             f.write('### GRPCRoute\n\n')
             f.write(gateway_grpc_table.to_markdown()+'\n\n')
             f.write('### TLSRoute\n\n')
@@ -100,6 +119,20 @@ def generate_conformance_tables(reports, currVersion):
         f.write("### HTTPRoute\n\n")
         f.write(mesh_http_table.to_markdown())
 
+        file_contents = f.getvalue()
+    finally:
+        f.close()
+
+    new_file = File(
+      src_dir=None,
+      dest_dir=mkdocsConfig['site_dir'],
+      path=f'implementations/{versionFile}.md',
+      use_directory_urls=mkdocsConfig['use_directory_urls'],
+    )
+    new_file.content_string = file_contents
+    new_file.generated_by = f'{__name__}'
+
+    return new_file
 
 def generate_profiles_report(reports, route,version):
 
@@ -114,7 +147,7 @@ def generate_profiles_report(reports, route,version):
                                'version','mode', 'extended.supportedFeatures']].T
     http_table.columns = http_table.iloc[0]
     http_table = http_table[1:].T
-    
+
     for row in http_table.itertuples():
         if type(row._4) is list:
             for feat in row._4:
@@ -130,8 +163,8 @@ def generate_profiles_report(reports, route,version):
 
 
 pathTemp = "conformance/reports/*/"
-allVersions = []
-reportedImplementationsPath = []
+allVersions = set()
+reportedImplementationsPath = set()
 
 # returns v1.0.0 and greater, since that's when reports started being generated in the comparison table
 
@@ -141,9 +174,10 @@ def getConformancePaths():
     report_path = versions[-1]+"**"
     for v in versions:
         vers = v.split(os.sep)[-2]
-        allVersions.append(vers)
-        reportedImplementationsPath.append(v+"**")
-    return reportedImplementationsPath
+        allVersions.add(vers)
+        reportedImplementationsPath.add(v+"**")
+
+    return sorted(list(reportedImplementationsPath))
 
 
 def getYaml(conf_path):
@@ -154,11 +188,12 @@ def getYaml(conf_path):
         if fnmatch(p, "*.yaml"):
 
             x = load_yaml(p)
-            profiles = pandas.json_normalize(
-                x, record_path=['profiles'], meta=["mode","implementation"], errors='ignore')
+            if 'profiles' in x:
+              profiles = pandas.json_normalize(
+                  x, record_path=['profiles'], meta=["mode","implementation"], errors='ignore')
 
-            implementation = pandas.json_normalize(profiles.implementation)
-            yamls.append(pandas.concat([implementation, profiles], axis=1))
+              implementation = pandas.json_normalize(profiles.implementation)
+              yamls.append(pandas.concat([implementation, profiles], axis=1))
 
     yamls = pandas.concat(yamls)
     return yamls

netlify.toml

@@ -3,7 +3,7 @@
     command = "make docs"
     publish = "site"
 [build.environment]
-    PYTHON_VERSION = "3.9"
+    PYTHON_VERSION = "3.12"
     GO_VERSION = "1.22.8"
 
 # Standard Netlify redirects
@@ -30,4 +30,4 @@
     from = "/geps/gep-1426"
     to = "/geps/gep-1294"
     status = 301
-    force = true
+    force = true