Improve settings test to check a wider scope of consistency

Author: eliericha

testsuite/drivers/shell.py

@@ -11,7 +11,7 @@ def run(self) -> None:
 
         # This takes care of failing the test in case the return code is
         # non-zero
-        check_call(
+        p = check_call(
             self,
             [self.working_dir("test.sh"), self.env.repo_base],
             parse_shebang=True,
@@ -24,3 +24,7 @@ def run(self) -> None:
             ignore_environ=False,
             timeout=15,  # seconds
         )
+
+        assert self.env.main_options
+        if self.env.main_options.verbose:
+            print(p.out)

testsuite/settings-doc/get-schema-properties.py

@@ -0,0 +1,24 @@
+import json
+import sys
+
+schema = json.load(sys.stdin)
+properties: dict[str, dict] = schema["properties"]
+settings: list[str] = []
+
+
+def process_property(name: str, value: dict):
+    """We want to print nested properties as <name1>.<name2>.<leafname>. This function
+    recurses into nested properties to achieve that.
+    """
+    if value.get("type", None) == "object" and "properties" in value:
+        for k, v in value["properties"].items():
+            process_property(f"{name}.{k}", v)
+    else:
+        settings.append(name)
+
+
+for k, v in properties.items():
+    process_property(k, v)
+
+settings = sorted(settings)
+print("\n".join(settings))

testsuite/settings-doc/test.sh

@@ -1,4 +1,49 @@
 #!/bin/sh
-python3 get-extension-settings.py <$1/integration/vscode/ada/package.json | grep ^ada. | sort >pkg.txt
-grep '^### ' $1/doc/settings.md | sed -e 's/..../ada./' | sort >doc.txt
+
+# Fail at the first error
+set -e
+
+# Collect settings from package.json and remove 'ada.' prefix
+python3 get-extension-settings.py <"$1/integration/vscode/ada/package.json" | grep ^ada. | sed -e 's/ada\.//' | sort >pkg.txt
+
+# Collect settings from documentation
+grep '^### ' "$1/doc/settings.md" | sed -e 's/....//' | sort >doc.txt
+
+# Collect settings from the JSON schema for .als.json files
+python3 get-schema-properties.py <"$1/integration/vscode/ada/schemas/als-settings-schema.json" >schema.txt
+
+# Collect settings read in the ALS implementation
+grep 'if Name = "[^"]\+"' "$1/source/ada/lsp-ada_configurations.adb" | sed -e 's/.*"\([^"]\+\)"/\1/' >impl.txt
+# Remove the following settings from the implementation list because they are
+# either hidden, or nested
+exclude="logThreshold onTypeFormatting indentOnly"
+for exc in $exclude; do
+    echo "$(grep -v "$exc" <impl.txt)" >impl.txt
+done
+# Add the following properties because they are nested
+add="onTypeFormatting.indentOnly"
+for a in $add; do
+    echo "$a" >>impl.txt
+done
+# Sort the list
+echo "$(sort <impl.txt)" >impl.txt
+
+# Check that all VS Code settings are documented
 diff -u pkg.txt doc.txt
+
+# The ada.trace.server setting exists only in VS Code and not in .als.json
+# files. So remove it before the comparison.
+#
+# We need to use a subshell because it's not allowed to read and write the same
+# file in one pipeline
+# echo "$(grep -v trace.server <doc.txt)" >doc.txt
+echo "$(grep -v trace.server <doc.txt)" >doc.txt
+
+# Check that all implemented settings are documented
+diff -u impl.txt doc.txt
+
+# Check that all implemented settings are defined in the JSON Schema
+diff -u impl.txt schema.txt
+
+# Check that all settings defined in the JSON Schema are documented
+diff -u schema.txt doc.txt