[libclang/python] Sync python kinds with Index.h enums (#143264)

Add tests to ensure that all C-enum variants are defined on Python side,
and that the Python bindings do not contain variants not defined on the C side

Author: DeinAlptraum

clang/bindings/python/clang/cindex.py

@@ -708,7 +708,6 @@ def is_unexposed(self):
         """Test if this is an unexposed kind."""
         return conf.lib.clang_isUnexposed(self)  # type: ignore [no-any-return]
 
-
     ###
     # Declaration Kinds
 
@@ -835,7 +834,6 @@ def is_unexposed(self):
     # A C++ access specifier decl.
     CXX_ACCESS_SPEC_DECL = 39
 
-
     ###
     # Reference Kinds
 
@@ -1436,12 +1434,60 @@ def is_unexposed(self):
     # OpenMP scope directive.
     OMP_SCOPE_DIRECTIVE = 306
 
+    # OpenMP reverse directive.
+    OMPReverseDirective = 307
+
+    # OpenMP interchange directive.
+    OMPInterchangeDirective = 308
+
+    # OpenMP assume directive.
+    OMPAssumeDirective = 309
+
     # OpenMP stripe directive.
     OMP_STRIPE_DIRECTIVE = 310
 
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
+    # OpenACC Loop Construct.
+    OpenACCLoopConstruct = 321
+
+    # OpenACC Combined Constructs.
+    OpenACCCombinedConstruct = 322
+
+    # OpenACC data Construct.
+    OpenACCDataConstruct = 323
+
+    # OpenACC enter data Construct.
+    OpenACCEnterDataConstruct = 324
+
+    # OpenACC exit data Construct.
+    OpenACCExitDataConstruct = 325
+
+    # OpenACC host_data Construct.
+    OpenACCHostDataConstruct = 326
+
+    # OpenACC wait Construct.
+    OpenACCWaitConstruct = 327
+
+    # OpenACC init Construct.
+    OpenACCInitConstruct = 328
+
+    # OpenACC shutdown Construct.
+    OpenACCShutdownConstruct = 329
+
+    # OpenACC set Construct.
+    OpenACCSetConstruct = 330
+
+    # OpenACC update Construct.
+    OpenACCUpdateConstruct = 331
+
+    # OpenACC atomic Construct.
+    OpenACCAtomicConstruct = 332
+
+    # OpenACC cache Construct.
+    OpenACCCacheConstruct = 333
+
     ###
     # Other Kinds
 
@@ -1560,6 +1606,7 @@ class ExceptionSpecificationKind(BaseEnumeration):
     UNEVALUATED = 6
     UNINSTANTIATED = 7
     UNPARSED = 8
+    NOTHROW = 9
 
 ### Cursors ###
 
@@ -2444,7 +2491,6 @@ class AccessSpecifier(BaseEnumeration):
     PUBLIC = 1
     PROTECTED = 2
     PRIVATE = 3
-    NONE = 4
 
 ### Type Kinds ###
 
@@ -2492,7 +2538,16 @@ def spelling(self):
     FLOAT128 = 30
     HALF = 31
     FLOAT16 = 32
+    SHORTACCUM = 33
+    ACCUM = 34
+    LONGACCUM = 35
+    USHORTACCUM = 36
+    UACCUM = 37
+    ULONGACCUM = 38
+    BFLOAT16 = 39
     IBM128 = 40
+    FIRSTBUILTIN = VOID
+    LASTBUILTIN = IBM128
     COMPLEX = 100
     POINTER = 101
     BLOCKPOINTER = 102
@@ -2576,6 +2631,10 @@ def spelling(self):
     ATOMIC = 177
     BTFTAGATTRIBUTED = 178
 
+    HLSLRESOURCE = 179
+    HLSLATTRIBUTEDRESOURCE = 180
+    HLSLINLINESPIRV = 181
+
 class RefQualifierKind(BaseEnumeration):
     """Describes a specific ref-qualifier of a type."""
 

clang/bindings/python/tests/cindex/test_enums.py

@@ -1,4 +1,5 @@
 import unittest
+from pathlib import Path
 
 from clang.cindex import (
     AccessSpecifier,
@@ -13,26 +14,15 @@
     TemplateArgumentKind,
     TLSKind,
     TokenKind,
+    TranslationUnit,
     TypeKind,
+    PrintingPolicyProperty,
+    BaseEnumeration,
 )
 
 
 class TestEnums(unittest.TestCase):
-    enums = [
-        TokenKind,
-        CursorKind,
-        TemplateArgumentKind,
-        ExceptionSpecificationKind,
-        AvailabilityKind,
-        AccessSpecifier,
-        TypeKind,
-        RefQualifierKind,
-        LanguageKind,
-        LinkageKind,
-        TLSKind,
-        StorageClass,
-        BinaryOperator,
-    ]
+    enums = BaseEnumeration.__subclasses__()
 
     def test_from_id(self):
         """Check that kinds can be constructed from valid IDs"""
@@ -44,10 +34,70 @@ def test_from_id(self):
             with self.assertRaises(ValueError):
                 enum.from_id(-1)
 
-    def test_duplicate_ids(self):
-        """Check that no two kinds have the same id"""
-        # for enum in self.enums:
+    def test_all_variants(self):
+        """Check that all libclang enum values are also defined in cindex.py"""
+        cenum_to_pythonenum = {
+            "CX_CXXAccessSpecifier": AccessSpecifier,
+            "CX_StorageClass": StorageClass,
+            "CXAvailabilityKind": AvailabilityKind,
+            "CXBinaryOperatorKind": BinaryOperator,
+            "CXCursorKind": CursorKind,
+            "CXCursor_ExceptionSpecificationKind": ExceptionSpecificationKind,
+            "CXLanguageKind": LanguageKind,
+            "CXLinkageKind": LinkageKind,
+            "CXPrintingPolicyProperty": PrintingPolicyProperty,
+            "CXRefQualifierKind": RefQualifierKind,
+            "CXTemplateArgumentKind": TemplateArgumentKind,
+            "CXTLSKind": TLSKind,
+            "CXTokenKind": TokenKind,
+            "CXTypeKind": TypeKind,
+        }
+
+        indexheader = (
+            Path(__file__).parent.parent.parent.parent.parent
+            / "include/clang-c/Index.h"
+        )
+        # FIXME: Index.h is a C file, but we read it as a C++ file because we
+        # don't get ENUM_CONSTANT_DECL cursors otherwise, which we need here
+        # See bug report: https://github.com/llvm/llvm-project/issues/159075
+        tu = TranslationUnit.from_source(indexheader, ["-x", "c++"])
+
+        enum_variant_map = {}
+        # For all enums in self.enums, extract all enum variants defined in Index.h
+        for cursor in tu.cursor.walk_preorder():
+            if cursor.kind == CursorKind.ENUM_CONSTANT_DECL:
+                python_enum = cenum_to_pythonenum.get(cursor.type.spelling)
+                if python_enum not in enum_variant_map:
+                    enum_variant_map[python_enum] = dict()
+                enum_variant_map[python_enum][cursor.enum_value] = cursor.spelling
+
         for enum in self.enums:
-            num_declared_variants = len(enum._member_map_.keys())
-            num_unique_variants = len(list(enum))
-            self.assertEqual(num_declared_variants, num_unique_variants)
+            with self.subTest(enum):
+                # This ensures only the custom assert message below is printed
+                self.longMessage = False
+
+                python_kinds = set([kind.value for kind in enum])
+                num_to_c_kind = enum_variant_map[enum]
+                c_kinds = set(num_to_c_kind.keys())
+                # Defined in Index.h but not in cindex.py
+                missing_python_kinds = c_kinds - python_kinds
+                missing_names = set(
+                    [num_to_c_kind[kind] for kind in missing_python_kinds]
+                )
+                self.assertEqual(
+                    missing_names,
+                    set(),
+                    f"{missing_names} variants are missing. "
+                    f"Please ensure these are defined in {enum} in cindex.py.",
+                )
+                # Defined in cindex.py but not in Index.h
+                superfluous_python_kinds = python_kinds - c_kinds
+                missing_names = set(
+                    [enum.from_id(kind) for kind in superfluous_python_kinds]
+                )
+                self.assertEqual(
+                    missing_names,
+                    set(),
+                    f"{missing_names} variants only exist in the Python bindings. "
+                    f"Please ensure that all {enum} kinds defined in cindex.py have an equivalent in Index.h",
+                )

clang/docs/ReleaseNotes.rst

@@ -125,6 +125,7 @@ Clang Python Bindings Potentially Breaking Changes
 - TypeKind ``ELABORATED`` is not used anymore, per clang AST changes removing
   ElaboratedTypes. The value becomes unused, and all the existing users should
   expect the former underlying type to be reported instead.
+- Remove ``AccessSpecifier.NONE`` kind. No libclang interfaces ever returned this kind.
 
 What's New in Clang |release|?
 ==============================