[randstruct] Also randomize composite function pointer structs (llvm#138385)

Check for struct members that are structs filled only with function
pointers by recursively examining it. Since the lamba
IsFunctionPointerOrForwardDecl cannot call itself directly, move it into
a helper function, EntirelyFunctionPointers, so it can be called from
the lambda.

Add test for composite function pointer structs getting automatically
randomized.

Add more tests for validating automatic randomization vs explicitly
annotated with "randomize_layout", and excluded with
"no_randomize_layout".

Reorder the "should we randomize?" "if" statement to check for
enablement before checking for Record details.

Fixes llvm#138355

Author: kees

clang/include/clang/Sema/Sema.h

@@ -6476,6 +6476,8 @@ class Sema final : public SemaBase {
   void setupImplicitSpecialMemberType(CXXMethodDecl *SpecialMem,
                                       QualType ResultTy,
                                       ArrayRef<QualType> Args);
+  // Helper for ActOnFields to check for all function pointer members.
+  bool EntirelyFunctionPointers(const RecordDecl *Record);
 
   // A cache representing if we've fully checked the various comparison category
   // types stored in ASTContext. The bit-index corresponds to the integer value

clang/lib/Sema/SemaDecl.cpp

@@ -19309,6 +19309,43 @@ static void ComputeSpecialMemberFunctionsEligiblity(Sema &S,
                      CXXSpecialMemberKind::MoveAssignment);
 }
 
+bool Sema::EntirelyFunctionPointers(const RecordDecl *Record) {
+  // Check to see if a FieldDecl is a pointer to a function.
+  auto IsFunctionPointerOrForwardDecl = [&](const Decl *D) {
+    const FieldDecl *FD = dyn_cast<FieldDecl>(D);
+    if (!FD) {
+      // Check whether this is a forward declaration that was inserted by
+      // Clang. This happens when a non-forward declared / defined type is
+      // used, e.g.:
+      //
+      //   struct foo {
+      //     struct bar *(*f)();
+      //     struct bar *(*g)();
+      //   };
+      //
+      // "struct bar" shows up in the decl AST as a "RecordDecl" with an
+      // incomplete definition.
+      if (const auto *TD = dyn_cast<TagDecl>(D))
+        return !TD->isCompleteDefinition();
+      return false;
+    }
+    QualType FieldType = FD->getType().getDesugaredType(Context);
+    if (isa<PointerType>(FieldType)) {
+      QualType PointeeType = cast<PointerType>(FieldType)->getPointeeType();
+      return PointeeType.getDesugaredType(Context)->isFunctionType();
+    }
+    // If a member is a struct entirely of function pointers, that counts too.
+    if (const RecordType *RT = FieldType->getAs<RecordType>()) {
+      const RecordDecl *Record = RT->getDecl();
+      if (Record->isStruct() && EntirelyFunctionPointers(Record))
+        return true;
+    }
+    return false;
+  };
+
+  return llvm::all_of(Record->decls(), IsFunctionPointerOrForwardDecl);
+}
+
 void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
                        ArrayRef<Decl *> Fields, SourceLocation LBrac,
                        SourceLocation RBrac,
@@ -19646,41 +19683,13 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
     // Handle attributes before checking the layout.
     ProcessDeclAttributeList(S, Record, Attrs);
 
-    // Check to see if a FieldDecl is a pointer to a function.
-    auto IsFunctionPointerOrForwardDecl = [&](const Decl *D) {
-      const FieldDecl *FD = dyn_cast<FieldDecl>(D);
-      if (!FD) {
-        // Check whether this is a forward declaration that was inserted by
-        // Clang. This happens when a non-forward declared / defined type is
-        // used, e.g.:
-        //
-        //   struct foo {
-        //     struct bar *(*f)();
-        //     struct bar *(*g)();
-        //   };
-        //
-        // "struct bar" shows up in the decl AST as a "RecordDecl" with an
-        // incomplete definition.
-        if (const auto *TD = dyn_cast<TagDecl>(D))
-          return !TD->isCompleteDefinition();
-        return false;
-      }
-      QualType FieldType = FD->getType().getDesugaredType(Context);
-      if (isa<PointerType>(FieldType)) {
-        QualType PointeeType = cast<PointerType>(FieldType)->getPointeeType();
-        return PointeeType.getDesugaredType(Context)->isFunctionType();
-      }
-      return false;
-    };
-
     // Maybe randomize the record's decls. We automatically randomize a record
     // of function pointers, unless it has the "no_randomize_layout" attribute.
-    if (!getLangOpts().CPlusPlus &&
+    if (!getLangOpts().CPlusPlus && !getLangOpts().RandstructSeed.empty() &&
+        !Record->isRandomized() && !Record->isUnion() &&
         (Record->hasAttr<RandomizeLayoutAttr>() ||
          (!Record->hasAttr<NoRandomizeLayoutAttr>() &&
-          llvm::all_of(Record->decls(), IsFunctionPointerOrForwardDecl))) &&
-        !Record->isUnion() && !getLangOpts().RandstructSeed.empty() &&
-        !Record->isRandomized()) {
+          EntirelyFunctionPointers(Record)))) {
       SmallVector<Decl *, 32> NewDeclOrdering;
       if (randstruct::randomizeStructureLayout(Context, Record,
                                                NewDeclOrdering))

clang/test/Sema/init-randomized-struct.c

@@ -75,3 +75,49 @@ struct enum_decl_test {
 } __attribute__((randomize_layout));
 
 struct enum_decl_test t13 = { BORK }; // Okay
+
+struct mixed {
+  int a;
+  short b;
+  unsigned c;
+  char d;
+} __attribute__((randomize_layout));
+
+struct mixed t14 = { 7 }; // expected-error {{a randomized struct can only be initialized with a designated initializer}}
+struct mixed t15 = { .b = 8 }; // Okay
+
+// This should be autodetected as randomized.
+struct funcs {
+  func_ptr a;
+  func_ptr b;
+  func_ptr c;
+  func_ptr d;
+};
+
+struct funcs t16 = { .c = foo }; // Okay
+struct funcs t17 = { foo }; // expected-error {{a randomized struct can only be initialized with a designated initializer}}
+
+// This should be forced off.
+struct funcs_unshuffled {
+  func_ptr a;
+  func_ptr b;
+  func_ptr c;
+  func_ptr d;
+} __attribute__((no_randomize_layout));
+
+struct funcs_unshuffled t18 = { .d = foo }; // Okay
+struct funcs_unshuffled t19 = { foo }; // Okay
+
+// This is still all function pointers.
+// https://github.com/llvm/llvm-project/issues/138355
+struct funcs_composite {
+  func_ptr a;
+  func_ptr b;
+  struct funcs inner;
+  func_ptr c;
+  func_ptr d;
+};
+
+struct funcs_composite t20 = { .a = foo }; // Okay
+struct funcs_composite t21 = { .inner.c = foo }; // Okay
+struct funcs_composite t22 = { foo }; // expected-error {{a randomized struct can only be initialized with a designated initializer}}