[RTTI] Add more error checking for malformed PE binaries

Fixes #7705

RTTI processing with malformed sections can still cause prolonged analysis, in extreme cases like the binary in the issue needing the user to stop the RTTI processing once it gets to the Itanium RTTI pass. More work to be done, I would like to solve this with some section heuristics using the sections/segments entropy.

Author: emesare

plugins/rtti/itanium.cpp

@@ -9,6 +9,7 @@ using namespace BinaryNinja::RTTI::Itanium;
 // TODO: Itanium doesnt really say anything about the sizing of these fields, i assume they are all u32 for thje most part.
 
 constexpr const char *TYPE_SOURCE_ITANIUM = "rtti_itanium";
+constexpr int MAX_FAILED_SCAN_ATTEMPTS = 10;
 
 
 Ref<Symbol> GetRealSymbol(BinaryView *view, uint64_t relocAddr, uint64_t symAddr)
@@ -737,6 +738,7 @@ void ItaniumRTTIProcessor::ProcessRTTI()
     uint64_t maxTypeInfoSize = TypeInfoSize(m_view);
 
     auto scan = [&](const Ref<Section> &section) {
+        int failedAttempts = 0;
         for (uint64_t currAddr = section->GetStart(); currAddr <= section->GetEnd() - maxTypeInfoSize; currAddr += addrSize)
         {
             if (bgTask->IsCancelled())
@@ -748,9 +750,14 @@ void ItaniumRTTIProcessor::ProcessRTTI()
             }
             catch (std::exception& e)
             {
+                if (failedAttempts++; failedAttempts > MAX_FAILED_SCAN_ATTEMPTS)
+                    break;
                 m_logger->LogWarnForException(e, "Failed to process object at %llx... skipping", currAddr);
             }
         }
+
+        if (failedAttempts > MAX_FAILED_SCAN_ATTEMPTS)
+            m_logger->LogWarn("Too many failed scans for section %llx... skipping", section->GetStart());
     };
 
     m_view->BeginBulkModifySymbols();
@@ -763,8 +770,16 @@ void ItaniumRTTIProcessor::ProcessRTTI()
         // Some RTTI unfortunately will get put into a DefaultSectionSemantics section, so we have to check those.
         if (sectionSemantics == ReadOnlyDataSectionSemantics || sectionSemantics == DefaultSectionSemantics)
         {
-            m_logger->LogDebug("Attempting to find RTTI in section %llx", section->GetStart());
-            scan(section);
+            // If a malformed binary makes the binary view set up unbacked sections we should not attempt to read in them.
+            if (m_view->ReadBuffer(section->GetStart(), 4).GetLength() == 4)
+            {
+                m_logger->LogDebug("Attempting to find RTTI in section %llx", section->GetStart());
+                scan(section);
+            }
+            else
+            {
+                m_logger->LogDebug("Unbacked start for section %llx... skipping", section->GetStart());
+            }
         }
     }
     m_view->EndBulkModifySymbols();

plugins/rtti/microsoft.cpp

@@ -694,14 +694,34 @@ void MicrosoftRTTIProcessor::ProcessRTTI()
     {
         if (segment->GetFlags() == (SegmentReadable | SegmentContainsData))
         {
+            // If a malformed binary makes the binary view set up unbacked segments we should not attempt to read in them.
+            if (m_view->ReadBuffer(segment->GetStart(), 4).GetLength() != 4)
+            {
+                m_logger->LogInfo("Unbacked start for segment %llx... skipping", segment->GetStart());
+                continue;
+            }
             m_logger->LogDebug("Attempting to find RTTI in segment %llx", segment->GetStart());
-            scan(segment);
+            try
+            {
+                scan(segment);
+            }
+            catch (std::exception &e)
+            {
+                m_logger->LogWarn("Unhandled exception in segment scan %llx %s", segment->GetStart(), e.what());
+            }
         }
         else if (checkWritableRData && rdataSection && rdataSection->GetStart() == segment->GetStart())
         {
             m_logger->LogDebug("Attempting to find RTTI in writable rdata segment %llx",
                                segment->GetStart());
-            scan(segment);
+            try
+            {
+                scan(segment);
+            }
+            catch (std::exception &e)
+            {
+                m_logger->LogWarn("Unhandled exception in writable segment scan %llx %s", segment->GetStart(), e.what());
+            }
         }
     }
 
@@ -757,13 +777,27 @@ void MicrosoftRTTIProcessor::ProcessVFT()
             if (segment->GetFlags() == (SegmentReadable | SegmentContainsData))
             {
                 m_logger->LogDebug("Attempting to find VirtualFunctionTables in segment %llx", segment->GetStart());
-                scan(segment);
+                try
+                {
+                    scan(segment);
+                }
+                catch (std::exception &e)
+                {
+                    m_logger->LogWarn("Unhandled exception in vtable segment scan %llx %s", segment->GetStart(), e.what());
+                }
             }
             else if (checkWritableRData && rdataSection && rdataSection->GetStart() == segment->GetStart())
             {
                 m_logger->LogDebug("Attempting to find VirtualFunctionTables in writable rdata segment %llx",
                                    segment->GetStart());
-                scan(segment);
+                try
+                {
+                    scan(segment);
+                }
+                catch (std::exception &e)
+                {
+                    m_logger->LogWarn("Unhandled exception in vtable writable segment scan %llx %s", segment->GetStart(), e.what());
+                }
             }
         }
     }