Implement Container V2 for fbstring

Author: JakeHillion

.circleci/config.yml

@@ -61,6 +61,8 @@ workflows:
           name: test-clang
           requires:
             - build-clang
+          # Tests disabled due to bad DWARF generated by the old clang compiler in CI
+          exclude_regex: "OilIntegration.fbstring_.*"
       - test:
           name: test-type-graph-clang
           requires:

oi/OITraceCode.cpp

@@ -81,6 +81,9 @@ class {
       index = (index + 1) % data.size();
     }
   }
+  bool add(const auto* p) {
+    return add((uintptr_t)p);
+  }
 } static pointers;
 
 }  // namespace
@@ -145,3 +148,8 @@ struct validate_offset {
 };
 
 enum class StubbedPointer : uintptr_t {};
+
+bool isStorageInline(const auto& c) {
+  return (uintptr_t)std::data(c) < (uintptr_t)(&c + sizeof(c)) &&
+         (uintptr_t)std::data(c) >= (uintptr_t)&c;
+}

test/integration/fbstring.toml

@@ -1,7 +1,6 @@
-includes = ["folly/FBString.h"]
+includes = ["folly/FBString.h", "utility"]
 [cases]
   [cases.empty]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322
     param_types = ["folly::fbstring&"]
     setup = "return {};"
     expect_json = '''
@@ -18,16 +17,22 @@ includes = ["folly/FBString.h"]
             "dynamicSize": 0,
             "exclusiveSize": 24,
             "length": 0,
-            "capacity": 23,
+            "capacity":23,
             "elementStaticSize": 1
           }
         ]
       }
     ]
     '''
+    expect_json_v2 = '''[{
+      "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
+      "staticSize": 24,
+      "exclusiveSize": 24,
+      "length": 0,
+      "capacity": 23
+    }]'''
 
   [cases.inline]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322
     param_types = ["folly::fbstring&"]
     setup = 'return {"012345"};'
     expect_json = '''
@@ -51,9 +56,15 @@ includes = ["folly/FBString.h"]
       }
     ]
     '''
+    expect_json_v2 = '''[{
+      "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
+      "staticSize": 24,
+      "exclusiveSize": 24,
+      "length": 6,
+      "capacity": 23
+    }]'''
 
   [cases.heap_allocated]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322
     param_types = ["folly::fbstring&"]
     setup = 'return {"abcdefghijklmnopqrstuvwxzy"};'
     expect_json = '''
@@ -77,9 +88,15 @@ includes = ["folly/FBString.h"]
       }
     ]
     '''
+    expect_json_v2 = '''[{
+      "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
+      "staticSize": 24,
+      "exclusiveSize": 50,
+      "length": 26,
+      "capacity": 26
+    }]'''
 
-  [cases.string_pooled]
-    oil_skip = 'not implemented for treebuilder v2' # https://github.com/facebookexperimental/object-introspection/issues/322
+  [cases.string_pooled_unique]
     param_types = ["folly::fbstring&"]
     setup = "return folly::fbstring(1024, 'c');"
     expect_json = '''
@@ -103,3 +120,36 @@ includes = ["folly/FBString.h"]
       }
     ]
     '''
+    expect_json_v2 = '''[{
+      "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
+      "staticSize": 24,
+      "exclusiveSize": 1056,
+      "length": 1024,
+      "capacity": 1024
+    }]'''
+  
+  [cases.string_pooled_shared]
+    param_types = ["std::pair<folly::fbstring, folly::fbstring>&"]
+    setup = """
+      folly::fbstring s(1024, 'c');
+      return {{s, s}};
+    """
+    expect_json_v2 = '''[{
+      "staticSize": 48,
+      "exclusiveSize": 0,
+      "members": [
+        {
+          "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
+          "staticSize": 24,
+          "exclusiveSize": 1056,
+          "length": 1024,
+          "capacity": 1024
+        }, {
+          "typeNames": ["folly::basic_fbstring<int8_t, std::char_traits<int8_t>, std::allocator<int8_t>, folly::fbstring_core<int8_t>>"],
+          "staticSize": 24,
+          "exclusiveSize": 24,
+          "length": 1024,
+          "capacity": 1024
+        }
+      ]
+    }]'''

types/fb_string_type.toml

@@ -69,3 +69,77 @@ struct TypeHandler<DB, %1%<T0, T1, T2, T3>> {
   }
 };
 """
+
+traversal_func = """
+// fbstring has inlining (SSO) and allocates large strings as
+// reference counted strings. Reference counted strings have an
+// overhead of a single std::atomic<size_t> at the beginning. To
+// correctly attribute the size in the processor we need the
+// following 4 categories as well as the usual metadata.
+
+enum class Category : uint8_t {
+  InlinedStorage,
+  OwnedHeapStorage,
+  ReferenceCountedStorage,
+  AlreadyAttributed,
+};
+
+constexpr static size_t minLargeSize = 255;
+size_t capacity = container.capacity();
+
+Category category;
+if (isStorageInline(container)) {
+  category = Category::InlinedStorage;
+} else if (capacity < minLargeSize) {
+  category = Category::OwnedHeapStorage;
+} else if (pointers.add(container.data())) {
+  category = Category::ReferenceCountedStorage;
+} else {
+  category = Category::AlreadyAttributed;
+}
+
+return returnArg.write((uintptr_t)container.data())
+  .write(capacity)
+  .write(container.size())
+  .write(static_cast<std::underlying_type_t<Category>>(category));
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.pointer = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+uint64_t capacity = std::get<ParsedData::VarInt>(d.val).value;
+el.container_stats.emplace(result::Element::ContainerStats { .capacity = capacity });
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+el.container_stats->length = std::get<ParsedData::VarInt>(d.val).value;
+"""
+
+[[codegen.processor]]
+type = "types::st::VarInt<DB>"
+func = """
+using CharType = T0;
+
+enum class Category : uint8_t {
+  InlinedStorage,
+  OwnedHeapStorage,
+  ReferenceCountedStorage,
+  AlreadyAttributed,
+};
+
+auto category = static_cast<Category>(std::get<ParsedData::VarInt>(d.val).value);
+if (category == Category::InlinedStorage || category == Category::AlreadyAttributed)
+  return;
+
+el.exclusive_size += el.container_stats->capacity * sizeof(CharType);
+if (category == Category::ReferenceCountedStorage)
+  el.exclusive_size += 8; // 8 bytes for std::atomic<size_t>
+"""