Skip to content

Commit b8b59c4

Browse files
mikeashmikeash
committed
[ExternalGenericMetadataBuilder] Add a test that validates built metadata against the runtime.
We run the builder, then use a small program that converts the JSON output into C code that generates the data. Compile that into a bundle, then load it as the prespecializations library. Then scan all the entries in the table and compare them with what the runtime builds dynamically.
1 parent 7b6f45c commit b8b59c4

File tree

9 files changed

+473
-25
lines changed

9 files changed

+473
-25
lines changed

‎include/swift/Runtime/LibPrespecialized.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,17 @@ Metadata *getLibPrespecializedMetadata(const TypeContextDescriptor *description,
5151

5252
} // namespace swift
5353

54+
// Validate the prespecialized metadata map by building each entry dynamically
55+
// and comparing. This should be called before any metadata is built for other
56+
// purposes, as any prespecialized entries that have already been cached will
57+
// not be rebuilt, so the validation will be comparing the prespecialized
58+
// metadata with itself.
59+
//
60+
// On return, outValidated is set to the total number of metadata records that
61+
// were validated (which is the total number in the table), and outFailed is set
62+
// to the number that failed validation.
63+
SWIFT_RUNTIME_EXPORT
64+
void _swift_validatePrespecializedMetadata(unsigned *outValidated,
65+
unsigned *outFailed);
66+
5467
#endif // SWIFT_LIB_PRESPECIALIZED_H

‎stdlib/public/runtime/GenericMetadataBuilder.cpp

Lines changed: 137 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "MetadataCache.h"
1919
#include "Private.h"
2020
#include "swift/ABI/Metadata.h"
21+
#include "swift/ABI/MetadataValues.h"
2122
#include "swift/ABI/TargetLayout.h"
2223
#include "swift/Runtime/EnvironmentVariables.h"
2324
#include "swift/Runtime/Metadata.h"
@@ -391,6 +392,141 @@ static bool equalVWTs(const ValueWitnessTable *a, const ValueWitnessTable *b) {
391392
return false;
392393
}
393394

395+
bool swift::compareGenericMetadata(const Metadata *original,
396+
const Metadata *newMetadata) {
397+
if (original == newMetadata)
398+
return true;
399+
400+
bool equal = true;
401+
402+
if (original->getKind() != newMetadata->getKind()) {
403+
validationLog(true, "Kinds do not match");
404+
equal = false;
405+
} else {
406+
auto originalDescriptor = original->getTypeContextDescriptor();
407+
auto newDescriptor = newMetadata->getTypeContextDescriptor();
408+
409+
if (originalDescriptor != newDescriptor) {
410+
validationLog(true, "Descriptors do not match");
411+
equal = false;
412+
} else if (!originalDescriptor->isGeneric()) {
413+
validationLog(true,
414+
"Descriptor is not generic and pointers are not identical");
415+
equal = false;
416+
} else {
417+
auto origVWT = asFullMetadata(original)->ValueWitnesses;
418+
auto newVWT = asFullMetadata(newMetadata)->ValueWitnesses;
419+
420+
if (!equalVWTs(origVWT, newVWT)) {
421+
validationLog(true, "VWTs do not match");
422+
equal = false;
423+
}
424+
425+
size_t originalSize = 0;
426+
size_t newSize = 0;
427+
428+
// Find the range of the generic arguments. They can't be compared with
429+
// bytewise equality.
430+
auto &genericContextHeader =
431+
originalDescriptor->getGenericContextHeader();
432+
auto *genericArgumentsPtr =
433+
originalDescriptor->getGenericArguments(original);
434+
uintptr_t genericArgumentsStart =
435+
(uintptr_t)genericArgumentsPtr - (uintptr_t)original;
436+
uintptr_t genericArgumentsEnd =
437+
genericArgumentsStart +
438+
genericContextHeader.NumKeyArguments * sizeof(void *);
439+
if (original->getKind() == MetadataKind::Class) {
440+
originalSize =
441+
reinterpret_cast<const ClassMetadata *>(original)->getSizeInWords();
442+
newSize = reinterpret_cast<const ClassMetadata *>(newMetadata)
443+
->getSizeInWords();
444+
} else {
445+
// Sizes are at least equal to genericArgumentsEnd.
446+
originalSize = newSize = genericArgumentsEnd;
447+
448+
if (original->getKind() == MetadataKind::Struct) {
449+
// If they're structs, try the trailing flags or field offsets.
450+
auto getSize = [](const Metadata *metadata, size_t previous) {
451+
auto *structMetadata =
452+
reinterpret_cast<const StructMetadata *>(metadata);
453+
454+
const void *end;
455+
if (auto *flags = structMetadata->getTrailingFlags())
456+
end = &flags[1];
457+
else if (auto *fieldOffsets = structMetadata->getFieldOffsets())
458+
end = &fieldOffsets[structMetadata->getDescription()->NumFields];
459+
else
460+
return previous;
461+
462+
return (uintptr_t)end - (uintptr_t)metadata;
463+
};
464+
465+
originalSize = getSize(original, originalSize);
466+
newSize = getSize(newMetadata, newSize);
467+
} else if (original->getKind() == MetadataKind::Enum) {
468+
// If they're enums, try the trailing flags.
469+
auto getSize = [](const Metadata *metadata, size_t previous) {
470+
auto *enumMetadata =
471+
reinterpret_cast<const EnumMetadata *>(metadata);
472+
473+
if (auto *flags = enumMetadata->getTrailingFlags())
474+
return (uintptr_t)&flags[1] - (uintptr_t)metadata;
475+
return previous;
476+
};
477+
478+
originalSize = getSize(original, originalSize);
479+
newSize = getSize(newMetadata, newSize);
480+
}
481+
}
482+
483+
if (originalSize != newSize) {
484+
validationLog(true, "Sizes do not match");
485+
equal = false;
486+
}
487+
488+
if (memcmp(original, newMetadata, genericArgumentsStart)) {
489+
validationLog(
490+
true,
491+
"Metadatas do not match in the part before generic arguments");
492+
equal = false;
493+
}
494+
495+
for (unsigned i = 0; i < genericContextHeader.NumKeyArguments; i++) {
496+
auto *originalArg =
497+
originalDescriptor->getGenericArguments(original)[i];
498+
auto *newArg = newDescriptor->getGenericArguments(newMetadata)[i];
499+
if (compareGenericMetadata(originalArg, newArg))
500+
continue;
501+
validationLog(true, "Generic argument %u does not match", i);
502+
equal = false;
503+
}
504+
505+
if (genericArgumentsEnd < originalSize) {
506+
if (memcmp((const char *)original + genericArgumentsEnd,
507+
(const char *)newMetadata + genericArgumentsEnd,
508+
originalSize - genericArgumentsEnd)) {
509+
validationLog(
510+
true,
511+
"Metadatas do not match in the part after generic arguments");
512+
}
513+
}
514+
}
515+
}
516+
517+
if (!equal) {
518+
validationLog(true, "Error: original and new metadata do not match!");
519+
validationLog(true, "Original metadata:");
520+
if (auto *error = dumpMetadata(original).getError())
521+
validationLog(true, "error dumping original metadata: %s", error->cStr());
522+
validationLog(true, "New metadata builder:");
523+
if (auto *error = dumpMetadata(newMetadata).getError())
524+
validationLog(true, "error dumping new metadata: %s", error->cStr());
525+
}
526+
527+
return equal;
528+
}
529+
394530
void swift::validateExternalGenericMetadataBuilder(
395531
const Metadata *original, const TypeContextDescriptor *description,
396532
const void * const *arguments) {
@@ -422,32 +558,8 @@ void swift::validateExternalGenericMetadataBuilder(
422558
if (!success)
423559
return;
424560

425-
auto origVWT = asFullMetadata(original)->ValueWitnesses;
426-
auto newVWT = asFullMetadata(newMetadata)->ValueWitnesses;
427-
428-
bool equal = true;
429-
if (!equalVWTs(origVWT, newVWT)) {
430-
validationLog(true, "VWTs do not match");
431-
equal = false;
432-
}
433-
size_t totalSize = sizeof(ValueMetadata) + *extraDataSize.getValue();
434-
if (memcmp(original, newMetadata, totalSize)) {
435-
validationLog(true, "Metadatas do not match");
436-
equal = false;
437-
}
438-
439-
if (!equal) {
440-
validationLog(true,
441-
"Error! Mismatch between new/old metadata builders!");
442-
validationLog(true, "Original metadata:");
443-
if (auto *error = dumpMetadata(original).getError())
444-
validationLog(true, "error dumping original metadata: %s",
445-
error->cStr());
446-
validationLog(true, "New metadata builder:");
447-
if (auto *error = dumpMetadata(newMetadata).getError())
448-
validationLog(true, "error dumping new metadata: %s", error->cStr());
561+
if (!compareGenericMetadata(original, newMetadata))
449562
swift::fatalError(0, "Fatal error: mismatched metadata.\n");
450-
}
451563

452564
auto typeName = swift_getTypeName(original, false);
453565
validationLog(false, "Validated generic metadata builder on %.*s",

‎stdlib/public/runtime/LibPrespecialized.cpp

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,14 @@ const LibPrespecializedData<InProcess> *swift::getLibPrespecializedData() {
6969
return SWIFT_LAZY_CONSTANT(findLibPrespecialized());
7070
}
7171

72+
static bool disableForValidation = false;
73+
7274
Metadata *
7375
swift::getLibPrespecializedMetadata(const TypeContextDescriptor *description,
7476
const void *const *arguments) {
77+
if (disableForValidation)
78+
return nullptr;
79+
7580
auto *data = getLibPrespecializedData();
7681
if (!data)
7782
return nullptr;
@@ -108,3 +113,50 @@ swift::getLibPrespecializedMetadata(const TypeContextDescriptor *description,
108113
result, (int)key.size(), key.data());
109114
return result;
110115
}
116+
117+
void _swift_validatePrespecializedMetadata(unsigned *outValidated,
118+
unsigned *outFailed) {
119+
if (outValidated)
120+
*outValidated = 0;
121+
if (outFailed)
122+
*outFailed = 0;
123+
124+
auto *data = getLibPrespecializedData();
125+
if (!data) {
126+
return;
127+
}
128+
129+
disableForValidation = true;
130+
131+
auto *metadataMap = data->getMetadataMap();
132+
auto metadataMapSize = metadataMap->arraySize;
133+
auto *array = metadataMap->array();
134+
for (uint64_t i = 0; i < metadataMapSize; i++) {
135+
auto &element = array[i];
136+
if (!element.key || !element.value)
137+
continue;
138+
139+
if (outValidated)
140+
(*outValidated)++;
141+
142+
const char *mangledName = element.key;
143+
// Skip the leading $.
144+
if (mangledName[0] == '$')
145+
mangledName++;
146+
147+
auto result = swift_getTypeByMangledName(MetadataState::Complete,
148+
mangledName, nullptr, {}, {});
149+
if (auto *error = result.getError()) {
150+
fprintf(stderr,
151+
"Prespecializations library validation: unable to build metadata "
152+
"for mangled name '%s'\n",
153+
mangledName);
154+
if (outFailed)
155+
(*outFailed)++;
156+
}
157+
158+
if (!compareGenericMetadata(result.getType().getMetadata(), element.value))
159+
if (outFailed)
160+
(*outFailed)++;
161+
}
162+
}

‎stdlib/public/runtime/Private.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,12 @@ class TypeInfo {
591591
SWIFT_RETURNS_NONNULL SWIFT_NODISCARD
592592
void *allocateMetadata(size_t size, size_t align);
593593

594+
// Compare two pieces of metadata that should be identical. Returns true if
595+
// they are, false if they are not equal. Dumps the metadata contents to
596+
// stderr if they are not equal.
597+
bool compareGenericMetadata(const Metadata *original,
598+
const Metadata *newMetadata);
599+
594600
void validateExternalGenericMetadataBuilder(
595601
const Metadata *original, const TypeContextDescriptor *description,
596602
const void * const *arguments);

‎test/ExternalGenericMetadataBuilder/Inputs/buildMetadataJSON.swift

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import ExternalGenericMetadataBuilder
2+
import Foundation
3+
4+
var args = CommandLine.arguments
5+
args.removeFirst() // Skip the program name.
6+
7+
let arch = args.removeFirst()
8+
let dylibs = args
9+
10+
let builder = swift_externalMetadataBuilder_create(1, arch)
11+
12+
guard var jsonIn = try! FileHandle.standardInput.readToEnd() else {
13+
fatalError("No data read from stdin, somehow")
14+
}
15+
16+
// NULL terminate.
17+
jsonIn.append(contentsOf: [0])
18+
19+
let readJSONErrorCStr = jsonIn.withUnsafeBytes {
20+
swift_externalMetadataBuilder_readNamesJSON(builder, $0)
21+
}
22+
23+
if let readJSONErrorCStr {
24+
fatalError("Could not read JSON input: \(String(cString: readJSONErrorCStr))")
25+
}
26+
27+
for dylib in dylibs {
28+
let url = URL(fileURLWithPath: dylib)
29+
let data = NSData(contentsOf: url)!
30+
31+
let machHeader = data.bytes.assumingMemoryBound(to: mach_header.self)
32+
let addDylibErrorCStr =
33+
swift_externalMetadataBuilder_addDylib(builder,
34+
url.lastPathComponent,
35+
machHeader,
36+
UInt64(data.length));
37+
if let addDylibErrorCStr {
38+
fatalError("Could not add dylib at \(dylib): \(String(cString: addDylibErrorCStr))")
39+
}
40+
}
41+
42+
let buildErrorCStr = swift_externalMetadataBuilder_buildMetadata(builder)
43+
if let buildErrorCStr {
44+
fatalError("Failed to build metadata: \(String(cString: buildErrorCStr))")
45+
}
46+
47+
let outputJSONCStr = swift_externalMetadataBuilder_getMetadataJSON(builder)
48+
if outputJSONCStr == nil {
49+
fatalError("JSON creation failed")
50+
}
51+
52+
fputs(outputJSONCStr, stdout)

0 commit comments

Comments
 (0)