Remove unused type annotations (microsoft#4689)

Types no longer in use, potentially due to type translation, are still kept
around just because they are referenced in the type annotation metadata.

This change adds a method DxilModule::RemoveUnusedTypeAnnotations() that
removes any type annotations not used by resources or function arguments
or return types.  Types outside this set should no longer need any type
annotations.  This will get rid of extra unused types often left over in
final dxil before.

Author: tex3d

include/dxc/DXIL/DxilModule.h

@@ -202,6 +202,7 @@ class DxilModule {
 
   bool StripReflection();
   void StripDebugRelatedCode();
+  void RemoveUnusedTypeAnnotations();
 
   // Helper to remove dx.* metadata with source and compile options.
   // If the parameter `bReplaceWithDummyData` is true, the named metadata

lib/DXIL/DxilModule.cpp

@@ -1824,6 +1824,55 @@ bool DxilModule::StripReflection() {
   return bChanged;
 }
 
+static void RemoveTypesFromSet(Type *Ty, SetVector<const StructType*> &typeSet) {
+  if (Ty->isPointerTy())
+    Ty = Ty->getPointerElementType();
+  while (Ty->isArrayTy())
+    Ty = Ty->getArrayElementType();
+  if (StructType *ST = dyn_cast<StructType>(Ty)) {
+    if (typeSet.count(ST)) {
+      typeSet.remove(ST);
+      for (unsigned i = 0; i < ST->getNumElements(); i++) {
+        RemoveTypesFromSet(ST->getElementType(i), typeSet);
+      }
+    }
+  }
+}
+
+template <typename TResource>
+static void
+RemoveUsedTypesFromSet(std::vector<std::unique_ptr<TResource>> &vec, SetVector<const StructType*> &typeSet) {
+  for (auto &p : vec) {
+    RemoveTypesFromSet(p->GetHLSLType(), typeSet);
+  }
+}
+
+void DxilModule::RemoveUnusedTypeAnnotations() {
+  // Collect annotated types
+  const DxilTypeSystem::StructAnnotationMap &SAMap = m_pTypeSystem->GetStructAnnotationMap();
+  SetVector<const StructType*> types;
+  for (const auto &it : SAMap)
+    types.insert(it.first);
+
+  // Iterate resource types and remove any HLSL types from set
+  RemoveUsedTypesFromSet(m_CBuffers, types);
+  RemoveUsedTypesFromSet(m_UAVs, types);
+  RemoveUsedTypesFromSet(m_SRVs, types);
+
+  // Iterate Function parameters and return types, removing any HLSL types found from set
+  for (Function &F : m_pModule->functions()) {
+    FunctionType *FT = F.getFunctionType();
+    RemoveTypesFromSet(FT->getReturnType(), types);
+    for (Type *PTy : FT->params())
+      RemoveTypesFromSet(PTy, types);
+  }
+
+  // Remove remaining set of types
+  for (const StructType *ST : types)
+    m_pTypeSystem->EraseStructAnnotation(ST);
+}
+
+
 void DxilModule::LoadDxilResources(const llvm::MDOperand &MDO) {
   if (MDO.get() == nullptr)
     return;

lib/HLSL/DxilPreparePasses.cpp

@@ -836,6 +836,9 @@ class DxilFinalizeModule : public ModulePass {
       // Strip parameters of entry function.
       StripEntryParameters(M, DM, IsLib);
 
+      // Remove unused types from type annotations
+      DM.RemoveUnusedTypeAnnotations();
+
       // Update flags to reflect any changes.
       DM.CollectShaderFlagsForModule();
 

tools/clang/test/HLSLFileCheck/hlsl/types/enum/enum6.hlsl

@@ -1,5 +1,4 @@
 // RUN: %dxc -E main -T vs_6_0 -HV 2017 %s | FileCheck %s
-// CHECK: %struct.PSInput = type { <4 x float>, <4 x float>, i16 }
 // CHECK: call void @dx.op.storeOutput.i16(i32 5, i32 2, i32 0, i8 0, i16 3)
 
 enum class E : min16int {

tools/clang/test/HLSLFileCheck/shader_targets/mesh/mesh.hlsl

@@ -9,7 +9,7 @@
 // CHECK: dx.op.emitIndices(i32 169,
 // CHECK: dx.op.storePrimitiveOutput.f32(i32 172,
 // CHECK: dx.op.storeVertexOutput.f32(i32 171,
-// CHECK: !"cullPrimitive", i32 3, i32 100, i32 4, !"SV_CullPrimitive", i32 7, i32 1}
+// CHECK: !{i32 5, !"SV_CullPrimitive", i8 1, i8 30, !{{[0-9]+}}, i8 1, i32 1, i8 1, i32 -1, i8 -1, !{{[0-9]+}}}
 
 #define MAX_VERT 32
 #define MAX_PRIM 16