diff --git a/Editor/AssetEditors/EditorHeaderHook.cs b/Editor/AssetEditors/EditorHeaderHook.cs
new file mode 100644
index 0000000..8ef480b
--- /dev/null
+++ b/Editor/AssetEditors/EditorHeaderHook.cs
@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using UnityEngine;
+using UnityEditor;
+using Object = UnityEngine.Object;
+
+[InitializeOnLoad]
+static class EditorHeaderHook
+{
+    private static Dictionary<Editor, (Object[] targets, MetadataEditor editor)> _targetsCache = new();
+    
+    static EditorHeaderHook()
+    {
+        Editor.finishedDefaultHeaderGUI += DisplayMetadata;
+    }
+
+    static void DisplayMetadata(Editor editor)
+    {
+        if (!EditorUtility.IsPersistent(editor.target))
+            return;
+        if (!GUI.enabled)
+            return;
+
+        if (_targetsCache.TryGetValue(editor, out var cache) == false || Equality(cache.targets, editor.targets) == false)
+        {
+            cache.editor?.Dispose();
+            _targetsCache[editor] = cache = (editor.targets, new MetadataEditor(editor.targets));
+        }
+
+        cache.editor.OnInspectorGUI();
+        
+        static bool Equality<T>(T[] a, T[] b)
+        {
+            if (a.Length != b.Length)
+                return false;
+
+            for (int i = 0; i < a.Length; i++)
+            {
+                if (ReferenceEquals(a[i], b[i]) == false)
+                    return false;
+            }
+
+            return true;
+        }
+    }
+}
\ No newline at end of file
diff --git a/Editor/AssetEditors/MaterialMetadataEditor.cs.meta b/Editor/AssetEditors/EditorHeaderHook.cs.meta
similarity index 100%
rename from Editor/AssetEditors/MaterialMetadataEditor.cs.meta
rename to Editor/AssetEditors/EditorHeaderHook.cs.meta
diff --git a/Editor/AssetEditors/MaterialMetadataEditor.cs b/Editor/AssetEditors/MaterialMetadataEditor.cs
deleted file mode 100644
index a2f63fa..0000000
--- a/Editor/AssetEditors/MaterialMetadataEditor.cs
+++ /dev/null
@@ -1,17 +0,0 @@
-using UnityEngine;
-using UnityEditor;
-
-[CustomEditor(typeof(Material)), CanEditMultipleObjects]
-public class MaterialMetadataEditor : MaterialEditor
-{
-    MetadataEditor metadataEditor;
-    public override void OnEnable() { base.OnEnable(); metadataEditor = new MetadataEditor(targets); }
-    public override void OnDisable() { base.OnDisable(); metadataEditor.Dispose(); metadataEditor = null; }
-    public override void OnInspectorGUI() 
-    { 
-        base.OnInspectorGUI();
-        if (!GUI.enabled)
-            return;
-        metadataEditor.OnInspectorGUI();
-    }
-}
diff --git a/Editor/Common/AssetMetadataUtility.cs b/Editor/Common/AssetMetadataUtility.cs
index 111fd2e..6f1d9d4 100644
--- a/Editor/Common/AssetMetadataUtility.cs
+++ b/Editor/Common/AssetMetadataUtility.cs
@@ -116,18 +116,30 @@ public static void GetAll(UnityEngine.Object target, List<CustomAssetMetadata> m
 		if (metadata == null) throw new NullReferenceException(nameof(metadata));
 		EnsureInitialized();
 		var assetPath = AssetDatabase.GetAssetPath(target);
-        if (assetPath == null ||
-            string.IsNullOrEmpty(assetPath))
+        if (assetPath == null || string.IsNullOrEmpty(assetPath))
             return;
 
-        var assets = AssetDatabase.LoadAllAssetsAtPath(assetPath);
-        foreach (var asset in assets)
+        if (target is SceneAsset)
         {
-            if (asset is CustomAssetMetadata additionalDataAsset)
+            // calling LoadAllAssetsAtPath with a scene throws, this doesn't.
+            // Still, right now scenes do not support adding metadata so this cannot be validated further
+            foreach (var allMetadataType in AllMetadataTypes)
             {
+                var additionalDataAsset = (CustomAssetMetadata)AssetDatabase.LoadAssetAtPath(assetPath, allMetadataType);
                 metadata.Add(additionalDataAsset);
             }
         }
+        else
+        {
+            var assets = AssetDatabase.LoadAllAssetsAtPath(assetPath);
+            foreach (var asset in assets)
+            {
+                if (asset is CustomAssetMetadata additionalDataAsset)
+                {
+                    metadata.Add(additionalDataAsset);
+                }
+            }
+        }
     }
 
     public static CustomAssetMetadata Add(UnityEngine.Object target, Type type)
diff --git a/Editor/Common/MetadataEditorInstance.cs b/Editor/Common/MetadataEditorInstance.cs
index 4cebd33..d98f490 100644
--- a/Editor/Common/MetadataEditorInstance.cs
+++ b/Editor/Common/MetadataEditorInstance.cs
@@ -260,11 +260,8 @@ static MetadataEditorInstance[] CreateEditors(UnityEngine.Object[] targets)
         return metadataEditors;
     }
 
-    static GUILayoutOption[] addMetadataButtonOptions = {
-		GUILayout.Width(230), GUILayout.Height(24), 
-        GUILayout.ExpandWidth(false)
-	};
-
+    private static GUIContent addMetadataButton = new GUIContent("+", "Add Metadata");
+    
     public void OnInspectorGUI()
     {
         using (new EditorGUI.DisabledScope(!canOpenForEdit))
@@ -283,18 +280,7 @@ public void OnInspectorGUI()
             // TODO: make it possible to re-order metadata like Components on GameObjects
 
             GUILayout.BeginVertical();
-            EditorGUILayout.Space();
-            // We seem to get the metadata in reverse order from the assetdatabase, compared to the order we add them
-            // So to make things feel more consistent, we reverse the order
-            for (int i = metadataEditors.Length - 1; i >= 0; i--)
-            {
-                metadataEditors[i].OnInspectorGUI();
-            }
 
-            EditorGUILayout.Space();
-
-			GUILayout.BeginHorizontal();
-			GUILayout.FlexibleSpace();
             List<Type> filteredMetadata = ListPool<Type>.Get();
             try
             {
@@ -311,7 +297,11 @@ public void OnInspectorGUI()
                 }
                 using (new EditorGUI.DisabledScope(!canAddMetadata))
                 {
-                    if (GUILayout.Button("Add Metadata", addMetadataButtonOptions))
+                    // Embed the button inside the previous rect
+                    var rect = GUILayoutUtility.GetRect(0,0,0,0);
+                    rect.width = rect.height = EditorGUIUtility.singleLineHeight;
+                    rect.y -= rect.height;
+                    if (GUI.Button(rect, addMetadataButton))
                     {
                         // TODO: have a nicer dropdownmenu, more like the "add components" menu
                         var menu = new GenericMenu();
@@ -336,10 +326,18 @@ public void OnInspectorGUI()
             {
                 ListPool<Type>.Release(filteredMetadata);
             }
-			GUILayout.FlexibleSpace();
-			GUILayout.EndHorizontal();
 
-			GUILayout.EndVertical();
+            if (metadataEditors.Length > 0)
+            {
+                // We seem to get the metadata in reverse order from the assetdatabase, compared to the order we add them
+                // So to make things feel more consistent, we reverse the order
+                for (int i = metadataEditors.Length - 1; i >= 0; i--)
+                {
+                    metadataEditors[i].OnInspectorGUI();
+                }
+            }
+
+            GUILayout.EndVertical();
         }
     }
 }
\ No newline at end of file