Fix #3108: illegal nested classes in enums throw off EnumValueDisplayMode handling.

Author: siegfriedpammer

ICSharpCode.Decompiler.Tests/TestCases/ILPretty/WeirdEnums.cs

@@ -10,6 +10,19 @@ public enum BooleanEnum : bool
 		Max = byte.MaxValue
 	}
 
+	public enum EnumWithNestedClass
+	{
+#pragma warning disable format
+		// error: nested types are not permitted in C#.
+		public class NestedClass
+		{
+		}
+		,
+#pragma warning enable format
+		Zero,
+		One
+	}
+
 	public enum NativeIntEnum : IntPtr
 	{
 		Zero = 0L,

ICSharpCode.Decompiler.Tests/TestCases/ILPretty/WeirdEnums.il

@@ -34,3 +34,33 @@
 	.field public static literal valuetype TestEnum.NativeIntEnum One = int64(1)
 	.field public static literal valuetype TestEnum.NativeIntEnum FortyTwo = int64(42)
 }
+
+.class nested public auto ansi sealed TestEnum.EnumWithNestedClass
+    extends [System.Runtime]System.Enum
+{
+        // Nested Types
+    .class nested public auto ansi beforefieldinit NestedClass
+        extends [mscorlib]System.Object
+    {
+        // Methods
+        .method public hidebysig specialname rtspecialname 
+            instance void .ctor () cil managed 
+        {
+            // Method begins at RVA 0x206c
+            // Code size 8 (0x8)
+            .maxstack 8
+
+            IL_0000: ldarg.0
+            IL_0001: call instance void [mscorlib]System.Object::.ctor()
+            IL_0006: nop
+            IL_0007: ret
+        } // end of method TestEnum.NestedClass::.ctor
+
+    } // end of class NestedClass
+
+    // Fields
+    .field public specialname rtspecialname int32 value__
+    .field public static literal valuetype TestEnum.EnumWithNestedClass Zero = int32(0)
+    .field public static literal valuetype TestEnum.EnumWithNestedClass One = int32(1)
+    
+} // end of class EnumWithNestedClass

ICSharpCode.Decompiler/CSharp/CSharpDecompiler.cs

@@ -1339,10 +1339,6 @@ EntityDeclaration DoDecompile(ITypeDefinition typeDef, DecompileRun decompileRun
 					}
 				}
 
-				decompileRun.EnumValueDisplayMode = typeDef.Kind == TypeKind.Enum
-					? DetectBestEnumValueDisplayMode(typeDef, module.PEFile)
-					: null;
-
 				// With C# 9 records, the relative order of fields and properties matters:
 				IEnumerable<IMember> fieldsAndProperties = recordDecompiler?.FieldsAndProperties
 					?? typeDef.Fields.Concat<IMember>(typeDef.Properties);
@@ -1406,7 +1402,9 @@ EntityDeclaration DoDecompile(ITypeDefinition typeDef, DecompileRun decompileRun
 				}
 				if (typeDecl.ClassType == ClassType.Enum)
 				{
-					switch (decompileRun.EnumValueDisplayMode)
+					Debug.Assert(typeDef.Kind == TypeKind.Enum);
+					EnumValueDisplayMode displayMode = DetectBestEnumValueDisplayMode(typeDef, module.PEFile);
+					switch (displayMode)
 					{
 						case EnumValueDisplayMode.FirstOnly:
 							foreach (var enumMember in typeDecl.Members.OfType<EnumMemberDeclaration>().Skip(1))
@@ -1425,13 +1423,33 @@ EntityDeclaration DoDecompile(ITypeDefinition typeDef, DecompileRun decompileRun
 							}
 							break;
 						case EnumValueDisplayMode.All:
-						case EnumValueDisplayMode.AllHex:
 							// nothing needs to be changed.
 							break;
+						case EnumValueDisplayMode.AllHex:
+							foreach (var enumMember in typeDecl.Members.OfType<EnumMemberDeclaration>())
+							{
+								var constantValue = (enumMember.GetSymbol() as IField).GetConstantValue();
+								if (constantValue == null || enumMember.Initializer is not PrimitiveExpression pe)
+								{
+									continue;
+								}
+								long initValue = (long)CSharpPrimitiveCast.Cast(TypeCode.Int64, constantValue, false);
+								if (initValue >= 10)
+								{
+									pe.Format = LiteralFormat.HexadecimalNumber;
+								}
+							}
+							break;
 						default:
 							throw new ArgumentOutOfRangeException();
 					}
-					decompileRun.EnumValueDisplayMode = null;
+					foreach (var item in typeDecl.Members)
+					{
+						if (item is not EnumMemberDeclaration)
+						{
+							typeDecl.InsertChildBefore(item, new Comment(" error: nested types are not permitted in C#."), Roles.Comment);
+						}
+					}
 				}
 				return typeDecl;
 			}
@@ -1927,13 +1945,7 @@ EntityDeclaration DoDecompile(IField field, DecompileRun decompileRun, ITypeReso
 					object constantValue = field.GetConstantValue();
 					if (constantValue != null)
 					{
-						long initValue = (long)CSharpPrimitiveCast.Cast(TypeCode.Int64, constantValue, false);
 						enumDec.Initializer = typeSystemAstBuilder.ConvertConstantValue(decompilationContext.CurrentTypeDefinition.EnumUnderlyingType, constantValue);
-						if (enumDec.Initializer is PrimitiveExpression primitive
-							&& initValue >= 10 && decompileRun.EnumValueDisplayMode == EnumValueDisplayMode.AllHex)
-						{
-							primitive.Format = LiteralFormat.HexadecimalNumber;
-						}
 					}
 					enumDec.Attributes.AddRange(field.GetAttributes().Select(a => new AttributeSection(typeSystemAstBuilder.ConvertAttribute(a))));
 					enumDec.AddAnnotation(new MemberResolveResult(null, field));

ICSharpCode.Decompiler/DecompileRun.cs

@@ -1,6 +1,23 @@
-using System;
+// Copyright (c) 2018 Siegfried Pammer
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this
+// software and associated documentation files (the "Software"), to deal in the Software
+// without restriction, including without limitation the rights to use, copy, modify, merge,
+// publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+// to whom the Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all copies or
+// substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+// PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+// FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+using System;
 using System.Collections.Generic;
-using System.Text;
 using System.Threading;
 
 using ICSharpCode.Decompiler.CSharp;
@@ -49,8 +66,6 @@ CSharp.TypeSystem.UsingScope CreateUsingScope(HashSet<string> requiredNamespaces
 			}
 			return usingScope;
 		}
-
-		public EnumValueDisplayMode? EnumValueDisplayMode { get; set; }
 	}
 
 	enum EnumValueDisplayMode