Generalize enum conversion logic - remove operation-specific special cases

Co-authored-by: hazzik <144791+hazzik@users.noreply.github.com>

Author: Copilot

src/DelegateDecompiler/Processor.cs

@@ -229,77 +229,18 @@ internal static BinaryExpression MakeBinaryExpression(Address left, Address righ
             left = AdjustBooleanConstant(left, rightType);
             right = AdjustBooleanConstant(right, leftType);
 
-            // For comparison, bitwise, and arithmetic operations, convert enums to handle type mismatches
-            // This handles cases where an enum with a non-int underlying type (e.g., byte, long) is used
-            // with an int constant loaded from IL
-            if (IsComparisonOperation(expressionType) || IsBitwiseOperation(expressionType) || IsArithmeticOperation(expressionType))
-            {
-                // Determine if we're dealing with long enums
-                var leftIsLongEnum = left.Type.IsEnum && (left.Type.GetEnumUnderlyingType() == typeof(long) || left.Type.GetEnumUnderlyingType() == typeof(ulong));
-                var rightIsLongEnum = right.Type.IsEnum && (right.Type.GetEnumUnderlyingType() == typeof(long) || right.Type.GetEnumUnderlyingType() == typeof(ulong));
-                
-                var hasLongEnum = leftIsLongEnum || rightIsLongEnum;
-                
-                left = ConvertEnumExpressionToInt(left);
-                right = ConvertEnumExpressionToInt(right);
-                
-                // Ensure type compatibility after conversions
-                if (!hasLongEnum)
-                {
-                    // For byte/short enums, ensure both are int
-                    if (left.Type == typeof(byte) || left.Type == typeof(sbyte) || left.Type == typeof(short) || left.Type == typeof(ushort))
-                        left = Expression.Convert(left, typeof(int));
-                    if (right.Type == typeof(byte) || right.Type == typeof(sbyte) || right.Type == typeof(short) || right.Type == typeof(ushort))
-                        right = Expression.Convert(right, typeof(int));
-                }
-            }
-            else
-            {
-                left = ConvertEnumExpressionToUnderlyingType(left);
-                right = ConvertEnumExpressionToUnderlyingType(right);
-            }
+            // Convert enums to their appropriate type for operations
+            // This uses ConvertEnumExpressionToInt which handles:
+            // - byte/short enums -> int
+            // - long/ulong enums -> long/ulong
+            // - Optimizes Convert(intConstant, long) -> longConstant
+            // - Handles Convert(Convert(enum, byte), int) -> Convert(enum, int)
+            left = ConvertEnumExpressionToInt(left);
+            right = ConvertEnumExpressionToInt(right);
 
             return Expression.MakeBinary(expressionType, left, right);
         }
 
-        static bool IsComparisonOperation(ExpressionType expressionType)
-        {
-            return expressionType == ExpressionType.Equal ||
-                   expressionType == ExpressionType.NotEqual ||
-                   expressionType == ExpressionType.LessThan ||
-                   expressionType == ExpressionType.LessThanOrEqual ||
-                   expressionType == ExpressionType.GreaterThan ||
-                   expressionType == ExpressionType.GreaterThanOrEqual;
-        }
-
-        static bool IsBitwiseOperation(ExpressionType expressionType)
-        {
-            return expressionType == ExpressionType.And ||
-                   expressionType == ExpressionType.Or ||
-                   expressionType == ExpressionType.ExclusiveOr;
-        }
-
-        static bool IsArithmeticOperation(ExpressionType expressionType)
-        {
-            return expressionType == ExpressionType.Add ||
-                   expressionType == ExpressionType.Subtract ||
-                   expressionType == ExpressionType.Multiply ||
-                   expressionType == ExpressionType.Divide ||
-                   expressionType == ExpressionType.Modulo;
-        }
-
-        static Expression UnwrapConstantConversion(Expression expression)
-        {
-            // If this is Convert(constant, targetType), unwrap it back to the constant
-            if (expression is UnaryExpression unary &&
-                unary.NodeType == ExpressionType.Convert &&
-                unary.Operand is ConstantExpression)
-            {
-                return unary.Operand;
-            }
-            return expression;
-        }
-
         internal static Expression ConvertEnumExpressionToInt(Expression expression)
         {
             // If the expression is already an int, return as-is
@@ -331,6 +272,21 @@ internal static Expression ConvertEnumExpressionToInt(Expression expression)
                     return Expression.Convert(operand, typeof(int));
                 }
 
+                // Optimize Convert(Convert(X, byte/short), Y) when going through unnecessary intermediate conversions
+                // This happens with operations like NOT that return int but IL converts to byte before enum
+                if (operand is UnaryExpression innerUnary && innerUnary.NodeType == ExpressionType.Convert)
+                {
+                    var innerOperand = innerUnary.Operand;
+                    // If we're converting from int through byte/short, skip the intermediate conversion
+                    if (innerOperand.Type == typeof(int) &&
+                        (innerUnary.Type == typeof(byte) || innerUnary.Type == typeof(sbyte) ||
+                         innerUnary.Type == typeof(short) || innerUnary.Type == typeof(ushort)))
+                    {
+                        // Keep just the inner operand, let the outer conversion happen
+                        return Expression.Convert(innerOperand, expression.Type);
+                    }
+                }
+                
                 // Optimize Convert(intConstant, long) to a direct long constant
                 if (operand is ConstantExpression constant &&
                     constant.Type == typeof(int) &&
@@ -347,7 +303,14 @@ internal static Expression ConvertEnumExpressionToInt(Expression expression)
         internal static Expression ConvertEnumExpressionToUnderlyingType(Expression expression)
         {
             if (expression.Type.IsEnum)
-                return Expression.Convert(expression, expression.Type.GetEnumUnderlyingType());
+            {
+                var underlyingType = expression.Type.GetEnumUnderlyingType();
+                // C# promotes byte/sbyte/short/ushort enums to int for operations
+                // Only long/ulong stay as their underlying type
+                if (underlyingType == typeof(long) || underlyingType == typeof(ulong))
+                    return Expression.Convert(expression, underlyingType);
+                return Expression.Convert(expression, typeof(int));
+            }
 
             return expression;
         }

src/DelegateDecompiler/Processors/BoxProcessor.cs

@@ -32,6 +32,21 @@ static Expression Box(Expression expression, Type type)
         if (expression.Type.IsEnum)
             return Expression.Convert(expression, type);
 
+        // Optimize Convert(Convert(int, byte/short), enum) -> Convert(int, enum)
+        // This happens when operations like NOT return int, IL converts to byte, then boxes to enum
+        if (type.IsEnum && expression is UnaryExpression unary && unary.NodeType == ExpressionType.Convert)
+        {
+            var operand = unary.Operand;
+            // Skip intermediate conversions from int to byte/short when boxing to enum
+            if (operand.Type == typeof(int) &&
+                (unary.Type == typeof(byte) || unary.Type == typeof(sbyte) ||
+                 unary.Type == typeof(short) || unary.Type == typeof(ushort)))
+            {
+                // Box the int directly to the enum
+                return Expression.Convert(operand, type);
+            }
+        }
+
         return expression;
     }
 }

src/DelegateDecompiler/Processors/ConvertProcessor.cs

@@ -23,6 +23,23 @@ public static void Register(Dictionary<OpCode, IProcessor> processors)
     public void Process(ProcessorState state, Instruction instruction)
     {
         var val = state.Stack.Pop();
-        state.Stack.Push(Expression.Convert(val, targetType));
+        Expression expr = val;
+        
+        // Optimize Convert(Convert(int, byte/short), X) -> Convert(int, X)
+        // This happens when operations like NOT return int, IL converts to byte, then to enum
+        if (expr is UnaryExpression unary && unary.NodeType == ExpressionType.Convert)
+        {
+            var operand = unary.Operand;
+            // Skip intermediate conversions from int to byte/short when going to another type
+            if (operand.Type == typeof(int) &&
+                (unary.Type == typeof(byte) || unary.Type == typeof(sbyte) ||
+                 unary.Type == typeof(short) || unary.Type == typeof(ushort)))
+            {
+                // Use the int directly, skip the intermediate byte/short conversion
+                expr = operand;
+            }
+        }
+        
+        state.Stack.Push(Expression.Convert(expr, targetType));
     }
 }

src/DelegateDecompiler/Processors/ConvertTypeProcessor.cs

@@ -15,8 +15,25 @@ public static void Register(Dictionary<OpCode, IProcessor> processors)
 
         public void Process(ProcessorState state, Instruction instruction)
         {
-            var expression = state.Stack.Pop();
-            state.Stack.Push(Expression.Convert(expression, (Type)instruction.Operand));
+            var address = state.Stack.Pop();
+            Expression expression = address;
+            var targetType = (Type)instruction.Operand;
+            
+            // Optimize Convert(Convert(int, byte/short), X) -> Convert(int, X)
+            // This happens when operations like NOT return int, IL converts to byte, then casts to enum
+            if (expression is UnaryExpression unary && unary.NodeType == ExpressionType.Convert)
+            {
+                var operand = unary.Operand;
+                // Skip intermediate conversions from int to byte/short
+                if (operand.Type == typeof(int) &&
+                    (unary.Type == typeof(byte) || unary.Type == typeof(sbyte) ||
+                     unary.Type == typeof(short) || unary.Type == typeof(ushort)))
+                {
+                    expression = operand;
+                }
+            }
+            
+            state.Stack.Push(Expression.Convert(expression, targetType));
         }
     }
 }

src/DelegateDecompiler/Processors/UnaryExpressionProcessor.cs

@@ -21,7 +21,16 @@ public void Process(ProcessorState state, Instruction instruction)
 
     static UnaryExpression MakeUnaryExpression(Expression operand, ExpressionType expressionType)
     {
-        operand = Processor.ConvertEnumExpressionToUnderlyingType(operand);
+        // For bitwise NOT on enums, convert to int (for byte/short) or underlying type (for long)
+        // to match the behavior of binary operations
+        if (expressionType == ExpressionType.Not && operand.Type.IsEnum)
+        {
+            operand = Processor.ConvertEnumExpressionToInt(operand);
+        }
+        else
+        {
+            operand = Processor.ConvertEnumExpressionToUnderlyingType(operand);
+        }
 
         return Expression.MakeUnary(expressionType, operand, operand.Type);
     }