Much improved support for marshaling const char* to string

* Changed support for marshaling "const char *"s as strings so that:

1) CppSharp.Types.Std.CSharpMarshalToNative generates setters that allocate unmanaged memory for the native bytes rather than an unpinned pointer into managed memory. Also set a null termination char.
2) Add tracking for when unmanaged memory is allocated for "const char *" strings. Free it when reassigned or dispsoed.
3) Added explicit support for Encoding.Default for ANSI support.
4) Allow setting a string value to null.

* Added test to prove that the unpinned ptr to managed memory approach wasn't working, and that the new approach appears to work.

* Change CSharpSources.GenerateDisposeMethods to free unmanaged memory held by IntPtr's corresponding to "const char *" strings.

* Changed copy constructor to deep-copy owned string refs to avoid ref counting.

* Update CSharpSources.GenerateFieldGetter to treat Char16 and Char32 the same as WideChar to avoid compilation errors on the generated sources if char32_t or char16_t are used.

* Added tests.

* Workaround for mac C++ compilation issue.

Author: jpnoir

src/Generator/Generators/CSharp/CSharpSources.cs

@@ -455,6 +455,12 @@ public override bool VisitClassDecl(Class @class)
                     var dict = $@"global::System.Collections.Concurrent.ConcurrentDictionary<IntPtr, {
                         printedClass}>";
                     WriteLine("internal static readonly {0} NativeToManagedMap = new {0}();", dict);
+
+                    // Add booleans to track who owns unmanaged memory for string fields
+                    foreach (var field in @class.Layout.Fields.Where(f => f.QualifiedType.Type.IsConstCharString()))
+                    {
+                        WriteLine($"private bool __{field.Name}_OwnsNativeMemory = false;");
+                    }
                 }
                 PopBlock(NewLineKind.BeforeNextBlock);
             }
@@ -871,7 +877,7 @@ private void GenerateClassField(Field field, bool @public = false)
             PopBlock(NewLineKind.BeforeNextBlock);
         }
 
-        #endregion
+#endregion
 
         private void GeneratePropertySetter<T>(T decl,
             Class @class, bool isAbstract = false, Property property = null)
@@ -1411,13 +1417,17 @@ private void GenerateFieldGetter(Field field, Class @class, QualifiedType return
                 if (templateSubstitution != null && returnType.Type.IsDependent)
                     Write($"({templateSubstitution.ReplacedParameter.Parameter.Name}) (object) ");
                 if ((final.IsPrimitiveType() && !final.IsPrimitiveType(PrimitiveType.Void) &&
-                    (!final.IsPrimitiveType(PrimitiveType.Char) &&
-                     !final.IsPrimitiveType(PrimitiveType.WideChar) ||
+                    ((!final.IsPrimitiveType(PrimitiveType.Char) &&
+                      !final.IsPrimitiveType(PrimitiveType.WideChar) &&
+                      !final.IsPrimitiveType(PrimitiveType.Char16) &&
+                      !final.IsPrimitiveType(PrimitiveType.Char32)) ||
                      (!Context.Options.MarshalCharAsManagedChar &&
                       !((PointerType) field.Type).QualifiedPointee.Qualifiers.IsConst)) &&
                     templateSubstitution == null) ||
                     (!((PointerType) field.Type).QualifiedPointee.Qualifiers.IsConst &&
-                     final.IsPrimitiveType(PrimitiveType.WideChar)))
+                      (final.IsPrimitiveType(PrimitiveType.WideChar) || 
+                       final.IsPrimitiveType(PrimitiveType.Char16) || 
+                       final.IsPrimitiveType(PrimitiveType.Char32))))
                     Write($"({field.Type.GetPointee().Desugar()}*) ");
             }
             WriteLine($"{@return};");
@@ -1626,7 +1636,7 @@ private void GenerateVariable(Class @class, Variable variable)
             PopBlock(NewLineKind.BeforeNextBlock);
         }
 
-        #region Virtual Tables
+#region Virtual Tables
 
         public List<VTableComponent> GetUniqueVTableMethodEntries(Class @class)
         {
@@ -2033,9 +2043,9 @@ public bool HasVirtualTables(Class @class)
             return @class.IsGenerated && @class.IsDynamic && GetUniqueVTableMethodEntries(@class).Count > 0;
         }
 
-        #endregion
+#endregion
 
-        #region Events
+#region Events
 
         public override bool VisitEvent(Event @event)
         {
@@ -2143,9 +2153,9 @@ private void GenerateEventRaiseWrapper(Event @event, string delegateInstance)
             UnindentAndWriteCloseBrace();
         }
 
-        #endregion
+#endregion
 
-        #region Constructors
+#region Constructors
 
         public void GenerateClassConstructors(Class @class)
         {
@@ -2266,6 +2276,21 @@ c is ClassTemplateSpecialization ?
                 }
             }
 
+            // If we have any fields holding references to unmanaged memory allocated here, free the
+            // referenced memory. Don't rely on testing if the field's IntPtr is IntPtr.Zero since 
+            // unmanaged memory isn't always initialized and/or a reference may be owned by the
+            // native side.
+            //
+            // TODO: We should delegate to the dispose methods of references we hold to other
+            // generated type instances since those instances could also hold references to
+            // unmanaged memory.
+            foreach (var field in @class.Layout.Fields.Where(f => f.QualifiedType.Type.IsConstCharString()))
+            {
+                var ptr = $"(({Helpers.InternalStruct}*){Helpers.InstanceIdentifier})->{field.Name}";
+                WriteLine($"if (__{field.Name}_OwnsNativeMemory)");
+                WriteLineIndent($"Marshal.FreeHGlobal({ptr});");
+            }
+
             WriteLine("if ({0})", Helpers.OwnsNativeInstanceIdentifier);
             WriteLineIndent("Marshal.FreeHGlobal({0});", Helpers.InstanceIdentifier);
 
@@ -2482,9 +2507,9 @@ private void GenerateClassConstructorBase(Class @class, Method method)
                 WriteLineIndent(": this()");
         }
 
-        #endregion
+#endregion
 
-        #region Methods / Functions
+#region Methods / Functions
 
         public void GenerateFunction(Function function, string parentName)
         {
@@ -2898,6 +2923,21 @@ private void GenerateClassConstructor(Method method, Class @class)
                     var classInternal = TypePrinter.PrintNative(@class);
                     WriteLine($@"*(({classInternal}*) {Helpers.InstanceIdentifier}) = *(({
                         classInternal}*) {method.Parameters[0].Name}.{Helpers.InstanceIdentifier});");
+
+                    // Copy any string references owned by the source to the new instance so we
+                    // don't have to ref count them.
+                    foreach (var field in @class.Fields.Where(f => f.QualifiedType.Type.IsConstCharString()))
+                    {
+                        var prop = @class.Properties.Where(p => p.Field == field).FirstOrDefault();
+                        // If there is no property or no setter then this instance can never own the native
+                        // memory. Worry about the case where there's only a setter (write-only) when we 
+                        // understand the use case and how it can occur.
+                        if (prop != null && prop.HasGetter && prop.HasSetter)
+                        {
+                            WriteLine($"if ({method.Parameters[0].Name}.__{field.OriginalName}_OwnsNativeMemory)");
+                            WriteLineIndent($@"this.{prop.Name} = {method.Parameters[0].Name}.{prop.Name};");
+                        }
+                    }
                 }
             }
             else
@@ -3230,7 +3270,7 @@ private string FormatMethodParameters(IEnumerable<Parameter> @params)
             return TypePrinter.VisitParameters(@params, true).Type;
         }
 
-        #endregion
+#endregion
 
         public override bool VisitTypedefNameDecl(TypedefNameDecl typedef)
         {

src/Generator/Types/Std/Stdlib.CSharp.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using CppSharp.AST;
 using CppSharp.AST.Extensions;
@@ -96,15 +97,18 @@ public override Type CSharpSignatureType(TypePrinterContext ctx)
                 return new CustomType(typePrinter.IntPtrType);
             }
 
-            var (enconding, _) = GetEncoding();
+            var (encoding, _) = GetEncoding();
 
-            if (enconding == Encoding.ASCII)
-                return new CustomType("[MarshalAs(UnmanagedType.LPStr)] string");
-            else if (enconding == Encoding.UTF8)
+            if (encoding == Encoding.ASCII || encoding == Encoding.Default)
+                // This is not really right. ASCII is 7-bit only - the 8th bit is stripped; ANSI has
+                // multi-byte support via a code page. MarshalAs(UnmanagedType.LPStr) marshals as ANSI.
+                // Perhaps we need a CppSharp.Runtime.ASCIIMarshaller?
+                return new CustomType("[MarshalAs(UnmanagedType.LPStr)] string");   
+            else if (encoding == Encoding.UTF8)
                 return new CustomType("[MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(CppSharp.Runtime.UTF8Marshaller))] string");
-            else if (enconding == Encoding.Unicode || enconding == Encoding.BigEndianUnicode)
+            else if (encoding == Encoding.Unicode || encoding == Encoding.BigEndianUnicode)
                 return new CustomType("[MarshalAs(UnmanagedType.LPWStr)] string");
-            else if (enconding == Encoding.UTF32)
+            else if (encoding == Encoding.UTF32)
                 return new CustomType("[MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(CppSharp.Runtime.UTF32Marshaller))] string");
 
             throw new System.NotSupportedException(
@@ -129,14 +133,63 @@ public override void CSharpMarshalToNative(CSharpMarshalContext ctx)
             if (substitution != null)
                 param = $"({substitution.Replacement}) (object) {param}";
 
-            string bytes = $"__bytes{ctx.ParameterIndex}";
-            string bytePtr = $"__bytePtr{ctx.ParameterIndex}";
-            ctx.Before.WriteLine($@"byte[] {bytes} = global::System.Text.Encoding.{
-                GetEncoding().Name}.GetBytes({param});");
-            ctx.Before.WriteLine($"fixed (byte* {bytePtr} = {bytes})");
-            ctx.HasCodeBlock = true;
-            ctx.Before.WriteOpenBraceAndIndent();
-            ctx.Return.Write($"new global::System.IntPtr({bytePtr})");
+            // Allow setting native field to null via setter property.
+            if (ctx.MarshalKind == MarshalKind.NativeField)
+            {
+                // Free memory if we're holding a pointer to unmanaged memory that we (think we)
+                // allocated. We can't simply compare with IntPtr.Zero since the reference could be
+                // owned by the native side.
+
+                // TODO: Surely, we can do better than stripping out the name of the field using
+                // string manipulation on the ReturnVarName, but I don't see it yet. Seems like it
+                // would be really helpful to have ctx hold a Decl property representing the
+                // "appropriate" Decl when we get here. When MarshalKind == NativeField, Decl would
+                // be set to the Field we're operating on.
+                var fieldName = ctx.ReturnVarName.Substring(ctx.ReturnVarName.LastIndexOf("->") + 2); 
+                
+                ctx.Before.WriteLine($"if (__{fieldName}_OwnsNativeMemory)");
+                ctx.Before.WriteLineIndent($"Marshal.FreeHGlobal({ctx.ReturnVarName});");
+                ctx.Before.WriteLine($"__{fieldName}_OwnsNativeMemory = true;");
+                ctx.Before.WriteLine($"if ({param} == null)");
+                ctx.Before.WriteOpenBraceAndIndent();
+                ctx.Before.WriteLine($"{ctx.ReturnVarName} = global::System.IntPtr.Zero;");
+                ctx.Before.WriteLine("return;");
+                ctx.Before.UnindentAndWriteCloseBrace();
+            }
+
+            var bytes = $"__bytes{ctx.ParameterIndex}";
+            var bytePtr = $"__bytePtr{ctx.ParameterIndex}";
+            var encodingName = GetEncoding().Name;
+
+            switch (encodingName)
+            {
+                case nameof(Encoding.Unicode):
+                    ctx.Before.WriteLine($@"var {bytePtr} = Marshal.StringToHGlobalUni({param});");
+                    break;
+                case nameof(Encoding.Default):
+                    ctx.Before.WriteLine($@"var {bytePtr} = Marshal.StringToHGlobalAnsi({param});");
+                    break;
+                default:
+                    {
+                        var encodingBytesPerChar = GetCharWidth() / 8;
+                        var writeNulMethod = encodingBytesPerChar switch
+                        {
+                            1 => nameof(Marshal.WriteByte),
+                            2 => nameof(Marshal.WriteInt16),
+                            4 => nameof(Marshal.WriteInt32),
+                            _ => throw new System.NotImplementedException(
+                                    $"Encoding bytes per char: {encodingBytesPerChar} is not implemented.")
+                        };
+
+                        ctx.Before.WriteLine($@"var {bytes} = global::System.Text.Encoding.{encodingName}.GetBytes({param});");
+                        ctx.Before.WriteLine($@"var {bytePtr} = Marshal.AllocHGlobal({bytes}.Length + {encodingBytesPerChar});");
+                        ctx.Before.WriteLine($"Marshal.Copy({bytes}, 0, {bytePtr}, {bytes}.Length);");
+                        ctx.Before.WriteLine($"Marshal.{writeNulMethod}({bytePtr} + {bytes}.Length, 0);");
+                    }
+                    break;
+            }
+
+            ctx.Return.Write($"{bytePtr}");
         }
 
         public override void CSharpMarshalToManaged(CSharpMarshalContext ctx)
@@ -168,6 +221,8 @@ public override void CSharpMarshalToManaged(CSharpMarshalContext ctx)
             switch (GetCharWidth())
             {
                 case 8:
+                    if (Context.Options.Encoding == Encoding.Default)   // aka ANSI with system default code page
+                        return (Context.Options.Encoding, nameof(Encoding.Default));
                     if (Context.Options.Encoding == Encoding.ASCII)
                         return (Context.Options.Encoding, nameof(Encoding.ASCII));
                     if (Context.Options.Encoding == Encoding.UTF8)