Hide feature behind flag, more testing

Author: WardBrian

src/driver/Entry.ml

@@ -31,6 +31,7 @@ let set_model_name model_name =
 
 let reset_mutable_states model_name (flags : Flags.t) =
   Common.Gensym.reset_danger_use_cautiously ();
+  Unicode.allow_unicode := flags.allow_unicode;
   Include_files.include_provider := flags.include_source;
   set_model_name model_name;
   Typechecker.check_that_all_functions_have_definition :=

src/driver/Flags.ml

@@ -7,6 +7,7 @@ type t =
   ; standalone_functions: bool
   ; use_opencl: bool
   ; include_source: Frontend.Include_files.t
+  ; allow_unicode: bool
   ; info: bool
   ; version: bool
   ; auto_format: bool
@@ -49,6 +50,7 @@ let default =
   ; standalone_functions= false
   ; use_opencl= false
   ; include_source= Frontend.Include_files.FileSystemPaths []
+  ; allow_unicode= false
   ; info= false
   ; version= false
   ; auto_format= false

src/driver/Flags.mli

@@ -9,6 +9,7 @@ type t =
   ; standalone_functions: bool
   ; use_opencl: bool
   ; include_source: Frontend.Include_files.t
+  ; allow_unicode: bool
         (* ------------------------- *)
         (* flags which switch compiler "modes" *)
   ; info: bool

src/frontend/Unicode.ml

@@ -1,3 +1,5 @@
+let allow_unicode = ref false
+
 let error ~loc msg =
   raise
     (Errors.SyntaxError
@@ -30,6 +32,10 @@ let validate_ascii_id ~loc id =
    https://www.unicode.org/reports/tr31 *)
 
 let validate_utf8_id ~loc id =
+  if not !allow_unicode then
+    error ~loc
+      "Unicode identifiers are not supported without the (experimental) \
+       allow-unicode flag";
   if not (String.is_valid_utf_8 id) then
     error ~loc "Identifier is not valid UTF-8 string";
   Debugging.lexer_logger ("unicode id: " ^ id);

src/stanc/CLI.ml

@@ -39,6 +39,12 @@ module Options = struct
        means the definition will be provided later as a C++ function." in
     Arg.(value & flag & info ["allow-undefined"] ~doc)
 
+  let allow_unicode =
+    let doc =
+      "$(i,(Experimental)) Allow unicode characters in the names of functions \
+       and variables" in
+    Arg.(value & flag & info ["allow-unicode"] ~doc)
+
   let auto_format =
     let doc =
       "Output a formatted version of the Stan program. The output can be \
@@ -359,6 +365,7 @@ module Conversion = struct
     let open Options in
     let+ optimization_level = optimization_level
     and+ allow_undefined = allow_undefined
+    and+ allow_unicode = allow_unicode
     and+ standalone_functions = standalone_functions
     and+ use_opencl = use_opencl
     and+ include_source = include_paths
@@ -374,6 +381,7 @@ module Conversion = struct
     Driver.Flags.
       { optimization_level
       ; allow_undefined
+      ; allow_unicode
       ; functions_only= false
       ; standalone_functions
       ; use_opencl

src/stancjs/stancjs.ml

@@ -140,6 +140,7 @@ let process_flags (flags : 'a Js.opt) includes : (Driver.Flags.t, string) result
              else if is_flag_set "Oexperimental" then Optimize.Oexperimental
              else Optimize.O0)
         ; allow_undefined= is_flag_set "allow-undefined"
+        ; allow_unicode= is_flag_set "allow-unicode"
         ; functions_only= is_flag_set "functions-only"
         ; standalone_functions= is_flag_set "standalone-functions"
         ; use_opencl= is_flag_set "use-opencl"

test/integration/bad/lang/stanc.expected

@@ -533,18 +533,6 @@ Semantic error in 'tparam_tuple_int.stan', line 2, column 2 to column 21:
    -------------------------------------------------
 
 (Transformed) Parameters cannot be integers.
-[exit 1]
-  $ ../../../../../install/default/bin/stanc unicode_normalization.stan
-Semantic error in 'unicode_normalization.stan', line 4, column 7 to column 13:
-   -------------------------------------------------
-     2:    real ñabc;
-     3:    // this is a different encoding than above, should be prevented still!
-     4:    real ñabc;
-                ^
-     5:  }
-   -------------------------------------------------
-
-Identifier 'ñabc' is already in use.
 [exit 1]
   $ ../../../../../install/default/bin/stanc unterminated_comment.stan
 Syntax error in 'unterminated_comment.stan', line 4, column -1, lexing error:

test/integration/bad/new/stanc.expected

@@ -1450,7 +1450,7 @@ Syntax error in 'lexing_error.stan', line 1, column 6, lexing error:
                 ^
    -------------------------------------------------
 
-Invalid character: '1'
+Unicode identifiers are not supported without the (experimental) allow-unicode flag
 [exit 1]
   $ ../../../../../install/default/bin/stanc location-scale-bad1.stan
 Syntax error in 'location-scale-bad1.stan', line 1, column 27 to column 32, parsing error:

test/integration/bad/unicode/dune

@@ -0,0 +1,15 @@
+(rule
+ (targets stanc.output)
+ (deps
+  (package stanc)
+  (:stanfiles
+   (glob_files *.stan)))
+ (action
+  (with-stdout-to
+   %{targets}
+   (run %{bin:run_bin_on_args} "%{bin:stanc} --allow-unicode" %{stanfiles}))))
+
+(rule
+ (alias runtest)
+ (action
+  (diff stanc.expected stanc.output)))

test/integration/bad/unicode/invalid-utf8-1.stan

@@ -0,0 +1,3 @@
+data {
+  int �(;
+}