From 7d6b3fb291951c2147ddd385b9b0e2d2adc15643 Mon Sep 17 00:00:00 2001 From: Gabriel Scherer Date: Tue, 29 Dec 2020 22:37:17 +0100 Subject: [PATCH 1/7] reimplement and document the handling of standalone tokens The previous approach to standalone-token handling would peek into a prefix of the line, and decide what to do based on the prefix. This relied on the property that we would have at most one standalone token per line. This is unfortunately not true, consider: ``` Begin. {{#foo}} {{#bar}} Middle. {{/bar}} {{/foo}} End. ``` The new approach processes the whole line at once, failing if it encounters non-whitespace non-standalone token. --- lib/mustache_lexer.mll | 115 +++++++++++++++++++++++++++----------- lib_test/test_mustache.ml | 20 +++++++ 2 files changed, 102 insertions(+), 33 deletions(-) diff --git a/lib/mustache_lexer.mll b/lib/mustache_lexer.mll index d32e462..3a52675 100644 --- a/lib/mustache_lexer.mll +++ b/lib/mustache_lexer.mll @@ -110,6 +110,41 @@ and mustache = parse | eof { EOF } { + (* Trim whitespace around standalone tags. + + The Mustache specification is careful with its treatment of + whitespace. In particular, tags that do not themselves expand to + visible content are defined as "standalone", with the + requirement that if one or several standalone tags "stand alone" + in a line (there is nothing else but whitespace), the whitespace + of this line should be ommitted. + + For example, this means that: + {{#foo}} + I can access {{var}} inside the section. + {{/foo} + takes, once rendered, only 1 line instead of 3: the newlines + after {{#foo}} and {{/foo}} are part of the "standalone + whitespace", so they are not included in the output. + + Note: if a line contains only whitespace, no standalone tag, + then the whitespace is preserved. + + We implement this by a post-processing past on the lexer token + stream. We split the token stream, one sub-stream per line, and + then for each token line we determine if satisfies the + standalone criterion. + + Another information collected at the same time, as it is also + part of whitespace processing, is the "indentation" of partials: + if a partial expands to multi-line content, and if it is + intended at the use-site (it is at a non-zero column with only + whitespace before it on the line), then the specification + mandates that all its lines should be indented by the same + amount. We collect this information during the whitespace + postprocessing of tokens, and store it in the Partial + constructor as the first parameter. + *) let handle_standalone lexer lexbuf = let ends_with_newline s = String.length s > 0 && @@ -148,40 +183,54 @@ and mustache = parse in loop 0 l in - let is_standalone toks = - let (skipped, toks) = skip_blanks toks in - match toks with - | ((OPEN_SECTION _ - | OPEN_INVERTED_SECTION _ - | CLOSE_SECTION _ - | PARTIAL _ - | COMMENT _), _, _) as tok :: toks' -> - let (_, toks_rest) = skip_blanks toks' in - begin match toks_rest with - | [] | [(EOF, _, _)] -> - let tok = - match tok with - | (PARTIAL (_, p), loc1, loc2) -> - (PARTIAL (skipped, p), loc1, loc2) - | _ -> tok - in - Some (tok, toks_rest) - | _ -> None - end - | _ -> None + let trim_standalone toks = + let toks = + (* if the line starts with a partial, + turn the skipped blank into partial indentation *) + let (skipped, toks_after_blank) = skip_blanks toks in + match toks_after_blank with + | (PARTIAL (_ , name), loc1, loc2) :: rest -> + (PARTIAL (skipped, name), loc1, loc2) :: rest + | _ -> toks + in + let toks = + (* if the line only contains whitespace and at least one standalone tags, + remove all whitespace *) + let rec standalone acc = function + | (RAW s, _, _) :: rest when is_blank s -> + (* omit whitespace *) + standalone acc rest + | ((OPEN_SECTION _ + | OPEN_INVERTED_SECTION _ + | CLOSE_SECTION _ + | PARTIAL _ + | COMMENT _), _, _) as tok :: rest -> + (* collect standalone tags *) + standalone (tok :: acc) rest + | [] | (EOF, _, _) :: _ -> + (* end of line *) + if (acc = []) then + (* if acc is empty, the line only contains whitespace, + which should be kept *) + None + else + Some (List.rev acc) + | _non_blank :: _rest -> + (* non-blank, non-standalone token *) + None + in + match standalone [] toks with + | None -> toks + | Some standalone_toks -> standalone_toks + in + assert (toks <> []); + toks in - let buffer = ref [] in fun () -> - match !buffer with - | tok :: toks -> - buffer := toks; tok - | [] -> - let toks = slurp_line () in - match is_standalone toks with - | Some (tok_standalone, toks_rest) -> - buffer := toks_rest; - tok_standalone - | None -> - buffer := List.tl toks; List.hd toks + let toks = match !buffer with + | (_ :: _) as toks -> toks + | [] -> trim_standalone (slurp_line ()) + in + buffer := List.tl toks; List.hd toks } diff --git a/lib_test/test_mustache.ml b/lib_test/test_mustache.ml index ed2bbec..2470550 100644 --- a/lib_test/test_mustache.ml +++ b/lib_test/test_mustache.ml @@ -103,6 +103,26 @@ let tests = [ , [ ( `O [ "a" , `String "foo" ], "foo" ) ] ) ; + ( (* check that a whitespace line is omitted + if it contains (several) standalone tokens *) +"Begin +{{#foo}} {{#bar}} +Middle +{{/bar}} {{/foo}} +End +" + , concat [ + raw "Begin\n"; + section ["foo"] (section ["bar"] (raw "Middle\n")); + raw "End\n"; + ] + , [ ( `O [ "foo" , `O []; "bar", `O [] ], +"Begin +Middle +End +" + ) ] ) ; + ] let mkloc (lnum_s, bol_s, cnum_s, lnum_e, bol_e, cnum_e) = From b7e36c31f63c394dda0e5899e0fcbcf25f62888d Mon Sep 17 00:00:00 2001 From: Gabriel Scherer Date: Tue, 29 Dec 2020 07:19:03 +0100 Subject: [PATCH 2/7] Implement template inheritance (~ partials with parameters) The implementation follows the spec proposal: https://github.com/mustache/spec/pull/75 which is supported by at least the following Mustache implementations: - hogan.js - mustache.php - mustache.java - GRMustache (Obj-C) and GRMustache.swift (Swift) - Text::Caml (Perl) - hxmustache (Haxe) - MuttonChop (Swift) --- CHANGES.md | 5 + bin/test/errors/parsing-errors.t | 21 ++- bin/test/inheritance.t/base.mustache | 6 + bin/test/inheritance.t/header.mustache | 3 + bin/test/inheritance.t/mypage.mustache | 10 ++ bin/test/inheritance.t/run.t | 14 ++ dune-project | 2 +- lib/mustache.ml | 238 ++++++++++++++++++------- lib/mustache.mli | 60 +++++-- lib/mustache_lexer.mll | 10 +- lib/mustache_parser.mly | 93 +++++++--- lib/mustache_types.ml | 31 +++- mustache.opam | 2 +- 13 files changed, 383 insertions(+), 112 deletions(-) create mode 100644 bin/test/inheritance.t/base.mustache create mode 100644 bin/test/inheritance.t/header.mustache create mode 100644 bin/test/inheritance.t/mypage.mustache create mode 100644 bin/test/inheritance.t/run.t diff --git a/CHANGES.md b/CHANGES.md index 06b89ed..1184484 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ ### 3.2.0 +* Support for "template inheritance" (partials with parameters) + `{{foo/bar}}" will include "foo/bar.mustache", relative to the current working directory. diff --git a/bin/test/errors/parsing-errors.t b/bin/test/errors/parsing-errors.t index 78555d8..8891cb6 100644 --- a/bin/test/errors/parsing-errors.t +++ b/bin/test/errors/parsing-errors.t @@ -28,7 +28,7 @@ Delimiter problems: $ PROBLEM=eof-before-section-end.mustache $ echo "{{#foo}} {{.}} {{/" > $PROBLEM $ mustache foo.json $PROBLEM - File "eof-before-section-end.mustache", line 2, character 0: ident expected. + File "eof-before-section-end.mustache", line 2, character 0: '}}' expected. [3] $ PROBLEM=eof-before-inverted-section.mustache @@ -84,7 +84,7 @@ Mismatch between section-start and section-end: $ echo "{{#foo}} {{.}} {{/bar}}" > $PROBLEM $ mustache foo.json $PROBLEM File "foo-bar.mustache", line 1, characters 0-23: - Section mismatch: {{#foo}} is closed by {{/bar}}. + Open/close tag mismatch: {{# foo }} is closed by {{/ bar }}. [3] $ PROBLEM=foo-not-closed.mustache @@ -97,9 +97,24 @@ Mismatch between section-start and section-end: $ echo "{{#bar}} {{#foo}} {{.}} {{/bar}} {{/foo}}" > $PROBLEM $ mustache foo.json $PROBLEM File "wrong-nesting.mustache", line 1, characters 9-32: - Section mismatch: {{#foo}} is closed by {{/bar}}. + Open/close tag mismatch: {{# foo }} is closed by {{/ bar }}. [3] + $ PROBLEM=wrong-nesting-variable.mustache + $ echo '{{#bar}} {{$foo}} {{.}} {{/bar}} {{/foo}}' > $PROBLEM + $ mustache foo.json $PROBLEM + File "wrong-nesting-variable.mustache", line 1, characters 9-32: + Open/close tag mismatch: {{$ foo }} is closed by {{/ bar }}. + [3] + + $ PROBLEM=wrong-nesting-partial.mustache + $ echo "{{#foo}} {{ $PROBLEM + $ mustache foo.json $PROBLEM + File "wrong-nesting-partial.mustache", line 1, characters 9-30: + Open/close tag mismatch: {{< foo-bar }} is closed by {{/ foo }}. + [3] + + Weird cases that may confuse our lexer or parser: diff --git a/bin/test/inheritance.t/base.mustache b/bin/test/inheritance.t/base.mustache new file mode 100644 index 0000000..c0544ec --- /dev/null +++ b/bin/test/inheritance.t/base.mustache @@ -0,0 +1,6 @@ + + {{$header}}{{/header}} + + {{$content}}{{/content}} + + diff --git a/bin/test/inheritance.t/header.mustache b/bin/test/inheritance.t/header.mustache new file mode 100644 index 0000000..7229aa8 --- /dev/null +++ b/bin/test/inheritance.t/header.mustache @@ -0,0 +1,3 @@ + + {{$title}}Default title{{/title}} + diff --git a/bin/test/inheritance.t/mypage.mustache b/bin/test/inheritance.t/mypage.mustache new file mode 100644 index 0000000..3e98ddf --- /dev/null +++ b/bin/test/inheritance.t/mypage.mustache @@ -0,0 +1,10 @@ +{{Hello world + {{/content}} +{{/base}} diff --git a/bin/test/inheritance.t/run.t b/bin/test/inheritance.t/run.t new file mode 100644 index 0000000..7d81999 --- /dev/null +++ b/bin/test/inheritance.t/run.t @@ -0,0 +1,14 @@ + $ echo "{}" > data.json + +This test is the reference example from the template-inheritance specification: +https://github.com/mustache/spec/pull/75 + + $ mustache data.json mypage.mustache + + + My page title + + +

Hello world

+ + diff --git a/dune-project b/dune-project index 27ebc61..9b11212 100644 --- a/dune-project +++ b/dune-project @@ -29,4 +29,4 @@ Contains the `mustache` command line utility for driving logic-less templates. (ezjsonm :with-test) (menhir (>= 20180703)) (cmdliner (>= 1.0.4)) - (ocaml (>= 4.06)))) + (ocaml (>= 4.08)))) diff --git a/lib/mustache.ml b/lib/mustache.ml index 57872f6..88b9810 100644 --- a/lib/mustache.ml +++ b/lib/mustache.ml @@ -42,10 +42,6 @@ module Json = struct let value: t -> value = fun t -> (t :> value) end -let option_map o f = match o with - | None -> None - | Some x -> Some (f x) - let escape_html s = let b = Buffer.create (String.length s) in String.iter ( function @@ -73,15 +69,24 @@ and erase_locs_desc = function | Locs.Section s -> No_locs.Section (erase_locs_section s) | Locs.Unescaped s -> No_locs.Unescaped s | Locs.Partial p -> No_locs.Partial (erase_locs_partial p) + | Locs.Param pa -> No_locs.Param (erase_locs_param pa) | Locs.Inverted_section s -> No_locs.Inverted_section (erase_locs_section s) | Locs.Concat l -> No_locs.Concat (List.map erase_locs l) | Locs.Comment s -> No_locs.Comment s -and erase_locs_section { Locs.name; Locs.contents } = - { No_locs.name; No_locs.contents = erase_locs contents } -and erase_locs_partial { Locs.indent; Locs.name; Locs.contents } = - { No_locs.indent; - No_locs.name; - No_locs.contents = lazy (option_map (Lazy.force contents) erase_locs) } +and erase_locs_section (s : Locs.section) : No_locs.section = { + name = s.name; + contents = erase_locs s.contents; +} +and erase_locs_partial (p : Locs.partial) : No_locs.partial = { + indent = p.indent; + name = p.name; + params = Option.map (List.map ~f:erase_locs_param) p.params; + contents = lazy (Option.map erase_locs (Lazy.force p.contents)) +} +and erase_locs_param (pa : Locs.param) : No_locs.param = { + name = pa.name; + contents = erase_locs pa.contents; +} let rec add_dummy_locs t = { Locs.loc = dummy_loc; @@ -92,16 +97,25 @@ and add_dummy_locs_desc = function | No_locs.Section s -> Locs.Section (add_dummy_locs_section s) | No_locs.Unescaped s -> Locs.Unescaped s | No_locs.Partial p -> Locs.Partial (add_dummy_locs_partial p) + | No_locs.Param pa -> Locs.Param (add_dummy_locs_param pa) | No_locs.Inverted_section s -> Locs.Inverted_section (add_dummy_locs_section s) | No_locs.Concat l -> Locs.Concat (List.map add_dummy_locs l) | No_locs.Comment s -> Locs.Comment s -and add_dummy_locs_section { No_locs.name; No_locs.contents } = - { Locs.name; Locs.contents = add_dummy_locs contents } -and add_dummy_locs_partial { No_locs.indent; No_locs.name; No_locs.contents } = - { Locs.indent; - Locs.name; - Locs.contents = lazy (option_map (Lazy.force contents) add_dummy_locs) } +and add_dummy_locs_section (s : No_locs.section) : Locs.section = { + name = s.name; + contents = add_dummy_locs s.contents; +} +and add_dummy_locs_partial (p : No_locs.partial) : Locs.partial = { + indent = p.indent; + name = p.name; + params = Option.map (List.map ~f:add_dummy_locs_param) p.params; + contents = lazy (Option.map add_dummy_locs (Lazy.force p.contents)); +} +and add_dummy_locs_param (pa : No_locs.param) : Locs.param = { + name = pa.name; + contents = add_dummy_locs pa.contents; +} (* Printing: defined on the ast without locations. *) @@ -126,7 +140,17 @@ let rec pp fmt = pp_dotted_name s.name pp s.contents pp_dotted_name s.name | Partial p -> - Format.fprintf fmt "{{> %s }}" p.name + begin match p.params with + | None -> Format.fprintf fmt "{{> %s }}" p.name + | Some params -> + Format.fprintf fmt "{{< %s }}%a{{/ %s }}" + p.name + (Format.pp_print_list pp_param) params + p.name + end + + | Param pa -> + Format.fprintf fmt "%a" pp_param pa | Comment s -> Format.fprintf fmt "{{! %s }}" s @@ -134,6 +158,12 @@ let rec pp fmt = | Concat s -> List.iter (pp fmt) s +and pp_param fmt pa = + Format.fprintf fmt "{{$%s}}%a{{/%s}}" + pa.name + pp pa.contents + pa.name + let to_string x = let b = Buffer.create 0 in let fmt = Format.formatter_of_buffer b in @@ -149,10 +179,7 @@ type template_parse_error = { and template_parse_error_kind = | Lexing of string | Parsing - | Mismatched_section of { - start_name: dotted_name; - end_name: dotted_name; - } + | Mismatched_names of name_mismatch_error exception Parse_error of template_parse_error @@ -172,8 +199,8 @@ let parse_lx (lexbuf: Lexing.lexbuf) : Locs.t = raise_err (loc_of lexbuf) (Lexing msg) | Mustache_parser.Error -> raise_err (loc_of lexbuf) Parsing - | Mismatched_section { loc; start_name; end_name } -> - raise_err loc (Mismatched_section { start_name; end_name }) + | Mismatched_names (loc, { name_kind; start_name; end_name }) -> + raise_err loc (Mismatched_names { name_kind; start_name; end_name }) let of_string s = parse_lx (Lexing.from_string s) @@ -218,10 +245,16 @@ let pp_template_parse_error ppf ({ loc; kind; } : template_parse_error) = p ppf "%s" msg | Parsing -> p ppf "syntax error" - | Mismatched_section { start_name; end_name } -> - p ppf "Section mismatch: {{#%a}} is closed by {{/%a}}" - pp_dotted_name start_name - pp_dotted_name end_name + | Mismatched_names { name_kind; start_name; end_name } -> + p ppf "Open/close tag mismatch: {{%c %s }} is closed by {{/ %s }}" + (match name_kind with + | Section_name -> '#' + | Inverted_section_name -> '^' + | Partial_with_params_name -> '<' + | Param_name -> '$' + ) + start_name + end_name end; p ppf ".@]" @@ -281,33 +314,75 @@ module Contexts : sig val top : t -> Json.value val add : t -> Json.value -> t val find_name : t -> string -> Json.value option + val add_param : t -> Locs.param -> t + val find_param : t -> string -> Locs.t option end = struct - (* a nonempty stack of contexts, most recent first *) - type t = Json.value * Json.value list + type t = { + (* nonempty stack of contexts, most recent first *) + stack: Json.value * Json.value list; + + (* an associative list of partial parameters + that have been defined *) + params: (string * Locs.t) list; + } - let start js = (js, []) + let start js = { + stack = (js, []); + params = []; + } - let top (js, _rest) = js + let top { stack = (js, _rest); _ } = js - let add (top, rest) ctx = (ctx, top::rest) + let add ctxs ctx = + let (top, rest) = ctxs.stack in + { ctxs with stack = (ctx, top::rest) } - let rec find_name ((top, rest) : t) name = + let rec find_name ctxs name = + let (top, _) = ctxs.stack in match top with | `Null | `Bool _ | `Float _ | `String _ | `A _ - -> find_in_rest rest name + -> find_in_rest ctxs name | `O dict -> match List.assoc name dict with - | exception Not_found -> find_in_rest rest name + | exception Not_found -> find_in_rest ctxs name | v -> Some v - and find_in_rest rest name = + and find_in_rest ctxs name = + let (_, rest) = ctxs.stack in match rest with | [] -> None - | top :: rest -> find_name (top, rest) name + | top :: rest -> find_name { ctxs with stack = (top, rest) } name + + (* Note: the template-inheritance specification for Mustache + (https://github.com/mustache/spec/pull/75) mandates that in case + of multi-level inclusion, the "topmost" definition of the + parameter wins. In other terms, when traversing the template + during rendering, the value defined first for this parameter has + precedence over later definitions. + + This is not a natural choice for our partial-with-arguments view, + where we would expect the parameter binding closest to the + use-site to win. This corresponds to an object-oriented view + where applying a partial-with-parameters is seen as "inheriting" + the parent/partial template, overriding a method for each + parameter. Multi-level inclusions correspond to inheritance + hierarchies (the parent template itself inherits from + a grandparent), and then late-binding mandates that the + definition "last" in the inheritance chain (so closest to the + start of the rendering) wins.*) + let add_param ctxs { Locs.name; contents } = + if List.mem_assoc name ctxs.params then + (* if the parameter is already bound, the existing binding has precedence *) + ctxs + else + {ctxs with params = (name, contents) :: ctxs.params} + + let find_param ctxs name = + List.assoc_opt name ctxs.params end let raise_err loc kind = @@ -376,6 +451,8 @@ module Lookup = struct | Some (`A [] | `Bool false | `Null) -> true | _ -> false + let param ctxs ~loc:_ ~key = + Contexts.find_param ctxs key end module Render = struct @@ -452,14 +529,35 @@ module Render = struct | elem -> enter elem end - | Partial { indent = partial_indent; name; contents } -> - begin match (Lazy.force contents, strict) with - | Some p, _ -> render (indent + partial_indent) p ctxs - | None, false -> () - | None, true -> - raise_err loc (Missing_partial { name }) + | Partial { indent = partial_indent; name; params; contents } -> + let partial = Lazy.force contents in + let ctxs = + match params with + | None -> ctxs + | Some params -> + List.fold_left ~f:Contexts.add_param ~init:ctxs params + in + begin match partial with + | None -> + if strict then + raise_err loc (Missing_partial { name }) + | Some partial -> + render (indent + partial_indent) partial ctxs end + | Param { name; contents } -> + let param = + match Lookup.param ctxs ~loc ~key:name with + | None -> + (* The "contents" of the partial parameter is to be used as + default content, if the parameter was not explicitly passed + by one of the partials in scope. *) + contents + | Some param -> + param + in + render indent param ctxs + | Comment _c -> () | Concat templates -> @@ -485,8 +583,8 @@ module Without_locations = struct let to_string = to_string - let rec fold ~string ~section ~escaped ~unescaped ~partial ~comment ~concat t = - let go = fold ~string ~section ~escaped ~unescaped ~partial ~comment ~concat in + let rec fold ~string ~section ~escaped ~unescaped ~partial ~param ~comment ~concat t = + let go = fold ~string ~section ~escaped ~unescaped ~partial ~param ~comment ~concat in match t with | String s -> string s | Escaped s -> escaped s @@ -498,7 +596,13 @@ module Without_locations = struct section ~inverted:true name (go contents) | Concat ms -> concat (List.map ms ~f:go) - | Partial p -> partial p.indent p.name p.contents + | Partial {indent; name; params; contents} -> + let params = + Option.map (List.map ~f:(fun {name; contents} -> (name, go contents))) params + in + partial indent name ?params contents + | Param { name; contents } -> + param name (go contents) module Infix = struct let (^) y x = Concat [x; y] @@ -509,7 +613,11 @@ module Without_locations = struct let unescaped s = Unescaped s let section n c = Section { name = n ; contents = c } let inverted_section n c = Inverted_section { name = n ; contents = c } - let partial ?(indent = 0) n c = Partial { indent ; name = n ; contents = c } + let partial ?(indent = 0) n ?params c = + let params = + Option.map (List.map ~f:(fun (name, contents) -> {name; contents})) params in + Partial { indent ; name = n ; params; contents = c } + let param n c = Param { name = n; contents = c } let concat t = Concat t let comment s = Comment s @@ -517,16 +625,16 @@ module Without_locations = struct let section ~inverted = if inverted then inverted_section else section in - let partial indent name contents = + let partial indent name ?params contents = let contents' = lazy ( match Lazy.force contents with - | None -> option_map (partials name) (expand_partials partials) + | None -> Option.map (expand_partials partials) (partials name) | Some t_opt -> Some t_opt ) in - partial ~indent name contents' + partial ~indent name ?params contents' in - fold ~string:raw ~section ~escaped ~unescaped ~partial ~comment ~concat + fold ~string:raw ~section ~escaped ~unescaped ~partial ~param ~comment ~concat let render_buf ?strict ?(partials = fun _ -> None) buf (m : t) (js : Json.t) = @@ -559,8 +667,8 @@ module With_locations = struct let to_string x = to_string (erase_locs x) - let rec fold ~string ~section ~escaped ~unescaped ~partial ~comment ~concat t = - let go = fold ~string ~section ~escaped ~unescaped ~partial ~comment ~concat in + let rec fold ~string ~section ~escaped ~unescaped ~partial ~param ~comment ~concat t = + let go = fold ~string ~section ~escaped ~unescaped ~partial ~param ~comment ~concat in let { desc; loc } = t in match desc with | String s -> string ~loc s @@ -573,7 +681,12 @@ module With_locations = struct section ~loc ~inverted:true name (go contents) | Concat ms -> concat ~loc (List.map ms ~f:go) - | Partial p -> partial ~loc p.indent p.name p.contents + | Partial p -> + let params = + Option.map (List.map ~f:(fun {name; contents} -> (name, go contents))) p.params in + partial ~loc p.indent p.name ?params p.contents + | Param { name; contents } -> + param ~loc name (go contents) module Infix = struct let (^) t1 t2 = { desc = Concat [t1; t2]; loc = dummy_loc } @@ -588,26 +701,31 @@ module With_locations = struct let inverted_section ~loc n c = { desc = Inverted_section { name = n; contents = c }; loc } - let partial ~loc ?(indent = 0) n c = - { desc = Partial { indent; name = n; contents = c }; + let partial ~loc ?(indent = 0) n ?params c = + let params = + Option.map (List.map ~f:(fun (name, contents) -> {name; contents})) params in + { desc = Partial { indent; name = n; params; contents = c }; loc } let concat ~loc t = { desc = Concat t; loc } let comment ~loc s = { desc = Comment s; loc } + let param ~loc n c = + { desc = Param { name = n; contents = c }; + loc } let rec expand_partials (partials : name -> t option) : t -> t = let section ~loc ~inverted = if inverted then inverted_section ~loc else section ~loc in - let partial ~loc indent name contents = + let partial ~loc indent name ?params contents = let contents' = lazy ( match Lazy.force contents with - | None -> option_map (partials name) (expand_partials partials) + | None -> Option.map (expand_partials partials) (partials name) | Some t_opt -> Some t_opt ) in - partial ~loc ~indent name contents' + partial ~loc ~indent name ?params contents' in - fold ~string:raw ~section ~escaped ~unescaped ~partial ~comment ~concat + fold ~string:raw ~section ~escaped ~unescaped ~partial ~param ~comment ~concat let render_buf ?strict ?(partials = fun _ -> None) buf (m : t) (js : Json.t) = let m = expand_partials partials m in diff --git a/lib/mustache.mli b/lib/mustache.mli index 281ed98..ea146a7 100644 --- a/lib/mustache.mli +++ b/lib/mustache.mli @@ -22,10 +22,11 @@ type dotted_name = string list type t = | String of string | Escaped of dotted_name - | Section of section | Unescaped of dotted_name - | Partial of partial + | Section of section | Inverted_section of section + | Partial of partial + | Param of param | Concat of t list | Comment of string and section = @@ -34,7 +35,11 @@ and section = and partial = { indent: int; name: name; + params: param list option; contents: t option Lazy.t } +and param = + { name: name; + contents: t } type loc = { loc_start: Lexing.position; @@ -129,13 +134,15 @@ val render : @param string Applied to each literal part of the template. @param escaped Applied to ["name"] for occurrences of [{{name}}]. @param unescaped Applied to ["name"] for occurrences of [{{{name}}}]. - @param partial Applied to ["box"] for occurrences of [{{> box}}]. + @param partial Applied to ["box"] for occurrences of [{{> box}}] or [{{< box}}]. + @param params Applied to ["param"] for occurrences of [{{$ param}}]. @param comment Applied to ["comment"] for occurrences of [{{! comment}}]. *) val fold : string: (string -> 'a) -> section: (inverted:bool -> dotted_name -> 'a -> 'a) -> escaped: (dotted_name -> 'a) -> unescaped: (dotted_name -> 'a) -> - partial: (int -> name -> t option Lazy.t -> 'a) -> + partial: (int -> name -> ?params:(name * 'a) list -> t option Lazy.t -> 'a) -> + param: (name -> 'a -> 'a) -> comment: (string -> 'a) -> concat:('a list -> 'a) -> t -> 'a @@ -170,8 +177,19 @@ val inverted_section : dotted_name -> t -> t (** [{{#person}} {{/person}}] *) val section : dotted_name -> t -> t -(** [{{> box}}] *) -val partial : ?indent:int -> name -> t option Lazy.t -> t +(** [{{> box}}] + or + {[ + {{< box}} + {{$param1}} default value for param1 {{/param1}} + {{$param2}} default value for param1 {{/param2}} + {{/box}} + ]} + *) +val partial : ?indent:int -> name -> ?params:(name * t) list -> t option Lazy.t -> t + +(** [{{$foo}} {{/foo}}] *) +val param: name -> t -> t (** [{{! this is a comment}}] *) val comment : string -> t @@ -189,10 +207,11 @@ module With_locations : sig type desc = | String of string | Escaped of dotted_name - | Section of section | Unescaped of dotted_name - | Partial of partial + | Section of section | Inverted_section of section + | Partial of partial + | Param of param | Concat of t list | Comment of string and section = @@ -201,7 +220,11 @@ module With_locations : sig and partial = { indent: int; name: name; + params: param list option; contents: t option Lazy.t } + and param = + { name: name; + contents: t } and t = { loc : loc; desc : desc } @@ -266,13 +289,15 @@ module With_locations : sig @param string Applied to each literal part of the template. @param escaped Applied to ["name"] for occurrences of [{{name}}]. @param unescaped Applied to ["name"] for occurrences of [{{{name}}}]. - @param partial Applied to ["box"] for occurrences of [{{> box}}]. + @param partial Applied to ["box"] for occurrences of [{{> box}}] or [{{< box}}]. + @param params Applied to ["param"] for occurrences of [{{$ param}}]. @param comment Applied to ["comment"] for occurrences of [{{! comment}}]. *) val fold : string: (loc:loc -> string -> 'a) -> section: (loc:loc -> inverted:bool -> dotted_name -> 'a -> 'a) -> escaped: (loc:loc -> dotted_name -> 'a) -> unescaped: (loc:loc -> dotted_name -> 'a) -> - partial: (loc:loc -> int -> name -> t option Lazy.t -> 'a) -> + partial: (loc:loc -> int -> name -> ?params:(name * 'a) list -> t option Lazy.t -> 'a) -> + param: (loc:loc -> name -> 'a -> 'a) -> comment: (loc:loc -> string -> 'a) -> concat:(loc:loc -> 'a list -> 'a) -> t -> 'a @@ -305,8 +330,19 @@ module With_locations : sig (** [{{#person}} {{/person}}] *) val section : loc:loc -> dotted_name -> t -> t - (** [{{> box}}] *) - val partial : loc:loc -> ?indent:int -> name -> t option Lazy.t -> t + (** [{{> box}}] + or + {[ + {{< box}} + {{$param1}} default value for param1 {{/param1}} + {{$param2}} default value for param1 {{/param2}} + {{/box}} + ]} + *) + val partial : loc:loc -> ?indent:int -> name -> ?params:(name * t) list -> t option Lazy.t -> t + + (** [{{$foo}} {{/foo}}] *) + val param: loc:loc -> name -> t -> t (** [{{! this is a comment}}] *) val comment : loc:loc -> string -> t diff --git a/lib/mustache_lexer.mll b/lib/mustache_lexer.mll index 3a52675..b258e1d 100644 --- a/lib/mustache_lexer.mll +++ b/lib/mustache_lexer.mll @@ -101,8 +101,10 @@ and mustache = parse | "{{&" { UNESCAPE (lex_tag lexbuf space ident (end_on "}}") |> split_ident) } | "{{#" { OPEN_SECTION (lex_tag lexbuf space ident (end_on "}}") |> split_ident) } | "{{^" { OPEN_INVERTED_SECTION (lex_tag lexbuf space ident (end_on "}}") |> split_ident) } - | "{{/" { CLOSE_SECTION (lex_tag lexbuf space ident (end_on "}}") |> split_ident) } + | "{{/" { CLOSE (lex_tag lexbuf space partial_name (end_on "}}")) } | "{{>" { PARTIAL (0, lex_tag lexbuf space partial_name (end_on "}}")) } + | "{{<" { OPEN_PARTIAL_WITH_PARAMS (0, lex_tag lexbuf space partial_name (end_on "}}")) } + | "{{$" { OPEN_PARAM (lex_tag lexbuf space ident (end_on "}}")) } | "{{!" { COMMENT (tok_arg lexbuf (comment [])) } | raw newline { new_line lexbuf; RAW (lexeme lexbuf) } | raw { RAW (lexeme lexbuf) } @@ -191,6 +193,8 @@ and mustache = parse match toks_after_blank with | (PARTIAL (_ , name), loc1, loc2) :: rest -> (PARTIAL (skipped, name), loc1, loc2) :: rest + | (OPEN_PARTIAL_WITH_PARAMS (_ , name), loc1, loc2) :: rest -> + (OPEN_PARTIAL_WITH_PARAMS (skipped, name), loc1, loc2) :: rest | _ -> toks in let toks = @@ -202,8 +206,10 @@ and mustache = parse standalone acc rest | ((OPEN_SECTION _ | OPEN_INVERTED_SECTION _ - | CLOSE_SECTION _ + | CLOSE _ | PARTIAL _ + | OPEN_PARTIAL_WITH_PARAMS _ + | OPEN_PARAM _ | COMMENT _), _, _) as tok :: rest -> (* collect standalone tags *) standalone (tok :: acc) rest diff --git a/lib/mustache_parser.mly b/lib/mustache_parser.mly index 263ec12..f1b7628 100644 --- a/lib/mustache_parser.mly +++ b/lib/mustache_parser.mly @@ -28,10 +28,12 @@ { loc_start = start_pos; loc_end = end_pos } - let parse_section loc start_name end_name contents = + let check_matching loc name_kind start_name end_name = if start_name <> end_name then - raise (Mismatched_section { loc = mkloc loc; start_name; end_name }); - { contents; name = start_name } + raise (Mismatched_names (mkloc loc, { name_kind; start_name; end_name })) + + let dotted name = + string_of_dotted_name name let with_loc loc desc = { loc = mkloc loc; desc } @@ -42,8 +44,10 @@ %token UNESCAPE %token OPEN_INVERTED_SECTION %token OPEN_SECTION -%token CLOSE_SECTION %token PARTIAL +%token OPEN_PARTIAL_WITH_PARAMS +%token OPEN_PARAM +%token CLOSE %token COMMENT %token RAW @@ -53,31 +57,46 @@ %% -section: - | ss = OPEN_INVERTED_SECTION - e = mustache_expr - se = CLOSE_SECTION { - with_loc $sloc - (Inverted_section (parse_section $sloc ss se e)) - } - | ss = OPEN_SECTION - e = mustache_expr - se = CLOSE_SECTION { - with_loc $sloc - (Section (parse_section $sloc ss se e)) - } - mustache_element: - | elt = UNESCAPE { with_loc $sloc (Unescaped elt) } | elt = ESCAPE { with_loc $sloc (Escaped elt) } - | elt = PARTIAL { + | elt = UNESCAPE { with_loc $sloc (Unescaped elt) } + | start_name = OPEN_SECTION + contents = mustache_expr + end_name = CLOSE { + check_matching $sloc Section_name (dotted start_name) end_name; + with_loc $sloc + (Section { name = start_name; contents }) + } + | start_name = OPEN_INVERTED_SECTION + contents = mustache_expr + end_name = CLOSE { + check_matching $sloc Inverted_section_name (dotted start_name) end_name; + with_loc $sloc + (Inverted_section { name = start_name; contents }) + } + | partial = PARTIAL { + let (indent, name) = partial in with_loc $sloc - (Partial { indent = fst elt; - name = snd elt; + (Partial { indent; name; params = None; contents = lazy None }) - } + } + | partial = OPEN_PARTIAL_WITH_PARAMS + params = params + end_name = CLOSE { + let (indent, start_name) = partial in + check_matching $sloc Partial_with_params_name start_name end_name; + with_loc $sloc + (Partial { indent; name = start_name; params = Some params; + contents = lazy None }) + } + | start_name = OPEN_PARAM + contents = mustache_expr + end_name = CLOSE { + check_matching $sloc Param_name start_name end_name; + with_loc $sloc + (Param { name = start_name; contents }) + } | s = COMMENT { with_loc $sloc (Comment s) } - | sec = section { sec } | s = RAW { with_loc $sloc (String s) } mustache_expr: @@ -88,6 +107,32 @@ mustache_expr: | xs -> with_loc $sloc (Concat xs) } +(* The template-inheritance specification describes partial-with-params + application of the form: + + {{ List.filter_map (function + | { loc = _; desc = Param param } -> Some param + | _ -> None + ) + } + mustache: | mexpr = mustache_expr EOF { mexpr } diff --git a/lib/mustache_types.ml b/lib/mustache_types.ml index 64647c7..e28a017 100644 --- a/lib/mustache_types.ml +++ b/lib/mustache_types.ml @@ -43,10 +43,11 @@ module Locs = struct type desc = | String of string | Escaped of dotted_name - | Section of section | Unescaped of dotted_name - | Partial of partial + | Section of section | Inverted_section of section + | Partial of partial + | Param of param | Concat of t list | Comment of string and section = @@ -55,7 +56,11 @@ module Locs = struct and partial = { indent: int; name: name; + params: param list option; contents: t option Lazy.t } + and param = + { name: name; + contents: t } and t = { loc : loc; desc : desc } @@ -67,10 +72,11 @@ module No_locs = struct type t = | String of string | Escaped of dotted_name - | Section of section | Unescaped of dotted_name - | Partial of partial + | Section of section | Inverted_section of section + | Partial of partial + | Param of param | Concat of t list | Comment of string and section = @@ -79,13 +85,20 @@ module No_locs = struct and partial = { indent: int; name: name; + params: param list option; contents: t option Lazy.t } + and param = + { name: name; + contents: t } end +type name_kind = Section_name | Inverted_section_name | Partial_with_params_name | Param_name +type name_mismatch_error = { + name_kind: name_kind; + start_name: name; + end_name: name; +} + (* this exception is used internally in the parser, never exposed to users *) -exception Mismatched_section of { - loc: loc; - start_name: dotted_name; - end_name: dotted_name; -} +exception Mismatched_names of loc * name_mismatch_error diff --git a/mustache.opam b/mustache.opam index 2704ad2..d5fedca 100644 --- a/mustache.opam +++ b/mustache.opam @@ -22,7 +22,7 @@ depends: [ "ezjsonm" {with-test} "menhir" {>= "20180703"} "cmdliner" {>= "1.0.4"} - "ocaml" {>= "4.06"} + "ocaml" {>= "4.08"} "odoc" {with-doc} ] build: [ From 0d905bc03f04ceb327947e0823bf3485eb3ead16 Mon Sep 17 00:00:00 2001 From: Gabriel Scherer Date: Tue, 29 Dec 2020 22:43:17 +0100 Subject: [PATCH 3/7] add inheritance specs to the specification testsuite The specs come from the semi-official specification https://github.com/mustache/spec/pull/75 --- lib_test/dune | 3 +- lib_test/spec_mustache.ml | 1 + specs/inheritance.json | 1 + specs/inheritance.yml | 192 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 specs/inheritance.json create mode 100644 specs/inheritance.yml diff --git a/lib_test/dune b/lib_test/dune index d5d047d..a55a42a 100644 --- a/lib_test/dune +++ b/lib_test/dune @@ -1,5 +1,4 @@ (tests (libraries mustache ounit2 ezjsonm) (names test_mustache spec_mustache) - (deps test_mustache.exe ../specs/comments.json ../specs/interpolation.json - ../specs/partials.json ../specs/sections.json ../specs/inverted.json)) + (deps test_mustache.exe (glob_files ../specs/*.json))) diff --git a/lib_test/spec_mustache.ml b/lib_test/spec_mustache.ml index 34ea939..af77877 100644 --- a/lib_test/spec_mustache.ml +++ b/lib_test/spec_mustache.ml @@ -94,6 +94,7 @@ let mktest test = let specs = [ "comments.json"; + "inheritance.json"; "interpolation.json"; "inverted.json"; "partials.json"; diff --git a/specs/inheritance.json b/specs/inheritance.json new file mode 100644 index 0000000..07b6311 --- /dev/null +++ b/specs/inheritance.json @@ -0,0 +1 @@ +{"overview":"Parent tags are used to expand an external template into the current template,\nwith optional parameters delimited by block tags.\n\nThese tags' content MUST be a non-whitespace character sequence NOT containing\nthe current closing delimiter; each Parent tag MUST be followed by an End\nSection tag with the same content within the matching parent tag.\n\nBlock tags are used inside of parent tags to assign data onto the context stack \nprior to rendering the parent template. Outside of parent tags, block tags are\nused to indicate where value set in the parent tag should be placed. If no value\nis set then the content in between the block tags, if any, is rendered.\n","tests":[{"name":"Default","desc":"Default content should be rendered if the block isn't overridden","data":{},"template":"{{$title}}Default title{{/title}}\n","expected":"Default title\n"},{"name":"Variable","desc":"Default content renders variables","data":{"bar":"baz"},"template":"{{$foo}}default {{bar}} content{{/foo}}\n","expected":"default baz content\n"},{"name":"Triple Mustache","desc":"Default content renders triple mustache variables","data":{"bar":""},"template":"{{$foo}}default {{{bar}}} content{{/foo}}\n","expected":"default content\n"},{"name":"Sections","desc":"Default content renders sections","data":{"bar":{"baz":"qux"}},"template":"{{$foo}}default {{#bar}}{{baz}}{{/bar}} content{{/foo}}\n","expected":"default qux content\n"},{"name":"Negative Sections","desc":"Default content renders negative sections","data":{"baz":"three"},"template":"{{$foo}}default {{^bar}}{{baz}}{{/bar}} content{{/foo}}\n","expected":"default three content\n"},{"name":"Mustache Injection","desc":"Mustache injection in default content","data":{"bar":{"baz":"{{qux}}"}},"template":"{{$foo}}default {{#bar}}{{baz}}{{/bar}} content{{/foo}}\n","expected":"default {{qux}} content\n"},{"name":"Inherit","desc":"Default content rendered inside included templates","data":{},"template":"{{include}}|{{' } + template: | + {{$foo}}default {{{bar}}} content{{/foo}} + expected: | + default content + + - name: Sections + desc: Default content renders sections + data: { bar: {baz: 'qux'} } + template: | + {{$foo}}default {{#bar}}{{baz}}{{/bar}} content{{/foo}} + expected: | + default qux content + + - name: Negative Sections + desc: Default content renders negative sections + data: { baz: 'three' } + template: | + {{$foo}}default {{^bar}}{{baz}}{{/bar}} content{{/foo}} + expected: | + default three content + + - name: Mustache Injection + desc: Mustache injection in default content + data: {bar: {baz: '{{qux}}'} } + template: | + {{$foo}}default {{#bar}}{{baz}}{{/bar}} content{{/foo}} + expected: | + default {{qux}} content + + - name: Inherit + desc: Default content rendered inside included templates + data: { } + template: | + {{include}}|{{ Date: Thu, 31 Dec 2020 16:50:53 +0100 Subject: [PATCH 4/7] document partials with parameters --- README.md | 14 ++++++++--- bin/mustache_cli.ml | 59 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 03afce8..1b96147 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,18 @@ let rendered = exit 2 ``` -Spec compliance ------------ +Supported template language +--------------------------- + +ocaml-mustache accepts the whole Mustache template language, except: +- it does not support setting delimiter tags to something else than '{{' and '}}'. +- it does not support lambdas inside the provided data -ocaml-mustache complies¹ to the latest [mustache specification](https://github.com/mustache/spec/tree/v1.1.3), and is automatically tested against it. +It is automatically tested against the latest +[mustache specification testsuite](https://github.com/mustache/spec/tree/v1.1.3). -¹: except for lambdas and set delimiters tags. +ocaml-mustache also supports template inheritance / partials with parameters, +tested against the [semi-official specification](https://github.com/mustache/spec/pull/75). Todo/Wish List ----------- diff --git a/bin/mustache_cli.ml b/bin/mustache_cli.ml index e046b30..7d9137e 100644 --- a/bin/mustache_cli.ml +++ b/bin/mustache_cli.ml @@ -150,6 +150,19 @@ let manpage = Cmdliner.[ (leftmost $(b,-I) option) has precedence, and the current working directory has precedence over include directories."; + `S "TEMPLATE INHERITANCE / PARTIALS WITH PARAMETERS"; + + `P "$(i,ocaml-mustache) supports a common extension to the original Mustache specification, + called 'template inheritance' or 'parent partials', or here 'partials with parameters'. + In addition to usual partials '{{>foo}}', which include a partial template, one can use + the syntax '{{ @@ -189,6 +202,47 @@ Mustache is: - simple - fun + + + +## Including a layount around a page; see $(b,PARTIALS WITH PARAMETERS). + +\$ cat new-post.json +{ + "title": "New Post", + "authors": "Foo and Bar", + "date": "today", + "content": "Shiny new content." +} + +\$ cat post.mustache +{{{{title}} +

{{content}}

+ {{/content}} +{{/post-layout}} + +\$ cat post-layout.mustache + + + {{\$page-title}}Default Title{{/page-title}} + + + {{\$content}}{{/content}} + + + +\$ $(tname) new-post.json post.mustache + + + Post: New Post + + +

New Post

+

Shiny new content.

+ |}; `S "CONFORMING TO"; @@ -200,6 +254,9 @@ Mustache is: `I ("Mustache specification testsuite", "https://github.com/mustache/spec"); + `I ("Semi-official specification of PARTIALS WITH PARAMETERS", + "https://github.com/mustache/spec/pull/75"); + `S "REPORTING BUGS"; `P "Report bugs on https://github.com/rgrinberg/ocaml-mustache/issues"; ] From d21c4d66e5a0d6ee1316bce8bac59562444711b6 Mon Sep 17 00:00:00 2001 From: Gabriel Scherer Date: Thu, 31 Dec 2020 16:51:05 +0100 Subject: [PATCH 5/7] test the examples from the manpage --- bin/test/manpage-examples.t/data.json | 2 + bin/test/manpage-examples.t/hello.mustache | 5 ++ bin/test/manpage-examples.t/new-post.json | 4 + .../manpage-examples.t/page-layout.mustache | 8 ++ bin/test/manpage-examples.t/page.mustache | 5 ++ bin/test/manpage-examples.t/post.mustache | 7 ++ bin/test/manpage-examples.t/run.t | 77 +++++++++++++++++++ 7 files changed, 108 insertions(+) create mode 100644 bin/test/manpage-examples.t/data.json create mode 100644 bin/test/manpage-examples.t/hello.mustache create mode 100644 bin/test/manpage-examples.t/new-post.json create mode 100644 bin/test/manpage-examples.t/page-layout.mustache create mode 100644 bin/test/manpage-examples.t/page.mustache create mode 100644 bin/test/manpage-examples.t/post.mustache create mode 100644 bin/test/manpage-examples.t/run.t diff --git a/bin/test/manpage-examples.t/data.json b/bin/test/manpage-examples.t/data.json new file mode 100644 index 0000000..adf0806 --- /dev/null +++ b/bin/test/manpage-examples.t/data.json @@ -0,0 +1,2 @@ +{ "name": "OCaml", + "qualities": [{"name": "simple"}, {"name": "fun"}] } diff --git a/bin/test/manpage-examples.t/hello.mustache b/bin/test/manpage-examples.t/hello.mustache new file mode 100644 index 0000000..d17b8b5 --- /dev/null +++ b/bin/test/manpage-examples.t/hello.mustache @@ -0,0 +1,5 @@ +Hello {{name}}! +Mustache is: +{{#qualities}} +- {{name}} +{{/qualities}} diff --git a/bin/test/manpage-examples.t/new-post.json b/bin/test/manpage-examples.t/new-post.json new file mode 100644 index 0000000..d553039 --- /dev/null +++ b/bin/test/manpage-examples.t/new-post.json @@ -0,0 +1,4 @@ +{ + "title": "New Post", + "content": "Shiny new content." +} diff --git a/bin/test/manpage-examples.t/page-layout.mustache b/bin/test/manpage-examples.t/page-layout.mustache new file mode 100644 index 0000000..c9f4236 --- /dev/null +++ b/bin/test/manpage-examples.t/page-layout.mustache @@ -0,0 +1,8 @@ + + + {{$page-title}}Default Title{{/page-title}} + + + {{$content}}{{/content}} + + diff --git a/bin/test/manpage-examples.t/page.mustache b/bin/test/manpage-examples.t/page.mustache new file mode 100644 index 0000000..1d0894e --- /dev/null +++ b/bin/test/manpage-examples.t/page.mustache @@ -0,0 +1,5 @@ + + + {{>hello}} + + diff --git a/bin/test/manpage-examples.t/post.mustache b/bin/test/manpage-examples.t/post.mustache new file mode 100644 index 0000000..7c15cc0 --- /dev/null +++ b/bin/test/manpage-examples.t/post.mustache @@ -0,0 +1,7 @@ +{{{{title}} +

{{content}}

+ {{/content}} +{{/page-layout}} diff --git a/bin/test/manpage-examples.t/run.t b/bin/test/manpage-examples.t/run.t new file mode 100644 index 0000000..a03891b --- /dev/null +++ b/bin/test/manpage-examples.t/run.t @@ -0,0 +1,77 @@ +Simple usage: + + $ cat data.json + { "name": "OCaml", + "qualities": [{"name": "simple"}, {"name": "fun"}] } + + $ cat hello.mustache + Hello {{name}}! + Mustache is: + {{#qualities}} + - {{name}} + {{/qualities}} + + $ mustache data.json hello.mustache + Hello OCaml! + Mustache is: + - simple + - fun + + +Using a partial to include a subpage: + + $ cat page.mustache + + + {{>hello}} + + + + $ mustache data.json page.mustache + + + Hello OCaml! + Mustache is: + - simple + - fun + + + + +Using a partial with parameters to include a layout around a page: + + $ cat new-post.json + { + "title": "New Post", + "content": "Shiny new content." + } + + $ cat post.mustache + {{{{title}} +

{{content}}

+ {{/content}} + {{/page-layout}} + + $ cat page-layout.mustache + + + {{$page-title}}Default Title{{/page-title}} + + + {{$content}}{{/content}} + + + + $ mustache new-post.json post.mustache + + + Post: New Post + + +

New Post

+

Shiny new content.

+ + From 7c3bc53befbd0458ba52c5695ce6434405c6a101 Mon Sep 17 00:00:00 2001 From: Gabriel Scherer Date: Thu, 31 Dec 2020 18:51:19 +0100 Subject: [PATCH 6/7] indentation tests (currently exhibiting poor results) --- bin/test/inheritance.t/run.t | 17 +++++++++++++++++ .../inheritance.t/test-indent-less.mustache | 6 ++++++ .../inheritance.t/test-indent-more.mustache | 6 ++++++ .../inheritance.t/test-indentation.mustache | 4 ++++ 4 files changed, 33 insertions(+) create mode 100644 bin/test/inheritance.t/test-indent-less.mustache create mode 100644 bin/test/inheritance.t/test-indent-more.mustache create mode 100644 bin/test/inheritance.t/test-indentation.mustache diff --git a/bin/test/inheritance.t/run.t b/bin/test/inheritance.t/run.t index 7d81999..56ff004 100644 --- a/bin/test/inheritance.t/run.t +++ b/bin/test/inheritance.t/run.t @@ -12,3 +12,20 @@ https://github.com/mustache/spec/pull/75

Hello world

+ + +We also test the indentation of parameter blocks. + + $ mustache data.json test-indent-more.mustache +

+ The test below should be indented in the same way as this line. + This text is not indented in the source, + it should be indented naturally in the output. +

+ + $ mustache data.json test-indent-less.mustache +

+ The test below should be indented in the same way as this line. + This text is very indented in the source, + it should be indented naturally in the output. +

diff --git a/bin/test/inheritance.t/test-indent-less.mustache b/bin/test/inheritance.t/test-indent-less.mustache new file mode 100644 index 0000000..b977054 --- /dev/null +++ b/bin/test/inheritance.t/test-indent-less.mustache @@ -0,0 +1,6 @@ +{{ + The test below should be indented in the same way as this line. + {{$indented-block}}{{/indented-block}} +

From 9e4daebaa4aeaf612a31bef2197a19bbb9645bd3 Mon Sep 17 00:00:00 2001 From: Gabriel Scherer Date: Fri, 1 Jan 2021 10:57:57 +0100 Subject: [PATCH 7/7] try harder to indent partial parameters as the user would naturally expect --- bin/test/inheritance.t/run.t | 14 ++-- lib/mustache.ml | 138 +++++++++++++++++++++-------------- lib/mustache.mli | 24 +++--- lib/mustache_lexer.mll | 74 ++++++++++++++----- lib/mustache_parser.mly | 7 +- lib/mustache_types.ml | 6 +- 6 files changed, 167 insertions(+), 96 deletions(-) diff --git a/bin/test/inheritance.t/run.t b/bin/test/inheritance.t/run.t index 56ff004..1e0496f 100644 --- a/bin/test/inheritance.t/run.t +++ b/bin/test/inheritance.t/run.t @@ -5,9 +5,9 @@ https://github.com/mustache/spec/pull/75 $ mustache data.json mypage.mustache - - My page title - + + My page title +

Hello world

@@ -19,13 +19,13 @@ We also test the indentation of parameter blocks. $ mustache data.json test-indent-more.mustache

The test below should be indented in the same way as this line. - This text is not indented in the source, - it should be indented naturally in the output. + This text is not indented in the source, + it should be indented naturally in the output.

$ mustache data.json test-indent-less.mustache

The test below should be indented in the same way as this line. - This text is very indented in the source, - it should be indented naturally in the output. + This text is very indented in the source, + it should be indented naturally in the output.

diff --git a/lib/mustache.ml b/lib/mustache.ml index 88b9810..a5cbd9c 100644 --- a/lib/mustache.ml +++ b/lib/mustache.ml @@ -84,6 +84,7 @@ and erase_locs_partial (p : Locs.partial) : No_locs.partial = { contents = lazy (Option.map erase_locs (Lazy.force p.contents)) } and erase_locs_param (pa : Locs.param) : No_locs.param = { + indent = pa.indent; name = pa.name; contents = erase_locs pa.contents; } @@ -113,6 +114,7 @@ and add_dummy_locs_partial (p : No_locs.partial) : Locs.partial = { contents = lazy (Option.map add_dummy_locs (Lazy.force p.contents)); } and add_dummy_locs_param (pa : No_locs.param) : Locs.param = { + indent = pa.indent; name = pa.name; contents = add_dummy_locs pa.contents; } @@ -315,7 +317,7 @@ module Contexts : sig val add : t -> Json.value -> t val find_name : t -> string -> Json.value option val add_param : t -> Locs.param -> t - val find_param : t -> string -> Locs.t option + val find_param : t -> string -> Locs.param option end = struct type t = { (* nonempty stack of contexts, most recent first *) @@ -323,7 +325,7 @@ end = struct (* an associative list of partial parameters that have been defined *) - params: (string * Locs.t) list; + params: Locs.param list; } let start js = { @@ -357,6 +359,9 @@ end = struct | [] -> None | top :: rest -> find_name { ctxs with stack = (top, rest) } name + + let param_has_name name (p : Locs.param) = String.equal p.name name + (* Note: the template-inheritance specification for Mustache (https://github.com/mustache/spec/pull/75) mandates that in case of multi-level inclusion, the "topmost" definition of the @@ -374,15 +379,15 @@ end = struct a grandparent), and then late-binding mandates that the definition "last" in the inheritance chain (so closest to the start of the rendering) wins.*) - let add_param ctxs { Locs.name; contents } = - if List.mem_assoc name ctxs.params then + let add_param ctxs (param : Locs.param) = + if List.exists (param_has_name param.name) ctxs.params then (* if the parameter is already bound, the existing binding has precedence *) ctxs else - {ctxs with params = (name, contents) :: ctxs.params} + {ctxs with params = param :: ctxs.params} let find_param ctxs name = - List.assoc_opt name ctxs.params + List.find_opt (param_has_name name) ctxs.params end let raise_err loc kind = @@ -474,34 +479,64 @@ module Render = struct ?(strict = true) (buf : Buffer.t) (m : Locs.t) (js : Json.t) = - let print_indent indent = - for _ = 0 to indent - 1 do - Buffer.add_char buf ' ' - done + let beginning_of_line = ref true in + + let print_indented buf indent line = + assert (indent >= 0); + if String.equal line "" + then () + else begin + for _i = 1 to indent do Buffer.add_char buf ' ' done; + Buffer.add_string buf line; + beginning_of_line := false; + end in - let beginning_of_line = ref true in + let print_dedented buf dedent line = + assert (dedent >= 0); + let rec print_from i = + if i = String.length line then () + else if i < dedent && (match line.[i] with ' ' | '\t' -> true | _ -> false) + then print_from (i + 1) + else begin + Buffer.add_substring buf line i (String.length line - i); + beginning_of_line := false; + end + in + print_from 0 + in - let align indent = - if !beginning_of_line then ( - print_indent indent; - beginning_of_line := false - ) + let print_line indent line = + if not !beginning_of_line then + Buffer.add_string buf line + else begin + if indent >= 0 + then print_indented buf indent line + else print_dedented buf (-indent) line; + end + in + + let print_newline buf = + Buffer.add_char buf '\n'; + beginning_of_line := true in let print_indented_string indent s = let lines = String.split_on_char '\n' s in - align indent; Buffer.add_string buf (List.hd lines); + print_line indent (List.hd lines); List.iter (fun line -> - Buffer.add_char buf '\n'; - beginning_of_line := true; - if line <> "" then ( - align indent; - Buffer.add_string buf line; - ) + print_newline buf; + print_line indent line ) (List.tl lines) in + let print_interpolated indent data = + (* per the specification, interpolated data should be spliced into the + document, with further lines *not* indented specifically; this effect + is obtained by calling print_line on the (possibly multiline) data. *) + print_line indent data + in + let rec render indent m (ctxs : Contexts.t) = let loc = m.loc in match m.desc with @@ -510,12 +545,12 @@ module Render = struct print_indented_string indent s | Escaped name -> - align indent; - Buffer.add_string buf (escape_html (Lookup.str ~strict ~loc ~key:name ctxs)) + print_interpolated indent + (escape_html (Lookup.str ~strict ~loc ~key:name ctxs)) | Unescaped name -> - align indent; - Buffer.add_string buf (Lookup.str ~strict ~loc ~key:name ctxs) + print_interpolated indent + (Lookup.str ~strict ~loc ~key:name ctxs) | Inverted_section s -> if Lookup.inverted ctxs ~loc ~key:s.name @@ -545,18 +580,13 @@ module Render = struct render (indent + partial_indent) partial ctxs end - | Param { name; contents } -> + | Param default_param -> let param = - match Lookup.param ctxs ~loc ~key:name with - | None -> - (* The "contents" of the partial parameter is to be used as - default content, if the parameter was not explicitly passed - by one of the partials in scope. *) - contents - | Some param -> - param + match Lookup.param ctxs ~loc ~key:default_param.name with + | Some passed_param -> passed_param + | None -> default_param in - render indent param ctxs + render (indent + default_param.indent - param.indent) param.contents ctxs | Comment _c -> () @@ -598,11 +628,11 @@ module Without_locations = struct concat (List.map ms ~f:go) | Partial {indent; name; params; contents} -> let params = - Option.map (List.map ~f:(fun {name; contents} -> (name, go contents))) params + Option.map (List.map ~f:(fun {indent; name; contents} -> (indent, name, go contents))) params in - partial indent name ?params contents - | Param { name; contents } -> - param name (go contents) + partial ?indent:(Some indent) name ?params contents + | Param { indent; name; contents } -> + param ?indent:(Some indent) name (go contents) module Infix = struct let (^) y x = Concat [x; y] @@ -615,9 +645,9 @@ module Without_locations = struct let inverted_section n c = Inverted_section { name = n ; contents = c } let partial ?(indent = 0) n ?params c = let params = - Option.map (List.map ~f:(fun (name, contents) -> {name; contents})) params in + Option.map (List.map ~f:(fun (indent, name, contents) -> {indent; name; contents})) params in Partial { indent ; name = n ; params; contents = c } - let param n c = Param { name = n; contents = c } + let param ?(indent=0) n c = Param { indent; name = n; contents = c } let concat t = Concat t let comment s = Comment s @@ -625,14 +655,14 @@ module Without_locations = struct let section ~inverted = if inverted then inverted_section else section in - let partial indent name ?params contents = + let partial ?indent name ?params contents = let contents' = lazy ( match Lazy.force contents with | None -> Option.map (expand_partials partials) (partials name) | Some t_opt -> Some t_opt ) in - partial ~indent name ?params contents' + partial ?indent name ?params contents' in fold ~string:raw ~section ~escaped ~unescaped ~partial ~param ~comment ~concat @@ -683,10 +713,10 @@ module With_locations = struct concat ~loc (List.map ms ~f:go) | Partial p -> let params = - Option.map (List.map ~f:(fun {name; contents} -> (name, go contents))) p.params in - partial ~loc p.indent p.name ?params p.contents - | Param { name; contents } -> - param ~loc name (go contents) + Option.map (List.map ~f:(fun {indent; name; contents} -> (indent, name, go contents))) p.params in + partial ~loc ?indent:(Some p.indent) p.name ?params p.contents + | Param { indent; name; contents } -> + param ~loc ?indent:(Some indent) name (go contents) module Infix = struct let (^) t1 t2 = { desc = Concat [t1; t2]; loc = dummy_loc } @@ -703,27 +733,27 @@ module With_locations = struct loc } let partial ~loc ?(indent = 0) n ?params c = let params = - Option.map (List.map ~f:(fun (name, contents) -> {name; contents})) params in + Option.map (List.map ~f:(fun (indent, name, contents) -> {indent; name; contents})) params in { desc = Partial { indent; name = n; params; contents = c }; loc } let concat ~loc t = { desc = Concat t; loc } let comment ~loc s = { desc = Comment s; loc } - let param ~loc n c = - { desc = Param { name = n; contents = c }; + let param ~loc ?(indent = 0) n c = + { desc = Param { indent; name = n; contents = c }; loc } let rec expand_partials (partials : name -> t option) : t -> t = let section ~loc ~inverted = if inverted then inverted_section ~loc else section ~loc in - let partial ~loc indent name ?params contents = + let partial ~loc ?indent name ?params contents = let contents' = lazy ( match Lazy.force contents with | None -> Option.map (expand_partials partials) (partials name) | Some t_opt -> Some t_opt ) in - partial ~loc ~indent name ?params contents' + partial ~loc ?indent name ?params contents' in fold ~string:raw ~section ~escaped ~unescaped ~partial ~param ~comment ~concat diff --git a/lib/mustache.mli b/lib/mustache.mli index ea146a7..6216543 100644 --- a/lib/mustache.mli +++ b/lib/mustache.mli @@ -38,7 +38,8 @@ and partial = params: param list option; contents: t option Lazy.t } and param = - { name: name; + { indent: int; + name: name; contents: t } type loc = @@ -141,8 +142,8 @@ val fold : string: (string -> 'a) -> section: (inverted:bool -> dotted_name -> 'a -> 'a) -> escaped: (dotted_name -> 'a) -> unescaped: (dotted_name -> 'a) -> - partial: (int -> name -> ?params:(name * 'a) list -> t option Lazy.t -> 'a) -> - param: (name -> 'a -> 'a) -> + partial: (?indent:int -> name -> ?params:(int * name * 'a) list -> t option Lazy.t -> 'a) -> + param: (?indent:int -> name -> 'a -> 'a) -> comment: (string -> 'a) -> concat:('a list -> 'a) -> t -> 'a @@ -186,10 +187,11 @@ val section : dotted_name -> t -> t {{/box}} ]} *) -val partial : ?indent:int -> name -> ?params:(name * t) list -> t option Lazy.t -> t +val partial : + ?indent:int -> name -> ?params:(int * name * t) list -> t option Lazy.t -> t (** [{{$foo}} {{/foo}}] *) -val param: name -> t -> t +val param : ?indent:int -> name -> t -> t (** [{{! this is a comment}}] *) val comment : string -> t @@ -223,7 +225,8 @@ module With_locations : sig params: param list option; contents: t option Lazy.t } and param = - { name: name; + { indent: int; + name: name; contents: t } and t = { loc : loc; @@ -296,8 +299,8 @@ module With_locations : sig section: (loc:loc -> inverted:bool -> dotted_name -> 'a -> 'a) -> escaped: (loc:loc -> dotted_name -> 'a) -> unescaped: (loc:loc -> dotted_name -> 'a) -> - partial: (loc:loc -> int -> name -> ?params:(name * 'a) list -> t option Lazy.t -> 'a) -> - param: (loc:loc -> name -> 'a -> 'a) -> + partial: (loc:loc -> ?indent:int -> name -> ?params:(int * name * 'a) list -> t option Lazy.t -> 'a) -> + param: (loc:loc -> ?indent:int -> name -> 'a -> 'a) -> comment: (loc:loc -> string -> 'a) -> concat:(loc:loc -> 'a list -> 'a) -> t -> 'a @@ -339,10 +342,11 @@ module With_locations : sig {{/box}} ]} *) - val partial : loc:loc -> ?indent:int -> name -> ?params:(name * t) list -> t option Lazy.t -> t + val partial : + loc:loc -> ?indent:int -> name -> ?params:(int * name * t) list -> t option Lazy.t -> t (** [{{$foo}} {{/foo}}] *) - val param: loc:loc -> name -> t -> t + val param : loc:loc -> ?indent:int -> name -> t -> t (** [{{! this is a comment}}] *) val comment : loc:loc -> string -> t diff --git a/lib/mustache_lexer.mll b/lib/mustache_lexer.mll index b258e1d..c8269a2 100644 --- a/lib/mustache_lexer.mll +++ b/lib/mustache_lexer.mll @@ -104,7 +104,7 @@ and mustache = parse | "{{/" { CLOSE (lex_tag lexbuf space partial_name (end_on "}}")) } | "{{>" { PARTIAL (0, lex_tag lexbuf space partial_name (end_on "}}")) } | "{{<" { OPEN_PARTIAL_WITH_PARAMS (0, lex_tag lexbuf space partial_name (end_on "}}")) } - | "{{$" { OPEN_PARAM (lex_tag lexbuf space ident (end_on "}}")) } + | "{{$" { OPEN_PARAM (0, lex_tag lexbuf space ident (end_on "}}")) } | "{{!" { COMMENT (tok_arg lexbuf (comment [])) } | raw newline { new_line lexbuf; RAW (lexeme lexbuf) } | raw { RAW (lexeme lexbuf) } @@ -159,23 +159,34 @@ and mustache = parse let loc_end = get_loc () in (tok, loc_start, loc_end) in - let slurp_line () = - let rec loop acc = - let tok = get_tok () in + let slurp_line lookahead = + let rec start = function + | None -> loop [] + | Some lookahead -> continue [] lookahead + and loop acc = + continue acc (get_tok ()) + and continue acc tok = match tok with - | EOF, _, _ -> tok :: acc - | RAW s, _, _ when ends_with_newline s -> tok :: acc + | EOF, _, _ -> (List.rev (tok :: acc), None) + | RAW s, _, _ when ends_with_newline s -> + let lookahead = get_tok () in + (List.rev (tok :: acc), Some lookahead) | _ -> loop (tok :: acc) in - List.rev (loop []) + start lookahead in - let is_blank s = - let ret = ref true in - for i = 0 to String.length s - 1 do - if not (List.mem s.[i] [' '; '\t'; '\r'; '\n']) then - ret := false + let count_indentation s = + let i = ref 0 in + let len = String.length s in + while (!i < len + && match s.[!i] with ' ' | '\t' | '\r' | '\n' -> true | _ -> false) + do + incr i done; - !ret + !i + in + let is_blank s = + count_indentation s = String.length s in let skip_blanks l = let rec loop skipped = function @@ -185,7 +196,7 @@ and mustache = parse in loop 0 l in - let trim_standalone toks = + let trim_standalone toks lookahead = let toks = (* if the line starts with a partial, turn the skipped blank into partial indentation *) @@ -195,6 +206,23 @@ and mustache = parse (PARTIAL (skipped, name), loc1, loc2) :: rest | (OPEN_PARTIAL_WITH_PARAMS (_ , name), loc1, loc2) :: rest -> (OPEN_PARTIAL_WITH_PARAMS (skipped, name), loc1, loc2) :: rest + | (OPEN_PARAM (_ , name), loc1, loc2) :: rest -> + (* we want to count the indentation of + {{$param}} + blah blah + {{/param}} + as the indentation of 'blah blah', not the indentation + of '{{$param}}' itself: using the parameter tag instead of the content + as indentation would result in the content being over-indented at each occurrence. + *) + let skipped = + match rest, lookahead with + | ((RAW end_of_line, _, _) :: _), + Some (RAW start_of_next_line, _, _) when ends_with_newline end_of_line -> + count_indentation start_of_next_line + | _ -> skipped + in + (OPEN_PARAM (skipped, name), loc1, loc2) :: rest | _ -> toks in let toks = @@ -232,11 +260,17 @@ and mustache = parse assert (toks <> []); toks in - let buffer = ref [] in + let line_rest = ref [] in + let lookahead = ref None in fun () -> - let toks = match !buffer with - | (_ :: _) as toks -> toks - | [] -> trim_standalone (slurp_line ()) - in - buffer := List.tl toks; List.hd toks + match !line_rest with + | next :: rest -> + line_rest := rest; + next + | [] -> + let next_line, next_lookahead = slurp_line !lookahead in + let next_line = trim_standalone next_line next_lookahead in + line_rest := List.tl next_line; + lookahead := next_lookahead; + List.hd next_line } diff --git a/lib/mustache_parser.mly b/lib/mustache_parser.mly index f1b7628..556106e 100644 --- a/lib/mustache_parser.mly +++ b/lib/mustache_parser.mly @@ -46,7 +46,7 @@ %token OPEN_SECTION %token PARTIAL %token OPEN_PARTIAL_WITH_PARAMS -%token OPEN_PARAM +%token OPEN_PARAM %token CLOSE %token COMMENT @@ -89,12 +89,13 @@ mustache_element: (Partial { indent; name = start_name; params = Some params; contents = lazy None }) } - | start_name = OPEN_PARAM + | param = OPEN_PARAM contents = mustache_expr end_name = CLOSE { + let (indent, start_name) = param in check_matching $sloc Param_name start_name end_name; with_loc $sloc - (Param { name = start_name; contents }) + (Param { indent; name = start_name; contents }) } | s = COMMENT { with_loc $sloc (Comment s) } | s = RAW { with_loc $sloc (String s) } diff --git a/lib/mustache_types.ml b/lib/mustache_types.ml index e28a017..b395df4 100644 --- a/lib/mustache_types.ml +++ b/lib/mustache_types.ml @@ -59,7 +59,8 @@ module Locs = struct params: param list option; contents: t option Lazy.t } and param = - { name: name; + { indent: int; + name: name; contents: t } and t = { loc : loc; @@ -88,7 +89,8 @@ module No_locs = struct params: param list option; contents: t option Lazy.t } and param = - { name: name; + { indent: int; + name: name; contents: t } end