Add filter support, add few escaping filters

tailhook · tailhook · commit b6404780280f · 2018-01-29T01:47:42.000+02:00
- feature: add filters support for template syntax
- feature: add `builtin.html_entities` filter
- feature: add `builtin.quoted_shell_argument` filter
diff --git a/doc/changelog.rst b/doc/changelog.rst
@@ -65,4 +65,4 @@ v0.3.6
   dictionaries can be used instead of multiple ``if`` statements or to refer
   to the same values multiple times
 * feature: Add python list syntax support: ``["item1", "item2"]``
-
+* feature: Add support for escaping of the variables no only validating
diff --git a/doc/index.rst b/doc/index.rst
@@ -65,6 +65,31 @@ Results in (note the indentation):
         }
     }
 
+Since v0.3.6 it also works well for HTML and has autoescaping,
+here is an example:
+
+.. code-block:: html
+
+    ## syntax: indent
+    ## filter default: builtin.html_entities  ### autoescape
+    ## validate n: [0-9]+
+    <!DOCTYPE html>
+    <html>
+        <body>
+            <h1>{{ title }}</h1>               ### auto escaped
+            <div class="user_canvas"
+                style="position: absolute;     ### note: it's unsafe to just
+                       left: {{ x | n }}px;    ### autoescape values here
+                       top: {{ y | n }}px;">
+            </div>
+        </body>
+    </html>
+
+While the validation here looks like excessive (you must prevalidate it in
+the app) some time ago we thought that escaping values against XSS manually is
+a normal practice, now we always use autoescape. We look at validation of other
+values directly in the template as another way to minimize human error.
+
 
 Indices and tables
 ==================
diff --git a/doc/template_syntax.rst b/doc/template_syntax.rst
@@ -176,6 +176,95 @@ safe to put anything except a quote, so for all variables printed in quotes we
 can add a ``quoted`` validator. See :ref:`front page <showcase>` for more
 practical example.
 
+.. _filter:
+.. index:: pair: Filter; Statement
+
+Filter Statement
+==================
+
+Similarly to validate statements there is filter statement. This is useful
+if you produce HTML output and escape (qoute) some characters in it. This
+works similarly to how filters work in any other template engine, except it
+requires to declare an alias to validator, at the top of the template::
+
+    ## filter h: builtin.html_entities
+    <html>
+        <body>
+            <h1>{{ title | h }}</h1>
+        </body>
+    </html>
+
+Of course, there is a default filter declared in the template too.
+
+Validators and filters are in the same namespace so you can override
+filter by a validator::
+
+    ## filter default: builtin.html_entities
+    ## validate ne: .*
+    <html>
+        <body>
+            <h1>{{ title }}</h1>  ### auto-escaped
+            {{ body | ne }}       ### a piece of HTML validated in advance
+        </body>
+    </html>
+
+
+Predefined validators
+---------------------
+
+
+.. index:: pair: html_entities; Builtin Filter
+
+:builtin.html_entities:
+
+    transforms ``<>"'`/`` into HTML entities. It's good idea to use it as a
+    ``filter default:`` in any HTML template.
+
+    .. note:: This is not a catch-all validator for HTML (like in any other
+        template), you can't put such variables in tag name, attribute name,
+        script or style tag.
+
+        This work the same in amost any other HTML template engine, though.
+        Consult OWASP_ for more info.
+
+    .. _owasp: https://www.owasp.org/index.php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet
+
+.. index:: pair: quoted_shell_argument; Builtin Filter
+
+:builtin.quoted_shell_argument:
+
+    escapes variable value into an argument that's safe to put onto shell
+    command-line. It's **not safe** to put argument in quotes.
+
+    Example:
+
+    .. code-block:: bash
+
+        ## filter arg: builtin.shell_argument
+        #!/bin/sh
+        some_cmd {{ value1 | arg }} {{ value2 | arg }}
+
+    If the value is a part of the argument you must stop quotes before the arg:
+
+    .. code-block:: bash
+
+        ## filter arg: builtin.shell_argument
+        #!/bin/sh
+        echo "Here's arg: "{{ value1 | arg }}". It's safe"
+
+    The quoting caveats and brittleness of shell-commands are the reason it's
+    not recommended to use it as a default argument for example:
+
+    .. code-block:: bash
+
+        ## validate alnum: [a-z0-9]+
+        #!/bin/sh
+        cp source/{{ name | alnum }} target/{{ name | alnum }}
+
+    The above, wouldn't be safe with just shell escaping as one could use
+    parent directory specifiers even if shell expansion would be escaped in
+    ``name``.
+
 
 .. index:: pair: If; Statement
 
diff --git a/src/escape.rs b/src/escape.rs
@@ -0,0 +1,28 @@
+pub fn html_entities(dest: &mut String, src: &str) {
+    for c in src.chars() {
+        match c {
+            '&' => dest.push_str("&amp;"),
+            '<' => dest.push_str("&lt;"),
+            '>' => dest.push_str("&gt;"),
+            '"' => dest.push_str("&quot;"),
+            '\'' => dest.push_str("&#x27;"),
+            '/' => dest.push_str("&#x2f;"),
+            '`' => dest.push_str("&#x96;"),
+            _ => dest.push(c),
+        }
+    }
+}
+
+pub fn quoted_shell_argument(dest: &mut String, src: &str) {
+    // Do same as python and bash: enclouse in single quotes
+    // and escape a single quote.
+    // Known quirk: zsh converges double backslash even in single-quotes
+    dest.push('\'');
+    for c in src.chars() {
+        match c {
+            '\'' => dest.push_str(r#"'"'"'"#),
+            _ => dest.push(c),
+        }
+    }
+    dest.push('\'');
+}
diff --git a/src/lib.rs b/src/lib.rs
@@ -16,6 +16,7 @@ extern crate regex;
 #[cfg(feature="json")] extern crate serde_json;
 
 mod compare;
+mod escape;
 mod grammar;
 mod helpers;
 mod indent;
@@ -70,8 +71,8 @@ pub struct Options {
     square: bool,
     round: bool,
 
-    default_validator: validators::Validator,
-    validators: HashMap<String, validators::Validator>,
+    default_filter: validators::Filter,
+    filters: HashMap<String, validators::Filter>,
 }
 
 /// Variable reference returned from methods of Variable trait
diff --git a/src/options.rs b/src/options.rs
@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 
 use preparser::Syntax;
-use validators::Validator;
+use validators::Filter;
 use {Options};
 
 impl Options {
@@ -13,8 +13,8 @@ impl Options {
             curly: false,
             square: false,
             round: false,
-            default_validator: Validator::Anything,
-            validators: HashMap::new(),
+            default_filter: Filter::NoFilter,
+            filters: HashMap::new(),
         }
     }
     /// Enables `oneline` syntax by default
diff --git a/src/parse_error.rs b/src/parse_error.rs
@@ -33,6 +33,9 @@ quick_error! {
             description("Validator regexp is invalid")
             display("Validator regex {:?} is invalid: {}", value, err)
         }
+        BadFilter(value: String) {
+            display("Filter {:?} is unknown", value)
+        }
     }
 }
 
diff --git a/src/preparser.rs b/src/preparser.rs
@@ -1,7 +1,7 @@
 use regex::{Regex, RegexSet};
 
 use parse_error::{ParseError, ParseErrorEnum};
-use validators::Validator;
+use validators::Filter;
 use {Options};
 
 
@@ -21,6 +21,7 @@ pub enum Syntax {
 pub enum Token {
     Syntax,
     Validate,
+    Filter,
     Comment,
 }
 
@@ -32,6 +33,8 @@ impl Preparser {
             (r"^##\s*syntax:\s*(\w+)(?:\n|$)", Syntax),
             (r"^##\s*validate\s+(\w+):[ \t]*(.*)\s*(?:\n|$)",
                 Validate),
+            (r"^##\s*filter\s+(\w+):[ \t]*(.*)\s*(?:\n|$)",
+                Filter),
             (r"^#.*(?:\n|$)", Comment),
             (r"^###.*(?:\n|$)", Comment),
             (r"^\s*\n", Comment),
@@ -99,11 +102,22 @@ impl Preparser {
                                 .map_err(|e| ParseErrorEnum::BadRegexValidator(
                                     regex.to_string(), e))?;
                             if name == "default" {
-                                options.default_validator =
-                                    Validator::Regex(regex);
+                                options.default_filter =
+                                    Filter::Validate(regex);
                             } else {
-                                options.validators.insert(
-                                    name.to_string(), Validator::Regex(regex));
+                                options.filters.insert(
+                                    name.to_string(), Filter::Validate(regex));
+                            }
+                        }
+                        Token::Filter => {
+                            let name = m.get(1).unwrap().as_str();
+                            let filter = m.get(2).unwrap().as_str().parse()?;
+                            if name == "default" {
+                                options.default_filter =
+                                    Filter::Escape(filter);
+                            } else {
+                                options.filters.insert(name.to_string(),
+                                    Filter::Escape(filter));
                             }
                         }
                         Token::Comment => {
@@ -120,7 +134,7 @@ impl Preparser {
 
 #[cfg(test)]
 mod test {
-    use validators::Validator;
+    use validators::Filter;
     use super::{Preparser, Syntax};
     use {Options};
 
@@ -129,17 +143,17 @@ mod test {
         let opt = Preparser::new().scan("## syntax: indent\n",
             Options::new().clone()).unwrap();
         assert_eq!(opt.syntax, Syntax::Indent);
-        assert!(matches!(opt.default_validator, Validator::Anything));
-        assert_eq!(opt.validators.len(), 0);
+        assert!(matches!(opt.default_filter, Filter::NoFilter));
+        assert_eq!(opt.filters.len(), 0);
     }
 
     #[test]
     fn oneline() {
         let opt = Preparser::new().scan("## syntax: oneline\n",
             Options::new().clone()).unwrap();
         assert_eq!(opt.syntax, Syntax::Oneline);
-        assert!(matches!(opt.default_validator, Validator::Anything));
-        assert_eq!(opt.validators.len(), 0);
+        assert!(matches!(opt.default_filter, Filter::NoFilter));
+        assert_eq!(opt.filters.len(), 0);
     }
 
     #[test]
diff --git a/src/render.rs b/src/render.rs
@@ -16,6 +16,7 @@ use preparser::Syntax::Oneline;
 use render_error::{RenderError, DataError};
 use varmap::{Context, SubContext, set, get};
 use vars::{UNDEFINED, TRUE, FALSE, Val, VarRef, RefVar};
+use validators::Filter;
 use {Pos, Variable, Var};
 
 
@@ -389,28 +390,40 @@ fn write_block<'x, 'render>(r: &mut Renderer,
                 let var = &eval_expr(r, root, &e);
                 match var.output() {
                     Ok(value) => {
-                        let start = r.buf.len();
-                        write!(&mut r.buf, "{}", value.0)?;
-                        let val = match *validator {
+                        let filter = match *validator {
                             Some(ref name) => {
-                                match r.template.options.validators.get(name) {
+                                match r.template.options.filters.get(name) {
                                     Some(val) => val,
                                     None => {
                                         r.errors.push((item.position.0,
                                             UnknownValidator(
                                                 name.to_string())));
-                                        &r.template.options.default_validator
+                                        &r.template.options.default_filter
                                     }
                                 }
                             }
                             None => {
-                                &r.template.options.default_validator
+                                &r.template.options.default_filter
                             }
                         };
-                        match val.validate(&r.buf[start..]) {
-                            Ok(()) => {}
-                            Err(e) => {
-                                r.errors.push((item.position.0, e));
+                        let start = r.buf.len();
+                        match *filter {
+                            Filter::NoFilter => {
+                                write!(&mut r.buf, "{}", value.0)?;
+                            }
+                            Filter::Validate(ref re) => {
+                                write!(&mut r.buf, "{}", value.0)?;
+                                if !re.is_match(&r.buf[start..]) {
+                                    r.errors.push((item.position.0,
+                                        DataError::RegexValidationError(
+                                            r.buf[start..].to_string(),
+                                            re.as_str().to_string())));
+                                }
+                            }
+                            Filter::Escape(ref escaper) => {
+                                let mut buf = String::with_capacity(1024);
+                                write!(&mut buf, "{}", value.0)?;
+                                escaper.escape(&mut r.buf, &buf);
                             }
                         }
                     }
diff --git a/src/tests/filter.rs b/src/tests/filter.rs
@@ -0,0 +1,36 @@
+use grammar::{Parser};
+use {Context};
+
+fn render_x(template: &str, x: &str) -> String {
+    let tpl = Parser::new().parse(template).unwrap();
+    let mut vars: Context = Context::new();
+    vars.set("x", &x);
+    tpl.render(&vars).unwrap()
+}
+
+#[test]
+fn filter_default() {
+    assert_diff!(
+        &render_x("## syntax: oneline\n\
+                   ## filter default: builtin.html_entities\n\
+                   {{ x }}", "<a>"),
+        "&lt;a&gt;", "\n", 0);
+}
+
+#[test]
+fn filter_html() {
+    assert_diff!(
+        &render_x("## syntax: oneline\n\
+                   ## filter h: builtin.html_entities\n\
+                   {{ x | h }}", "<a>"),
+        "&lt;a&gt;", "\n", 0);
+}
+
+#[test]
+fn filter_shell_argument() {
+    assert_diff!(
+        &render_x("## syntax: oneline\n\
+                   ## filter arg: builtin.quoted_shell_argument\n\
+                   echo {{ x | arg }}", "don't crash"),
+        r#"echo 'don'"'"'t crash'"#, "\n", 0);
+}
diff --git a/src/tests/mod.rs b/src/tests/mod.rs
@@ -8,5 +8,6 @@ mod vars;
 mod boolean;
 mod math;
 mod validate;
+mod filter;
 
 pub use self::diff::assert_eq;
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
diff --git a/src/validators.rs b/src/validators.rs

Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,9 @@ quick_error! {`
`33`	`33`	`description("Validator regexp is invalid")`
`34`	`34`	`display("Validator regex {:?} is invalid: {}", value, err)`
`35`	`35`	`}`
	`36`	`+ BadFilter(value: String) {`
	`37`	`+ display("Filter {:?} is unknown", value)`
	`38`	`+ }`
`36`	`39`	`}`
`37`	`40`	`}`
`38`	`41`