Merge pull request #1263 from HubSpot/handle-escaping-and-unescaping-in-tokenizer

Fix helper token escape handling and unescaping when unquoting strings

Author: jasmith-hs

src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java

@@ -692,7 +692,7 @@ public Object resolveObject(String variable, int lineNumber, int startPosition)
       return "";
     }
     if (WhitespaceUtils.isQuoted(variable)) {
-      return WhitespaceUtils.unquote(variable);
+      return WhitespaceUtils.unquoteAndUnescape(variable);
     } else {
       Object val = retraceVariable(variable, lineNumber, startPosition);
       if (val == null) {

src/main/java/com/hubspot/jinjava/lib/tag/BlockTag.java

@@ -70,7 +70,7 @@ public OutputNode interpretOutput(TagNode tagNode, JinjavaInterpreter interprete
       );
     }
 
-    String blockName = WhitespaceUtils.unquote(tagData.next());
+    String blockName = WhitespaceUtils.unquoteAndUnescape(tagData.next());
 
     interpreter.addBlock(
       blockName,

src/main/java/com/hubspot/jinjava/util/HelperStringTokenizer.java

@@ -36,6 +36,7 @@ public class HelperStringTokenizer extends AbstractIterator<String> {
   private boolean useComma = false;
   private char quoteChar = 0;
   private boolean inQuote = false;
+  private boolean isEscaped = false;
 
   public HelperStringTokenizer(String s) {
     value = s.toCharArray();
@@ -71,7 +72,8 @@ protected String computeNext() {
 
   private String makeToken() {
     char c = value[currPost++];
-    if (c == '"' || c == '\'') {
+
+    if ((c == '"' || c == '\'') && !isEscaped) {
       if (inQuote) {
         if (quoteChar == c) {
           inQuote = false;
@@ -81,6 +83,9 @@ private String makeToken() {
         quoteChar = c;
       }
     }
+
+    isEscaped = (c == '\\' && !isEscaped);
+
     if ((Character.isWhitespace(c) || (useComma && c == ',')) && !inQuote) {
       return newToken();
     }

src/main/java/com/hubspot/jinjava/util/WhitespaceUtils.java

@@ -104,7 +104,6 @@ public static String unquote(String s) {
     return s.trim();
   }
 
-  // TODO see if all usages of unquote can use this method instead
   public static String unquoteAndUnescape(String s) {
     if (Strings.isNullOrEmpty(s)) {
       return "";

src/test/java/com/hubspot/jinjava/lib/tag/CycleTagTest.java

@@ -32,4 +32,11 @@ public void itDefaultsMultipleNullToImageUsingAs() {
       "{% for item in [0,1] %}{% cycle {{foo}},{{bar}} as var %}{% cycle var %}{% endfor %}";
     assertThat(interpreter.render(template)).isEqualTo("{{foo}}{{bar}}");
   }
+
+  @Test
+  public void itHandlesEscapedQuotes() {
+    String template =
+      "{% for item in [0,1] %}{% cycle 'a','class=\\'foo bar\\'' %}.{% endfor %}";
+    assertThat(interpreter.render(template)).isEqualTo("a.class='foo bar'.");
+  }
 }

src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerCycleTagTest.java

@@ -10,6 +10,7 @@
 import com.hubspot.jinjava.lib.tag.Tag;
 import com.hubspot.jinjava.mode.EagerExecutionMode;
 import com.hubspot.jinjava.tree.parse.TagToken;
+import java.util.List;
 import java.util.Optional;
 import org.junit.After;
 import org.junit.Before;
@@ -65,9 +66,22 @@ public void itAddCycleTagAsADeferredToken() {
 
   @Test
   public void itHandlesDeferredCycle() {
-    interpreter.getContext().put("deferred", DeferredValue.instance());
     String template =
       "{% set l = [] %}{% for item in deferred %}{% cycle l.append(deferred),5 %}{% endfor %}{{ l }}";
     assertThat(interpreter.render(template)).isEqualTo(template);
   }
+
+  @Test
+  public void iitHandlesEscapedQuotesInVariable() {
+    String template =
+      "{% set class = \"class='foo bar'\" %}{% for item in deferred %}{% cycle 'item-1',class %}.{% endfor %}";
+    String firstPass = interpreter.render(template);
+    assertThat(firstPass)
+      .isEqualTo(
+        "{% for item in deferred %}{% cycle 'item-1','class=\\'foo bar\\'' %}.{% endfor %}"
+      );
+    interpreter.getContext().put("deferred", List.of(0, 1));
+    String secondPass = interpreter.render(firstPass);
+    assertThat(secondPass).isEqualTo("item-1.class='foo bar'.");
+  }
 }

src/test/java/com/hubspot/jinjava/util/HelperStringTokenizerTest.java

@@ -112,4 +112,30 @@ public void itDoesntReturnTrailingNull() {
       .containsExactly("product", "in", "collections.frontpage.products")
       .doesNotContainNull();
   }
+
+  @Test
+  public void itHandlesEscapedQuotesWithinQuotedStrings() {
+    assertThat(
+      new HelperStringTokenizer("'hi','y\\'all don\\'t'").splitComma(true).allTokens()
+    )
+      .containsExactly("'hi'", "'y\\'all don\\'t'");
+  }
+
+  @Test
+  public void itHandlesEscapedDoubleQuotesWithinQuotedStrings() {
+    assertThat(
+      new HelperStringTokenizer("\"hi\",\"say \\\"hello\\\"\"")
+        .splitComma(true)
+        .allTokens()
+    )
+      .containsExactly("\"hi\"", "\"say \\\"hello\\\"\"");
+  }
+
+  @Test
+  public void itHandlesEscapedBackslashes() {
+    assertThat(
+      new HelperStringTokenizer("'path\\\\to\\file'").splitComma(true).allTokens()
+    )
+      .containsExactly("'path\\\\to\\file'");
+  }
 }