Parser: ensure error on tokens after doc end

Author: biojppm

changelog/current.md

@@ -10,7 +10,7 @@
     {map: }
     : bad
     ```
-  - [PR#586](https://github.com/biojppm/rapidyaml/pull/586): colon on newline generally ([play](https://play.yaml.com/?show=xd#c2NhbGFyCjogYmFkCi0tLQpbc2VxXQo6IGJhZAotLS0KW21hcF0KOiBiYWQK)):
+  - [PR#586](https://github.com/biojppm/rapidyaml/pull/586): tokens after explicit document endcolon on newline:
     ```yaml
     bad cases:
       scalar
@@ -20,3 +20,8 @@
       {map: }
         : bad
     ```
+  - [PR#586](https://github.com/biojppm/rapidyaml/pull/586): tokens after explicit document end:
+    ```yaml
+    foo: bar
+    ... bad tokens
+    ```

src/c4/yml/parse_engine.def.hpp

@@ -1786,6 +1786,17 @@ void ParseEngine<EventHandler>::_end_doc_suddenly()
     addrem_flags(RUNK|RTOP|NDOC, RMAP|RSEQ|RDOC);
 }
 
+template<class EventHandler>
+void ParseEngine<EventHandler>::_check_doc_end_tokens() const
+{
+    csubstr rem = m_evt_handler->m_curr->line_contents.rem;
+    _RYML_ASSERT_PARSE_(m_evt_handler->m_stack.m_callbacks, !rem.begins_with_any(". \t"), m_evt_handler->m_curr->pos);
+    if(C4_UNLIKELY(rem.len && !rem.begins_with('#')))
+    {
+        _c4err("parse error");
+    }
+}
+
 template<class EventHandler>
 void ParseEngine<EventHandler>::_start_doc_suddenly()
 {
@@ -5964,6 +5975,23 @@ void ParseEngine<EventHandler>::_handle_seq_block()
             _maybe_skip_whitespace_tokens();
             goto seqblck_finish;
         }
+        else if(first == '.')
+        {
+            _c4dbgp("seqblck[RVAL]: maybe end doc?");
+            csubstr rs = m_evt_handler->m_curr->line_contents.rem.sub(1);
+            if(rs == ".." || rs.begins_with(".. "))
+            {
+                _c4dbgp("seqblck[RVAL]: end+start doc");
+                _end_doc_suddenly();
+                _line_progressed(3);
+                _maybe_skip_whitespace_tokens();
+                _check_doc_end_tokens();
+            }
+            else
+            {
+                _c4err("parse error");
+            }
+        }
         else
         {
             _c4err("parse error");
@@ -6083,10 +6111,11 @@ void ParseEngine<EventHandler>::_handle_seq_block()
             csubstr rs = rem.sub(1);
             if(rs == ".." || rs.begins_with(".. "))
             {
-                _c4dbgp("seqblck[RNXT]: end+start doc");
+                _c4dbgp("seqblck[RNXT]: end doc");
                 _end_doc_suddenly();
                 _line_progressed(3);
                 _maybe_skip_whitespace_tokens();
+                _check_doc_end_tokens();
                 goto seqblck_finish;
             }
             else
@@ -6352,6 +6381,7 @@ void ParseEngine<EventHandler>::_handle_map_block()
                 _end_doc_suddenly();
                 _line_progressed(3);
                 _maybe_skip_whitespace_tokens();
+                _check_doc_end_tokens();
                 goto mapblck_finish;
             }
             else
@@ -6795,6 +6825,7 @@ void ParseEngine<EventHandler>::_handle_map_block()
                 _end_doc_suddenly();
                 _line_progressed(3);
                 _maybe_skip_whitespace_tokens();
+                _check_doc_end_tokens();
                 goto mapblck_finish;
             }
             else
@@ -7233,6 +7264,8 @@ bool ParseEngine<EventHandler>::_handle_map_block_qmrk()
             _c4dbgp("mapblck[QMRK]: end+start doc");
             _end_doc_suddenly();
             _line_progressed(3);
+            _maybe_skip_whitespace_tokens();
+            _check_doc_end_tokens();
             return false; // finish mapblck
         }
         else
@@ -7314,6 +7347,8 @@ bool ParseEngine<EventHandler>::_handle_map_block_rkcl()
             _c4dbgp("mapblck[RKCL]: end+start doc");
             _end_doc_suddenly();
             _line_progressed(3);
+            _maybe_skip_whitespace_tokens();
+            _check_doc_end_tokens();
             return false; // finish mapblck
         }
         else
@@ -7514,6 +7549,7 @@ void ParseEngine<EventHandler>::_handle_unk()
                 addrem_flags(NDOC|RUNK, RDOC);
                 _line_progressed(3u);
                 _maybe_skip_whitespace_tokens();
+                _check_doc_end_tokens();
                 return;
             }
         }

src/c4/yml/parse_engine.hpp

@@ -516,6 +516,7 @@ class ParseEngine
     void  _begin2_doc_expl();
     void  _end2_doc();
     void  _end2_doc_expl();
+    void  _check_doc_end_tokens() const;
 
     void  _maybe_begin_doc();
     void  _maybe_end_doc();

test/test_engine_1_doc.cpp

@@ -174,6 +174,131 @@ ENGINE_TEST(DocEmptyTerm4_2,
 }
 
 
+ENGINE_TEST(DocEmptyTermErr,
+            "...\n...\n"
+            ,
+            ""
+            ,
+            "+STR\n"
+            "-STR\n")
+{
+    ___(ps.begin_stream());
+    ___(ps.end_stream());
+}
+
+
+//-----------------------------------------------------------------------------
+
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err0,
+                   Location(2, 5),
+                   "foo: bar\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err1,
+                   Location(2, 5),
+                   "foo: bar\n"
+                   "... [bad]\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err2,
+                   Location(2, 5),
+                   "foo: bar\n"
+                   "... {bad: yes}\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err3,
+                   Location(2, 5),
+                   "foo:\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err4,
+                   Location(2, 5),
+                   "foo:\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err5,
+                   Location(2, 5),
+                   "?\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_MapBlck_err6,
+                   Location(2, 5),
+                   "? foo\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST(DocTerm_MapBlck_ok,
+            "foo: bar\n"
+            "... #bad\n"
+            ,
+            "foo: bar\n"
+            ,
+            "+STR\n"
+            "+DOC\n"
+            "+MAP\n"
+            "=VAL :foo\n"
+            "=VAL :bar\n"
+            "-MAP\n"
+            "-DOC ...\n"
+            "-STR\n")
+{
+    ___(ps.begin_stream());
+    ___(ps.begin_doc());
+    ___(ps.begin_map_val_block());
+    ___(ps.set_key_scalar_plain("foo"));
+    ___(ps.set_val_scalar_plain("bar"));
+    ___(ps.end_map_block());
+    ___(ps.end_doc_expl());
+    ___(ps.end_stream());
+}
+
+ENGINE_TEST_ERRLOC(DocTerm_SeqBlck_err0,
+                   Location(2, 5),
+                   "- foo\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_SeqBlck_err1,
+                   Location(2, 5),
+                   "-\n"
+                   "... bad\n"
+                   )
+
+ENGINE_TEST_ERRLOC(DocTerm_MapFlow_err0,
+                   Location(2, 5),
+                   "{key: val}\n"
+                   "... bad\n"
+                   )
+
+ENGINE_TEST_ERRLOC(DocTerm_SeqFlow_err0,
+                   Location(2, 5),
+                   "{key: val}\n"
+                   "... bad\n"
+                   )
+
+ENGINE_TEST_ERRLOC(DocTerm_ScalarPlain_err0,
+                   Location(2, 5),
+                   "scalar\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_ScalarSquo_err0,
+                   Location(2, 5),
+                   "'scalar'\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_ScalarDquo_err0,
+                   Location(2, 5),
+                   "\"scalar\"\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_ScalarLit_err0,
+                   Location(3, 5),
+                   "|\n  scalar\n"
+                   "... bad\n"
+                   )
+ENGINE_TEST_ERRLOC(DocTerm_FoldedLit_err0,
+                   Location(3, 5),
+                   ">\n  scalar\n"
+                   "... bad\n"
+                   )
+
+
 //-----------------------------------------------------------------------------
 
 ENGINE_TEST(DocEmptyExplMult,

test/test_lib/test_case.cpp

@@ -458,7 +458,10 @@ void ExpectError::check_error_parse(Tree *tree, fntestref fn, Location const& ex
                 "---------------\n");
     }, )
     EXPECT_NE(context.m_error, ExpectedErrorType::err_none);
-    EXPECT_EQ(context.m_error, ExpectedErrorType::err_parse);
+    if(context.m_error != ExpectedErrorType::err_none)
+    {
+        EXPECT_EQ(context.m_error, ExpectedErrorType::err_parse);
+    }
 }
 
 void ExpectError::check_error_visit(Tree *tree, fntestref fn, id_type id)
@@ -498,7 +501,10 @@ void ExpectError::check_error_visit(Tree *tree, fntestref fn, id_type id)
         std::cout << "---------------\n";
     }, )
     EXPECT_NE(context.m_error, ExpectedErrorType::err_none);
-    EXPECT_EQ(context.m_error, ExpectedErrorType::err_visit);
+    if(context.m_error != ExpectedErrorType::err_none)
+    {
+        EXPECT_EQ(context.m_error, ExpectedErrorType::err_visit);
+    }
 }
 
 void ExpectError::check_assert(ExpectedErrorType errtype, Tree *tree, fntestref fn, Location const& loc)

test/testsuite.cpp

@@ -770,6 +770,7 @@ struct TestSequenceData
         //SKIP_IF(has_container_keys); // DO IT!
         if(m_expected_error_to_tree_checked)
             return;
+        SCOPED_TRACE("check_expected_error");
         ExpectError::check_error_parse(&levels[0].tree_parsed_from_src, [this]{
             levels[0].parse_yaml_to_tree();
         });
@@ -781,6 +782,7 @@ struct TestSequenceData
         //SKIP_IF(has_container_keys); // DO IT!
         if(m_expected_error_to_events_checked)
             return;
+        SCOPED_TRACE("check_expected_error_events");
         ExpectError::check_error_parse([this]{
             levels[0].parse_yaml_to_events();
         });
@@ -792,6 +794,7 @@ struct TestSequenceData
         //SKIP_IF(has_container_keys); // DO IT!
         if(m_expected_error_to_events_ints_checked)
             return;
+        SCOPED_TRACE("check_expected_error_events");
         ExpectError::check_error_parse([this]{
             levels[0].parse_yaml_to_events_ints();
         });
@@ -924,18 +927,21 @@ bool g_do_subcases = true;
 TEST(which##_errors, check_expected_error_src_to_tree)                  \
 {                                                                       \
     SKIP_IF(!g_suite_case->test_case_expects_error);                    \
+    SCOPED_TRACE("check_expected_error_src_to_tree");                   \
     g_suite_case->which.check_expected_error();                         \
 }                                                                       \
                                                                         \
 TEST(which##_errors, check_expected_error_src_to_events)                \
 {                                                                       \
     SKIP_IF(!g_suite_case->test_case_expects_error);                    \
+    SCOPED_TRACE("check_expected_error_src_to_events");                 \
     g_suite_case->which.check_expected_error_events();                  \
 }                                                                       \
                                                                         \
 TEST(which##_errors, check_expected_error_src_to_events_ints)           \
 {                                                                       \
     SKIP_IF(!g_suite_case->test_case_expects_error);                    \
+    SCOPED_TRACE("check_expected_error_src_to_events_ints");            \
     g_suite_case->which.check_expected_error_events_ints();             \
 }                                                                       \
                                                                         \

test/testsuite/testsuite_parts.cpp

@@ -29,7 +29,6 @@ constexpr const AllowedFailure allowed_failures[] = {
     // These tests are temporarily skipped, and cover issues that must be fixed.
 
     // These invalid YAML cases do not materialize parse errors
-    _("3HFZ-error"              , "should not accept scalar after ..."),
     _("4EJS-error"              , "should not accept tabs as indendation in a mapping"),
     _("5TRB-error"              , "should not accept document-end marker in double quoted string"),
     _("5U3A-error"              , "should not accept opening a sequence on same line as map key"),