Fixes #509

Author: yuin

_test/extra.txt

@@ -843,3 +843,21 @@ text" /></p>
 </code></pre>
 </blockquote>
 //= = = = = = = = = = = = = = = = = = = = = = = =//
+
+68: HTML comments in list items
+//- - - - - - - - -//
+- test
+  <!--
+    Test
+  -->
+- test2
+//- - - - - - - - -//
+<ul>
+<li>test
+<!--
+  Test
+-->
+</li>
+<li>test2</li>
+</ul>
+//= = = = = = = = = = = = = = = = = = = = = = = =//

parser/code_block.go

@@ -37,7 +37,7 @@ func (b *codeBlockParser) Open(parent ast.Node, reader text.Reader, pc Context)
 	}
 	segment.ForceNewline = true
 	node.Lines().Append(segment)
-	reader.Advance(segment.Len() - 1)
+	reader.AdvanceToEOL()
 	return node, NoChildren
 
 }
@@ -62,7 +62,7 @@ func (b *codeBlockParser) Continue(node ast.Node, reader text.Reader, pc Context
 
 	segment.ForceNewline = true
 	node.Lines().Append(segment)
-	reader.Advance(segment.Len() - 1)
+	reader.AdvanceToEOL()
 	return Continue | NoChildren
 }
 

parser/html_block.go

@@ -150,7 +150,7 @@ func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context)
 		}
 	}
 	if node != nil {
-		reader.Advance(segment.Len() - util.TrimRightSpaceLength(line))
+		reader.AdvanceToEOL()
 		node.Lines().Append(segment)
 		return node, NoChildren
 	}
@@ -173,7 +173,7 @@ func (b *htmlBlockParser) Continue(node ast.Node, reader text.Reader, pc Context
 		}
 		if htmlBlockType1CloseRegexp.Match(line) {
 			htmlBlock.ClosureLine = segment
-			reader.Advance(segment.Len() - util.TrimRightSpaceLength(line))
+			reader.AdvanceToEOL()
 			return Close
 		}
 	case ast.HTMLBlockType2:
@@ -202,7 +202,7 @@ func (b *htmlBlockParser) Continue(node ast.Node, reader text.Reader, pc Context
 		}
 		if bytes.Contains(line, closurePattern) {
 			htmlBlock.ClosureLine = segment
-			reader.Advance(segment.Len())
+			reader.AdvanceToEOL()
 			return Close
 		}
 
@@ -212,7 +212,7 @@ func (b *htmlBlockParser) Continue(node ast.Node, reader text.Reader, pc Context
 		}
 	}
 	node.Lines().Append(segment)
-	reader.Advance(segment.Len() - util.TrimRightSpaceLength(line))
+	reader.AdvanceToEOL()
 	return Continue | NoChildren
 }
 

parser/list_item.go

@@ -53,7 +53,7 @@ func (b *listItemParser) Open(parent ast.Node, reader text.Reader, pc Context) (
 func (b *listItemParser) Continue(node ast.Node, reader text.Reader, pc Context) State {
 	line, _ := reader.PeekLine()
 	if util.IsBlank(line) {
-		reader.Advance(len(line) - 1)
+		reader.AdvanceToEOL()
 		return Continue | HasChildren
 	}
 

parser/paragraph.go

@@ -29,7 +29,7 @@ func (b *paragraphParser) Open(parent ast.Node, reader text.Reader, pc Context)
 	}
 	node := ast.NewParagraph()
 	node.Lines().Append(segment)
-	reader.Advance(segment.Len() - 1)
+	reader.AdvanceToEOL()
 	return node, NoChildren
 }
 
@@ -39,7 +39,7 @@ func (b *paragraphParser) Continue(node ast.Node, reader text.Reader, pc Context
 		return Close
 	}
 	node.Lines().Append(segment)
-	reader.Advance(segment.Len() - 1)
+	reader.AdvanceToEOL()
 	return Continue | NoChildren
 }
 

parser/thematic_break.go

@@ -50,9 +50,9 @@ func (b *thematicBreakPraser) Trigger() []byte {
 }
 
 func (b *thematicBreakPraser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) {
-	line, segment := reader.PeekLine()
+	line, _ := reader.PeekLine()
 	if isThematicBreak(line, reader.LineOffset()) {
-		reader.Advance(segment.Len() - 1)
+		reader.AdvanceToEOL()
 		return ast.NewThematicBreak(), NoChildren
 	}
 	return nil, NoChildren

text/reader.go

@@ -55,6 +55,11 @@ type Reader interface {
 	// reader.
 	AdvanceAndSetPadding(int, int)
 
+	// AdvanceToEOL advances the internal pointer to the end of line.
+	// If the line ends with a newline, it will be included in the segment.
+	// If the line ends with EOF, it will not be included in the segment.
+	AdvanceToEOL()
+
 	// AdvanceLine advances the internal pointer to the next line head.
 	AdvanceLine()
 
@@ -220,16 +225,44 @@ func (r *reader) AdvanceAndSetPadding(n, padding int) {
 	}
 }
 
+func (r *reader) AdvanceToEOL() {
+	if r.pos.Start >= r.sourceLength {
+		return
+	}
+
+	r.lineOffset = -1
+	i := -1
+	if r.peekedLine != nil {
+		r.pos.Start += len(r.peekedLine) - r.pos.Padding - 1
+		if r.source[r.pos.Start] == '\n' {
+			i = 0
+		}
+	}
+	if i == -1 {
+		i = bytes.IndexByte(r.source[r.pos.Start:], '\n')
+	}
+	r.peekedLine = nil
+	if i != -1 {
+		r.pos.Start += i
+	} else {
+		r.pos.Start = r.sourceLength
+	}
+	r.pos.Padding = 0
+}
+
 func (r *reader) AdvanceLine() {
 	r.lineOffset = -1
 	r.peekedLine = nil
 	r.pos.Start = r.pos.Stop
 	r.head = r.pos.Start
-	if r.pos.Start < 0 {
+	if r.pos.Start < 0 || r.pos.Start >= r.sourceLength {
 		return
 	}
 	r.pos.Stop = r.sourceLength
-	i := bytes.IndexByte(r.source[r.pos.Start:], '\n')
+	i := 0
+	if r.source[r.pos.Start] != '\n' {
+		i = bytes.IndexByte(r.source[r.pos.Start:], '\n')
+	}
 	if i != -1 {
 		r.pos.Stop = r.pos.Start + i + 1
 	}
@@ -441,6 +474,17 @@ func (r *blockReader) AdvanceAndSetPadding(n, padding int) {
 	}
 }
 
+func (r *blockReader) AdvanceToEOL() {
+	r.lineOffset = -1
+	r.pos.Padding = 0
+	c := r.source[r.pos.Stop-1]
+	if c == '\n' {
+		r.pos.Start = r.pos.Stop - 1
+	} else {
+		r.pos.Start = r.pos.Stop
+	}
+}
+
 func (r *blockReader) AdvanceLine() {
 	r.SetPosition(r.line+1, NewSegment(invalidValue, invalidValue))
 	r.head = r.pos.Start