Implement backwards iteration for token.Cursor (#429)

This adds methods to `token.Cursor` to implement backwards iteration
with `Prev` and `PrevSkippable`. Methods `Pop` and `PopSkippable` have
been renamed to `Next` and `NextSkippable` to match the backward
counterparts. The constructor has been altered to `NewCursorAt` which
takes a single starting cursor. The bounds of the cursor iteration are
now delimited by the open and closing token when iterating.

These methods will be useful for reading attached and trailing comments.

Author: emcfarlane

experimental/ast/path.go

@@ -107,7 +107,11 @@ func (p Path) Components(yield func(PathComponent) bool) {
 
 	var sep token.Token
 	var broken bool
-	token.NewCursor(first, p.raw.End.In(p.Context())).Rest()(func(tok token.Token) bool {
+	token.NewCursorAt(first).Rest()(func(tok token.Token) bool {
+		if tok.ID() > p.raw.End {
+			// We've reached the end of the path.
+			return false
+		}
 		if tok.Text() == "." || tok.Text() == "/" {
 			if !sep.IsZero() {
 				// Uh-oh, empty path component!

experimental/parser/parse_decl.go

@@ -43,7 +43,7 @@ func parseDecl(p *parser, c *token.Cursor, in taxa.Noun) ast.DeclAny {
 
 	var unexpected []token.Token
 	for !c.Done() && !canStartDecl(first) {
-		unexpected = append(unexpected, c.Pop())
+		unexpected = append(unexpected, c.Next())
 		first = c.Peek()
 	}
 	switch len(unexpected) {
@@ -64,15 +64,15 @@ func parseDecl(p *parser, c *token.Cursor, in taxa.Noun) ast.DeclAny {
 	}
 
 	if first.Text() == ";" {
-		c.Pop()
+		c.Next()
 
 		// This is an empty decl.
 		return p.NewDeclEmpty(first).AsAny()
 	}
 
 	// This is a bare declaration body.
 	if canStartBody(first) {
-		return parseBody(p, c.Pop(), in).AsAny()
+		return parseBody(p, c.Next(), in).AsAny()
 	}
 
 	// We need to parse a path here. At this point, we need to generate a
@@ -273,7 +273,7 @@ func parseBody(p *parser, braces token.Token, in taxa.Noun) ast.DeclBody {
 // parseRange parses a reserved/extensions range.
 func parseRange(p *parser, c *token.Cursor) ast.DeclRange {
 	// Consume the keyword token.
-	kw := c.Pop()
+	kw := c.Next()
 
 	in := taxa.Extensions
 	if kw.Text() == "reserved" {
@@ -391,7 +391,7 @@ func tryParseOptions(p *parser, c *token.Cursor, in taxa.Noun) ast.CompactOption
 	if !canStartOptions(c.Peek()) {
 		return ast.CompactOptions{}
 	}
-	return parseOptions(p, c.Pop(), in)
+	return parseOptions(p, c.Next(), in)
 }
 
 // parseOptions parses a ([]-delimited) compact options list.
@@ -421,7 +421,7 @@ func parseOptions(p *parser, brackets token.Token, _ taxa.Noun) ast.CompactOptio
 				)
 				fallthrough
 			case "=":
-				c.Pop()
+				c.Next()
 			default:
 				p.Error(errUnexpected{
 					what:  eq,

experimental/parser/parse_def.go

@@ -184,7 +184,7 @@ func (defInputs) what(*defParser) taxa.Noun  { return taxa.MethodIns }
 func (defInputs) canStart(p *defParser) bool { return p.c.Peek().Text() == "(" }
 
 func (defInputs) parse(p *defParser) report.Span {
-	next := p.c.Pop()
+	next := p.c.Next()
 	if next.IsLeaf() {
 		return report.Span{} // Diagnosed by the lexer.
 	}
@@ -205,7 +205,7 @@ func (defOutputs) canStart(p *defParser) bool { return p.c.Peek().Text() == "ret
 func (defOutputs) parse(p *defParser) report.Span {
 	// Note that the inputs and outputs of a method are parsed
 	// separately, so foo(bar) and foo returns (bar) are both possible.
-	returns := p.c.Pop()
+	returns := p.c.Next()
 
 	var ty ast.TypeAny
 	list, err := p.Punct(p.c, "(", taxa.KeywordReturns.After())
@@ -314,7 +314,7 @@ func (defOptions) what(*defParser) taxa.Noun  { return taxa.CompactOptions }
 func (defOptions) canStart(p *defParser) bool { return canStartOptions(p.c.Peek()) }
 
 func (defOptions) parse(p *defParser) report.Span {
-	next := p.c.Pop()
+	next := p.c.Next()
 	if next.IsLeaf() {
 		return report.Span{} // Diagnosed by the lexer.
 	}
@@ -333,7 +333,7 @@ func (defBody) what(*defParser) taxa.Noun  { return taxa.Body }
 func (defBody) canStart(p *defParser) bool { return canStartBody(p.c.Peek()) }
 
 func (defBody) parse(p *defParser) report.Span {
-	next := p.c.Pop()
+	next := p.c.Next()
 	if next.IsLeaf() {
 		return report.Span{} // Diagnosed by the lexer.
 	}

experimental/parser/parse_delimited.go

@@ -74,7 +74,7 @@ func (d delimited[T]) iter(yield func(value T, delim token.Token) bool) {
 	var latest int // The index of the most recently seen delimiter.
 
 	if next := d.c.Peek(); slices.Contains(d.delims, next.Text()) {
-		_ = d.c.Pop()
+		_ = d.c.Next()
 		latest = slices.Index(d.delims, next.Text())
 
 		d.p.Error(errUnexpected{
@@ -98,13 +98,13 @@ func (d delimited[T]) iter(yield func(value T, delim token.Token) bool) {
 				break
 			}
 
-			first := d.c.Pop()
+			first := d.c.Next()
 			var last token.Token
 			for !d.c.Done() && !d.start(d.c.Peek()) {
 				if d.stop != nil && d.stop(d.c.Peek()) {
 					break
 				}
-				last = d.c.Pop()
+				last = d.c.Next()
 			}
 
 			want := d.what.AsSet()
@@ -151,7 +151,7 @@ func (d delimited[T]) iter(yield func(value T, delim token.Token) bool) {
 			}
 			latest = which
 
-			next := d.c.Pop()
+			next := d.c.Next()
 			if delim.IsZero() {
 				delim = next
 				continue

experimental/parser/parse_expr.go

@@ -55,7 +55,7 @@ func parseExprInfix(p *parser, c *token.Cursor, where taxa.Place, lhs ast.ExprAn
 			case ":":
 				return p.NewExprField(ast.ExprFieldArgs{
 					Key:   lhs,
-					Colon: c.Pop(),
+					Colon: c.Next(),
 					Value: parseExprInfix(p, c, where, ast.ExprAny{}, prec+1),
 				}).AsAny()
 
@@ -105,7 +105,7 @@ func parseExprInfix(p *parser, c *token.Cursor, where taxa.Place, lhs ast.ExprAn
 		case "to":
 			return p.NewExprRange(ast.ExprRangeArgs{
 				Start: lhs,
-				To:    c.Pop(),
+				To:    c.Next(),
 				End:   parseExprInfix(p, c, taxa.KeywordTo.After(), ast.ExprAny{}, prec),
 			}).AsAny()
 		}
@@ -128,7 +128,7 @@ func parseExprPrefix(p *parser, c *token.Cursor, where taxa.Place) ast.ExprAny {
 		return ast.ExprAny{}
 
 	case next.Text() == "-":
-		c.Pop()
+		c.Next()
 		inner := parseExprPrefix(p, c, taxa.Minus.After())
 		return p.NewExprPrefixed(ast.ExprPrefixedArgs{
 			Prefix: next,
@@ -151,13 +151,13 @@ func parseExprSolo(p *parser, c *token.Cursor, where taxa.Place) ast.ExprAny {
 		return ast.ExprAny{}
 
 	case next.Kind() == token.String, next.Kind() == token.Number:
-		return ast.ExprLiteral{Token: c.Pop()}.AsAny()
+		return ast.ExprLiteral{Token: c.Next()}.AsAny()
 
 	case canStartPath(next):
 		return ast.ExprPath{Path: parsePath(p, c)}.AsAny()
 
 	case (next.Text() == "{" || next.Text() == "<" || next.Text() == "[") && !next.IsLeaf():
-		body := c.Pop()
+		body := c.Next()
 		in := taxa.Dict
 		if next.Text() == "[" {
 			in = taxa.Array
@@ -225,7 +225,7 @@ func parseExprSolo(p *parser, c *token.Cursor, where taxa.Place) ast.ExprAny {
 func peekTokenExpr(p *parser, c *token.Cursor) token.Token {
 	next := c.Peek()
 	if next.IsZero() {
-		token, span := c.JustAfter()
+		token, span := c.SeekToEnd()
 		err := errUnexpected{
 			what:  span,
 			where: taxa.Expr.In(),

experimental/parser/parse_path.go

@@ -39,7 +39,7 @@ func parsePath(p *parser, c *token.Cursor) ast.Path {
 
 	var prevSeparator token.Token
 	if start.Text() == "." || start.Text() == "/" {
-		prevSeparator = c.Pop()
+		prevSeparator = c.Next()
 	}
 
 	var done bool
@@ -60,7 +60,7 @@ func parsePath(p *parser, c *token.Cursor) ast.Path {
 					want:  taxa.NewSet(taxa.Ident, taxa.Parens),
 				})
 			}
-			prevSeparator = c.Pop()
+			prevSeparator = c.Next()
 
 		case next.Kind() == token.Ident:
 			if !first && prevSeparator.IsZero() {
@@ -72,7 +72,7 @@ func parsePath(p *parser, c *token.Cursor) ast.Path {
 
 			end = next
 			prevSeparator = token.Zero
-			c.Pop()
+			c.Next()
 
 		case next.Text() == "(":
 			if !first && prevSeparator.IsZero() {
@@ -98,7 +98,7 @@ func parsePath(p *parser, c *token.Cursor) ast.Path {
 
 			end = next
 			prevSeparator = token.Zero
-			c.Pop()
+			c.Next()
 
 		default:
 			if prevSeparator.IsZero() {

experimental/parser/parse_state.go

@@ -35,12 +35,12 @@ type parser struct {
 func (p *parser) Punct(c *token.Cursor, want string, where taxa.Place) (token.Token, report.Diagnose) {
 	next := c.Peek()
 	if next.Text() == want {
-		return c.Pop(), nil
+		return c.Next(), nil
 	}
 
 	wanted := taxa.NewSet(taxa.Punct(want, false))
 	if next.IsZero() {
-		tok, span := c.JustAfter()
+		tok, span := c.SeekToEnd()
 		err := errUnexpected{
 			what:  span,
 			where: where,

experimental/parser/parse_type.go

@@ -167,7 +167,7 @@ func parseTypeImpl(p *parser, c *token.Cursor, where taxa.Place, pathAfter bool)
 	// Next, look for some angle brackets. We need to do this before draining
 	// mods, because angle brackets bind more tightly than modifiers.
 	if angles := c.Peek(); angles.Text() == "<" && !angles.IsLeaf() {
-		c.Pop() // Consume the angle brackets.
+		c.Next() // Consume the angle brackets.
 		generic := p.NewTypeGeneric(ast.TypeGenericArgs{
 			Path:          tyPath,
 			AngleBrackets: angles,