Add helper for converting token IDs to indexes

experimental/token/cursor.go

@@ -59,7 +59,7 @@ func NewCursorAt(tok Token) *Cursor {
 
 	return &Cursor{
 		withContext: tok.withContext,
-		idx:         int(tok.ID() - 1), // Convert to 0-based index.
+		idx:         tok.ID().naturalIndex(), // Convert to 0-based index.
 	}
 }
 
@@ -178,7 +178,7 @@ func (c *Cursor) PrevSkippable() Token {
 	if c.IsSynthetic() {
 		c.idx--
 	} else {
-		c.idx = int(tok.ID() - 1)
+		c.idx = tok.ID().naturalIndex()
 	}
 	return tok
 }

experimental/token/raw.go

@@ -55,10 +55,35 @@ func (t ID) String() string {
 		return "Token(<nil>)"
 	}
 	if t < 0 {
-		return fmt.Sprintf("Token(^%d)", ^int(t))
+		return fmt.Sprintf("Token(%d)", t.syntheticIndex())
 	}
+	return fmt.Sprintf("Token(%d)", t.naturalIndex())
+}
+
+// naturalIndex returns the index of this token in the natural stream.
+func (t ID) naturalIndex() int {
+	if t.IsZero() {
+		panic("protocompile/token: called naturalIndex on zero token")
+	}
+	if t < 0 {
+		panic("protocompile/token: called naturalIndex on synthetic token")
+	}
+	// Need to subtract off one, because the zeroth
+	// ID is used as a "missing" sentinel.
+	return int(t) - 1
+}
 
-	return fmt.Sprintf("Token(%d)", int(t)-1)
+// syntheticIndex returns the index of this token in the synthetic stream.
+func (t ID) syntheticIndex() int {
+	if t.IsZero() {
+		panic("protocompile/token: called syntheticIndex on zero token")
+	}
+	if t > 0 {
+		panic("protocompile/token: called syntheticIndex on natural token")
+	}
+	// Need to invert the bits, because synthetic tokens are
+	// stored as negative numbers.
+	return ^int(t)
 }
 
 // Constants for extracting the parts of tokenImpl.kindAndOffset.

experimental/token/token.go

@@ -325,7 +325,7 @@ func (t Token) Children() *Cursor {
 		start, _ := t.StartEnd()
 		return &Cursor{
 			withContext: t.withContext,
-			idx:         int(start.id), // Skip the start!
+			idx:         start.id.naturalIndex() + 1, // Skip the start!
 		}
 	}
 
@@ -497,14 +497,12 @@ func (t Token) nat() *nat {
 	if t.IsSynthetic() {
 		return nil
 	}
-	// Need to subtract off one, because the zeroth
-	// ID is used as a "missing" sentinel.
-	return &t.Context().Stream().nats[t.id-1]
+	return &t.Context().Stream().nats[t.id.naturalIndex()]
 }
 
 func (t Token) synth() *synth {
 	if !t.IsSynthetic() {
 		return nil
 	}
-	return &t.Context().Stream().synths[^t.id]
+	return &t.Context().Stream().synths[t.id.syntheticIndex()]
 }