Merge branch 'master' of github.com:kevin-valerio/go-arithmetic-panik

Author: kevin-valerio

api/next/74958.txt

@@ -0,0 +1 @@
+pkg go/scanner, method (*Scanner) End() token.Pos #74958

doc/next/6-stdlib/99-minor/go/scanner/74958.md

@@ -0,0 +1 @@
+The scanner now allows retrieving the end position of a token via the new [Scanner.End] method.

src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go

@@ -342,7 +342,6 @@ var depsRules = `
 	< internal/gover
 	< go/version
 	< go/token
-	< go/internal/scannerhooks
 	< go/scanner
 	< go/ast
 	< go/internal/typeparams;

src/cmd/compile/internal/ssa/opGen.go

@@ -81,6 +81,8 @@ func expectedErrors(fset *token.FileSet, filename string, src []byte) map[token.
 
 	for {
 		pos, tok, lit := s.Scan()
+		end := s.End()
+
 		switch tok {
 		case token.EOF:
 			return errors
@@ -104,13 +106,7 @@ func expectedErrors(fset *token.FileSet, filename string, src []byte) map[token.
 			fallthrough
 		default:
 			prev = pos
-			var l int // token length
-			if tok.IsLiteral() {
-				l = len(lit)
-			} else {
-				l = len(tok.String())
-			}
-			here = prev + token.Pos(l)
+			here = end
 		}
 	}
 }

src/go/build/deps_test.go

@@ -28,7 +28,6 @@ import (
 	"fmt"
 	"go/ast"
 	"go/build/constraint"
-	"go/internal/scannerhooks"
 	"go/scanner"
 	"go/token"
 	"strings"
@@ -53,10 +52,9 @@ type parser struct {
 	goVersion   string            // minimum Go version found in //go:build comment
 
 	// Next token
-	pos       token.Pos   // token position
-	tok       token.Token // one token look-ahead
-	lit       string      // token literal
-	stringEnd token.Pos   // position immediately after token; STRING only
+	pos token.Pos   // token position
+	tok token.Token // one token look-ahead
+	lit string      // token literal
 
 	// Error recovery
 	// (used to limit the number of calls to parser.advance
@@ -87,6 +85,11 @@ func (p *parser) init(file *token.File, src []byte, mode Mode) {
 	p.next()
 }
 
+// end returns the end position of the current token
+func (p *parser) end() token.Pos {
+	return p.scanner.End()
+}
+
 // ----------------------------------------------------------------------------
 // Parsing support
 
@@ -165,10 +168,6 @@ func (p *parser) next0() {
 				continue
 			}
 		} else {
-			if p.tok == token.STRING {
-				p.stringEnd = scannerhooks.StringEnd(&p.scanner)
-			}
-
 			// Found a non-comment; top of file is over.
 			p.top = false
 		}
@@ -726,7 +725,7 @@ func (p *parser) parseFieldDecl() *ast.Field {
 
 	var tag *ast.BasicLit
 	if p.tok == token.STRING {
-		tag = &ast.BasicLit{ValuePos: p.pos, ValueEnd: p.stringEnd, Kind: p.tok, Value: p.lit}
+		tag = &ast.BasicLit{ValuePos: p.pos, ValueEnd: p.end(), Kind: p.tok, Value: p.lit}
 		p.next()
 	}
 
@@ -1480,11 +1479,7 @@ func (p *parser) parseOperand() ast.Expr {
 		return x
 
 	case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
-		end := p.pos + token.Pos(len(p.lit))
-		if p.tok == token.STRING {
-			end = p.stringEnd
-		}
-		x := &ast.BasicLit{ValuePos: p.pos, ValueEnd: end, Kind: p.tok, Value: p.lit}
+		x := &ast.BasicLit{ValuePos: p.pos, ValueEnd: p.end(), Kind: p.tok, Value: p.lit}
 		p.next()
 		return x
 
@@ -2525,7 +2520,7 @@ func (p *parser) parseImportSpec(doc *ast.CommentGroup, _ token.Token, _ int) as
 	var path string
 	if p.tok == token.STRING {
 		path = p.lit
-		end = p.stringEnd
+		end = p.end()
 		p.next()
 	} else if p.tok.IsLiteral() {
 		p.error(pos, "import path must be a string")

src/go/internal/scannerhooks/hooks.go

@@ -10,7 +10,6 @@ package scanner
 import (
 	"bytes"
 	"fmt"
-	"go/internal/scannerhooks"
 	"go/token"
 	"path/filepath"
 	"strconv"
@@ -42,19 +41,14 @@ type Scanner struct {
 	lineOffset int       // current line offset
 	insertSemi bool      // insert a semicolon before next newline
 	nlPos      token.Pos // position of newline in preceding comment
-	stringEnd  token.Pos // end position; defined only for STRING tokens
+
+	endPosValid bool
+	endPos      token.Pos // overrides the offset as the default end position
 
 	// public state - ok to modify
 	ErrorCount int // number of errors encountered
 }
 
-// Provide go/parser with backdoor access to the StringEnd information.
-func init() {
-	scannerhooks.StringEnd = func(scanner any) token.Pos {
-		return scanner.(*Scanner).stringEnd
-	}
-}
-
 const (
 	bom = 0xFEFF // byte order mark, only permitted as very first character
 	eof = -1     // end of file
@@ -154,7 +148,9 @@ func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode
 		err:  err,
 		mode: mode,
 
-		ch: ' ',
+		ch:          ' ',
+		endPosValid: true,
+		endPos:      token.NoPos,
 	}
 
 	s.next()
@@ -700,7 +696,7 @@ func stripCR(b []byte, comment bool) []byte {
 	return c[:i]
 }
 
-func (s *Scanner) scanRawString() (string, int) {
+func (s *Scanner) scanRawString() string {
 	// '`' opening already consumed
 	offs := s.offset - 1
 
@@ -721,12 +717,11 @@ func (s *Scanner) scanRawString() (string, int) {
 	}
 
 	lit := s.src[offs:s.offset]
-	rawLen := len(lit)
 	if hasCR {
 		lit = stripCR(lit, false)
 	}
 
-	return string(lit), rawLen
+	return string(lit)
 }
 
 func (s *Scanner) skipWhitespace() {
@@ -777,6 +772,21 @@ func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Tok
 	return tok0
 }
 
+// End returns the position immediately after the last scanned token.
+// If [Scanner.Scan] has not been called yet, End returns [token.NoPos].
+func (s *Scanner) End() token.Pos {
+	// Handles special case:
+	// - Makes sure we return [token.NoPos], even when [Scanner.Init] has consumed a BOM.
+	// - When the previous token was a synthetic [token.SEMICOLON] inside a multi-line
+	//   comment, we make sure End returns its ending position (i.e. prevPos+len("\n")).
+	if s.endPosValid {
+		return s.endPos
+	}
+
+	// Normal case: s.file.Pos(s.offset) represents the end of the token
+	return s.file.Pos(s.offset)
+}
+
 // Scan scans the next token and returns the token position, the token,
 // and its literal string if applicable. The source end is indicated by
 // [token.EOF].
@@ -790,7 +800,9 @@ func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Tok
 // If the returned token is [token.SEMICOLON], the corresponding
 // literal string is ";" if the semicolon was present in the source,
 // and "\n" if the semicolon was inserted because of a newline or
-// at EOF.
+// at EOF. If the newline is within a /*...*/ comment, the SEMICOLON token
+// is synthesized immediately after the COMMENT token; its position is that
+// of the actual newline within the comment.
 //
 // If the returned token is [token.ILLEGAL], the literal string is the
 // offending character.
@@ -809,10 +821,13 @@ func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Tok
 // and thus relative to the file set.
 func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
 scanAgain:
+	s.endPosValid = false
 	if s.nlPos.IsValid() {
 		// Return artificial ';' token after /*...*/ comment
 		// containing newline, at position of first newline.
 		pos, tok, lit = s.nlPos, token.SEMICOLON, "\n"
+		s.endPos = pos + 1
+		s.endPosValid = true
 		s.nlPos = token.NoPos
 		return
 	}
@@ -860,17 +875,14 @@ scanAgain:
 			insertSemi = true
 			tok = token.STRING
 			lit = s.scanString()
-			s.stringEnd = pos + token.Pos(len(lit))
 		case '\'':
 			insertSemi = true
 			tok = token.CHAR
 			lit = s.scanRune()
 		case '`':
 			insertSemi = true
 			tok = token.STRING
-			var rawLen int
-			lit, rawLen = s.scanRawString()
-			s.stringEnd = pos + token.Pos(rawLen)
+			lit = s.scanRawString()
 		case ':':
 			tok = s.switch2(token.COLON, token.DEFINE)
 		case '.':