Skip to content

Commit 99c3021

Browse files
griesemergriesemer
committed
go/scanner: recognize //line and /*line directives incl. columns
This change updates go/scanner to recognize the extended line directives that are now also handled by cmd/compile: //line filename:line //line filename:line:column /*line filename:line*/ /*line filename:line:column*/ As before, //-style line directives must start in column 1. /*-style line directives may be placed anywhere in the code. In both cases, the specified position applies to the character immediately following the comment; for line comments that is the first character on the next line (after the newline of the comment). The go/token API is extended by a new method File.AddLineColumnInfo(offset int, filename string, line, column int) which extends the existing File.AddLineInfo(offset int, filename string, line int) by adding a column parameter. Adjusted token.Position computation is changed to take into account column information if provided via a line directive: A (line-directive) relative position will have a non-zero column iff the line directive specified a column; if the position is on the same line as the line directive, the column is relative to the specified column (otherwise it is relative to the line beginning). See also #24183. Finally, Position.String() has been adjusted to not print a column value if the column is unknown (== 0). Fixes #24143. Change-Id: I5518c825ad94443365c049a95677407b46ba55a1 Reviewed-on: https://go-review.googlesource.com/97795 Reviewed-by: Matthew Dempsky <[email protected]>
1 parent 2004602 commit 99c3021

File tree

3 files changed

+194
-71
lines changed

3 files changed

+194
-71
lines changed

‎src/go/scanner/scanner.go‎

Lines changed: 114 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -141,46 +141,26 @@ func (s *Scanner) error(offs int, msg string) {
141141
s.ErrorCount++
142142
}
143143

144-
var prefix = []byte("//line ")
145-
146-
func (s *Scanner) interpretLineComment(text []byte) {
147-
if bytes.HasPrefix(text, prefix) {
148-
// get filename and line number, if any
149-
if i := bytes.LastIndex(text, []byte{':'}); i > 0 {
150-
if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
151-
// valid //line filename:line comment
152-
filename := string(bytes.TrimSpace(text[len(prefix):i]))
153-
if filename != "" {
154-
filename = filepath.Clean(filename)
155-
if !filepath.IsAbs(filename) {
156-
// make filename relative to current directory
157-
filename = filepath.Join(s.dir, filename)
158-
}
159-
}
160-
// update scanner position
161-
s.file.AddLineInfo(s.lineOffset+len(text)+1, filename, line) // +len(text)+1 since comment applies to next line
162-
}
163-
}
164-
}
165-
}
166-
167144
func (s *Scanner) scanComment() string {
168145
// initial '/' already consumed; s.ch == '/' || s.ch == '*'
169146
offs := s.offset - 1 // position of initial '/'
170-
hasCR := false
147+
next := -1 // position immediately following the comment; < 0 means invalid comment
148+
numCR := 0
171149

172150
if s.ch == '/' {
173151
//-style comment
152+
// (the final '\n' is not considered part of the comment)
174153
s.next()
175154
for s.ch != '\n' && s.ch >= 0 {
176155
if s.ch == '\r' {
177-
hasCR = true
156+
numCR++
178157
}
179158
s.next()
180159
}
181-
if offs == s.lineOffset {
182-
// comment starts at the beginning of the current line
183-
s.interpretLineComment(s.src[offs:s.offset])
160+
// if we are at '\n', the position following the comment is afterwards
161+
next = s.offset
162+
if s.ch == '\n' {
163+
next++
184164
}
185165
goto exit
186166
}
@@ -190,11 +170,12 @@ func (s *Scanner) scanComment() string {
190170
for s.ch >= 0 {
191171
ch := s.ch
192172
if ch == '\r' {
193-
hasCR = true
173+
numCR++
194174
}
195175
s.next()
196176
if ch == '*' && s.ch == '/' {
197177
s.next()
178+
next = s.offset
198179
goto exit
199180
}
200181
}
@@ -203,13 +184,116 @@ func (s *Scanner) scanComment() string {
203184

204185
exit:
205186
lit := s.src[offs:s.offset]
206-
if hasCR {
187+
188+
// On Windows, a (//-comment) line may end in "\r\n".
189+
// Remove the final '\r' before analyzing the text for
190+
// line directives (matching the compiler). Remove any
191+
// other '\r' afterwards (matching the pre-existing be-
192+
// havior of the scanner).
193+
if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' {
194+
lit = lit[:len(lit)-1]
195+
numCR--
196+
}
197+
198+
// interpret line directives
199+
// (//line directives must start at the beginning of the current line)
200+
if next >= 0 /* implies valid comment */ && (lit[1] == '*' || offs == s.lineOffset) && bytes.HasPrefix(lit[2:], prefix) {
201+
s.updateLineInfo(next, offs, lit)
202+
}
203+
204+
if numCR > 0 {
207205
lit = stripCR(lit, lit[1] == '*')
208206
}
209207

210208
return string(lit)
211209
}
212210

211+
var prefix = []byte("line ")
212+
213+
// updateLineInfo parses the incoming comment text at offset offs
214+
// as a line directive. If successful, it updates the line info table
215+
// for the position next per the line directive.
216+
func (s *Scanner) updateLineInfo(next, offs int, text []byte) {
217+
// the existing code used to ignore incorrect line/column values
218+
// TODO(gri) adjust once we agree on the directive syntax (issue #24183)
219+
reportErrors := false
220+
221+
// extract comment text
222+
if text[1] == '*' {
223+
text = text[:len(text)-2] // lop off trailing "*/"
224+
}
225+
text = text[7:] // lop off leading "//line " or "/*line "
226+
offs += 7
227+
228+
i, n, ok := trailingDigits(text)
229+
if i == 0 {
230+
return // ignore (not a line directive)
231+
}
232+
// i > 0
233+
234+
if !ok {
235+
// text has a suffix :xxx but xxx is not a number
236+
if reportErrors {
237+
s.error(offs+i, "invalid line number: "+string(text[i:]))
238+
}
239+
return
240+
}
241+
242+
var line, col int
243+
i2, n2, ok2 := trailingDigits(text[:i-1])
244+
if ok2 {
245+
//line filename:line:col
246+
i, i2 = i2, i
247+
line, col = n2, n
248+
if col == 0 {
249+
if reportErrors {
250+
s.error(offs+i2, "invalid column number: "+string(text[i2:]))
251+
}
252+
return
253+
}
254+
text = text[:i2-1] // lop off ":col"
255+
} else {
256+
//line filename:line
257+
line = n
258+
}
259+
260+
if line == 0 {
261+
if reportErrors {
262+
s.error(offs+i, "invalid line number: "+string(text[i:]))
263+
}
264+
return
265+
}
266+
267+
// the existing code used to trim whitespace around filenames
268+
// TODO(gri) adjust once we agree on the directive syntax (issue #24183)
269+
filename := string(bytes.TrimSpace(text[:i-1])) // lop off ":line", and trim white space
270+
271+
// If we have a column (//line filename:line:col form),
272+
// an empty filename means to use the previous filename.
273+
if filename != "" {
274+
filename = filepath.Clean(filename)
275+
if !filepath.IsAbs(filename) {
276+
// make filename relative to current directory
277+
filename = filepath.Join(s.dir, filename)
278+
}
279+
} else if ok2 {
280+
// use existing filename
281+
filename = s.file.Position(s.file.Pos(offs)).Filename
282+
}
283+
284+
s.file.AddLineColumnInfo(next, filename, line, col)
285+
}
286+
287+
func trailingDigits(text []byte) (int, int, bool) {
288+
i := bytes.LastIndexByte(text, ':') // look from right (Windows filenames may contain ':')
289+
if i < 0 {
290+
return 0, 0, false // no ":"
291+
}
292+
// i >= 0
293+
n, err := strconv.ParseUint(string(text[i+1:]), 10, 0)
294+
return i + 1, int(n), err == nil
295+
}
296+
213297
func (s *Scanner) findLineEnd() bool {
214298
// initial '/' already consumed
215299

‎src/go/scanner/scanner_test.go‎

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -503,39 +503,52 @@ func TestSemis(t *testing.T) {
503503
}
504504

505505
type segment struct {
506-
srcline string // a line of source text
507-
filename string // filename for current token
508-
line int // line number for current token
506+
srcline string // a line of source text
507+
filename string // filename for current token
508+
line, column int // line number for current token
509509
}
510510

511511
var segments = []segment{
512512
// exactly one token per line since the test consumes one token per segment
513-
{" line1", filepath.Join("dir", "TestLineComments"), 1},
514-
{"\nline2", filepath.Join("dir", "TestLineComments"), 2},
515-
{"\nline3 //line File1.go:100", filepath.Join("dir", "TestLineComments"), 3}, // bad line comment, ignored
516-
{"\nline4", filepath.Join("dir", "TestLineComments"), 4},
517-
{"\n//line File1.go:100\n line100", filepath.Join("dir", "File1.go"), 100},
518-
{"\n//line \t :42\n line1", "", 42},
519-
{"\n//line File2.go:200\n line200", filepath.Join("dir", "File2.go"), 200},
520-
{"\n//line foo\t:42\n line42", filepath.Join("dir", "foo"), 42},
521-
{"\n //line foo:42\n line44", filepath.Join("dir", "foo"), 44}, // bad line comment, ignored
522-
{"\n//line foo 42\n line46", filepath.Join("dir", "foo"), 46}, // bad line comment, ignored
523-
{"\n//line foo:42 extra text\n line48", filepath.Join("dir", "foo"), 48}, // bad line comment, ignored
524-
{"\n//line ./foo:42\n line42", filepath.Join("dir", "foo"), 42},
525-
{"\n//line a/b/c/File1.go:100\n line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100},
513+
{" line1", filepath.Join("dir", "TestLineDirectives"), 1, 3},
514+
{"\nline2", filepath.Join("dir", "TestLineDirectives"), 2, 1},
515+
{"\nline3 //line File1.go:100", filepath.Join("dir", "TestLineDirectives"), 3, 1}, // bad line comment, ignored
516+
{"\nline4", filepath.Join("dir", "TestLineDirectives"), 4, 1},
517+
{"\n//line File1.go:100\n line100", filepath.Join("dir", "File1.go"), 100, 0},
518+
{"\n//line \t :42\n line1", "", 42, 0},
519+
{"\n//line File2.go:200\n line200", filepath.Join("dir", "File2.go"), 200, 0},
520+
{"\n//line foo\t:42\n line42", filepath.Join("dir", "foo"), 42, 0},
521+
{"\n //line foo:42\n line44", filepath.Join("dir", "foo"), 44, 0}, // bad line comment, ignored
522+
{"\n//line foo 42\n line46", filepath.Join("dir", "foo"), 46, 0}, // bad line comment, ignored
523+
{"\n//line foo:42 extra text\n line48", filepath.Join("dir", "foo"), 48, 0}, // bad line comment, ignored
524+
{"\n//line ./foo:42\n line42", filepath.Join("dir", "foo"), 42, 0},
525+
{"\n//line a/b/c/File1.go:100\n line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100, 0},
526+
527+
// tests for new line directive syntax
528+
{"\n//line :100\na1", "", 100, 0}, // missing filename means empty filename
529+
{"\n//line bar:100\nb1", filepath.Join("dir", "bar"), 100, 0},
530+
{"\n//line :100:10\nc1", filepath.Join("dir", "bar"), 100, 10}, // missing filename means current filename
531+
{"\n//line foo:100:10\nd1", filepath.Join("dir", "foo"), 100, 10},
532+
533+
{"\n/*line :100*/a2", "", 100, 0}, // missing filename means empty filename
534+
{"\n/*line bar:100*/b2", filepath.Join("dir", "bar"), 100, 0},
535+
{"\n/*line :100:10*/c2", filepath.Join("dir", "bar"), 100, 10}, // missing filename means current filename
536+
{"\n/*line foo:100:10*/d2", filepath.Join("dir", "foo"), 100, 10},
537+
{"\n/*line foo:100:10*/ e2", filepath.Join("dir", "foo"), 100, 14}, // line-directive relative column
538+
{"\n/*line foo:100:10*/\n\nf2", filepath.Join("dir", "foo"), 102, 1}, // absolute column since on new line
526539
}
527540

528541
var unixsegments = []segment{
529-
{"\n//line /bar:42\n line42", "/bar", 42},
542+
{"\n//line /bar:42\n line42", "/bar", 42, 0},
530543
}
531544

532545
var winsegments = []segment{
533-
{"\n//line c:\\bar:42\n line42", "c:\\bar", 42},
534-
{"\n//line c:\\dir\\File1.go:100\n line100", "c:\\dir\\File1.go", 100},
546+
{"\n//line c:\\bar:42\n line42", "c:\\bar", 42, 0},
547+
{"\n//line c:\\dir\\File1.go:100\n line100", "c:\\dir\\File1.go", 100, 0},
535548
}
536549

537-
// Verify that comments of the form "//line filename:line" are interpreted correctly.
538-
func TestLineComments(t *testing.T) {
550+
// Verify that line directives are interpreted correctly.
551+
func TestLineDirectives(t *testing.T) {
539552
segs := segments
540553
if runtime.GOOS == "windows" {
541554
segs = append(segs, winsegments...)
@@ -551,16 +564,16 @@ func TestLineComments(t *testing.T) {
551564

552565
// verify scan
553566
var S Scanner
554-
file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
555-
S.Init(file, []byte(src), nil, dontInsertSemis)
567+
file := fset.AddFile(filepath.Join("dir", "TestLineDirectives"), fset.Base(), len(src))
568+
S.Init(file, []byte(src), func(pos token.Position, msg string) { t.Error(Error{pos, msg}) }, dontInsertSemis)
556569
for _, s := range segs {
557570
p, _, lit := S.Scan()
558571
pos := file.Position(p)
559572
checkPos(t, lit, p, token.Position{
560573
Filename: s.filename,
561574
Offset: pos.Offset,
562575
Line: s.line,
563-
Column: pos.Column,
576+
Column: s.column,
564577
})
565578
}
566579

‎src/go/token/position.go‎

Lines changed: 43 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ func (pos *Position) IsValid() bool { return pos.Line > 0 }
3030
// String returns a string in one of several forms:
3131
//
3232
// file:line:column valid position with file name
33+
// file:line valid position with file name but no column (column == 0)
3334
// line:column valid position without file name
35+
// line valid position without file name and no column (column == 0)
3436
// file invalid position with file name
3537
// - invalid position without file name
3638
//
@@ -40,7 +42,10 @@ func (pos Position) String() string {
4042
if s != "" {
4143
s += ":"
4244
}
43-
s += fmt.Sprintf("%d:%d", pos.Line, pos.Column)
45+
s += fmt.Sprintf("%d", pos.Line)
46+
if pos.Column != 0 {
47+
s += fmt.Sprintf(":%d", pos.Column)
48+
}
4449
}
4550
if s == "" {
4651
s = "-"
@@ -204,28 +209,36 @@ func (f *File) SetLinesForContent(content []byte) {
204209
f.mutex.Unlock()
205210
}
206211

207-
// A lineInfo object describes alternative file and line number
208-
// information (such as provided via a //line comment in a .go
209-
// file) for a given file offset.
212+
// A lineInfo object describes alternative file, line, and column
213+
// number information (such as provided via a //line directive)
214+
// for a given file offset.
210215
type lineInfo struct {
211216
// fields are exported to make them accessible to gob
212-
Offset int
213-
Filename string
214-
Line int
217+
Offset int
218+
Filename string
219+
Line, Column int
215220
}
216221

217-
// AddLineInfo adds alternative file and line number information for
218-
// a given file offset. The offset must be larger than the offset for
219-
// the previously added alternative line info and smaller than the
220-
// file size; otherwise the information is ignored.
221-
//
222-
// AddLineInfo is typically used to register alternative position
223-
// information for //line filename:line comments in source files.
222+
// AddLineInfo is like AddLineColumnInfo with a column = 1 argument.
223+
// It is here for backward-compatibility for code prior to Go 1.11.
224224
//
225225
func (f *File) AddLineInfo(offset int, filename string, line int) {
226+
f.AddLineColumnInfo(offset, filename, line, 1)
227+
}
228+
229+
// AddLineColumnInfo adds alternative file, line, and column number
230+
// information for a given file offset. The offset must be larger
231+
// than the offset for the previously added alternative line info
232+
// and smaller than the file size; otherwise the information is
233+
// ignored.
234+
//
235+
// AddLineColumnInfo is typically used to register alternative position
236+
// information for line directives such as //line filename:line:column.
237+
//
238+
func (f *File) AddLineColumnInfo(offset int, filename string, line, column int) {
226239
f.mutex.Lock()
227240
if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size {
228-
f.infos = append(f.infos, lineInfo{offset, filename, line})
241+
f.infos = append(f.infos, lineInfo{offset, filename, line, column})
229242
}
230243
f.mutex.Unlock()
231244
}
@@ -275,12 +288,25 @@ func (f *File) unpack(offset int, adjusted bool) (filename string, line, column
275288
line, column = i+1, offset-f.lines[i]+1
276289
}
277290
if adjusted && len(f.infos) > 0 {
278-
// almost no files have extra line infos
291+
// few files have extra line infos
279292
if i := searchLineInfos(f.infos, offset); i >= 0 {
280293
alt := &f.infos[i]
281294
filename = alt.Filename
282295
if i := searchInts(f.lines, alt.Offset); i >= 0 {
283-
line += alt.Line - i - 1
296+
// i+1 is the line at which the alternative position was recorded
297+
d := line - (i + 1) // line distance from alternative position base
298+
line = alt.Line + d
299+
if alt.Column == 0 {
300+
// alternative column is unknown => relative column is unknown
301+
// (the current specification for line directives requires
302+
// this to apply until the next PosBase/line directive,
303+
// not just until the new newline)
304+
column = 0
305+
} else if d == 0 {
306+
// the alternative position base is on the current line
307+
// => column is relative to alternative column
308+
column = alt.Column + (offset - alt.Offset)
309+
}
284310
}
285311
}
286312
}

0 commit comments

Comments
 (0)