Fix Scanner.getNextToken0() to gracefully handle end of content (#4675)

- add checks to stop when the end of source is reached when handling
  a markdown comment
- also do not attempt to access the source with the EOF index
- add new test to ScannerTest
- fixes #4674

Author: jjohnstn

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/parser/Scanner.java

@@ -1730,7 +1730,7 @@ protected TerminalToken getNextToken0() throws InvalidInputException {
 								}
 								isUnicode = false;
 								previous = this.currentPosition;
-								if (((this.currentCharacter = this.source[this.currentPosition++]) == '\\')
+								if ((this.currentPosition < this.eofPosition && (this.currentCharacter = this.source[this.currentPosition++]) == '\\')
 									&& (this.source[this.currentPosition] == 'u')) {
 									//-------------unicode traitement ------------
 									getNextUnicodeChar();
@@ -1746,8 +1746,8 @@ protected TerminalToken getNextToken0() throws InvalidInputException {
 								//loop as long as lines start with ///
 								int firstTag = 0;
 								while(true) {
-									if (this.currentPosition > this.eofPosition) {
-										throw unterminatedComment();
+									if (this.currentPosition == this.eofPosition) {
+										break;
 									}
 									if ((this.currentCharacter == '\r') || (this.currentCharacter == '\n')) {
 										if (this.recordLineSeparator) {

org.eclipse.jdt.core.tests.compiler/src/org/eclipse/jdt/core/tests/compiler/regression/ScannerTest.java

@@ -1848,4 +1848,38 @@ public void testTerminalTokensAPIs() {
 		assertTrue(TerminalToken.getRestrictedKeyword("When".toCharArray()) == TerminalToken.TokenNameNotAToken);
 		assertTrue(TerminalToken.getRestrictedKeyword("blah".toCharArray()) == TerminalToken.TokenNameNotAToken);
 	}
+
+	// https://github.com/eclipse-jdt/eclipse.jdt.core/issues/4674
+	public void testIssue4674() {
+		IScanner scanner = ToolFactory.createScanner(true, true, true, "23", "23");
+		final char[] source = "/// @return a string".toCharArray();
+		scanner.setSource(source);
+		final StringBuilder buffer = new StringBuilder();
+		try {
+			int token;
+			boolean foundMarkdown = false;
+			boolean foundOther = false;
+			while ((token = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
+				try {
+					switch(token) {
+						case ITerminalSymbols.TokenNameCOMMENT_MARKDOWN :
+							foundMarkdown = true;
+							break;
+						default :
+							foundOther = true;
+							buffer.append(scanner.getCurrentTokenSource());
+							break;
+					}
+				} catch (ArrayIndexOutOfBoundsException e) {
+					e.printStackTrace();
+				}
+			}
+			assertTrue("Should have found markdown comment", foundMarkdown);
+			assertTrue("Should have found EOF token", token == ITerminalSymbols.TokenNameEOF);
+			assertFalse("Should not have found other", foundOther);
+		} catch (InvalidInputException e) {
+			assertTrue("Should not have InvalidInputException", false);
+		}
+	}
+
 }