X-Git-Url: http://git.phpeclipse.com diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java index bef873a..e313f23 100644 --- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java +++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java @@ -12,11 +12,15 @@ package net.sourceforge.phpdt.internal.compiler.parser; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Stack; + import net.sourceforge.phpdt.core.compiler.CharOperation; import net.sourceforge.phpdt.core.compiler.IScanner; import net.sourceforge.phpdt.core.compiler.ITerminalSymbols; import net.sourceforge.phpdt.core.compiler.InvalidInputException; -import net.sourceforge.phpdt.internal.compiler.ast.StringLiteral; +import net.sourceforge.phpeclipse.internal.compiler.ast.StringLiteral; + + public class Scanner implements IScanner, ITerminalSymbols { /* * APIs ares - getNextToken() which return the current type of the token @@ -32,6 +36,7 @@ public class Scanner implements IScanner, ITerminalSymbols { public boolean containsAssertKeyword = false; public boolean recordLineSeparator; public boolean phpMode = false; + public Stack encapsedStringStack = null; public char currentCharacter; public int startPosition; public int currentPosition; @@ -40,6 +45,7 @@ public class Scanner implements IScanner, ITerminalSymbols { // source public boolean tokenizeComments; public boolean tokenizeWhiteSpace; + public boolean tokenizeStrings; //source should be viewed as a window (aka a part) //of a entire very large stream public char source[]; @@ -49,11 +55,11 @@ public class Scanner implements IScanner, ITerminalSymbols { //when == 0 ==> no unicode in the current token public boolean unicodeAsBackSlash = false; public boolean scanningFloatLiteral = false; - //support for /** comments - //public char[][] comments = new char[10][]; - public int[] commentStops = new int[10]; - public int[] commentStarts = new int[10]; - public int commentPtr = -1; // no comment test with commentPtr value -1 +//support for /** comments + public int[] commentStops = new int[10]; + public int[] commentStarts = new int[10]; + public int commentPtr = -1; // no comment test with commentPtr value -1 + protected int lastCommentLinePosition = -1; //diet parsing support - jump over some method body when requested public boolean diet = false; //support for the poor-line-debuggers .... @@ -131,12 +137,8 @@ public class Scanner implements IScanner, ITerminalSymbols { public char[][] taskTags = null; public char[][] taskPriorities = null; public static final boolean DEBUG = false; - public Scanner() { - this(false, false); - } - public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) { - this(tokenizeComments, tokenizeWhiteSpace, false); - } + public static final boolean TRACE = false; + /** * Determines if the specified character is permissible as the first * character in a PHP identifier @@ -675,66 +677,430 @@ public class Scanner implements IScanner, ITerminalSymbols { return false; } } - public int getNextToken() throws InvalidInputException { - int htmlPosition = currentPosition; + public int getCastOrParen() { + int tempPosition = currentPosition; + char tempCharacter = currentCharacter; + int tempToken = TokenNameLPAREN; + boolean found = false; + StringBuffer buf = new StringBuffer(); try { - while (!phpMode) { + do { currentCharacter = source[currentPosition++]; - if (currentCharacter == '<') { - if (getNextChar('?')) { - currentCharacter = source[currentPosition++]; - if ((currentCharacter == ' ') - || Character.isWhitespace(currentCharacter)) { - // = 'a' && currentCharacter <= 'z') + || (currentCharacter >= 'A' && currentCharacter <= 'Z')) { + buf.append(currentCharacter); + currentCharacter = source[currentPosition++]; + } + if (buf.length() >= 3 && buf.length() <= 7) { + char[] data = buf.toString().toCharArray(); + int index = 0; + switch (data.length) { + case 3 : + // int + if ((data[index] == 'i') && (data[++index] == 'n') + && (data[++index] == 't')) { + found = true; + tempToken = TokenNameintCAST; + } + break; + case 4 : + // bool real + if ((data[index] == 'b') && (data[++index] == 'o') + && (data[++index] == 'o') && (data[++index] == 'l')) { + found = true; + tempToken = TokenNameboolCAST; + } else { + index = 0; + if ((data[index] == 'r') && (data[++index] == 'e') + && (data[++index] == 'a') && (data[++index] == 'l')) { + found = true; + tempToken = TokenNamedoubleCAST; + } + } + break; + case 5 : + // array unset float + if ((data[index] == 'a') && (data[++index] == 'r') + && (data[++index] == 'r') && (data[++index] == 'a') + && (data[++index] == 'y')) { + found = true; + tempToken = TokenNamearrayCAST; + } else { + index = 0; + if ((data[index] == 'u') && (data[++index] == 'n') + && (data[++index] == 's') && (data[++index] == 'e') + && (data[++index] == 't')) { + found = true; + tempToken = TokenNameunsetCAST; + } else { + index = 0; + if ((data[index] == 'f') && (data[++index] == 'l') + && (data[++index] == 'o') && (data[++index] == 'a') + && (data[++index] == 't')) { + found = true; + tempToken = TokenNamedoubleCAST; + } } + } + break; + case 6 : + // object string double + if ((data[index] == 'o') && (data[++index] == 'b') + && (data[++index] == 'j') && (data[++index] == 'e') + && (data[++index] == 'c') && (data[++index] == 't')) { + found = true; + tempToken = TokenNameobjectCAST; } else { - boolean phpStart = (currentCharacter == 'P') - || (currentCharacter == 'p'); - if (phpStart) { - int test = getNextChar('H', 'h'); - if (test >= 0) { - test = getNextChar('P', 'p'); - if (test >= 0) { - // source.length ? TokenNameEOF : TokenNameRBRACE; } try { - while (true) { //loop for jumping over comments + while (true) { withoutUnicodePtr = 0; - //start with a new token (even comment written with unicode ) + //start with a new token + char encapsedChar = ' '; + if (!encapsedStringStack.isEmpty()) { + encapsedChar = ((Character) encapsedStringStack.peek()).charValue(); + } + if (encapsedChar != '$' && encapsedChar != ' ') { + currentCharacter = source[currentPosition++]; + if (currentCharacter == encapsedChar) { + switch (currentCharacter) { + case '`' : + return TokenNameEncapsedString0; + case '\'' : + return TokenNameEncapsedString1; + case '"' : + return TokenNameEncapsedString2; + } + } + while (currentCharacter != encapsedChar) { + /** ** in PHP \r and \n are valid in string literals *** */ + switch (currentCharacter) { + case '\\' : + int escapeSize = currentPosition; + boolean backSlashAsUnicodeInString = unicodeAsBackSlash; + //scanEscapeCharacter make a side effect on this value and + // we need the previous value few lines down this one + scanDoubleQuotedEscapeCharacter(); + escapeSize = currentPosition - escapeSize; + if (withoutUnicodePtr == 0) { + //buffer all the entries that have been left aside.... + withoutUnicodePtr = currentPosition - escapeSize - 1 + - startPosition; + System.arraycopy(source, startPosition, + withoutUnicodeBuffer, 1, withoutUnicodePtr); + withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; + } else { //overwrite the / in the buffer + withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter; + if (backSlashAsUnicodeInString) { //there are TWO \ in + withoutUnicodePtr--; + } + } + break; + case '$' : + if (isPHPIdentifierStart(source[currentPosition]) + || source[currentPosition] == '{') { + currentPosition--; + encapsedStringStack.push(new Character('$')); + return TokenNameSTRING; + } + break; + case '{' : + if (source[currentPosition] == '$') { // CURLY_OPEN + currentPosition--; + encapsedStringStack.push(new Character('$')); + return TokenNameSTRING; + } + } + // consume next character + unicodeAsBackSlash = false; + currentCharacter = source[currentPosition++]; + if (withoutUnicodePtr != 0) { + withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; + } + // } + } // end while + currentPosition--; + return TokenNameSTRING; + } // ---------Consume white space and handles startPosition--------- int whiteStart = currentPosition; - boolean isWhiteSpace; - do { + startPosition = currentPosition; + currentCharacter = source[currentPosition++]; + if (encapsedChar == '$') { + switch (currentCharacter) { + case '\\' : + currentCharacter = source[currentPosition++]; + return TokenNameSTRING; + case '{' : + if (encapsedChar == '$') { + if (getNextChar('$')) + return TokenNameCURLY_OPEN; + } + return TokenNameLBRACE; + case '}' : + return TokenNameRBRACE; + case '[' : + return TokenNameLBRACKET; + case ']' : + return TokenNameRBRACKET; + case '\'' : + if (tokenizeStrings) { + consumeStringConstant(); + return TokenNameStringSingleQuote; + } + return TokenNameEncapsedString1; + case '"' : + return TokenNameEncapsedString2; + case '`' : + if (tokenizeStrings) { + consumeStringInterpolated(); + return TokenNameStringInterpolated; + } + return TokenNameEncapsedString0; + case '-' : + if (getNextChar('>')) + return TokenNameMINUS_GREATER; + return TokenNameSTRING; + default : + if (currentCharacter == '$') { + int oldPosition = currentPosition; + try { + currentCharacter = source[currentPosition++]; + if (currentCharacter == '{') { + return TokenNameDOLLAR_LBRACE; + } + if (isPHPIdentifierStart(currentCharacter)) { + return scanIdentifierOrKeyword(true); + } else { + currentPosition = oldPosition; + return TokenNameSTRING; + } + } catch (IndexOutOfBoundsException e) { + currentPosition = oldPosition; + return TokenNameSTRING; + } + } + if (isPHPIdentifierStart(currentCharacter)) + return scanIdentifierOrKeyword(false); + if (Character.isDigit(currentCharacter)) + return scanNumber(false); + return TokenNameERROR; + } + } + // boolean isWhiteSpace; + + while ((currentCharacter == ' ') + || Character.isWhitespace(currentCharacter)) { startPosition = currentPosition; currentCharacter = source[currentPosition++]; // if (((currentCharacter = source[currentPosition++]) == '\\') @@ -765,10 +1262,10 @@ public class Scanner implements IScanner, ITerminalSymbols { currentLine = null; } } - isWhiteSpace = (currentCharacter == ' ') - || Character.isWhitespace(currentCharacter); + // isWhiteSpace = (currentCharacter == ' ') + // || Character.isWhitespace(currentCharacter); // } - } while (isWhiteSpace); + } if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) { // reposition scanner in case we are interested by spaces as tokens currentPosition--; @@ -781,7 +1278,7 @@ public class Scanner implements IScanner, ITerminalSymbols { // ---------Identify the next token------------- switch (currentCharacter) { case '(' : - return TokenNameLPAREN; + return getCastOrParen(); case ')' : return TokenNameRPAREN; case '{' : @@ -797,6 +1294,8 @@ public class Scanner implements IScanner, ITerminalSymbols { case ',' : return TokenNameCOMMA; case '.' : + if (getNextChar('=')) + return TokenNameDOT_EQUAL; if (getNextCharAsDigit()) return scanNumber(true); return TokenNameDOT; @@ -842,56 +1341,69 @@ public class Scanner implements IScanner, ITerminalSymbols { return TokenNameREMAINDER; case '<' : { - int test; - if ((test = getNextChar('=', '<')) == 0) - return TokenNameLESS_EQUAL; - if (test > 0) { - if (getNextChar('=')) - return TokenNameLEFT_SHIFT_EQUAL; - if (getNextChar('<')) { - int heredocStart = currentPosition; - int heredocLength = 0; - currentCharacter = source[currentPosition++]; - if (isPHPIdentifierStart(currentCharacter)) { - currentCharacter = source[currentPosition++]; - } else { - return TokenNameERROR; - } - while (isPHPIdentifierPart(currentCharacter)) { + int oldPosition = currentPosition; + try { + currentCharacter = source[currentPosition++]; + } catch (IndexOutOfBoundsException e) { + currentPosition = oldPosition; + return TokenNameLESS; + } + switch (currentCharacter) { + case '=' : + return TokenNameLESS_EQUAL; + case '>' : + return TokenNameNOT_EQUAL; + case '<' : + if (getNextChar('=')) + return TokenNameLEFT_SHIFT_EQUAL; + if (getNextChar('<')) { currentCharacter = source[currentPosition++]; - } - heredocLength = currentPosition - heredocStart - 1; - // heredoc end-tag determination - boolean endTag = true; - char ch; - do { - ch = source[currentPosition++]; - if (ch == '\r' || ch == '\n') { - if (recordLineSeparator) { - pushLineSeparator(); - } else { - currentLine = null; - } - for (int i = 0; i < heredocLength; i++) { - if (source[currentPosition + i] != source[heredocStart - + i]) { - endTag = false; - break; + while (Character.isWhitespace(currentCharacter)) { + currentCharacter = source[currentPosition++]; + } + int heredocStart = currentPosition - 1; + int heredocLength = 0; + if (isPHPIdentifierStart(currentCharacter)) { + currentCharacter = source[currentPosition++]; + } else { + return TokenNameERROR; + } + while (isPHPIdentifierPart(currentCharacter)) { + currentCharacter = source[currentPosition++]; + } + heredocLength = currentPosition - heredocStart - 1; + // heredoc end-tag determination + boolean endTag = true; + char ch; + do { + ch = source[currentPosition++]; + if (ch == '\r' || ch == '\n') { + if (recordLineSeparator) { + pushLineSeparator(); + } else { + currentLine = null; + } + for (int i = 0; i < heredocLength; i++) { + if (source[currentPosition + i] != source[heredocStart + + i]) { + endTag = false; + break; + } + } + if (endTag) { + currentPosition += heredocLength - 1; + currentCharacter = source[currentPosition++]; + break; // do...while loop + } else { + endTag = true; } } - if (endTag) { - currentPosition += heredocLength - 1; - currentCharacter = source[currentPosition++]; - break; // do...while loop - } else { - endTag = true; - } - } - } while (true); - return TokenNameHEREDOC; - } - return TokenNameLEFT_SHIFT; + } while (true); + return TokenNameHEREDOC; + } + return TokenNameLEFT_SHIFT; } + currentPosition = oldPosition; return TokenNameLESS; } case '>' : @@ -941,7 +1453,11 @@ public class Scanner implements IScanner, ITerminalSymbols { case '?' : if (getNextChar('>')) { phpMode = false; - return TokenNameStopPHP; + if (currentPosition == source.length) { + phpMode = true; + return TokenNameINLINE_HTML; + } + return getInlinedHTML(currentPosition - 2); } return TokenNameQUESTION; case ':' : @@ -950,390 +1466,32 @@ public class Scanner implements IScanner, ITerminalSymbols { return TokenNameCOLON; case '@' : return TokenNameAT; - // case '\'' : - // { - // int test; - // if ((test = getNextChar('\n', '\r')) == 0) { - // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); - // } - // if (test > 0) { - // // relocate if finding another quote fairly close: thus unicode - // '/u000D' will be fully consumed - // for (int lookAhead = 0; - // lookAhead < 3; - // lookAhead++) { - // if (currentPosition + lookAhead - // == source.length) - // break; - // if (source[currentPosition + lookAhead] - // == '\n') - // break; - // if (source[currentPosition + lookAhead] - // == '\'') { - // currentPosition += lookAhead + 1; - // break; - // } - // } - // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); - // } - // } - // if (getNextChar('\'')) { - // // relocate if finding another quote fairly close: thus unicode - // '/u000D' will be fully consumed - // for (int lookAhead = 0; - // lookAhead < 3; - // lookAhead++) { - // if (currentPosition + lookAhead - // == source.length) - // break; - // if (source[currentPosition + lookAhead] - // == '\n') - // break; - // if (source[currentPosition + lookAhead] - // == '\'') { - // currentPosition += lookAhead + 1; - // break; - // } - // } - // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); - // } - // if (getNextChar('\\')) - // scanEscapeCharacter(); - // else { // consume next character - // unicodeAsBackSlash = false; - // if (((currentCharacter = source[currentPosition++]) - // == '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - // if (withoutUnicodePtr != 0) { - // withoutUnicodeBuffer[++withoutUnicodePtr] = - // currentCharacter; - // } - // } - // } - // // if (getNextChar('\'')) - // // return TokenNameCharacterLiteral; - // // relocate if finding another quote fairly close: thus unicode - // '/u000D' will be fully consumed - // for (int lookAhead = 0; lookAhead < 20; lookAhead++) { - // if (currentPosition + lookAhead == source.length) - // break; - // if (source[currentPosition + lookAhead] == '\n') - // break; - // if (source[currentPosition + lookAhead] == '\'') { - // currentPosition += lookAhead + 1; - // break; - // } - // } - // throw new InvalidInputException(INVALID_CHARACTER_CONSTANT); case '\'' : - try { - // consume next character - unicodeAsBackSlash = false; - currentCharacter = source[currentPosition++]; - // if (((currentCharacter = source[currentPosition++]) == '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - // if (withoutUnicodePtr != 0) { - // withoutUnicodeBuffer[++withoutUnicodePtr] = - // currentCharacter; - // } - // } - while (currentCharacter != '\'') { - /** ** in PHP \r and \n are valid in string literals *** */ - // if ((currentCharacter == '\n') - // || (currentCharacter == '\r')) { - // // relocate if finding another quote fairly close: thus - // unicode '/u000D' will be fully consumed - // for (int lookAhead = 0; lookAhead < 50; lookAhead++) { - // if (currentPosition + lookAhead == source.length) - // break; - // if (source[currentPosition + lookAhead] == '\n') - // break; - // if (source[currentPosition + lookAhead] == '\"') { - // currentPosition += lookAhead + 1; - // break; - // } - // } - // throw new InvalidInputException(INVALID_CHAR_IN_STRING); - // } - if (currentCharacter == '\\') { - int escapeSize = currentPosition; - boolean backSlashAsUnicodeInString = unicodeAsBackSlash; - //scanEscapeCharacter make a side effect on this value and - // we need the previous value few lines down this one - scanSingleQuotedEscapeCharacter(); - escapeSize = currentPosition - escapeSize; - if (withoutUnicodePtr == 0) { - //buffer all the entries that have been left aside.... - withoutUnicodePtr = currentPosition - escapeSize - 1 - - startPosition; - System.arraycopy(source, startPosition, - withoutUnicodeBuffer, 1, withoutUnicodePtr); - withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; - } else { //overwrite the / in the buffer - withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter; - if (backSlashAsUnicodeInString) { //there are TWO \ in - // the stream where - // only one is correct - withoutUnicodePtr--; - } - } - } - // consume next character - unicodeAsBackSlash = false; - currentCharacter = source[currentPosition++]; - // if (((currentCharacter = source[currentPosition++]) == - // '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - if (withoutUnicodePtr != 0) { - withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; - } - // } - } - } catch (IndexOutOfBoundsException e) { - throw new InvalidInputException(UNTERMINATED_STRING); - } catch (InvalidInputException e) { - if (e.getMessage().equals(INVALID_ESCAPE)) { - // relocate if finding another quote fairly close: thus - // unicode '/u000D' will be fully consumed - for (int lookAhead = 0; lookAhead < 50; lookAhead++) { - if (currentPosition + lookAhead == source.length) - break; - if (source[currentPosition + lookAhead] == '\n') - break; - if (source[currentPosition + lookAhead] == '\'') { - currentPosition += lookAhead + 1; - break; - } - } - } - throw e; // rethrow - } - if (checkNonExternalizedStringLiterals) { // check for presence - // of NLS tags - // //$NON-NLS-?$ where - // ? is an int. - if (currentLine == null) { - currentLine = new NLSLine(); - lines.add(currentLine); - } - currentLine.add(new StringLiteral( - getCurrentTokenSourceString(), startPosition, - currentPosition - 1)); - } - return TokenNameStringConstant; + consumeStringConstant(); + return TokenNameStringSingleQuote; case '"' : - try { - // consume next character - unicodeAsBackSlash = false; - currentCharacter = source[currentPosition++]; - // if (((currentCharacter = source[currentPosition++]) == '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - // if (withoutUnicodePtr != 0) { - // withoutUnicodeBuffer[++withoutUnicodePtr] = - // currentCharacter; - // } - // } - while (currentCharacter != '"') { - /** ** in PHP \r and \n are valid in string literals *** */ - // if ((currentCharacter == '\n') - // || (currentCharacter == '\r')) { - // // relocate if finding another quote fairly close: thus - // unicode '/u000D' will be fully consumed - // for (int lookAhead = 0; lookAhead < 50; lookAhead++) { - // if (currentPosition + lookAhead == source.length) - // break; - // if (source[currentPosition + lookAhead] == '\n') - // break; - // if (source[currentPosition + lookAhead] == '\"') { - // currentPosition += lookAhead + 1; - // break; - // } - // } - // throw new InvalidInputException(INVALID_CHAR_IN_STRING); - // } - if (currentCharacter == '\\') { - int escapeSize = currentPosition; - boolean backSlashAsUnicodeInString = unicodeAsBackSlash; - //scanEscapeCharacter make a side effect on this value and - // we need the previous value few lines down this one - scanDoubleQuotedEscapeCharacter(); - escapeSize = currentPosition - escapeSize; - if (withoutUnicodePtr == 0) { - //buffer all the entries that have been left aside.... - withoutUnicodePtr = currentPosition - escapeSize - 1 - - startPosition; - System.arraycopy(source, startPosition, - withoutUnicodeBuffer, 1, withoutUnicodePtr); - withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; - } else { //overwrite the / in the buffer - withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter; - if (backSlashAsUnicodeInString) { //there are TWO \ in - // the stream where - // only one is correct - withoutUnicodePtr--; - } - } - } - // consume next character - unicodeAsBackSlash = false; - currentCharacter = source[currentPosition++]; - // if (((currentCharacter = source[currentPosition++]) == - // '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - if (withoutUnicodePtr != 0) { - withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; - } - // } - } - } catch (IndexOutOfBoundsException e) { - throw new InvalidInputException(UNTERMINATED_STRING); - } catch (InvalidInputException e) { - if (e.getMessage().equals(INVALID_ESCAPE)) { - // relocate if finding another quote fairly close: thus - // unicode '/u000D' will be fully consumed - for (int lookAhead = 0; lookAhead < 50; lookAhead++) { - if (currentPosition + lookAhead == source.length) - break; - if (source[currentPosition + lookAhead] == '\n') - break; - if (source[currentPosition + lookAhead] == '\"') { - currentPosition += lookAhead + 1; - break; - } - } - } - throw e; // rethrow - } - if (checkNonExternalizedStringLiterals) { // check for presence - // of NLS tags - // //$NON-NLS-?$ where - // ? is an int. - if (currentLine == null) { - currentLine = new NLSLine(); - lines.add(currentLine); - } - currentLine.add(new StringLiteral( - getCurrentTokenSourceString(), startPosition, - currentPosition - 1)); + if (tokenizeStrings) { + consumeStringLiteral(); + return TokenNameStringDoubleQuote; } - return TokenNameStringLiteral; + return TokenNameEncapsedString2; case '`' : - try { - // consume next character - unicodeAsBackSlash = false; - currentCharacter = source[currentPosition++]; - // if (((currentCharacter = source[currentPosition++]) == '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - // if (withoutUnicodePtr != 0) { - // withoutUnicodeBuffer[++withoutUnicodePtr] = - // currentCharacter; - // } - // } - while (currentCharacter != '`') { - /** ** in PHP \r and \n are valid in string literals *** */ - // if ((currentCharacter == '\n') - // || (currentCharacter == '\r')) { - // // relocate if finding another quote fairly close: thus - // unicode '/u000D' will be fully consumed - // for (int lookAhead = 0; lookAhead < 50; lookAhead++) { - // if (currentPosition + lookAhead == source.length) - // break; - // if (source[currentPosition + lookAhead] == '\n') - // break; - // if (source[currentPosition + lookAhead] == '\"') { - // currentPosition += lookAhead + 1; - // break; - // } - // } - // throw new InvalidInputException(INVALID_CHAR_IN_STRING); - // } - if (currentCharacter == '\\') { - int escapeSize = currentPosition; - boolean backSlashAsUnicodeInString = unicodeAsBackSlash; - //scanEscapeCharacter make a side effect on this value and - // we need the previous value few lines down this one - scanDoubleQuotedEscapeCharacter(); - escapeSize = currentPosition - escapeSize; - if (withoutUnicodePtr == 0) { - //buffer all the entries that have been left aside.... - withoutUnicodePtr = currentPosition - escapeSize - 1 - - startPosition; - System.arraycopy(source, startPosition, - withoutUnicodeBuffer, 1, withoutUnicodePtr); - withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; - } else { //overwrite the / in the buffer - withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter; - if (backSlashAsUnicodeInString) { //there are TWO \ in - // the stream where - // only one is correct - withoutUnicodePtr--; - } - } - } - // consume next character - unicodeAsBackSlash = false; - currentCharacter = source[currentPosition++]; - // if (((currentCharacter = source[currentPosition++]) == - // '\\') - // && (source[currentPosition] == 'u')) { - // getNextUnicodeChar(); - // } else { - if (withoutUnicodePtr != 0) { - withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter; - } - // } - } - } catch (IndexOutOfBoundsException e) { - throw new InvalidInputException(UNTERMINATED_STRING); - } catch (InvalidInputException e) { - if (e.getMessage().equals(INVALID_ESCAPE)) { - // relocate if finding another quote fairly close: thus - // unicode '/u000D' will be fully consumed - for (int lookAhead = 0; lookAhead < 50; lookAhead++) { - if (currentPosition + lookAhead == source.length) - break; - if (source[currentPosition + lookAhead] == '\n') - break; - if (source[currentPosition + lookAhead] == '`') { - currentPosition += lookAhead + 1; - break; - } - } - } - throw e; // rethrow - } - if (checkNonExternalizedStringLiterals) { // check for presence - // of NLS tags - // //$NON-NLS-?$ where - // ? is an int. - if (currentLine == null) { - currentLine = new NLSLine(); - lines.add(currentLine); - } - currentLine.add(new StringLiteral( - getCurrentTokenSourceString(), startPosition, - currentPosition - 1)); + if (tokenizeStrings) { + consumeStringInterpolated(); + return TokenNameStringInterpolated; } - return TokenNameStringInterpolated; + return TokenNameEncapsedString0; case '#' : case '/' : { + char startChar = currentCharacter; + if (getNextChar('=')) { + return TokenNameDIVIDE_EQUAL; + } int test; - if ((currentCharacter == '#') - || (test = getNextChar('/', '*')) == 0) { + if ((startChar == '#') || (test = getNextChar('/', '*')) == 0) { //line comment + this.lastCommentLinePosition = this.currentPosition; int endPositionForLineComment = 0; try { //get the next char currentCharacter = source[currentPosition++]; @@ -1362,7 +1520,8 @@ public class Scanner implements IScanner, ITerminalSymbols { // Character.getNumericValue(source[currentPosition++])) // > 15 // || c4 < 0) { - // throw new InvalidInputException(INVALID_UNICODE_ESCAPE); + // throw new + // InvalidInputException(INVALID_UNICODE_ESCAPE); // } else { // currentCharacter = // (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4); @@ -1375,11 +1534,12 @@ public class Scanner implements IScanner, ITerminalSymbols { // } //jump over the \\ boolean isUnicode = false; while (currentCharacter != '\r' && currentCharacter != '\n') { + this.lastCommentLinePosition = this.currentPosition; if (currentCharacter == '?') { if (getNextChar('>')) { startPosition = currentPosition - 2; phpMode = false; - return TokenNameStopPHP; + return TokenNameINLINE_HTML; } } //get the next char @@ -1432,8 +1592,10 @@ public class Scanner implements IScanner, ITerminalSymbols { } else { endPositionForLineComment = currentPosition - 1; } - recordComment(false); - if ((currentCharacter == '\r') +// recordComment(false); + recordComment(TokenNameCOMMENT_LINE); + if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition); + if ((currentCharacter == '\r') || (currentCharacter == '\n')) { checkNonExternalizeString(); if (recordLineSeparator) { @@ -1535,19 +1697,25 @@ public class Scanner implements IScanner, ITerminalSymbols { // currentPosition++; // } //jump over the \\ } - recordComment(isJavadoc); + //recordComment(isJavadoc); + if (isJavadoc) { + recordComment(TokenNameCOMMENT_PHPDOC); + } else { + recordComment(TokenNameCOMMENT_BLOCK); + } + if (tokenizeComments) { if (isJavadoc) return TokenNameCOMMENT_PHPDOC; return TokenNameCOMMENT_BLOCK; } } catch (IndexOutOfBoundsException e) { +// reset end position for error reporting + currentPosition-=2; throw new InvalidInputException(UNTERMINATED_COMMENT); } break; } - if (getNextChar('=')) - return TokenNameDIVIDE_EQUAL; return TokenNameDIVIDE; } case '\u001a' : @@ -1558,13 +1726,19 @@ public class Scanner implements IScanner, ITerminalSymbols { throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$ default : if (currentCharacter == '$') { - while ((currentCharacter = source[currentPosition++]) == '$') { + int oldPosition = currentPosition; + try { + currentCharacter = source[currentPosition++]; + if (isPHPIdentifierStart(currentCharacter)) { + return scanIdentifierOrKeyword(true); + } else { + currentPosition = oldPosition; + return TokenNameDOLLAR; + } + } catch (IndexOutOfBoundsException e) { + currentPosition = oldPosition; + return TokenNameDOLLAR; } - if (currentCharacter == '{') - return TokenNameDOLLAR_LBRACE; - if (isPHPIdentifierStart(currentCharacter)) - return scanIdentifierOrKeyword(true); - return TokenNameERROR; } if (isPHPIdentifierStart(currentCharacter)) return scanIdentifierOrKeyword(false); @@ -1579,6 +1753,96 @@ public class Scanner implements IScanner, ITerminalSymbols { } return TokenNameEOF; } + + private int getInlinedHTML(int start) throws InvalidInputException { + int token = getInlinedHTMLToken(start); + if (token == TokenNameINLINE_HTML) { +// Stack stack = new Stack(); +// // scan html for errors +// Source inlinedHTMLSource = new Source(new String(source, startPosition, currentPosition - startPosition)); +// int lastPHPEndPos=0; +// for (Iterator i=inlinedHTMLSource.getNextTagIterator(0); i.hasNext();) { +// Tag tag=(Tag)i.next(); +// +// if (tag instanceof StartTag) { +// StartTag startTag=(StartTag)tag; +// // System.out.println("startTag: "+tag); +// if (startTag.isServerTag()) { +// // TODO : what to do with a server tag ? +// } else { +// // do whatever with HTML start tag +// // use startTag.getElement() to find the element corresponding +// // to this start tag which may be useful if you implement code +// // folding etc +// stack.push(startTag); +// } +// } else { +// EndTag endTag=(EndTag)tag; +// StartTag stag = (StartTag) stack.peek(); +//// System.out.println("endTag: "+tag); +// // do whatever with HTML end tag. +// } +// } + } + return token; + } + /** + * @return @throws + * InvalidInputException + */ + private int getInlinedHTMLToken(int start) throws InvalidInputException { + // int htmlPosition = start; + if (currentPosition > source.length) { + currentPosition = source.length; + return TokenNameEOF; + } + startPosition = start; + try { + while (!phpMode) { + currentCharacter = source[currentPosition++]; + if (currentCharacter == '<') { + if (getNextChar('?')) { + currentCharacter = source[currentPosition++]; + if ((currentCharacter == ' ') + || Character.isWhitespace(currentCharacter)) { + // = 0) { + test = getNextChar('P', 'p'); + if (test >= 0) { + // TokenNameKEYWORD); + } final char[] optimizedCurrentTokenSource1() { //return always the same char[] build only once //optimization at no speed cost of 99.5 % of the singleCharIdentifier @@ -2347,26 +2614,48 @@ public class Scanner implements IScanner, ITerminalSymbols { } } } - public final void recordComment(boolean isJavadoc) { - // a new annotation comment is recorded - try { - commentStops[++commentPtr] = isJavadoc - ? currentPosition - : -currentPosition; - } catch (IndexOutOfBoundsException e) { - int oldStackLength = commentStops.length; - int[] oldStack = commentStops; - commentStops = new int[oldStackLength + 30]; - System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength); - commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition; - //grows the positions buffers too - int[] old = commentStarts; - commentStarts = new int[oldStackLength + 30]; - System.arraycopy(old, 0, commentStarts, 0, oldStackLength); - } - //the buffer is of a correct size here - commentStarts[commentPtr] = startPosition; - } + public void recordComment(int token) { + // compute position + int stopPosition = this.currentPosition; + switch (token) { + case TokenNameCOMMENT_LINE: + stopPosition = -this.lastCommentLinePosition; + break; + case TokenNameCOMMENT_BLOCK: + stopPosition = -this.currentPosition; + break; + } + + // a new comment is recorded + int length = this.commentStops.length; + if (++this.commentPtr >= length) { + System.arraycopy(this.commentStops, 0, this.commentStops = new int[length + 30], 0, length); + //grows the positions buffers too + System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[length + 30], 0, length); + } + this.commentStops[this.commentPtr] = stopPosition; + this.commentStarts[this.commentPtr] = this.startPosition; +} +// public final void recordComment(boolean isJavadoc) { +// // a new annotation comment is recorded +// try { +// commentStops[++commentPtr] = isJavadoc +// ? currentPosition +// : -currentPosition; +// } catch (IndexOutOfBoundsException e) { +// int oldStackLength = commentStops.length; +// int[] oldStack = commentStops; +// commentStops = new int[oldStackLength + 30]; +// System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength); +// commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition; +// //grows the positions buffers too +// int[] old = commentStarts; +// commentStarts = new int[oldStackLength + 30]; +// System.arraycopy(old, 0, commentStarts, 0, oldStackLength); +// } +// //the buffer is of a correct size here +// commentStarts[commentPtr] = startPosition; +// } public void resetTo(int begin, int end) { //reset the scanner to a given position where it may rescan again diet = false; @@ -2508,9 +2797,9 @@ public class Scanner implements IScanner, ITerminalSymbols { while (getNextCharAsJavaIdentifierPart()) { }; if (isVariable) { - if (new String(getCurrentTokenSource()).equals("$this")) { - return TokenNamethis; - } + // if (new String(getCurrentTokenSource()).equals("$this")) { + // return TokenNamethis; + // } return TokenNameVariable; } int index, length; @@ -2600,7 +2889,7 @@ public class Scanner implements IScanner, ITerminalSymbols { case 3 : //and if ((data[++index] == 'n') && (data[++index] == 'd')) { - return TokenNameAND; + return TokenNameand; } else { return TokenNameIdentifier; } @@ -2635,7 +2924,7 @@ public class Scanner implements IScanner, ITerminalSymbols { return TokenNameIdentifier; } case 'c' : - //case catch class const continue + //case catch class clone const continue switch (length) { case 4 : if ((data[++index] == 'a') && (data[++index] == 's') @@ -2647,10 +2936,16 @@ public class Scanner implements IScanner, ITerminalSymbols { if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h')) return TokenNamecatch; - if ((data[index] == 'l') && (data[++index] == 'a') + index = 0; + if ((data[++index] == 'l') && (data[++index] == 'a') && (data[++index] == 's') && (data[++index] == 's')) return TokenNameclass; - if ((data[index] == 'o') && (data[++index] == 'n') + index = 0; + if ((data[++index] == 'l') && (data[++index] == 'o') + && (data[++index] == 'n') && (data[++index] == 'e')) + return TokenNameclone; + index = 0; + if ((data[++index] == 'o') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't')) return TokenNameconst; else @@ -2793,9 +3088,9 @@ public class Scanner implements IScanner, ITerminalSymbols { else return TokenNameIdentifier; case 5 : -// if ((data[++index] == 'a') && (data[++index] == 'l') -// && (data[++index] == 's') && (data[++index] == 'e')) -// return TokenNamefalse; + // if ((data[++index] == 'a') && (data[++index] == 'l') + // && (data[++index] == 's') && (data[++index] == 'e')) + // return TokenNamefalse; if ((data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 'a') && (data[++index] == 'l')) return TokenNamefinal; @@ -2912,12 +3207,12 @@ public class Scanner implements IScanner, ITerminalSymbols { return TokenNamenew; else return TokenNameIdentifier; -// case 4 : -// if ((data[++index] == 'u') && (data[++index] == 'l') -// && (data[++index] == 'l')) -// return TokenNamenull; -// else -// return TokenNameIdentifier; + // case 4 : + // if ((data[++index] == 'u') && (data[++index] == 'l') + // && (data[++index] == 'l')) + // return TokenNamenull; + // else + // return TokenNameIdentifier; default : return TokenNameIdentifier; } @@ -2925,7 +3220,7 @@ public class Scanner implements IScanner, ITerminalSymbols { // or old_function if (length == 2) { if (data[++index] == 'r') { - return TokenNameOR; + return TokenNameor; } } // if (length == 12) { @@ -3029,12 +3324,12 @@ public class Scanner implements IScanner, ITerminalSymbols { return TokenNametry; else return TokenNameIdentifier; -// case 4 : -// if ((data[++index] == 'r') && (data[++index] == 'u') -// && (data[++index] == 'e')) -// return TokenNametrue; -// else -// return TokenNameIdentifier; + // case 4 : + // if ((data[++index] == 'r') && (data[++index] == 'u') + // && (data[++index] == 'e')) + // return TokenNametrue; + // else + // return TokenNameIdentifier; case 5 : if ((data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'w')) @@ -3095,7 +3390,7 @@ public class Scanner implements IScanner, ITerminalSymbols { switch (length) { case 3 : if ((data[++index] == 'o') && (data[++index] == 'r')) - return TokenNameXOR; + return TokenNamexor; else return TokenNameIdentifier; default : @@ -3287,6 +3582,7 @@ public class Scanner implements IScanner, ITerminalSymbols { initialPosition = currentPosition = 0; containsAssertKeyword = false; withoutUnicodeBuffer = new char[this.source.length]; + encapsedStringStack = new Stack(); } public String toString() { if (startPosition == source.length) @@ -3317,14 +3613,16 @@ public class Scanner implements IScanner, ITerminalSymbols { case TokenNameERROR : return "ScannerError"; // + new String(getCurrentTokenSource()) + ")"; // //$NON-NLS-1$ - case TokenNameStopPHP : - return "StopPHP(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ + case TokenNameINLINE_HTML : + return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameIdentifier : return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameVariable : return "Variable(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameabstract : return "abstract"; //$NON-NLS-1$ + case TokenNameand : + return "AND"; //$NON-NLS-1$ case TokenNamearray : return "array"; //$NON-NLS-1$ case TokenNameas : @@ -3335,6 +3633,14 @@ public class Scanner implements IScanner, ITerminalSymbols { return "case"; //$NON-NLS-1$ case TokenNameclass : return "class"; //$NON-NLS-1$ + case TokenNamecatch : + return "catch"; //$NON-NLS-1$ + case TokenNameclone : + //$NON-NLS-1$ + return "clone"; + case TokenNameconst : + //$NON-NLS-1$ + return "const"; case TokenNamecontinue : return "continue"; //$NON-NLS-1$ case TokenNamedefault : @@ -3359,10 +3665,12 @@ public class Scanner implements IScanner, ITerminalSymbols { return "endswitch"; //$NON-NLS-1$ case TokenNameendwhile : return "endwhile"; //$NON-NLS-1$ + case TokenNameexit: + return "exit"; case TokenNameextends : return "extends"; //$NON-NLS-1$ -// case TokenNamefalse : -// return "false"; //$NON-NLS-1$ + // case TokenNamefalse : + // return "false"; //$NON-NLS-1$ case TokenNamefinal : return "final"; //$NON-NLS-1$ case TokenNamefor : @@ -3381,14 +3689,20 @@ public class Scanner implements IScanner, ITerminalSymbols { return "include"; //$NON-NLS-1$ case TokenNameinclude_once : return "include_once"; //$NON-NLS-1$ + case TokenNameinstanceof : + return "instanceof"; //$NON-NLS-1$ case TokenNameinterface : return "interface"; //$NON-NLS-1$ + case TokenNameisset : + return "isset"; //$NON-NLS-1$ case TokenNamelist : return "list"; //$NON-NLS-1$ case TokenNamenew : return "new"; //$NON-NLS-1$ -// case TokenNamenull : -// return "null"; //$NON-NLS-1$ + // case TokenNamenull : + // return "null"; //$NON-NLS-1$ + case TokenNameor : + return "OR"; //$NON-NLS-1$ case TokenNameprint : return "print"; //$NON-NLS-1$ case TokenNameprivate : @@ -3407,27 +3721,37 @@ public class Scanner implements IScanner, ITerminalSymbols { return "static"; //$NON-NLS-1$ case TokenNameswitch : return "switch"; //$NON-NLS-1$ -// case TokenNametrue : -// return "true"; //$NON-NLS-1$ + // case TokenNametrue : + // return "true"; //$NON-NLS-1$ case TokenNameunset : return "unset"; //$NON-NLS-1$ case TokenNamevar : return "var"; //$NON-NLS-1$ case TokenNamewhile : return "while"; //$NON-NLS-1$ - case TokenNamethis : - return "$this"; //$NON-NLS-1$ + case TokenNamexor : + return "XOR"; //$NON-NLS-1$ + // case TokenNamethis : + // return "$this"; //$NON-NLS-1$ case TokenNameIntegerLiteral : return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameDoubleLiteral : return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ - case TokenNameStringLiteral : - return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ - case TokenNameStringConstant : + case TokenNameStringDoubleQuote : + return "StringLiteral(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ + case TokenNameStringSingleQuote : return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameStringInterpolated : return "StringInterpolated(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ + case TokenNameEncapsedString0 : + return "`"; //$NON-NLS-1$ + case TokenNameEncapsedString1 : + return "\'"; //$NON-NLS-1$ + case TokenNameEncapsedString2 : + return "\""; //$NON-NLS-1$ + case TokenNameSTRING : + return "STRING(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$ case TokenNameHEREDOC : return "HEREDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ case TokenNamePLUS_PLUS : @@ -3468,6 +3792,8 @@ public class Scanner implements IScanner, ITerminalSymbols { return "^="; //$NON-NLS-1$ case TokenNameREMAINDER_EQUAL : return "%="; //$NON-NLS-1$ + case TokenNameDOT_EQUAL : + return ".="; //$NON-NLS-1$ case TokenNameLEFT_SHIFT_EQUAL : return "<<="; //$NON-NLS-1$ case TokenNameRIGHT_SHIFT_EQUAL : @@ -3530,6 +3856,8 @@ public class Scanner implements IScanner, ITerminalSymbols { return "="; //$NON-NLS-1$ case TokenNameAT : return "@"; + case TokenNameDOLLAR : + return "$"; case TokenNameDOLLAR_LBRACE : return "${"; case TokenNameEOF : @@ -3542,8 +3870,9 @@ public class Scanner implements IScanner, ITerminalSymbols { return "COMMENT_BLOCK(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ case TokenNameCOMMENT_PHPDOC : return "COMMENT_PHPDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ - case TokenNameHTML : - return "HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ + // case TokenNameHTML : + // return "HTML(" + new String(getCurrentTokenSource()) + ")"; + // //$NON-NLS-1$ case TokenNameFILE : return "__FILE__"; //$NON-NLS-1$ case TokenNameLINE : @@ -3553,12 +3882,29 @@ public class Scanner implements IScanner, ITerminalSymbols { case TokenNameMETHOD_C : return "__METHOD__"; //$NON-NLS-1$ case TokenNameFUNC_C : - return "__FUNCTION__"; //$NON-NLS-1$ + return "__FUNCTION__"; //$NON-NLS-1 + case TokenNameboolCAST : + return "( bool )"; //$NON-NLS-1$ + case TokenNameintCAST : + return "( int )"; //$NON-NLS-1$ + case TokenNamedoubleCAST : + return "( double )"; //$NON-NLS-1$ + case TokenNameobjectCAST : + return "( object )"; //$NON-NLS-1$ + case TokenNamestringCAST : + return "( string )"; //$NON-NLS-1$ default : return "not-a-token(" + (new Integer(act)) + ") " + new String(getCurrentTokenSource()); //$NON-NLS-1$ } } + + public Scanner() { + this(false, false); + } + public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) { + this(tokenizeComments, tokenizeWhiteSpace, false); + } public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals) { this(tokenizeComments, tokenizeWhiteSpace, @@ -3566,11 +3912,23 @@ public class Scanner implements IScanner, ITerminalSymbols { } public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, boolean assertMode) { + this(tokenizeComments, tokenizeWhiteSpace, + checkNonExternalizedStringLiterals, assertMode, false, null, null); + } + public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, + boolean checkNonExternalizedStringLiterals, boolean assertMode, + boolean tokenizeStrings, + char[][] taskTags, + char[][] taskPriorities) { this.eofPosition = Integer.MAX_VALUE; this.tokenizeComments = tokenizeComments; this.tokenizeWhiteSpace = tokenizeWhiteSpace; + this.tokenizeStrings = tokenizeStrings; this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals; this.assertMode = assertMode; + this.encapsedStringStack = null; + this.taskTags = taskTags; + this.taskPriorities = taskPriorities; } private void checkNonExternalizeString() throws InvalidInputException { if (currentLine == null)