* permissible as the first character in a PHP identifier
*/
public static boolean isPHPIdentifierStart(char ch) {
- return Character.isLetter(ch) || (ch == '_');
+ return Character.isLetter(ch)
+ || (ch == '_')
+ || (0x7F <= ch && ch <= 0xFF);
}
/**
* other than the first character
*/
public static boolean isPHPIdentifierPart(char ch) {
- return Character.isLetterOrDigit(ch) || (ch == '_');
+ return Character.isLetterOrDigit(ch)
+ || (ch == '_')
+ || (0x7F <= ch && ch <= 0xFF);
}
public final boolean atEnd() {
}
public int getNextToken() throws InvalidInputException {
-
+ int htmlPosition = currentPosition;
try {
- int htmlPosition = currentPosition;
while (!phpMode) {
currentCharacter = source[currentPosition++];
if (currentCharacter == '<') {
}
} //-----------------end switch while try--------------------
catch (IndexOutOfBoundsException e) {
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ }
return TokenNameEOF;
}