* permissible as the first character in a PHP identifier
*/
public static boolean isPHPIdentifierStart(char ch) {
- return Character.isLetter(ch) || (ch == '_');
+ return Character.isLetter(ch)
+ || (ch == '_')
+ || (0x7F <= ch && ch <= 0xFF);
}
/**
* other than the first character
*/
public static boolean isPHPIdentifierPart(char ch) {
- return Character.isLetterOrDigit(ch) || (ch == '_');
+ return Character.isLetterOrDigit(ch)
+ || (ch == '_')
+ || (0x7F <= ch && ch <= 0xFF);
}
public final boolean atEnd() {
}
public int getNextToken() throws InvalidInputException {
+ int htmlPosition = currentPosition;
try {
while (!phpMode) {
- startPosition = currentPosition;
currentCharacter = source[currentPosition++];
if (currentCharacter == '<') {
if (getNextChar('?')) {
// <?
startPosition = currentPosition;
phpMode = true;
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ return TokenNameHTML;
+ }
} else {
boolean phpStart =
(currentCharacter == 'P') || (currentCharacter == 'p');
// <?PHP <?php
startPosition = currentPosition;
phpMode = true;
+
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ return TokenNameHTML;
+ }
}
}
}
}
} //-----------------end switch while try--------------------
catch (IndexOutOfBoundsException e) {
+ if (tokenizeWhiteSpace) {
+ // && (whiteStart != currentPosition - 1)) {
+ // reposition scanner in case we are interested by spaces as tokens
+ startPosition = htmlPosition;
+ }
return TokenNameEOF;
}
default :
if (currentCharacter == '$') {
- while ( (currentCharacter = source[currentPosition++])=='$') {
+ while ((currentCharacter = source[currentPosition++]) == '$') {
}
if (currentCharacter == '{')
return TokenNameDOLLAR_LBRACE;
return new char[] { charOne };
}
}
+
final char[] optimizedCurrentTokenSource2() {
//try to return the same char[] build only once
newEntry2 = max;
return r;
}
+
final char[] optimizedCurrentTokenSource3() {
//try to return the same char[] build only once
newEntry3 = max;
return r;
}
+
final char[] optimizedCurrentTokenSource4() {
//try to return the same char[] build only once
return r;
}
+
final char[] optimizedCurrentTokenSource5() {
//try to return the same char[] build only once
return r;
}
+
final char[] optimizedCurrentTokenSource6() {
//try to return the same char[] build only once
newEntry6 = max;
return r;
}
+
public final void pushLineSeparator() throws InvalidInputException {
//see comment on isLineDelimiter(char) for the use of '\n' and '\r'
final int INCREMENT = 250;
//disptach on the second char :-)...cool....but fast !
useAssertAsAnIndentifier = false;
-
+
while (getNextCharAsJavaIdentifierPart()) {
};