import net.sourceforge.phpdt.core.compiler.IScanner;
import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
import net.sourceforge.phpdt.core.compiler.InvalidInputException;
+import net.sourceforge.phpdt.internal.compiler.env.ICompilationUnit;
import net.sourceforge.phpeclipse.internal.compiler.ast.StringLiteral;
public class Scanner implements IScanner, ITerminalSymbols {
charArray_v = new char[] { 'v' }, charArray_w = new char[] { 'w' }, charArray_x = new char[] { 'x' },
charArray_y = new char[] { 'y' }, charArray_z = new char[] { 'z' };
+ static final char[] charArray_va = new char[] { '$', 'a' }, charArray_vb = new char[] { '$', 'b' }, charArray_vc = new char[] {
+ '$',
+ 'c' }, charArray_vd = new char[] { '$', 'd' }, charArray_ve = new char[] { '$', 'e' },
+ charArray_vf = new char[] { '$', 'f' }, charArray_vg = new char[] { '$', 'g' }, charArray_vh = new char[] { '$', 'h' },
+ charArray_vi = new char[] { '$', 'i' }, charArray_vj = new char[] { '$', 'j' }, charArray_vk = new char[] { '$', 'k' },
+ charArray_vl = new char[] { '$', 'l' }, charArray_vm = new char[] { '$', 'm' }, charArray_vn = new char[] { '$', 'n' },
+ charArray_vo = new char[] { '$', 'o' }, charArray_vp = new char[] { '$', 'p' }, charArray_vq = new char[] { '$', 'q' },
+ charArray_vr = new char[] { '$', 'r' }, charArray_vs = new char[] { '$', 's' }, charArray_vt = new char[] { '$', 't' },
+ charArray_vu = new char[] { '$', 'u' }, charArray_vv = new char[] { '$', 'v' }, charArray_vw = new char[] { '$', 'w' },
+ charArray_vx = new char[] { '$', 'x' }, charArray_vy = new char[] { '$', 'y' }, charArray_vz = new char[] { '$', 'z' };
+
static final char[] initCharArray = new char[] { '\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000' };
static final int TableSize = 30, InternalTableSize = 6;
public char[][] taskPriorities = null;
+ public boolean isTaskCaseSensitive = true;
+
public static final boolean DEBUG = false;
public static final boolean TRACE = false;
+ public ICompilationUnit compilationUnit = null;
+
/**
- * Determines if the specified character is permissible as the first character in a PHP identifier
+ * Determines if the specified character is permissible as the first character in a PHP identifier or
+ * variable
+ *
+ * The '$' character for PHP variables is regarded as a correct first character !
+ *
+ */
+ public static boolean isPHPIdentOrVarStart(char ch) {
+ return Character.isLetter(ch) || (ch == '$') ||(ch == '_') || (0x7F <= ch && ch <= 0xFF);
+ }
+
+ /**
+ * Determines if the specified character is permissible as the first character in a PHP identifier.
+ *
+ * The '$' character for PHP variables isn't regarded as the first character !
*/
public static boolean isPHPIdentifierStart(char ch) {
return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
}
return result;
}
-
+
+ public final char[] getRawTokenSourceEnd() {
+ int length = this.eofPosition - this.currentPosition - 1;
+ char[] sourceEnd = new char[length];
+ System.arraycopy(this.source, this.currentPosition, sourceEnd, 0, length);
+ return sourceEnd;
+}
public int getCurrentTokenStartPosition() {
return this.startPosition;
}
public final char[] getCurrentStringLiteralSource() {
// Return the token REAL source (aka unicodes are precomputed)
+ if (startPosition + 1 >= currentPosition) {
+ return new char[0];
+ }
char[] result;
int length;
System.arraycopy(source, startPosition + 1, result = new char[length = currentPosition - startPosition - 2], 0, length);
return result;
}
+ public final char[] getCurrentStringLiteralSource(int startPos) {
+ // Return the token REAL source (aka unicodes are precomputed)
+ char[] result;
+ int length;
+ System.arraycopy(source, startPos + 1, result = new char[length = currentPosition - startPos - 2], 0, length);
+ // }
+ return result;
+ }
+
/*
* Search the source position corresponding to the end of a given line number
*
case '#':
case '/': {
char startChar = currentCharacter;
- if (getNextChar('=')) {
+ if (getNextChar('=') && startChar == '/') {
return TokenNameDIVIDE_EQUAL;
}
int test;
return TokenNameCOMMENT_PHPDOC;
return TokenNameCOMMENT_BLOCK;
}
+
+ if (this.taskTags != null) {
+ checkTaskTag(this.startPosition, this.currentPosition);
+ }
} catch (IndexOutOfBoundsException e) {
// reset end position for error reporting
currentPosition -= 2;
* @throws InvalidInputException
*/
private int getInlinedHTMLToken(int start) throws InvalidInputException {
- // int htmlPosition = start;
if (currentPosition > source.length) {
currentPosition = source.length;
return TokenNameEOF;
if (currentCharacter == '<') {
if (getNextChar('?')) {
currentCharacter = source[currentPosition++];
- if ((currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
+ if ((currentCharacter != 'P') && (currentCharacter != 'p')) {
+ currentPosition--;
+ // (currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
// <?
if (ignorePHPOneLiner) {
if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
return TokenNameINLINE_HTML;
}
} else {
- boolean phpStart = (currentCharacter == 'P') || (currentCharacter == 'p');
- if (phpStart) {
- int test = getNextChar('H', 'h');
+ // boolean phpStart = (currentCharacter == 'P') || (currentCharacter == 'p');
+ // if (phpStart) {
+ int test = getNextChar('H', 'h');
+ if (test >= 0) {
+ test = getNextChar('P', 'p');
if (test >= 0) {
- test = getNextChar('P', 'p');
- if (test >= 0) {
- // <?PHP <?php
- if (ignorePHPOneLiner) {
- if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
- phpMode = true;
- return TokenNameINLINE_HTML;
- }
- } else {
+ // <?PHP <?php
+ if (ignorePHPOneLiner) {
+ if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
phpMode = true;
return TokenNameINLINE_HTML;
}
+ } else {
+ phpMode = true;
+ return TokenNameINLINE_HTML;
}
}
}
+ // }
}
}
}
}
final char[] optimizedCurrentTokenSource2() {
- //try to return the same char[] build only once
char c0, c1;
- int hash = (((c0 = source[startPosition]) << 6) + (c1 = source[startPosition + 1])) % TableSize;
+ c0 = source[startPosition];
+ c1 = source[startPosition + 1];
+ if (c0 == '$') {
+ //return always the same char[] build only once
+ //optimization at no speed cost of 99.5 % of the singleCharIdentifier
+ switch (c1) {
+ case 'a':
+ return charArray_va;
+ case 'b':
+ return charArray_vb;
+ case 'c':
+ return charArray_vc;
+ case 'd':
+ return charArray_vd;
+ case 'e':
+ return charArray_ve;
+ case 'f':
+ return charArray_vf;
+ case 'g':
+ return charArray_vg;
+ case 'h':
+ return charArray_vh;
+ case 'i':
+ return charArray_vi;
+ case 'j':
+ return charArray_vj;
+ case 'k':
+ return charArray_vk;
+ case 'l':
+ return charArray_vl;
+ case 'm':
+ return charArray_vm;
+ case 'n':
+ return charArray_vn;
+ case 'o':
+ return charArray_vo;
+ case 'p':
+ return charArray_vp;
+ case 'q':
+ return charArray_vq;
+ case 'r':
+ return charArray_vr;
+ case 's':
+ return charArray_vs;
+ case 't':
+ return charArray_vt;
+ case 'u':
+ return charArray_vu;
+ case 'v':
+ return charArray_vv;
+ case 'w':
+ return charArray_vw;
+ case 'x':
+ return charArray_vx;
+ case 'y':
+ return charArray_vy;
+ case 'z':
+ return charArray_vz;
+ }
+ }
+ //try to return the same char[] build only once
+ int hash = ((c0 << 6) + c1) % TableSize;
char[][] table = charArray_length[0][hash];
int i = newEntry2;
while (++i < InternalTableSize) {
if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'd') && (data[++index] == 'e')
&& (data[++index] == 'c') && (data[++index] == 'l') && (data[++index] == 'a') && (data[++index] == 'r')
&& (data[++index] == 'e'))
- return TokenNameendforeach;
+ return TokenNameenddeclare;
index = 0;
if ((data[++index] == 'n') // endforeach
&& (data[++index] == 'd') && (data[++index] == 'f') && (data[++index] == 'o') && (data[++index] == 'r')
}
public final void setSource(char[] source) {
+ setSource(null, source);
+ }
+
+ public final void setSource(ICompilationUnit compilationUnit, char[] source) {
//the source-buffer is set to sourceString
+ this.compilationUnit = compilationUnit;
if (source == null) {
this.source = new char[0];
} else {
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals,
boolean assertMode) {
- this(tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, assertMode, false, null, null);
+ this(tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, assertMode, false, null, null, true);
}
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals,
- boolean assertMode, boolean tokenizeStrings, char[][] taskTags, char[][] taskPriorities) {
+ boolean assertMode, boolean tokenizeStrings, char[][] taskTags, char[][] taskPriorities, boolean isTaskCaseSensitive) {
this.eofPosition = Integer.MAX_VALUE;
this.tokenizeComments = tokenizeComments;
this.tokenizeWhiteSpace = tokenizeWhiteSpace;
}
}
- // chech presence of task: tags
+ //chech presence of task: tags
+ //TODO (frederic) see if we need to take unicode characters into account...
public void checkTaskTag(int commentStart, int commentEnd) {
+ char[] src = this.source;
+
// only look for newer task: tags
if (this.foundTaskCount > 0 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
return;
}
int foundTaskIndex = this.foundTaskCount;
- nextChar: for (int i = commentStart; i < commentEnd && i < this.eofPosition; i++) {
+ char previous = src[commentStart + 1]; // should be '*' or '/'
+ nextChar: for (int i = commentStart + 2; i < commentEnd && i < this.eofPosition; i++) {
char[] tag = null;
char[] priority = null;
- // check for tag occurrence
- nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
- tag = this.taskTags[itag];
- priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
- int tagLength = tag.length;
- for (int t = 0; t < tagLength; t++) {
- if (this.source[i + t] != tag[t])
+ // check for tag occurrence only if not ambiguous with javadoc tag
+ if (previous != '@') {
+ nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
+ tag = this.taskTags[itag];
+ int tagLength = tag.length;
+ if (tagLength == 0)
continue nextTag;
+
+ // ensure tag is not leaded with letter if tag starts with a letter
+ if (Scanner.isPHPIdentifierStart(tag[0])) {
+ if (Scanner.isPHPIdentifierPart(previous)) {
+ continue nextTag;
+ }
+ }
+
+ for (int t = 0; t < tagLength; t++) {
+ char sc, tc;
+ int x = i + t;
+ if (x >= this.eofPosition || x >= commentEnd)
+ continue nextTag;
+ if ((sc = src[i + t]) != (tc = tag[t])) { // case sensitive check
+ if (this.isTaskCaseSensitive || (Character.toLowerCase(sc) != Character.toLowerCase(tc))) { // case insensitive check
+ continue nextTag;
+ }
+ }
+ }
+ // ensure tag is not followed with letter if tag finishes with a letter
+ if (i + tagLength < commentEnd && Scanner.isPHPIdentifierPart(src[i + tagLength - 1])) {
+ if (Scanner.isPHPIdentifierPart(src[i + tagLength]))
+ continue nextTag;
+ }
+ if (this.foundTaskTags == null) {
+ this.foundTaskTags = new char[5][];
+ this.foundTaskMessages = new char[5][];
+ this.foundTaskPriorities = new char[5][];
+ this.foundTaskPositions = new int[5][];
+ } else if (this.foundTaskCount == this.foundTaskTags.length) {
+ System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0,
+ this.foundTaskCount);
+ System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
+ this.foundTaskCount);
+ System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0,
+ this.foundTaskCount);
+ System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
+ this.foundTaskCount);
+ }
+
+ priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
+
+ this.foundTaskTags[this.foundTaskCount] = tag;
+ this.foundTaskPriorities[this.foundTaskCount] = priority;
+ this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
+ this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
+ this.foundTaskCount++;
+ i += tagLength - 1; // will be incremented when looping
+ break nextTag;
}
- if (this.foundTaskTags == null) {
- this.foundTaskTags = new char[5][];
- this.foundTaskMessages = new char[5][];
- this.foundTaskPriorities = new char[5][];
- this.foundTaskPositions = new int[5][];
- } else if (this.foundTaskCount == this.foundTaskTags.length) {
- System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
- System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
- this.foundTaskCount);
- System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0,
- this.foundTaskCount);
- System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
- this.foundTaskCount);
- }
- this.foundTaskTags[this.foundTaskCount] = tag;
- this.foundTaskPriorities[this.foundTaskCount] = priority;
- this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
- this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
- this.foundTaskCount++;
- i += tagLength - 1; // will be incremented when looping
}
+ previous = src[i];
}
for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
// retrieve message start and end positions
int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1 : commentEnd - 1;
// at most beginning of next task
- if (max_value < msgStart)
+ if (max_value < msgStart) {
max_value = msgStart; // would only occur if tag is before EOF.
+ }
int end = -1;
char c;
for (int j = msgStart; j < max_value; j++) {
- if ((c = this.source[j]) == '\n' || c == '\r') {
+ if ((c = src[j]) == '\n' || c == '\r') {
end = j - 1;
break;
}
}
if (end == -1) {
for (int j = max_value; j > msgStart; j--) {
- if ((c = this.source[j]) == '*') {
+ if ((c = src[j]) == '*') {
end = j - 1;
break;
}
if (msgStart == end)
continue; // empty
// trim the message
- while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
+ while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
end--;
- while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
+ while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
msgStart++;
// update the end position of the task
this.foundTaskPositions[i][1] = end;
// get the message source
final int messageLength = end - msgStart + 1;
char[] message = new char[messageLength];
- System.arraycopy(source, msgStart, message, 0, messageLength);
+ System.arraycopy(src, msgStart, message, 0, messageLength);
this.foundTaskMessages[i] = message;
}
}
+
+ // chech presence of task: tags
+ // public void checkTaskTag(int commentStart, int commentEnd) {
+ // // only look for newer task: tags
+ // if (this.foundTaskCount > 0 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
+ // return;
+ // }
+ // int foundTaskIndex = this.foundTaskCount;
+ // nextChar: for (int i = commentStart; i < commentEnd && i < this.eofPosition; i++) {
+ // char[] tag = null;
+ // char[] priority = null;
+ // // check for tag occurrence
+ // nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
+ // tag = this.taskTags[itag];
+ // priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
+ // int tagLength = tag.length;
+ // for (int t = 0; t < tagLength; t++) {
+ // if (this.source[i + t] != tag[t])
+ // continue nextTag;
+ // }
+ // if (this.foundTaskTags == null) {
+ // this.foundTaskTags = new char[5][];
+ // this.foundTaskMessages = new char[5][];
+ // this.foundTaskPriorities = new char[5][];
+ // this.foundTaskPositions = new int[5][];
+ // } else if (this.foundTaskCount == this.foundTaskTags.length) {
+ // System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
+ // System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
+ // this.foundTaskCount);
+ // System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities = new char[this.foundTaskCount * 2][], 0,
+ // this.foundTaskCount);
+ // System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
+ // this.foundTaskCount);
+ // }
+ // this.foundTaskTags[this.foundTaskCount] = tag;
+ // this.foundTaskPriorities[this.foundTaskCount] = priority;
+ // this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
+ // this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
+ // this.foundTaskCount++;
+ // i += tagLength - 1; // will be incremented when looping
+ // }
+ // }
+ // for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
+ // // retrieve message start and end positions
+ // int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
+ // int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1 : commentEnd - 1;
+ // // at most beginning of next task
+ // if (max_value < msgStart)
+ // max_value = msgStart; // would only occur if tag is before EOF.
+ // int end = -1;
+ // char c;
+ // for (int j = msgStart; j < max_value; j++) {
+ // if ((c = this.source[j]) == '\n' || c == '\r') {
+ // end = j - 1;
+ // break;
+ // }
+ // }
+ // if (end == -1) {
+ // for (int j = max_value; j > msgStart; j--) {
+ // if ((c = this.source[j]) == '*') {
+ // end = j - 1;
+ // break;
+ // }
+ // }
+ // if (end == -1)
+ // end = max_value;
+ // }
+ // if (msgStart == end)
+ // continue; // empty
+ // // trim the message
+ // while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
+ // end--;
+ // while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
+ // msgStart++;
+ // // update the end position of the task
+ // this.foundTaskPositions[i][1] = end;
+ // // get the message source
+ // final int messageLength = end - msgStart + 1;
+ // char[] message = new char[messageLength];
+ // System.arraycopy(source, msgStart, message, 0, messageLength);
+ // this.foundTaskMessages[i] = message;
+ // }
+ // }
}
\ No newline at end of file
git://git.phpeclipse.com / phpeclipse.git / blobdiff