import java.util.Iterator;
import java.util.List;
import java.util.Stack;
+
import net.sourceforge.phpdt.core.compiler.CharOperation;
import net.sourceforge.phpdt.core.compiler.IScanner;
import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
import net.sourceforge.phpdt.core.compiler.InvalidInputException;
-import net.sourceforge.phpdt.internal.compiler.ast.StringLiteral;
+import net.sourceforge.phpeclipse.internal.compiler.ast.StringLiteral;
+
+
public class Scanner implements IScanner, ITerminalSymbols {
/*
* APIs ares - getNextToken() which return the current type of the token
//when == 0 ==> no unicode in the current token
public boolean unicodeAsBackSlash = false;
public boolean scanningFloatLiteral = false;
- //support for /** comments
- //public char[][] comments = new char[10][];
- public int[] commentStops = new int[10];
- public int[] commentStarts = new int[10];
- public int commentPtr = -1; // no comment test with commentPtr value -1
+//support for /** comments
+ public int[] commentStops = new int[10];
+ public int[] commentStarts = new int[10];
+ public int commentPtr = -1; // no comment test with commentPtr value -1
+ protected int lastCommentLinePosition = -1;
//diet parsing support - jump over some method body when requested
public boolean diet = false;
//support for the poor-line-debuggers ....
public char[][] taskPriorities = null;
public static final boolean DEBUG = false;
public static final boolean TRACE = false;
- public Scanner() {
- this(false, false);
- }
- public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
- this(tokenizeComments, tokenizeWhiteSpace, false);
- }
+
/**
* Determines if the specified character is permissible as the first
* character in a PHP identifier
case '\'' :
if (tokenizeStrings) {
consumeStringConstant();
- return TokenNameStringConstant;
+ return TokenNameStringSingleQuote;
}
return TokenNameEncapsedString1;
case '"' :
return TokenNameAT;
case '\'' :
consumeStringConstant();
- return TokenNameStringConstant;
+ return TokenNameStringSingleQuote;
case '"' :
if (tokenizeStrings) {
consumeStringLiteral();
- return TokenNameStringLiteral;
+ return TokenNameStringDoubleQuote;
}
return TokenNameEncapsedString2;
case '`' :
int test;
if ((startChar == '#') || (test = getNextChar('/', '*')) == 0) {
//line comment
+ this.lastCommentLinePosition = this.currentPosition;
int endPositionForLineComment = 0;
try { //get the next char
currentCharacter = source[currentPosition++];
// } //jump over the \\
boolean isUnicode = false;
while (currentCharacter != '\r' && currentCharacter != '\n') {
+ this.lastCommentLinePosition = this.currentPosition;
if (currentCharacter == '?') {
if (getNextChar('>')) {
startPosition = currentPosition - 2;
} else {
endPositionForLineComment = currentPosition - 1;
}
- recordComment(false);
- if ((currentCharacter == '\r')
+// recordComment(false);
+ recordComment(TokenNameCOMMENT_LINE);
+ if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
+ if ((currentCharacter == '\r')
|| (currentCharacter == '\n')) {
checkNonExternalizeString();
if (recordLineSeparator) {
// currentPosition++;
// } //jump over the \\
}
- recordComment(isJavadoc);
+ //recordComment(isJavadoc);
+ if (isJavadoc) {
+ recordComment(TokenNameCOMMENT_PHPDOC);
+ } else {
+ recordComment(TokenNameCOMMENT_BLOCK);
+ }
+
if (tokenizeComments) {
if (isJavadoc)
return TokenNameCOMMENT_PHPDOC;
}
return TokenNameEOF;
}
+
+ private int getInlinedHTML(int start) throws InvalidInputException {
+ int token = getInlinedHTMLToken(start);
+ if (token == TokenNameINLINE_HTML) {
+// Stack stack = new Stack();
+// // scan html for errors
+// Source inlinedHTMLSource = new Source(new String(source, startPosition, currentPosition - startPosition));
+// int lastPHPEndPos=0;
+// for (Iterator i=inlinedHTMLSource.getNextTagIterator(0); i.hasNext();) {
+// Tag tag=(Tag)i.next();
+//
+// if (tag instanceof StartTag) {
+// StartTag startTag=(StartTag)tag;
+// // System.out.println("startTag: "+tag);
+// if (startTag.isServerTag()) {
+// // TODO : what to do with a server tag ?
+// } else {
+// // do whatever with HTML start tag
+// // use startTag.getElement() to find the element corresponding
+// // to this start tag which may be useful if you implement code
+// // folding etc
+// stack.push(startTag);
+// }
+// } else {
+// EndTag endTag=(EndTag)tag;
+// StartTag stag = (StartTag) stack.peek();
+//// System.out.println("endTag: "+tag);
+// // do whatever with HTML end tag.
+// }
+// }
+ }
+ return token;
+ }
/**
* @return @throws
* InvalidInputException
*/
- private int getInlinedHTML(int start) throws InvalidInputException {
+ private int getInlinedHTMLToken(int start) throws InvalidInputException {
// int htmlPosition = start;
if (currentPosition > source.length) {
currentPosition = source.length;
}
}
}
- public final void recordComment(boolean isJavadoc) {
- // a new annotation comment is recorded
- try {
- commentStops[++commentPtr] = isJavadoc
- ? currentPosition
- : -currentPosition;
- } catch (IndexOutOfBoundsException e) {
- int oldStackLength = commentStops.length;
- int[] oldStack = commentStops;
- commentStops = new int[oldStackLength + 30];
- System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength);
- commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition;
- //grows the positions buffers too
- int[] old = commentStarts;
- commentStarts = new int[oldStackLength + 30];
- System.arraycopy(old, 0, commentStarts, 0, oldStackLength);
- }
- //the buffer is of a correct size here
- commentStarts[commentPtr] = startPosition;
- }
+ public void recordComment(int token) {
+ // compute position
+ int stopPosition = this.currentPosition;
+ switch (token) {
+ case TokenNameCOMMENT_LINE:
+ stopPosition = -this.lastCommentLinePosition;
+ break;
+ case TokenNameCOMMENT_BLOCK:
+ stopPosition = -this.currentPosition;
+ break;
+ }
+
+ // a new comment is recorded
+ int length = this.commentStops.length;
+ if (++this.commentPtr >= length) {
+ System.arraycopy(this.commentStops, 0, this.commentStops = new int[length + 30], 0, length);
+ //grows the positions buffers too
+ System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[length + 30], 0, length);
+ }
+ this.commentStops[this.commentPtr] = stopPosition;
+ this.commentStarts[this.commentPtr] = this.startPosition;
+}
+// public final void recordComment(boolean isJavadoc) {
+// // a new annotation comment is recorded
+// try {
+// commentStops[++commentPtr] = isJavadoc
+// ? currentPosition
+// : -currentPosition;
+// } catch (IndexOutOfBoundsException e) {
+// int oldStackLength = commentStops.length;
+// int[] oldStack = commentStops;
+// commentStops = new int[oldStackLength + 30];
+// System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength);
+// commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition;
+// //grows the positions buffers too
+// int[] old = commentStarts;
+// commentStarts = new int[oldStackLength + 30];
+// System.arraycopy(old, 0, commentStarts, 0, oldStackLength);
+// }
+// //the buffer is of a correct size here
+// commentStarts[commentPtr] = startPosition;
+// }
public void resetTo(int begin, int end) {
//reset the scanner to a given position where it may rescan again
diet = false;
return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameDoubleLiteral :
return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
- case TokenNameStringLiteral :
+ case TokenNameStringDoubleQuote :
return "StringLiteral(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
- case TokenNameStringConstant :
+ case TokenNameStringSingleQuote :
return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
case TokenNameStringInterpolated :
return "StringInterpolated(" + new String(getCurrentTokenSource())
+ new String(getCurrentTokenSource()); //$NON-NLS-1$
}
}
+
+ public Scanner() {
+ this(false, false);
+ }
+ public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
+ this(tokenizeComments, tokenizeWhiteSpace, false);
+ }
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
boolean checkNonExternalizedStringLiterals) {
this(tokenizeComments, tokenizeWhiteSpace,
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
boolean checkNonExternalizedStringLiterals, boolean assertMode) {
this(tokenizeComments, tokenizeWhiteSpace,
- checkNonExternalizedStringLiterals, assertMode, false);
+ checkNonExternalizedStringLiterals, assertMode, false, null, null);
}
public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
boolean checkNonExternalizedStringLiterals, boolean assertMode,
- boolean tokenizeStrings) {
+ boolean tokenizeStrings,
+ char[][] taskTags,
+ char[][] taskPriorities) {
this.eofPosition = Integer.MAX_VALUE;
this.tokenizeComments = tokenizeComments;
this.tokenizeWhiteSpace = tokenizeWhiteSpace;
this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
this.assertMode = assertMode;
this.encapsedStringStack = null;
+ this.taskTags = taskTags;
+ this.taskPriorities = taskPriorities;
}
private void checkNonExternalizeString() throws InvalidInputException {
if (currentLine == null)
git://git.phpeclipse.com / phpeclipse.git / blobdiff