X-Git-Url: http://git.phpeclipse.com diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java index 10a8ea8..af7f65a 100644 --- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java +++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java @@ -13,11 +13,14 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Stack; + import net.sourceforge.phpdt.core.compiler.CharOperation; import net.sourceforge.phpdt.core.compiler.IScanner; import net.sourceforge.phpdt.core.compiler.ITerminalSymbols; import net.sourceforge.phpdt.core.compiler.InvalidInputException; -import net.sourceforge.phpdt.internal.compiler.ast.StringLiteral; +import net.sourceforge.phpeclipse.internal.compiler.ast.StringLiteral; + + public class Scanner implements IScanner, ITerminalSymbols { /* * APIs ares - getNextToken() which return the current type of the token @@ -52,11 +55,11 @@ public class Scanner implements IScanner, ITerminalSymbols { //when == 0 ==> no unicode in the current token public boolean unicodeAsBackSlash = false; public boolean scanningFloatLiteral = false; - //support for /** comments - //public char[][] comments = new char[10][]; - public int[] commentStops = new int[10]; - public int[] commentStarts = new int[10]; - public int commentPtr = -1; // no comment test with commentPtr value -1 +//support for /** comments + public int[] commentStops = new int[10]; + public int[] commentStarts = new int[10]; + public int commentPtr = -1; // no comment test with commentPtr value -1 + protected int lastCommentLinePosition = -1; //diet parsing support - jump over some method body when requested public boolean diet = false; //support for the poor-line-debuggers .... @@ -135,12 +138,7 @@ public class Scanner implements IScanner, ITerminalSymbols { public char[][] taskPriorities = null; public static final boolean DEBUG = false; public static final boolean TRACE = false; - public Scanner() { - this(false, false); - } - public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) { - this(tokenizeComments, tokenizeWhiteSpace, false); - } + /** * Determines if the specified character is permissible as the first * character in a PHP identifier @@ -1487,6 +1485,7 @@ public class Scanner implements IScanner, ITerminalSymbols { int test; if ((startChar == '#') || (test = getNextChar('/', '*')) == 0) { //line comment + this.lastCommentLinePosition = this.currentPosition; int endPositionForLineComment = 0; try { //get the next char currentCharacter = source[currentPosition++]; @@ -1529,6 +1528,7 @@ public class Scanner implements IScanner, ITerminalSymbols { // } //jump over the \\ boolean isUnicode = false; while (currentCharacter != '\r' && currentCharacter != '\n') { + this.lastCommentLinePosition = this.currentPosition; if (currentCharacter == '?') { if (getNextChar('>')) { startPosition = currentPosition - 2; @@ -1586,8 +1586,10 @@ public class Scanner implements IScanner, ITerminalSymbols { } else { endPositionForLineComment = currentPosition - 1; } - recordComment(false); - if ((currentCharacter == '\r') +// recordComment(false); + recordComment(TokenNameCOMMENT_LINE); + if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition); + if ((currentCharacter == '\r') || (currentCharacter == '\n')) { checkNonExternalizeString(); if (recordLineSeparator) { @@ -1689,7 +1691,13 @@ public class Scanner implements IScanner, ITerminalSymbols { // currentPosition++; // } //jump over the \\ } - recordComment(isJavadoc); + //recordComment(isJavadoc); + if (isJavadoc) { + recordComment(TokenNameCOMMENT_PHPDOC); + } else { + recordComment(TokenNameCOMMENT_BLOCK); + } + if (tokenizeComments) { if (isJavadoc) return TokenNameCOMMENT_PHPDOC; @@ -1737,11 +1745,44 @@ public class Scanner implements IScanner, ITerminalSymbols { } return TokenNameEOF; } + + private int getInlinedHTML(int start) throws InvalidInputException { + int token = getInlinedHTMLToken(start); + if (token == TokenNameINLINE_HTML) { +// Stack stack = new Stack(); +// // scan html for errors +// Source inlinedHTMLSource = new Source(new String(source, startPosition, currentPosition - startPosition)); +// int lastPHPEndPos=0; +// for (Iterator i=inlinedHTMLSource.getNextTagIterator(0); i.hasNext();) { +// Tag tag=(Tag)i.next(); +// +// if (tag instanceof StartTag) { +// StartTag startTag=(StartTag)tag; +// // System.out.println("startTag: "+tag); +// if (startTag.isServerTag()) { +// // TODO : what to do with a server tag ? +// } else { +// // do whatever with HTML start tag +// // use startTag.getElement() to find the element corresponding +// // to this start tag which may be useful if you implement code +// // folding etc +// stack.push(startTag); +// } +// } else { +// EndTag endTag=(EndTag)tag; +// StartTag stag = (StartTag) stack.peek(); +//// System.out.println("endTag: "+tag); +// // do whatever with HTML end tag. +// } +// } + } + return token; + } /** * @return @throws * InvalidInputException */ - private int getInlinedHTML(int start) throws InvalidInputException { + private int getInlinedHTMLToken(int start) throws InvalidInputException { // int htmlPosition = start; if (currentPosition > source.length) { currentPosition = source.length; @@ -2562,26 +2603,48 @@ public class Scanner implements IScanner, ITerminalSymbols { } } } - public final void recordComment(boolean isJavadoc) { - // a new annotation comment is recorded - try { - commentStops[++commentPtr] = isJavadoc - ? currentPosition - : -currentPosition; - } catch (IndexOutOfBoundsException e) { - int oldStackLength = commentStops.length; - int[] oldStack = commentStops; - commentStops = new int[oldStackLength + 30]; - System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength); - commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition; - //grows the positions buffers too - int[] old = commentStarts; - commentStarts = new int[oldStackLength + 30]; - System.arraycopy(old, 0, commentStarts, 0, oldStackLength); - } - //the buffer is of a correct size here - commentStarts[commentPtr] = startPosition; - } + public void recordComment(int token) { + // compute position + int stopPosition = this.currentPosition; + switch (token) { + case TokenNameCOMMENT_LINE: + stopPosition = -this.lastCommentLinePosition; + break; + case TokenNameCOMMENT_BLOCK: + stopPosition = -this.currentPosition; + break; + } + + // a new comment is recorded + int length = this.commentStops.length; + if (++this.commentPtr >= length) { + System.arraycopy(this.commentStops, 0, this.commentStops = new int[length + 30], 0, length); + //grows the positions buffers too + System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[length + 30], 0, length); + } + this.commentStops[this.commentPtr] = stopPosition; + this.commentStarts[this.commentPtr] = this.startPosition; +} +// public final void recordComment(boolean isJavadoc) { +// // a new annotation comment is recorded +// try { +// commentStops[++commentPtr] = isJavadoc +// ? currentPosition +// : -currentPosition; +// } catch (IndexOutOfBoundsException e) { +// int oldStackLength = commentStops.length; +// int[] oldStack = commentStops; +// commentStops = new int[oldStackLength + 30]; +// System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength); +// commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition; +// //grows the positions buffers too +// int[] old = commentStarts; +// commentStarts = new int[oldStackLength + 30]; +// System.arraycopy(old, 0, commentStarts, 0, oldStackLength); +// } +// //the buffer is of a correct size here +// commentStarts[commentPtr] = startPosition; +// } public void resetTo(int begin, int end) { //reset the scanner to a given position where it may rescan again diet = false; @@ -3824,6 +3887,13 @@ public class Scanner implements IScanner, ITerminalSymbols { + new String(getCurrentTokenSource()); //$NON-NLS-1$ } } + + public Scanner() { + this(false, false); + } + public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) { + this(tokenizeComments, tokenizeWhiteSpace, false); + } public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals) { this(tokenizeComments, tokenizeWhiteSpace, @@ -3832,11 +3902,13 @@ public class Scanner implements IScanner, ITerminalSymbols { public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, boolean assertMode) { this(tokenizeComments, tokenizeWhiteSpace, - checkNonExternalizedStringLiterals, assertMode, false); + checkNonExternalizedStringLiterals, assertMode, false, null, null); } public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals, boolean assertMode, - boolean tokenizeStrings) { + boolean tokenizeStrings, + char[][] taskTags, + char[][] taskPriorities) { this.eofPosition = Integer.MAX_VALUE; this.tokenizeComments = tokenizeComments; this.tokenizeWhiteSpace = tokenizeWhiteSpace; @@ -3844,6 +3916,8 @@ public class Scanner implements IScanner, ITerminalSymbols { this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals; this.assertMode = assertMode; this.encapsedStringStack = null; + this.taskTags = taskTags; + this.taskPriorities = taskPriorities; } private void checkNonExternalizeString() throws InvalidInputException { if (currentLine == null)