From fb516ef5ba630982a1fe5600fc377070c7f57242 Mon Sep 17 00:00:00 2001 From: musk Date: Sat, 12 Apr 2003 16:12:30 +0000 Subject: [PATCH] Fixed partitioning bug causing an assertion exception and failing to build phppart when two partitions of the same type follow each other with no partition in between e.g. --- .../phpeditor/php/PHPPartitionScanner.java | 718 ++++++++++++-------- 1 files changed, 417 insertions(+), 301 deletions(-) diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpeclipse/phpeditor/php/PHPPartitionScanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpeclipse/phpeditor/php/PHPPartitionScanner.java index 54eaa7d..d1a64ad 100644 --- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpeclipse/phpeditor/php/PHPPartitionScanner.java +++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpeclipse/phpeditor/php/PHPPartitionScanner.java @@ -6,7 +6,7 @@ * Created on 05.03.2003 * * @author Stefan Langer (musk) - * @version $Revision: 1.15 $ + * @version $Revision: 1.16 $ */ package net.sourceforge.phpeclipse.phpeditor.php; @@ -18,360 +18,476 @@ import org.eclipse.jface.text.rules.*; /** * */ -public class PHPPartitionScanner implements IPartitionTokenScanner { - private static final boolean DEBUG = false; - private IDocument fDocument = null; - private int fOffset = -1; - private String fContentType = IPHPPartitionScannerConstants.HTML; - private String fPrevContentType; - - private boolean partitionBorder = false; - private int fTokenOffset; - private int fEnd = -1; - private int fLength; - private Map tokens = new HashMap(); - - public PHPPartitionScanner() { - this.tokens.put( - IPHPPartitionScannerConstants.PHP, - new Token(IPHPPartitionScannerConstants.PHP)); - this.tokens.put( - IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT, - new Token(IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT)); - this.tokens.put( - IPHPPartitionScannerConstants.HTML, - new Token(IPHPPartitionScannerConstants.HTML)); - this.tokens.put( - IPHPPartitionScannerConstants.HTML_MULTILINE_COMMENT, - new Token(IPHPPartitionScannerConstants.HTML_MULTILINE_COMMENT)); - this.tokens.put( - IDocument.DEFAULT_CONTENT_TYPE, - new Token(IDocument.DEFAULT_CONTENT_TYPE)); - } - - private IToken getToken(String type) { - fLength = fOffset - fTokenOffset; - IToken token = (IToken) this.tokens.get(type); - Assert.isNotNull(token, "Token for type \"" + type + "\" not found!"); - if (DEBUG) { - System.out.println( - "Partition: fTokenOffset=" - + fTokenOffset - + " fContentType=" - + fContentType - + " fLength=" - + fLength); +public class PHPPartitionScanner implements IPartitionTokenScanner +{ + private static final boolean DEBUG = true; + private boolean fInString = false; + private boolean fInDoubString = false; + private IDocument fDocument = null; + private int fOffset = -1; + private String fContentType = IPHPPartitionScannerConstants.HTML; + private String fPrevContentType = IPHPPartitionScannerConstants.HTML; + + private boolean partitionBorder = false; + private int fTokenOffset; + private int fEnd = -1; + private int fLength; + private int fCurrentLength; + private Map tokens = new HashMap(); + + public PHPPartitionScanner() + { + this.tokens.put( + IPHPPartitionScannerConstants.PHP, + new Token(IPHPPartitionScannerConstants.PHP)); + this.tokens.put( + IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT, + new Token(IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT)); + this.tokens.put( + IPHPPartitionScannerConstants.HTML, + new Token(IPHPPartitionScannerConstants.HTML)); + this.tokens.put( + IPHPPartitionScannerConstants.HTML_MULTILINE_COMMENT, + new Token(IPHPPartitionScannerConstants.HTML_MULTILINE_COMMENT)); + this.tokens.put( + IDocument.DEFAULT_CONTENT_TYPE, + new Token(IDocument.DEFAULT_CONTENT_TYPE)); } - return token; - } -/* (non-Javadoc) - * @see org.eclipse.jface.text.rules.IPartitionTokenScanner#setPartialRange(org.eclipse.jface.text.IDocument, int, int, java.lang.String, int) - */ -public void setPartialRange( - IDocument document, - int offset, - int length, - String contentType, - int partitionOffset) -{ - this.setRange(document, offset, length); - if (DEBUG) + private IToken getToken(String type) { - System.out.println( - "PartialRange: contentType=" - + contentType - + " partitionOffset=" - + partitionOffset); + fLength = fCurrentLength; + if (DEBUG) + { + + try + { + if (fLength <= 0) + { + int line = fDocument.getLineOfOffset(fOffset); + System.err.println( + "Error at " + + line + + " offset:" + + String.valueOf( + fOffset - fDocument.getLineOffset(line))); + } + } + catch (BadLocationException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + Assert.isTrue(fLength > 0, "Partition length <= 0!"); + fCurrentLength = 0; + // String can never cross partition borders so reset string detection + fInString = false; + fInDoubString = false; + IToken token = (IToken) this.tokens.get(type); + Assert.isNotNull(token, "Token for type \"" + type + "\" not found!"); + if (DEBUG) + { + System.out.println( + "Partition: fTokenOffset=" + + fTokenOffset + + " fContentType=" + + type + + " fLength=" + + fLength); + } + return token; } - - if(offset == partitionOffset) - { - try + + /* (non-Javadoc) + * @see org.eclipse.jface.text.rules.IPartitionTokenScanner#setPartialRange(org.eclipse.jface.text.IDocument, int, int, java.lang.String, int) + */ + public void setPartialRange( + IDocument document, + int offset, + int length, + String contentType, + int partitionOffset) + { + if (DEBUG) + { + System.out.println( + "PartialRange: contentType=" + + contentType + + " partitionOffset=" + + partitionOffset); + } + + try { - fContentType = fDocument.getContentType(offset); + if (partitionOffset > -1) + { + partitionBorder = false; + // because of strings we have to parse the whole partition + this.setRange( + document, + partitionOffset, + offset - partitionOffset + length); + // sometimes we get a wrong partition so we retrieve the partition + // directly from the document + fContentType = fDocument.getContentType(partitionOffset); + //TODO determine the previouse contenttypes as a stack + //if(partitionOffset > 1) + // fPrevContentType = fDocument.getContentType(partitionOffset-1); + } + else + this.setRange(document, offset, length); + } catch (BadLocationException e) { - //should never happen + // should never happen + // TODO print stack trace to log + // fall back just scan the whole document again + this.setRange(document, 0, fDocument.getLength()); } - } - else if (this.tokens.containsKey(contentType)) - fContentType = contentType; - // TODO Calculate previouse contenttype - if (partitionOffset > -1) - { - partitionBorder = false; - fTokenOffset = partitionOffset; - } -} - /* (non-Javadoc) - * @see org.eclipse.jface.text.rules.ITokenScanner#getTokenLength() - */ - public int getTokenLength() { - return fLength; - } - - /* (non-Javadoc) - * @see org.eclipse.jface.text.rules.ITokenScanner#getTokenOffset() - */ - public int getTokenOffset() { - return fTokenOffset; - } - -/* (non-Javadoc) - * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken() - */ -public IToken nextToken() -{ - int c; + } - // check if we are not allready at the end of the - // file - if ((c = read()) == ICharacterScanner.EOF) + /* (non-Javadoc) + * @see org.eclipse.jface.text.rules.ITokenScanner#getTokenLength() + */ + public int getTokenLength() { - partitionBorder = false; - return Token.EOF; + return fLength; } - else - unread(); - if (partitionBorder) + /* (non-Javadoc) + * @see org.eclipse.jface.text.rules.ITokenScanner#getTokenOffset() + */ + public int getTokenOffset() { - fTokenOffset = fOffset; - partitionBorder = false; + return fTokenOffset; } - while ((c = read()) != ICharacterScanner.EOF) + /* (non-Javadoc) + * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken() + */ + public IToken nextToken() { - switch (c) + int c; + + // check if we are not allready at the end of the + // file + if ((c = read()) == ICharacterScanner.EOF) { - case '<' : - if (fContentType - != IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT - && checkPattern(new char[] { '?', 'p', 'h', 'p' }, true)) - { - if (fContentType != IPHPPartitionScannerConstants.PHP - && fOffset - 5 > 0) + partitionBorder = false; + return Token.EOF; + } + else + unread(); + + if (partitionBorder) + { + fTokenOffset = fOffset; + partitionBorder = false; + } + + while ((c = read()) != ICharacterScanner.EOF) + { + switch (c) + { + case '<' : + if (!isInString(IPHPPartitionScannerConstants.PHP) + && fContentType + != IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT + && checkPattern(new char[] { '?', 'p', 'h', 'p' }, true)) { - fOffset -= 5; - IToken token = getToken(fContentType); - // save previouse contenttype - fPrevContentType = fContentType; + if (fContentType != IPHPPartitionScannerConstants.PHP + && fCurrentLength > 5) + { + unread(5); + IToken token = getToken(fContentType); + // save previouse contenttype + //TODO build stack for previouse contenttype + fPrevContentType = fContentType; - fContentType = IPHPPartitionScannerConstants.PHP; + fContentType = IPHPPartitionScannerConstants.PHP; - return token; - } - else - fContentType = IPHPPartitionScannerConstants.PHP; + return token; + } + else + fContentType = IPHPPartitionScannerConstants.PHP; - // remember offset of this partition - fTokenOffset = fOffset - 5; - } - else if ( - fContentType - != IPHPPartitionScannerConstants.PHP_MULTILINE_COMMENT - && checkPattern(new char[] { '?' }, false)) - { - if (fContentType != IPHPPartitionScannerConstants.PHP - && fOffset - 2 > 0) + // remember offset of this partition + fTokenOffset = fOffset - 5; + fCurrentLength = 5; + } + else if ( + !isInString(IPHPPartitionScannerConstants.PHP) + && fContentType + != IPHPPartitionScannerConstants + .PHP_MULTILINE_COMMENT + && checkPattern(new char[] { '?' }, false)) { - fOffset -= 2; + if (fContentType != IPHPPartitionScannerConstants.PHP + && fCurrentLength > 2) + { + unread(2); IToken token = getToken(fContentType); - // save previouse contenttype - fPrevContentType = - fContentType; + // save previouse contenttype + fPrevContentType = fContentType; fContentType = IPHPPartitionScannerConstants.PHP; return token; - } - else - fContentType = IPHPPartitionScannerConstants.PHP; - // remember offset of this partition + } + else + fContentType = IPHPPartitionScannerConstants.PHP; + // remember offset of this partition fTokenOffset = fOffset - 2; - } - else if (checkPattern(new char[] { '!', '-', '-' })) - { // return previouse partition - if ( - fContentType - != IPHPPartitionScannerConstants.HTML_MULTILINE_COMMENT - && fOffset - 4 > 0) - { - fOffset -= 4; + fCurrentLength = 2; + } + else if ( + !isInString(IPHPPartitionScannerConstants.PHP) + && checkPattern(new char[] { '!', '-', '-' })) + { // return previouse partition + if (fContentType + != IPHPPartitionScannerConstants + .HTML_MULTILINE_COMMENT + && fCurrentLength > 4) + { + unread(4); IToken token = getToken(fContentType); fContentType = IPHPPartitionScannerConstants .HTML_MULTILINE_COMMENT; return token; - } - else - fContentType = - IPHPPartitionScannerConstants - .HTML_MULTILINE_COMMENT; - fTokenOffset = fOffset - 4; - } - break; case '?' : - if (fContentType == IPHPPartitionScannerConstants.PHP) - { - if ((c = read()) == '>') - { // TODO Actually calculate the previouse contenttype from the document - if ( - fPrevContentType - != null) - fContentType = fPrevContentType; else + } + else + fContentType = + IPHPPartitionScannerConstants + .HTML_MULTILINE_COMMENT; + + fTokenOffset = fOffset - 4; + fCurrentLength = 4; + } + break; + case '?' : + if (!isInString(IPHPPartitionScannerConstants.PHP) + && fContentType == IPHPPartitionScannerConstants.PHP) + { + if ((c = read()) == '>') + { + if (fPrevContentType != null) + fContentType = fPrevContentType; + else fContentType = IPHPPartitionScannerConstants.HTML; - partitionBorder = true; - return getToken( - IPHPPartitionScannerConstants.PHP); - } - else if (c != ICharacterScanner.EOF) - unread(); } - break; case '-' : - if (fContentType - == IPHPPartitionScannerConstants.HTML_MULTILINE_COMMENT - && checkPattern(new char[] { '-', '>' })) - { - fContentType = IPHPPartitionScannerConstants.HTML; + partitionBorder = true; + return getToken(IPHPPartitionScannerConstants.PHP); + } + else if (c != ICharacterScanner.EOF) + unread(); + } + break; + case '-' : + if (!isInString(IPHPPartitionScannerConstants.PHP) + && fContentType + == IPHPPartitionScannerConstants + .HTML_MULTILINE_COMMENT + && checkPattern(new char[] { '-', '>' })) + { + fContentType = IPHPPartitionScannerConstants.HTML; partitionBorder = true; return getToken( IPHPPartitionScannerConstants .HTML_MULTILINE_COMMENT); - } - break; case '/' : - if ((c = read()) == '*') - { // MULTINE COMMENT JAVASCRIPT, CSS, PHP - if ( - fContentType == IPHPPartitionScannerConstants.PHP - && fOffset - 2 > 0) - { - fOffset -= 2; + } + break; + case '/' : + if (!isInString(IPHPPartitionScannerConstants.PHP) && (c = read()) == '*') + { // MULTINE COMMENT JAVASCRIPT, CSS, PHP + if (fContentType == IPHPPartitionScannerConstants.PHP + && fCurrentLength > 2) + { + unread(2); IToken token = getToken(fContentType); fContentType = IPHPPartitionScannerConstants .PHP_MULTILINE_COMMENT; return token; - } - else if ( - fContentType - == IPHPPartitionScannerConstants - .PHP_MULTILINE_COMMENT) - { + } + else if ( + fContentType + == IPHPPartitionScannerConstants + .PHP_MULTILINE_COMMENT) + { - fTokenOffset = fOffset - 2; } + fTokenOffset = fOffset - 2; + fCurrentLength = 2; + } - } - else if (c != ICharacterScanner.EOF) - unread(); break; case '*' : - if ((c = read()) == '/') + } + else if (!isInString(IPHPPartitionScannerConstants.PHP) && c != ICharacterScanner.EOF) + unread(); + break; + case '*' : + if (!isInString(IPHPPartitionScannerConstants.PHP) && (c = read()) == '/') + { + if (fContentType + == IPHPPartitionScannerConstants + .PHP_MULTILINE_COMMENT) { - if (fContentType + fContentType = IPHPPartitionScannerConstants.PHP; + partitionBorder = true; + return getToken( + IPHPPartitionScannerConstants + .PHP_MULTILINE_COMMENT); + } + else if ( + fContentType == IPHPPartitionScannerConstants - .PHP_MULTILINE_COMMENT) - { - fContentType = - IPHPPartitionScannerConstants.PHP; - partitionBorder = true; - return getToken( - IPHPPartitionScannerConstants - .PHP_MULTILINE_COMMENT); - } - else if ( - fContentType - == IPHPPartitionScannerConstants - .CSS_MULTILINE_COMMENT) - { - } - else if ( - fContentType - == IPHPPartitionScannerConstants - .JS_MULTILINE_COMMENT) - { - } + .CSS_MULTILINE_COMMENT) + { } - else if (c != ICharacterScanner.EOF) - unread(); break; } - } // end of file reached but we have to return the - // last partition. - return getToken(fContentType); + else if ( + fContentType + == IPHPPartitionScannerConstants + .JS_MULTILINE_COMMENT) + { + } + } + else if (!isInString(IPHPPartitionScannerConstants.PHP) && c != ICharacterScanner.EOF) + unread(); + break; + case '\'' : + if (!fInDoubString) + fInString = !fInString; + break; + case '"' : + // toggle String mode + if (!fInString) + fInDoubString = !fInDoubString; + break; + } + } // end of file reached but we have to return the + // last partition. + return getToken(fContentType); + } + /* (non-Javadoc) + * @see org.eclipse.jface.text.rules.ITokenScanner#setRange(org.eclipse.jface.text.IDocument, int, int) + */ + public void setRange(IDocument document, int offset, int length) + { + if (DEBUG) + { + System.out.println( + "SET RANGE: offset=" + offset + " length=" + length); + } + + fDocument = document; + fOffset = offset; + fTokenOffset = offset; + fCurrentLength = 0; + fLength = 0; + fEnd = fOffset + length; + fInString = false; + fInDoubString = false; + //partitionBorder = false; + } + + private int read() + { + try + { + if (fOffset < fEnd) + { + fCurrentLength++; + return fDocument.getChar(fOffset++); + } + return ICharacterScanner.EOF; + } + catch (BadLocationException e) + { + // should never happen + // TODO write stacktrace to log + fOffset = fEnd; + return ICharacterScanner.EOF; } - /* (non-Javadoc) - * @see org.eclipse.jface.text.rules.ITokenScanner#setRange(org.eclipse.jface.text.IDocument, int, int) - */ - public void setRange(IDocument document, int offset, int length) { - if (DEBUG) { - System.out.println("SET RANGE: offset=" + offset + " length=" + length); } - fDocument = document; - fOffset = offset; - fTokenOffset = offset; - fLength = 0; - fEnd = fOffset + length; - //partitionBorder = false; - } - - private int read() { - try { - if (fOffset < fEnd) { - return fDocument.getChar(fOffset++); - } - return ICharacterScanner.EOF; - } catch (BadLocationException e) { - // should never happen - // TODO write stacktrace to log - fOffset = fEnd; - return ICharacterScanner.EOF; + private void unread() + { + --fOffset; + --fCurrentLength; } - } - - private void unread() { - --fOffset; - } - - private boolean checkPattern(char[] pattern) { - return checkPattern(pattern, false); - } - - /** - * Check if next character sequence read from document is equals to - * the provided pattern. Pattern is read from left to right until the - * first character read doesn't match. If this happens all read characters are - * unread. - * @param pattern The pattern to check. - * @return true if pattern is equals else returns false. - */ - private boolean checkPattern(char[] pattern, boolean ignoreCase) { - int prevOffset = fOffset; - for (int i = 0; i < pattern.length; i++) { - int c = read(); - - if (c == ICharacterScanner.EOF - || !letterEquals(c, pattern[i], ignoreCase)) { - fOffset = prevOffset; - return false; - } + + private void unread(int num) + { + fOffset -= num; + fCurrentLength -= num; } - return true; - } - - private boolean letterEquals(int test, char letter, boolean ignoreCase) { - if (test == letter) - return true; - else if ( - ignoreCase - && Character.isLowerCase(letter) - && test == Character.toUpperCase(letter)) - return true; - else if ( - ignoreCase - && Character.isUpperCase(letter) - && test == Character.toLowerCase(letter)) - return true; - - return false; - } + private boolean checkPattern(char[] pattern) + { + return checkPattern(pattern, false); + } + + /** + * Check if next character sequence read from document is equals to + * the provided pattern. Pattern is read from left to right until the + * first character read doesn't match. If this happens all read characters are + * unread. + * @param pattern The pattern to check. + * @return true if pattern is equals else returns false. + */ + private boolean checkPattern(char[] pattern, boolean ignoreCase) + { + int prevOffset = fOffset; + int prevLength = fCurrentLength; + for (int i = 0; i < pattern.length; i++) + { + int c = read(); + + if (c == ICharacterScanner.EOF + || !letterEquals(c, pattern[i], ignoreCase)) + { + fOffset = prevOffset; + fCurrentLength = prevLength; + return false; + } + } + + return true; + } + + private boolean letterEquals(int test, char letter, boolean ignoreCase) + { + if (test == letter) + return true; + else if ( + ignoreCase + && Character.isLowerCase(letter) + && test == Character.toUpperCase(letter)) + return true; + else if ( + ignoreCase + && Character.isUpperCase(letter) + && test == Character.toLowerCase(letter)) + return true; + + return false; + } + + /** + * Checks wether the offset is in a String and the specified + * contenttype is the current content type. + * Strings are delimited, mutual exclusive, by a " or by a '. + * + * @param contentType The contenttype to check. + * @return true if the current offset is in a string else + * returns false. + */ + private boolean isInString(String contentType) + { + if(fContentType == contentType) + return (fInString || fInDoubString); + else + return false; + } } -- 1.7.1