changed isPHPIdentifierStart, isPHPIdentifierPart

[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / compiler / parser / Scanner.java

diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
index 04c3935..c897fdb 100644 (file)
--- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
+++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
@@ -170,7 +170,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
    * permissible as the first character in a PHP identifier
    */
   public static boolean isPHPIdentifierStart(char ch) {
-    return Character.isLetter(ch) || (ch == '_');
+    return Character.isLetter(ch)
+      || (ch == '_')
+      || (0x7F <= ch && ch <= 0xFF);
   }
 
   /**
@@ -178,7 +180,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
    * other than the first character
    */
   public static boolean isPHPIdentifierPart(char ch) {
-    return Character.isLetterOrDigit(ch) || (ch == '_');
+    return Character.isLetterOrDigit(ch)
+      || (ch == '_')
+      || (0x7F <= ch && ch <= 0xFF);
   }
 
   public final boolean atEnd() {
@@ -719,9 +723,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
   }
 
   public int getNextToken() throws InvalidInputException {
+    int htmlPosition = currentPosition;
     try {
       while (!phpMode) {
-        startPosition = currentPosition;
         currentCharacter = source[currentPosition++];
         if (currentCharacter == '<') {
           if (getNextChar('?')) {
@@ -731,6 +735,12 @@ public class Scanner implements IScanner, ITerminalSymbols {
               // <?
               startPosition = currentPosition;
               phpMode = true;
+              if (tokenizeWhiteSpace) {
+                // && (whiteStart != currentPosition - 1)) {
+                // reposition scanner in case we are interested by spaces as tokens
+                startPosition = htmlPosition;
+                return TokenNameHTML;
+              }
             } else {
               boolean phpStart =
                 (currentCharacter == 'P') || (currentCharacter == 'p');
@@ -742,6 +752,13 @@ public class Scanner implements IScanner, ITerminalSymbols {
                     // <?PHP  <?php
                     startPosition = currentPosition;
                     phpMode = true;
+
+                    if (tokenizeWhiteSpace) {
+                      // && (whiteStart != currentPosition - 1)) {
+                      // reposition scanner in case we are interested by spaces as tokens
+                      startPosition = htmlPosition;
+                      return TokenNameHTML;
+                    }
                   }
                 }
               }
@@ -759,6 +776,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
       }
     } //-----------------end switch while try--------------------
     catch (IndexOutOfBoundsException e) {
+      if (tokenizeWhiteSpace) {
+        // && (whiteStart != currentPosition - 1)) {
+        // reposition scanner in case we are interested by spaces as tokens
+        startPosition = htmlPosition;
+      }
       return TokenNameEOF;
     }