X-Git-Url: http://git.phpeclipse.com

diff --git a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
index 83f8e87..c897fdb 100644
--- a/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
+++ b/net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java
@@ -170,7 +170,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
    * permissible as the first character in a PHP identifier
    */
   public static boolean isPHPIdentifierStart(char ch) {
-    return Character.isLetter(ch) || (ch == '_');
+    return Character.isLetter(ch)
+      || (ch == '_')
+      || (0x7F <= ch && ch <= 0xFF);
   }
 
   /**
@@ -178,7 +180,9 @@ public class Scanner implements IScanner, ITerminalSymbols {
    * other than the first character
    */
   public static boolean isPHPIdentifierPart(char ch) {
-    return Character.isLetterOrDigit(ch) || (ch == '_');
+    return Character.isLetterOrDigit(ch)
+      || (ch == '_')
+      || (0x7F <= ch && ch <= 0xFF);
   }
 
   public final boolean atEnd() {
@@ -719,9 +723,8 @@ public class Scanner implements IScanner, ITerminalSymbols {
   }
 
   public int getNextToken() throws InvalidInputException {
-
+    int htmlPosition = currentPosition;
     try {
-      int htmlPosition = currentPosition;
       while (!phpMode) {
         currentCharacter = source[currentPosition++];
         if (currentCharacter == '<') {
@@ -773,6 +776,11 @@ public class Scanner implements IScanner, ITerminalSymbols {
       }
     } //-----------------end switch while try--------------------
     catch (IndexOutOfBoundsException e) {
+      if (tokenizeWhiteSpace) {
+        // && (whiteStart != currentPosition - 1)) {
+        // reposition scanner in case we are interested by spaces as tokens
+        startPosition = htmlPosition;
+      }
       return TokenNameEOF;
     }