net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2001, 2002 International Business Machines Corp. and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v0.5
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v05.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  ******************************************************************************/
  11 package net.sourceforge.phpdt.internal.compiler.parser;
  12 import java.util.ArrayList;
  13 import java.util.Iterator;
  14 import java.util.List;
  15 import java.util.Stack;
  16
  17 import net.sourceforge.phpdt.core.compiler.CharOperation;
  18 import net.sourceforge.phpdt.core.compiler.IScanner;
  19 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
  20 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
  21 import net.sourceforge.phpeclipse.internal.compiler.ast.StringLiteral;
  22
  23
  24 public class Scanner implements IScanner, ITerminalSymbols {
  25   /*
  26    * APIs ares - getNextToken() which return the current type of the token
  27    * (this value is not memorized by the scanner) - getCurrentTokenSource()
  28    * which provides with the token "REAL" source (aka all unicode have been
  29    * transformed into a correct char) - sourceStart gives the position into the
  30    * stream - currentPosition-1 gives the sourceEnd position into the stream
  31    */
  32   // 1.4 feature
  33   private boolean assertMode;
  34   public boolean useAssertAsAnIndentifier = false;
  35   //flag indicating if processed source contains occurrences of keyword assert
  36   public boolean containsAssertKeyword = false;
  37   public boolean recordLineSeparator;
  38   public boolean phpMode = false;
  39   public Stack encapsedStringStack = null;
  40   public char currentCharacter;
  41   public int startPosition;
  42   public int currentPosition;
  43   public int initialPosition, eofPosition;
  44   // after this position eof are generated instead of real token from the
  45   // source
  46   public boolean tokenizeComments;
  47   public boolean tokenizeWhiteSpace;
  48   public boolean tokenizeStrings;
  49   //source should be viewed as a window (aka a part)
  50   //of a entire very large stream
  51   public char source[];
  52   //unicode support
  53   public char[] withoutUnicodeBuffer;
  54   public int withoutUnicodePtr;
  55   //when == 0 ==> no unicode in the current token
  56   public boolean unicodeAsBackSlash = false;
  57   public boolean scanningFloatLiteral = false;
  58 //support for /** comments
  59         public int[] commentStops = new int[10];
  60         public int[] commentStarts = new int[10];
  61         public int commentPtr = -1; // no comment test with commentPtr value -1
  62         protected int lastCommentLinePosition = -1;
  63   //diet parsing support - jump over some method body when requested
  64   public boolean diet = false;
  65   //support for the poor-line-debuggers ....
  66   //remember the position of the cr/lf
  67   public int[] lineEnds = new int[250];
  68   public int linePtr = -1;
  69   public boolean wasAcr = false;
  70   public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
  71   public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
  72   public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
  73   public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
  74   public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
  75   public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
  76   public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
  77   public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
  78   public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
  79   public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
  80   public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
  81   public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
  82   //----------------optimized identifier managment------------------
  83   static final char[] charArray_a = new char[]{'a'},
  84       charArray_b = new char[]{'b'}, charArray_c = new char[]{'c'},
  85       charArray_d = new char[]{'d'}, charArray_e = new char[]{'e'},
  86       charArray_f = new char[]{'f'}, charArray_g = new char[]{'g'},
  87       charArray_h = new char[]{'h'}, charArray_i = new char[]{'i'},
  88       charArray_j = new char[]{'j'}, charArray_k = new char[]{'k'},
  89       charArray_l = new char[]{'l'}, charArray_m = new char[]{'m'},
  90       charArray_n = new char[]{'n'}, charArray_o = new char[]{'o'},
  91       charArray_p = new char[]{'p'}, charArray_q = new char[]{'q'},
  92       charArray_r = new char[]{'r'}, charArray_s = new char[]{'s'},
  93       charArray_t = new char[]{'t'}, charArray_u = new char[]{'u'},
  94       charArray_v = new char[]{'v'}, charArray_w = new char[]{'w'},
  95       charArray_x = new char[]{'x'}, charArray_y = new char[]{'y'},
  96       charArray_z = new char[]{'z'};
  97   static final char[] initCharArray = new char[]{'\u0000', '\u0000', '\u0000',
  98       '\u0000', '\u0000', '\u0000'};
  99   static final int TableSize = 30, InternalTableSize = 6;
 100   //30*6 = 180 entries
 101   public static final int OptimizedLength = 6;
 102   public/* static */
 103   final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][];
 104   // support for detecting non-externalized string literals
 105   int currentLineNr = -1;
 106   int previousLineNr = -1;
 107   NLSLine currentLine = null;
 108   List lines = new ArrayList();
 109   public static final String TAG_PREFIX = "//$NON-NLS-"; //$NON-NLS-1$
 110   public static final int TAG_PREFIX_LENGTH = TAG_PREFIX.length();
 111   public static final String TAG_POSTFIX = "$"; //$NON-NLS-1$
 112   public static final int TAG_POSTFIX_LENGTH = TAG_POSTFIX.length();
 113   public StringLiteral[] nonNLSStrings = null;
 114   public boolean checkNonExternalizedStringLiterals = true;
 115   public boolean wasNonExternalizedStringLiteral = false;
 116   /* static */{
 117     for (int i = 0; i < 6; i++) {
 118       for (int j = 0; j < TableSize; j++) {
 119         for (int k = 0; k < InternalTableSize; k++) {
 120           charArray_length[i][j][k] = initCharArray;
 121         }
 122       }
 123     }
 124   }
 125   static int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0, newEntry5 = 0,
 126       newEntry6 = 0;
 127   public static final int RoundBracket = 0;
 128   public static final int SquareBracket = 1;
 129   public static final int CurlyBracket = 2;
 130   public static final int BracketKinds = 3;
 131   // task tag support
 132   public char[][] foundTaskTags = null;
 133   public char[][] foundTaskMessages;
 134   public char[][] foundTaskPriorities = null;
 135   public int[][] foundTaskPositions;
 136   public int foundTaskCount = 0;
 137   public char[][] taskTags = null;
 138   public char[][] taskPriorities = null;
 139   public static final boolean DEBUG = false;
 140   public static final boolean TRACE = false;
 141
 142   /**
 143    * Determines if the specified character is permissible as the first
 144    * character in a PHP identifier
 145    */
 146   public static boolean isPHPIdentifierStart(char ch) {
 147     return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
 148   }
 149   /**
 150    * Determines if the specified character may be part of a PHP identifier as
 151    * other than the first character
 152    */
 153   public static boolean isPHPIdentifierPart(char ch) {
 154     return Character.isLetterOrDigit(ch) || (ch == '_')
 155         || (0x7F <= ch && ch <= 0xFF);
 156   }
 157   public final boolean atEnd() {
 158     // This code is not relevant if source is
 159     // Only a part of the real stream input
 160     return source.length == currentPosition;
 161   }
 162   public char[] getCurrentIdentifierSource() {
 163     //return the token REAL source (aka unicodes are precomputed)
 164     char[] result;
 165     //    if (withoutUnicodePtr != 0)
 166     //      //0 is used as a fast test flag so the real first char is in position 1
 167     //      System.arraycopy(
 168     //        withoutUnicodeBuffer,
 169     //        1,
 170     //        result = new char[withoutUnicodePtr],
 171     //        0,
 172     //        withoutUnicodePtr);
 173     //    else {
 174     int length = currentPosition - startPosition;
 175     switch (length) { // see OptimizedLength
 176       case 1 :
 177         return optimizedCurrentTokenSource1();
 178       case 2 :
 179         return optimizedCurrentTokenSource2();
 180       case 3 :
 181         return optimizedCurrentTokenSource3();
 182       case 4 :
 183         return optimizedCurrentTokenSource4();
 184       case 5 :
 185         return optimizedCurrentTokenSource5();
 186       case 6 :
 187         return optimizedCurrentTokenSource6();
 188     }
 189     //no optimization
 190     System.arraycopy(source, startPosition, result = new char[length], 0,
 191         length);
 192     //   }
 193     return result;
 194   }
 195   public int getCurrentTokenEndPosition() {
 196     return this.currentPosition - 1;
 197   }
 198   public final char[] getCurrentTokenSource() {
 199     // Return the token REAL source (aka unicodes are precomputed)
 200     char[] result;
 201     //    if (withoutUnicodePtr != 0)
 202     //      // 0 is used as a fast test flag so the real first char is in position 1
 203     //      System.arraycopy(
 204     //        withoutUnicodeBuffer,
 205     //        1,
 206     //        result = new char[withoutUnicodePtr],
 207     //        0,
 208     //        withoutUnicodePtr);
 209     //    else {
 210     int length;
 211     System.arraycopy(source, startPosition,
 212         result = new char[length = currentPosition - startPosition], 0, length);
 213     //    }
 214     return result;
 215   }
 216   public final char[] getCurrentTokenSource(int startPos) {
 217     // Return the token REAL source (aka unicodes are precomputed)
 218     char[] result;
 219     //    if (withoutUnicodePtr != 0)
 220     //      // 0 is used as a fast test flag so the real first char is in position 1
 221     //      System.arraycopy(
 222     //        withoutUnicodeBuffer,
 223     //        1,
 224     //        result = new char[withoutUnicodePtr],
 225     //        0,
 226     //        withoutUnicodePtr);
 227     //    else {
 228     int length;
 229     System.arraycopy(source, startPos,
 230         result = new char[length = currentPosition - startPos], 0, length);
 231     //  }
 232     return result;
 233   }
 234   public final char[] getCurrentTokenSourceString() {
 235     //return the token REAL source (aka unicodes are precomputed).
 236     //REMOVE the two " that are at the beginning and the end.
 237     char[] result;
 238     if (withoutUnicodePtr != 0)
 239       //0 is used as a fast test flag so the real first char is in position 1
 240       System.arraycopy(withoutUnicodeBuffer, 2,
 241       //2 is 1 (real start) + 1 (to jump over the ")
 242           result = new char[withoutUnicodePtr - 2], 0, withoutUnicodePtr - 2);
 243     else {
 244       int length;
 245       System.arraycopy(source, startPosition + 1,
 246           result = new char[length = currentPosition - startPosition - 2], 0,
 247           length);
 248     }
 249     return result;
 250   }
 251   public int getCurrentTokenStartPosition() {
 252     return this.startPosition;
 253   }
 254   public final char[] getCurrentStringLiteralSource() {
 255     // Return the token REAL source (aka unicodes are precomputed)
 256     char[] result;
 257     int length;
 258     System.arraycopy(source, startPosition + 1,
 259         result = new char[length = currentPosition - startPosition - 2], 0,
 260         length);
 261     //    }
 262     return result;
 263   }
 264   /*
 265    * Search the source position corresponding to the end of a given line number
 266    *
 267    * Line numbers are 1-based, and relative to the scanner initialPosition.
 268    * Character positions are 0-based.
 269    *
 270    * In case the given line number is inconsistent, answers -1.
 271    */
 272   public final int getLineEnd(int lineNumber) {
 273     if (lineEnds == null)
 274       return -1;
 275     if (lineNumber >= lineEnds.length)
 276       return -1;
 277     if (lineNumber <= 0)
 278       return -1;
 279     if (lineNumber == lineEnds.length - 1)
 280       return eofPosition;
 281     return lineEnds[lineNumber - 1];
 282     // next line start one character behind the lineEnd of the previous line
 283   }
 284   /**
 285    * Search the source position corresponding to the beginning of a given line
 286    * number
 287    *
 288    * Line numbers are 1-based, and relative to the scanner initialPosition.
 289    * Character positions are 0-based.
 290    *
 291    * e.g. getLineStart(1) --> 0 i.e. first line starts at character 0.
 292    *
 293    * In case the given line number is inconsistent, answers -1.
 294    */
 295   public final int getLineStart(int lineNumber) {
 296     if (lineEnds == null)
 297       return -1;
 298     if (lineNumber >= lineEnds.length)
 299       return -1;
 300     if (lineNumber <= 0)
 301       return -1;
 302     if (lineNumber == 1)
 303       return initialPosition;
 304     return lineEnds[lineNumber - 2] + 1;
 305     // next line start one character behind the lineEnd of the previous line
 306   }
 307   public final boolean getNextChar(char testedChar) {
 308     //BOOLEAN
 309     //handle the case of unicode.
 310     //when a unicode appears then we must use a buffer that holds char
 311     // internal values
 312     //At the end of this method currentCharacter holds the new visited char
 313     //and currentPosition points right next after it
 314     //Both previous lines are true if the currentCharacter is == to the
 315     // testedChar
 316     //On false, no side effect has occured.
 317     //ALL getNextChar.... ARE OPTIMIZED COPIES
 318     int temp = currentPosition;
 319     try {
 320       currentCharacter = source[currentPosition++];
 321       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 322       //        && (source[currentPosition] == 'u')) {
 323       //        //-------------unicode traitement ------------
 324       //        int c1, c2, c3, c4;
 325       //        int unicodeSize = 6;
 326       //        currentPosition++;
 327       //        while (source[currentPosition] == 'u') {
 328       //          currentPosition++;
 329       //          unicodeSize++;
 330       //        }
 331       //
 332       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 333       //          || c1 < 0)
 334       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 335       //            || c2 < 0)
 336       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 337       //            || c3 < 0)
 338       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 339       //            || c4 < 0)) {
 340       //          currentPosition = temp;
 341       //          return false;
 342       //        }
 343       //
 344       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 345       //        if (currentCharacter != testedChar) {
 346       //          currentPosition = temp;
 347       //          return false;
 348       //        }
 349       //        unicodeAsBackSlash = currentCharacter == '\\';
 350       //
 351       //        //need the unicode buffer
 352       //        if (withoutUnicodePtr == 0) {
 353       //          //buffer all the entries that have been left aside....
 354       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 355       //          System.arraycopy(
 356       //            source,
 357       //            startPosition,
 358       //            withoutUnicodeBuffer,
 359       //            1,
 360       //            withoutUnicodePtr);
 361       //        }
 362       //        //fill the buffer with the char
 363       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 364       //        return true;
 365       //
 366       //      } //-------------end unicode traitement--------------
 367       //      else {
 368       if (currentCharacter != testedChar) {
 369         currentPosition = temp;
 370         return false;
 371       }
 372       unicodeAsBackSlash = false;
 373       //        if (withoutUnicodePtr != 0)
 374       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 375       return true;
 376       //      }
 377     } catch (IndexOutOfBoundsException e) {
 378       unicodeAsBackSlash = false;
 379       currentPosition = temp;
 380       return false;
 381     }
 382   }
 383   public final int getNextChar(char testedChar1, char testedChar2) {
 384     //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
 385     //test can be done with (x==0) for the first and (x>0) for the second
 386     //handle the case of unicode.
 387     //when a unicode appears then we must use a buffer that holds char
 388     // internal values
 389     //At the end of this method currentCharacter holds the new visited char
 390     //and currentPosition points right next after it
 391     //Both previous lines are true if the currentCharacter is == to the
 392     // testedChar1/2
 393     //On false, no side effect has occured.
 394     //ALL getNextChar.... ARE OPTIMIZED COPIES
 395     int temp = currentPosition;
 396     try {
 397       int result;
 398       currentCharacter = source[currentPosition++];
 399       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 400       //        && (source[currentPosition] == 'u')) {
 401       //        //-------------unicode traitement ------------
 402       //        int c1, c2, c3, c4;
 403       //        int unicodeSize = 6;
 404       //        currentPosition++;
 405       //        while (source[currentPosition] == 'u') {
 406       //          currentPosition++;
 407       //          unicodeSize++;
 408       //        }
 409       //
 410       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 411       //          || c1 < 0)
 412       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 413       //            || c2 < 0)
 414       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 415       //            || c3 < 0)
 416       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 417       //            || c4 < 0)) {
 418       //          currentPosition = temp;
 419       //          return 2;
 420       //        }
 421       //
 422       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 423       //        if (currentCharacter == testedChar1)
 424       //          result = 0;
 425       //        else if (currentCharacter == testedChar2)
 426       //          result = 1;
 427       //        else {
 428       //          currentPosition = temp;
 429       //          return -1;
 430       //        }
 431       //
 432       //        //need the unicode buffer
 433       //        if (withoutUnicodePtr == 0) {
 434       //          //buffer all the entries that have been left aside....
 435       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 436       //          System.arraycopy(
 437       //            source,
 438       //            startPosition,
 439       //            withoutUnicodeBuffer,
 440       //            1,
 441       //            withoutUnicodePtr);
 442       //        }
 443       //        //fill the buffer with the char
 444       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 445       //        return result;
 446       //      } //-------------end unicode traitement--------------
 447       //      else {
 448       if (currentCharacter == testedChar1)
 449         result = 0;
 450       else if (currentCharacter == testedChar2)
 451         result = 1;
 452       else {
 453         currentPosition = temp;
 454         return -1;
 455       }
 456       //        if (withoutUnicodePtr != 0)
 457       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 458       return result;
 459       //     }
 460     } catch (IndexOutOfBoundsException e) {
 461       currentPosition = temp;
 462       return -1;
 463     }
 464   }
 465   public final boolean getNextCharAsDigit() {
 466     //BOOLEAN
 467     //handle the case of unicode.
 468     //when a unicode appears then we must use a buffer that holds char
 469     // internal values
 470     //At the end of this method currentCharacter holds the new visited char
 471     //and currentPosition points right next after it
 472     //Both previous lines are true if the currentCharacter is a digit
 473     //On false, no side effect has occured.
 474     //ALL getNextChar.... ARE OPTIMIZED COPIES
 475     int temp = currentPosition;
 476     try {
 477       currentCharacter = source[currentPosition++];
 478       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 479       //        && (source[currentPosition] == 'u')) {
 480       //        //-------------unicode traitement ------------
 481       //        int c1, c2, c3, c4;
 482       //        int unicodeSize = 6;
 483       //        currentPosition++;
 484       //        while (source[currentPosition] == 'u') {
 485       //          currentPosition++;
 486       //          unicodeSize++;
 487       //        }
 488       //
 489       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 490       //          || c1 < 0)
 491       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 492       //            || c2 < 0)
 493       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 494       //            || c3 < 0)
 495       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 496       //            || c4 < 0)) {
 497       //          currentPosition = temp;
 498       //          return false;
 499       //        }
 500       //
 501       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 502       //        if (!Character.isDigit(currentCharacter)) {
 503       //          currentPosition = temp;
 504       //          return false;
 505       //        }
 506       //
 507       //        //need the unicode buffer
 508       //        if (withoutUnicodePtr == 0) {
 509       //          //buffer all the entries that have been left aside....
 510       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 511       //          System.arraycopy(
 512       //            source,
 513       //            startPosition,
 514       //            withoutUnicodeBuffer,
 515       //            1,
 516       //            withoutUnicodePtr);
 517       //        }
 518       //        //fill the buffer with the char
 519       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 520       //        return true;
 521       //      } //-------------end unicode traitement--------------
 522       //      else {
 523       if (!Character.isDigit(currentCharacter)) {
 524         currentPosition = temp;
 525         return false;
 526       }
 527       //        if (withoutUnicodePtr != 0)
 528       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 529       return true;
 530       //      }
 531     } catch (IndexOutOfBoundsException e) {
 532       currentPosition = temp;
 533       return false;
 534     }
 535   }
 536   public final boolean getNextCharAsDigit(int radix) {
 537     //BOOLEAN
 538     //handle the case of unicode.
 539     //when a unicode appears then we must use a buffer that holds char
 540     // internal values
 541     //At the end of this method currentCharacter holds the new visited char
 542     //and currentPosition points right next after it
 543     //Both previous lines are true if the currentCharacter is a digit base on
 544     // radix
 545     //On false, no side effect has occured.
 546     //ALL getNextChar.... ARE OPTIMIZED COPIES
 547     int temp = currentPosition;
 548     try {
 549       currentCharacter = source[currentPosition++];
 550       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 551       //        && (source[currentPosition] == 'u')) {
 552       //        //-------------unicode traitement ------------
 553       //        int c1, c2, c3, c4;
 554       //        int unicodeSize = 6;
 555       //        currentPosition++;
 556       //        while (source[currentPosition] == 'u') {
 557       //          currentPosition++;
 558       //          unicodeSize++;
 559       //        }
 560       //
 561       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 562       //          || c1 < 0)
 563       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 564       //            || c2 < 0)
 565       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 566       //            || c3 < 0)
 567       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 568       //            || c4 < 0)) {
 569       //          currentPosition = temp;
 570       //          return false;
 571       //        }
 572       //
 573       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 574       //        if (Character.digit(currentCharacter, radix) == -1) {
 575       //          currentPosition = temp;
 576       //          return false;
 577       //        }
 578       //
 579       //        //need the unicode buffer
 580       //        if (withoutUnicodePtr == 0) {
 581       //          //buffer all the entries that have been left aside....
 582       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 583       //          System.arraycopy(
 584       //            source,
 585       //            startPosition,
 586       //            withoutUnicodeBuffer,
 587       //            1,
 588       //            withoutUnicodePtr);
 589       //        }
 590       //        //fill the buffer with the char
 591       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 592       //        return true;
 593       //      } //-------------end unicode traitement--------------
 594       //      else {
 595       if (Character.digit(currentCharacter, radix) == -1) {
 596         currentPosition = temp;
 597         return false;
 598       }
 599       //        if (withoutUnicodePtr != 0)
 600       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 601       return true;
 602       //      }
 603     } catch (IndexOutOfBoundsException e) {
 604       currentPosition = temp;
 605       return false;
 606     }
 607   }
 608   public boolean getNextCharAsJavaIdentifierPart() {
 609     //BOOLEAN
 610     //handle the case of unicode.
 611     //when a unicode appears then we must use a buffer that holds char
 612     // internal values
 613     //At the end of this method currentCharacter holds the new visited char
 614     //and currentPosition points right next after it
 615     //Both previous lines are true if the currentCharacter is a
 616     // JavaIdentifierPart
 617     //On false, no side effect has occured.
 618     //ALL getNextChar.... ARE OPTIMIZED COPIES
 619     int temp = currentPosition;
 620     try {
 621       currentCharacter = source[currentPosition++];
 622       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 623       //        && (source[currentPosition] == 'u')) {
 624       //        //-------------unicode traitement ------------
 625       //        int c1, c2, c3, c4;
 626       //        int unicodeSize = 6;
 627       //        currentPosition++;
 628       //        while (source[currentPosition] == 'u') {
 629       //          currentPosition++;
 630       //          unicodeSize++;
 631       //        }
 632       //
 633       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 634       //          || c1 < 0)
 635       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 636       //            || c2 < 0)
 637       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 638       //            || c3 < 0)
 639       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 640       //            || c4 < 0)) {
 641       //          currentPosition = temp;
 642       //          return false;
 643       //        }
 644       //
 645       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 646       //        if (!isPHPIdentifierPart(currentCharacter)) {
 647       //          currentPosition = temp;
 648       //          return false;
 649       //        }
 650       //
 651       //        //need the unicode buffer
 652       //        if (withoutUnicodePtr == 0) {
 653       //          //buffer all the entries that have been left aside....
 654       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 655       //          System.arraycopy(
 656       //            source,
 657       //            startPosition,
 658       //            withoutUnicodeBuffer,
 659       //            1,
 660       //            withoutUnicodePtr);
 661       //        }
 662       //        //fill the buffer with the char
 663       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 664       //        return true;
 665       //      } //-------------end unicode traitement--------------
 666       //      else {
 667       if (!isPHPIdentifierPart(currentCharacter)) {
 668         currentPosition = temp;
 669         return false;
 670       }
 671       //        if (withoutUnicodePtr != 0)
 672       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 673       return true;
 674       //      }
 675     } catch (IndexOutOfBoundsException e) {
 676       currentPosition = temp;
 677       return false;
 678     }
 679   }
 680   public int getCastOrParen() {
 681     int tempPosition = currentPosition;
 682     char tempCharacter = currentCharacter;
 683     int tempToken = TokenNameLPAREN;
 684     boolean found = false;
 685     StringBuffer buf = new StringBuffer();
 686     try {
 687       do {
 688         currentCharacter = source[currentPosition++];
 689       } while (currentCharacter == ' ' || currentCharacter == '\t');
 690       while ((currentCharacter >= 'a' && currentCharacter <= 'z')
 691           || (currentCharacter >= 'A' && currentCharacter <= 'Z')) {
 692         buf.append(currentCharacter);
 693         currentCharacter = source[currentPosition++];
 694       }
 695       if (buf.length() >= 3 && buf.length() <= 7) {
 696         char[] data = buf.toString().toCharArray();
 697         int index = 0;
 698         switch (data.length) {
 699           case 3 :
 700             // int
 701             if ((data[index] == 'i') && (data[++index] == 'n')
 702                 && (data[++index] == 't')) {
 703               found = true;
 704               tempToken = TokenNameintCAST;
 705             }
 706             break;
 707           case 4 :
 708             // bool real
 709             if ((data[index] == 'b') && (data[++index] == 'o')
 710                 && (data[++index] == 'o') && (data[++index] == 'l')) {
 711               found = true;
 712               tempToken = TokenNameboolCAST;
 713             } else {
 714               index = 0;
 715               if ((data[index] == 'r') && (data[++index] == 'e')
 716                   && (data[++index] == 'a') && (data[++index] == 'l')) {
 717                 found = true;
 718                 tempToken = TokenNamedoubleCAST;
 719               }
 720             }
 721             break;
 722           case 5 :
 723             // array unset float
 724             if ((data[index] == 'a') && (data[++index] == 'r')
 725                 && (data[++index] == 'r') && (data[++index] == 'a')
 726                 && (data[++index] == 'y')) {
 727               found = true;
 728               tempToken = TokenNamearrayCAST;
 729             } else {
 730               index = 0;
 731               if ((data[index] == 'u') && (data[++index] == 'n')
 732                   && (data[++index] == 's') && (data[++index] == 'e')
 733                   && (data[++index] == 't')) {
 734                 found = true;
 735                 tempToken = TokenNameunsetCAST;
 736               } else {
 737                 index = 0;
 738                 if ((data[index] == 'f') && (data[++index] == 'l')
 739                     && (data[++index] == 'o') && (data[++index] == 'a')
 740                     && (data[++index] == 't')) {
 741                   found = true;
 742                   tempToken = TokenNamedoubleCAST;
 743                 }
 744               }
 745             }
 746             break;
 747           case 6 :
 748             // object string double
 749             if ((data[index] == 'o') && (data[++index] == 'b')
 750                 && (data[++index] == 'j') && (data[++index] == 'e')
 751                 && (data[++index] == 'c') && (data[++index] == 't')) {
 752               found = true;
 753               tempToken = TokenNameobjectCAST;
 754             } else {
 755               index = 0;
 756               if ((data[index] == 's') && (data[++index] == 't')
 757                   && (data[++index] == 'r') && (data[++index] == 'i')
 758                   && (data[++index] == 'n') && (data[++index] == 'g')) {
 759                 found = true;
 760                 tempToken = TokenNamestringCAST;
 761               } else {
 762                 index = 0;
 763                 if ((data[index] == 'd') && (data[++index] == 'o')
 764                     && (data[++index] == 'u') && (data[++index] == 'b')
 765                     && (data[++index] == 'l') && (data[++index] == 'e')) {
 766                   found = true;
 767                   tempToken = TokenNamedoubleCAST;
 768                 }
 769               }
 770             }
 771             break;
 772           case 7 :
 773             // boolean integer
 774             if ((data[index] == 'b') && (data[++index] == 'o')
 775                 && (data[++index] == 'o') && (data[++index] == 'l')
 776                 && (data[++index] == 'e') && (data[++index] == 'a')
 777                 && (data[++index] == 'n')) {
 778               found = true;
 779               tempToken = TokenNameboolCAST;
 780             } else {
 781               index = 0;
 782               if ((data[index] == 'i') && (data[++index] == 'n')
 783                   && (data[++index] == 't') && (data[++index] == 'e')
 784                   && (data[++index] == 'g') && (data[++index] == 'e')
 785                   && (data[++index] == 'r')) {
 786                 found = true;
 787                 tempToken = TokenNameintCAST;
 788               }
 789             }
 790             break;
 791         }
 792         if (found) {
 793           while (currentCharacter == ' ' || currentCharacter == '\t') {
 794             currentCharacter = source[currentPosition++];
 795           }
 796           if (currentCharacter == ')') {
 797             return tempToken;
 798           }
 799         }
 800       }
 801     } catch (IndexOutOfBoundsException e) {
 802     }
 803     currentCharacter = tempCharacter;
 804     currentPosition = tempPosition;
 805     return TokenNameLPAREN;
 806   }
 807   public void consumeStringInterpolated() throws InvalidInputException {
 808     try {
 809       // consume next character
 810       unicodeAsBackSlash = false;
 811       currentCharacter = source[currentPosition++];
 812       //                if (((currentCharacter = source[currentPosition++]) == '\\')
 813       //                  && (source[currentPosition] == 'u')) {
 814       //                  getNextUnicodeChar();
 815       //                } else {
 816       //                  if (withoutUnicodePtr != 0) {
 817       //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
 818       //                      currentCharacter;
 819       //                  }
 820       //                }
 821       while (currentCharacter != '`') {
 822         /** ** in PHP \r and \n are valid in string literals *** */
 823         //                if ((currentCharacter == '\n')
 824         //                  || (currentCharacter == '\r')) {
 825         //                  // relocate if finding another quote fairly close: thus unicode
 826         // '/u000D' will be fully consumed
 827         //                  for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
 828         //                    if (currentPosition + lookAhead == source.length)
 829         //                      break;
 830         //                    if (source[currentPosition + lookAhead] == '\n')
 831         //                      break;
 832         //                    if (source[currentPosition + lookAhead] == '\"') {
 833         //                      currentPosition += lookAhead + 1;
 834         //                      break;
 835         //                    }
 836         //                  }
 837         //                  throw new InvalidInputException(INVALID_CHAR_IN_STRING);
 838         //                }
 839         if (currentCharacter == '\\') {
 840           int escapeSize = currentPosition;
 841           boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
 842           //scanEscapeCharacter make a side effect on this value and we need
 843           // the previous value few lines down this one
 844           scanDoubleQuotedEscapeCharacter();
 845           escapeSize = currentPosition - escapeSize;
 846           if (withoutUnicodePtr == 0) {
 847             //buffer all the entries that have been left aside....
 848             withoutUnicodePtr = currentPosition - escapeSize - 1
 849                 - startPosition;
 850             System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
 851                 withoutUnicodePtr);
 852             withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 853           } else { //overwrite the / in the buffer
 854             withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
 855             if (backSlashAsUnicodeInString) { //there are TWO \ in the stream
 856                                               // where only one is correct
 857               withoutUnicodePtr--;
 858             }
 859           }
 860         }
 861         // consume next character
 862         unicodeAsBackSlash = false;
 863         currentCharacter = source[currentPosition++];
 864         //                  if (((currentCharacter = source[currentPosition++]) == '\\')
 865         //                    && (source[currentPosition] == 'u')) {
 866         //                    getNextUnicodeChar();
 867         //                  } else {
 868         if (withoutUnicodePtr != 0) {
 869           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 870         }
 871         //                  }
 872       }
 873     } catch (IndexOutOfBoundsException e) {
 874       throw new InvalidInputException(UNTERMINATED_STRING);
 875     } catch (InvalidInputException e) {
 876       if (e.getMessage().equals(INVALID_ESCAPE)) {
 877         // relocate if finding another quote fairly close: thus unicode
 878         // '/u000D' will be fully consumed
 879         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
 880           if (currentPosition + lookAhead == source.length)
 881             break;
 882           if (source[currentPosition + lookAhead] == '\n')
 883             break;
 884           if (source[currentPosition + lookAhead] == '`') {
 885             currentPosition += lookAhead + 1;
 886             break;
 887           }
 888         }
 889       }
 890       throw e; // rethrow
 891     }
 892     if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
 893                                               // //$NON-NLS-?$ where ? is an
 894                                               // int.
 895       if (currentLine == null) {
 896         currentLine = new NLSLine();
 897         lines.add(currentLine);
 898       }
 899       currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
 900           startPosition, currentPosition - 1));
 901     }
 902   }
 903   public void consumeStringConstant() throws InvalidInputException {
 904     try {
 905       // consume next character
 906       unicodeAsBackSlash = false;
 907       currentCharacter = source[currentPosition++];
 908       //                if (((currentCharacter = source[currentPosition++]) == '\\')
 909       //                  && (source[currentPosition] == 'u')) {
 910       //                  getNextUnicodeChar();
 911       //                } else {
 912       //                  if (withoutUnicodePtr != 0) {
 913       //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
 914       //                      currentCharacter;
 915       //                  }
 916       //                }
 917       while (currentCharacter != '\'') {
 918         /** ** in PHP \r and \n are valid in string literals *** */
 919         //                  if ((currentCharacter == '\n')
 920         //                    || (currentCharacter == '\r')) {
 921         //                    // relocate if finding another quote fairly close: thus unicode
 922         // '/u000D' will be fully consumed
 923         //                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
 924         //                      if (currentPosition + lookAhead == source.length)
 925         //                        break;
 926         //                      if (source[currentPosition + lookAhead] == '\n')
 927         //                        break;
 928         //                      if (source[currentPosition + lookAhead] == '\"') {
 929         //                        currentPosition += lookAhead + 1;
 930         //                        break;
 931         //                      }
 932         //                    }
 933         //                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
 934         //                  }
 935         if (currentCharacter == '\\') {
 936           int escapeSize = currentPosition;
 937           boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
 938           //scanEscapeCharacter make a side effect on this value and we need
 939           // the previous value few lines down this one
 940           scanSingleQuotedEscapeCharacter();
 941           escapeSize = currentPosition - escapeSize;
 942           if (withoutUnicodePtr == 0) {
 943             //buffer all the entries that have been left aside....
 944             withoutUnicodePtr = currentPosition - escapeSize - 1
 945                 - startPosition;
 946             System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
 947                 withoutUnicodePtr);
 948             withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 949           } else { //overwrite the / in the buffer
 950             withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
 951             if (backSlashAsUnicodeInString) { //there are TWO \ in the stream
 952                                               // where only one is correct
 953               withoutUnicodePtr--;
 954             }
 955           }
 956         }
 957         // consume next character
 958         unicodeAsBackSlash = false;
 959         currentCharacter = source[currentPosition++];
 960         //                  if (((currentCharacter = source[currentPosition++]) == '\\')
 961         //                    && (source[currentPosition] == 'u')) {
 962         //                    getNextUnicodeChar();
 963         //                  } else {
 964         if (withoutUnicodePtr != 0) {
 965           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 966         }
 967         //                  }
 968       }
 969     } catch (IndexOutOfBoundsException e) {
 970       throw new InvalidInputException(UNTERMINATED_STRING);
 971     } catch (InvalidInputException e) {
 972       if (e.getMessage().equals(INVALID_ESCAPE)) {
 973         // relocate if finding another quote fairly close: thus unicode
 974         // '/u000D' will be fully consumed
 975         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
 976           if (currentPosition + lookAhead == source.length)
 977             break;
 978           if (source[currentPosition + lookAhead] == '\n')
 979             break;
 980           if (source[currentPosition + lookAhead] == '\'') {
 981             currentPosition += lookAhead + 1;
 982             break;
 983           }
 984         }
 985       }
 986       throw e; // rethrow
 987     }
 988     if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
 989                                               // //$NON-NLS-?$ where ? is an
 990                                               // int.
 991       if (currentLine == null) {
 992         currentLine = new NLSLine();
 993         lines.add(currentLine);
 994       }
 995       currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
 996           startPosition, currentPosition - 1));
 997     }
 998   }
 999   public void consumeStringLiteral() throws InvalidInputException {
1000     try {
1001       // consume next character
1002       unicodeAsBackSlash = false;
1003       currentCharacter = source[currentPosition++];
1004       //                if (((currentCharacter = source[currentPosition++]) == '\\')
1005       //                  && (source[currentPosition] == 'u')) {
1006       //                  getNextUnicodeChar();
1007       //                } else {
1008       //                  if (withoutUnicodePtr != 0) {
1009       //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
1010       //                      currentCharacter;
1011       //                  }
1012       //                }
1013       while (currentCharacter != '"') {
1014         /** ** in PHP \r and \n are valid in string literals *** */
1015         //                  if ((currentCharacter == '\n')
1016         //                    || (currentCharacter == '\r')) {
1017         //                    // relocate if finding another quote fairly close: thus unicode
1018         // '/u000D' will be fully consumed
1019         //                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1020         //                      if (currentPosition + lookAhead == source.length)
1021         //                        break;
1022         //                      if (source[currentPosition + lookAhead] == '\n')
1023         //                        break;
1024         //                      if (source[currentPosition + lookAhead] == '\"') {
1025         //                        currentPosition += lookAhead + 1;
1026         //                        break;
1027         //                      }
1028         //                    }
1029         //                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1030         //                  }
1031         if (currentCharacter == '\\') {
1032           int escapeSize = currentPosition;
1033           boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1034           //scanEscapeCharacter make a side effect on this value and we need
1035           // the previous value few lines down this one
1036           scanDoubleQuotedEscapeCharacter();
1037           escapeSize = currentPosition - escapeSize;
1038           if (withoutUnicodePtr == 0) {
1039             //buffer all the entries that have been left aside....
1040             withoutUnicodePtr = currentPosition - escapeSize - 1
1041                 - startPosition;
1042             System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1,
1043                 withoutUnicodePtr);
1044             withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1045           } else { //overwrite the / in the buffer
1046             withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1047             if (backSlashAsUnicodeInString) { //there are TWO \ in the stream
1048                                               // where only one is correct
1049               withoutUnicodePtr--;
1050             }
1051           }
1052         }
1053         // consume next character
1054         unicodeAsBackSlash = false;
1055         currentCharacter = source[currentPosition++];
1056         //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1057         //                    && (source[currentPosition] == 'u')) {
1058         //                    getNextUnicodeChar();
1059         //                  } else {
1060         if (withoutUnicodePtr != 0) {
1061           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1062         }
1063         //                  }
1064       }
1065     } catch (IndexOutOfBoundsException e) {
1066       throw new InvalidInputException(UNTERMINATED_STRING);
1067     } catch (InvalidInputException e) {
1068       if (e.getMessage().equals(INVALID_ESCAPE)) {
1069         // relocate if finding another quote fairly close: thus unicode
1070         // '/u000D' will be fully consumed
1071         for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1072           if (currentPosition + lookAhead == source.length)
1073             break;
1074           if (source[currentPosition + lookAhead] == '\n')
1075             break;
1076           if (source[currentPosition + lookAhead] == '\"') {
1077             currentPosition += lookAhead + 1;
1078             break;
1079           }
1080         }
1081       }
1082       throw e; // rethrow
1083     }
1084     if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags
1085                                               // //$NON-NLS-?$ where ? is an
1086                                               // int.
1087       if (currentLine == null) {
1088         currentLine = new NLSLine();
1089         lines.add(currentLine);
1090       }
1091       currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
1092           startPosition, currentPosition - 1));
1093     }
1094   }
1095   public int getNextToken() throws InvalidInputException {
1096     if (!phpMode) {
1097       return getInlinedHTML(currentPosition);
1098     }
1099     if (phpMode) {
1100       this.wasAcr = false;
1101       if (diet) {
1102         jumpOverMethodBody();
1103         diet = false;
1104         return currentPosition > source.length ? TokenNameEOF : TokenNameRBRACE;
1105       }
1106       try {
1107         while (true) {
1108           withoutUnicodePtr = 0;
1109           //start with a new token
1110           char encapsedChar = ' ';
1111           if (!encapsedStringStack.isEmpty()) {
1112             encapsedChar = ((Character) encapsedStringStack.peek()).charValue();
1113           }
1114           if (encapsedChar != '$' && encapsedChar != ' ') {
1115             currentCharacter = source[currentPosition++];
1116             if (currentCharacter == encapsedChar) {
1117               switch (currentCharacter) {
1118                 case '`' :
1119                   return TokenNameEncapsedString0;
1120                 case '\'' :
1121                   return TokenNameEncapsedString1;
1122                 case '"' :
1123                   return TokenNameEncapsedString2;
1124               }
1125             }
1126             while (currentCharacter != encapsedChar) {
1127               /** ** in PHP \r and \n are valid in string literals *** */
1128               switch (currentCharacter) {
1129                 case '\\' :
1130                   int escapeSize = currentPosition;
1131                   boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1132                   //scanEscapeCharacter make a side effect on this value and
1133                   // we need the previous value few lines down this one
1134                   scanDoubleQuotedEscapeCharacter();
1135                   escapeSize = currentPosition - escapeSize;
1136                   if (withoutUnicodePtr == 0) {
1137                     //buffer all the entries that have been left aside....
1138                     withoutUnicodePtr = currentPosition - escapeSize - 1
1139                         - startPosition;
1140                     System.arraycopy(source, startPosition,
1141                         withoutUnicodeBuffer, 1, withoutUnicodePtr);
1142                     withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1143                   } else { //overwrite the / in the buffer
1144                     withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1145                     if (backSlashAsUnicodeInString) { //there are TWO \ in
1146                       withoutUnicodePtr--;
1147                     }
1148                   }
1149                   break;
1150                 case '$' :
1151                   if (isPHPIdentifierStart(source[currentPosition])
1152                       || source[currentPosition] == '{') {
1153                     currentPosition--;
1154                     encapsedStringStack.push(new Character('$'));
1155                     return TokenNameSTRING;
1156                   }
1157                   break;
1158                 case '{' :
1159                   if (source[currentPosition] == '$') { // CURLY_OPEN
1160                     currentPosition--;
1161                     encapsedStringStack.push(new Character('$'));
1162                     return TokenNameSTRING;
1163                   }
1164               }
1165               // consume next character
1166               unicodeAsBackSlash = false;
1167               currentCharacter = source[currentPosition++];
1168               if (withoutUnicodePtr != 0) {
1169                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1170               }
1171               //                  }
1172             } // end while
1173             currentPosition--;
1174             return TokenNameSTRING;
1175           }
1176           // ---------Consume white space and handles startPosition---------
1177           int whiteStart = currentPosition;
1178           startPosition = currentPosition;
1179           currentCharacter = source[currentPosition++];
1180           if (encapsedChar == '$') {
1181             switch (currentCharacter) {
1182               case '\\' :
1183                 currentCharacter = source[currentPosition++];
1184                 return TokenNameSTRING;
1185               case '{' :
1186                 if (encapsedChar == '$') {
1187                   if (getNextChar('$'))
1188                     return TokenNameCURLY_OPEN;
1189                 }
1190                 return TokenNameLBRACE;
1191               case '}' :
1192                 return TokenNameRBRACE;
1193               case '[' :
1194                 return TokenNameLBRACKET;
1195               case ']' :
1196                 return TokenNameRBRACKET;
1197               case '\'' :
1198                 if (tokenizeStrings) {
1199                   consumeStringConstant();
1200                   return TokenNameStringSingleQuote;
1201                 }
1202                 return TokenNameEncapsedString1;
1203               case '"' :
1204                 return TokenNameEncapsedString2;
1205               case '`' :
1206                 if (tokenizeStrings) {
1207                   consumeStringInterpolated();
1208                   return TokenNameStringInterpolated;
1209                 }
1210                 return TokenNameEncapsedString0;
1211               case '-' :
1212                 if (getNextChar('>'))
1213                   return TokenNameMINUS_GREATER;
1214                 return TokenNameSTRING;
1215               default :
1216                 if (currentCharacter == '$') {
1217                   int oldPosition = currentPosition;
1218                   try {
1219                     currentCharacter = source[currentPosition++];
1220                     if (currentCharacter == '{') {
1221                       return TokenNameDOLLAR_LBRACE;
1222                     }
1223                     if (isPHPIdentifierStart(currentCharacter)) {
1224                       return scanIdentifierOrKeyword(true);
1225                     } else {
1226                       currentPosition = oldPosition;
1227                       return TokenNameSTRING;
1228                     }
1229                   } catch (IndexOutOfBoundsException e) {
1230                     currentPosition = oldPosition;
1231                     return TokenNameSTRING;
1232                   }
1233                 }
1234                 if (isPHPIdentifierStart(currentCharacter))
1235                   return scanIdentifierOrKeyword(false);
1236                 if (Character.isDigit(currentCharacter))
1237                   return scanNumber(false);
1238                 return TokenNameERROR;
1239             }
1240           }
1241           //          boolean isWhiteSpace;
1242
1243           while ((currentCharacter == ' ')
1244               || Character.isWhitespace(currentCharacter)) {
1245             startPosition = currentPosition;
1246             currentCharacter = source[currentPosition++];
1247             //            if (((currentCharacter = source[currentPosition++]) == '\\')
1248             //              && (source[currentPosition] == 'u')) {
1249             //              isWhiteSpace = jumpOverUnicodeWhiteSpace();
1250             //            } else {
1251             if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1252               checkNonExternalizeString();
1253               if (recordLineSeparator) {
1254                 pushLineSeparator();
1255               } else {
1256                 currentLine = null;
1257               }
1258             }
1259             //            isWhiteSpace = (currentCharacter == ' ')
1260             //                || Character.isWhitespace(currentCharacter);
1261             //            }
1262           }
1263           if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
1264             // reposition scanner in case we are interested by spaces as tokens
1265             currentPosition--;
1266             startPosition = whiteStart;
1267             return TokenNameWHITESPACE;
1268           }
1269           //little trick to get out in the middle of a source compuation
1270           if (currentPosition > eofPosition)
1271             return TokenNameEOF;
1272           // ---------Identify the next token-------------
1273           switch (currentCharacter) {
1274             case '(' :
1275               return getCastOrParen();
1276             case ')' :
1277               return TokenNameRPAREN;
1278             case '{' :
1279               return TokenNameLBRACE;
1280             case '}' :
1281               return TokenNameRBRACE;
1282             case '[' :
1283               return TokenNameLBRACKET;
1284             case ']' :
1285               return TokenNameRBRACKET;
1286             case ';' :
1287               return TokenNameSEMICOLON;
1288             case ',' :
1289               return TokenNameCOMMA;
1290             case '.' :
1291               if (getNextChar('='))
1292                 return TokenNameDOT_EQUAL;
1293               if (getNextCharAsDigit())
1294                 return scanNumber(true);
1295               return TokenNameDOT;
1296             case '+' :
1297               {
1298                 int test;
1299                 if ((test = getNextChar('+', '=')) == 0)
1300                   return TokenNamePLUS_PLUS;
1301                 if (test > 0)
1302                   return TokenNamePLUS_EQUAL;
1303                 return TokenNamePLUS;
1304               }
1305             case '-' :
1306               {
1307                 int test;
1308                 if ((test = getNextChar('-', '=')) == 0)
1309                   return TokenNameMINUS_MINUS;
1310                 if (test > 0)
1311                   return TokenNameMINUS_EQUAL;
1312                 if (getNextChar('>'))
1313                   return TokenNameMINUS_GREATER;
1314                 return TokenNameMINUS;
1315               }
1316             case '~' :
1317               if (getNextChar('='))
1318                 return TokenNameTWIDDLE_EQUAL;
1319               return TokenNameTWIDDLE;
1320             case '!' :
1321               if (getNextChar('=')) {
1322                 if (getNextChar('=')) {
1323                   return TokenNameNOT_EQUAL_EQUAL;
1324                 }
1325                 return TokenNameNOT_EQUAL;
1326               }
1327               return TokenNameNOT;
1328             case '*' :
1329               if (getNextChar('='))
1330                 return TokenNameMULTIPLY_EQUAL;
1331               return TokenNameMULTIPLY;
1332             case '%' :
1333               if (getNextChar('='))
1334                 return TokenNameREMAINDER_EQUAL;
1335               return TokenNameREMAINDER;
1336             case '<' :
1337               {
1338                 int oldPosition = currentPosition;
1339                 try {
1340                   currentCharacter = source[currentPosition++];
1341                 } catch (IndexOutOfBoundsException e) {
1342                   currentPosition = oldPosition;
1343                   return TokenNameLESS;
1344                 }
1345                 switch (currentCharacter) {
1346                   case '=' :
1347                     return TokenNameLESS_EQUAL;
1348                   case '>' :
1349                     return TokenNameNOT_EQUAL;
1350                   case '<' :
1351                     if (getNextChar('='))
1352                       return TokenNameLEFT_SHIFT_EQUAL;
1353                     if (getNextChar('<')) {
1354                       currentCharacter = source[currentPosition++];
1355                       while (Character.isWhitespace(currentCharacter)) {
1356                         currentCharacter = source[currentPosition++];
1357                       }
1358                       int heredocStart = currentPosition - 1;
1359                       int heredocLength = 0;
1360                       if (isPHPIdentifierStart(currentCharacter)) {
1361                         currentCharacter = source[currentPosition++];
1362                       } else {
1363                         return TokenNameERROR;
1364                       }
1365                       while (isPHPIdentifierPart(currentCharacter)) {
1366                         currentCharacter = source[currentPosition++];
1367                       }
1368                       heredocLength = currentPosition - heredocStart - 1;
1369                       // heredoc end-tag determination
1370                       boolean endTag = true;
1371                       char ch;
1372                       do {
1373                         ch = source[currentPosition++];
1374                         if (ch == '\r' || ch == '\n') {
1375                           if (recordLineSeparator) {
1376                             pushLineSeparator();
1377                           } else {
1378                             currentLine = null;
1379                           }
1380                           for (int i = 0; i < heredocLength; i++) {
1381                             if (source[currentPosition + i] != source[heredocStart
1382                                 + i]) {
1383                               endTag = false;
1384                               break;
1385                             }
1386                           }
1387                           if (endTag) {
1388                             currentPosition += heredocLength - 1;
1389                             currentCharacter = source[currentPosition++];
1390                             break; // do...while loop
1391                           } else {
1392                             endTag = true;
1393                           }
1394                         }
1395                       } while (true);
1396                       return TokenNameHEREDOC;
1397                     }
1398                     return TokenNameLEFT_SHIFT;
1399                 }
1400                 currentPosition = oldPosition;
1401                 return TokenNameLESS;
1402               }
1403             case '>' :
1404               {
1405                 int test;
1406                 if ((test = getNextChar('=', '>')) == 0)
1407                   return TokenNameGREATER_EQUAL;
1408                 if (test > 0) {
1409                   if ((test = getNextChar('=', '>')) == 0)
1410                     return TokenNameRIGHT_SHIFT_EQUAL;
1411                   return TokenNameRIGHT_SHIFT;
1412                 }
1413                 return TokenNameGREATER;
1414               }
1415             case '=' :
1416               if (getNextChar('=')) {
1417                 if (getNextChar('=')) {
1418                   return TokenNameEQUAL_EQUAL_EQUAL;
1419                 }
1420                 return TokenNameEQUAL_EQUAL;
1421               }
1422               if (getNextChar('>'))
1423                 return TokenNameEQUAL_GREATER;
1424               return TokenNameEQUAL;
1425             case '&' :
1426               {
1427                 int test;
1428                 if ((test = getNextChar('&', '=')) == 0)
1429                   return TokenNameAND_AND;
1430                 if (test > 0)
1431                   return TokenNameAND_EQUAL;
1432                 return TokenNameAND;
1433               }
1434             case '|' :
1435               {
1436                 int test;
1437                 if ((test = getNextChar('|', '=')) == 0)
1438                   return TokenNameOR_OR;
1439                 if (test > 0)
1440                   return TokenNameOR_EQUAL;
1441                 return TokenNameOR;
1442               }
1443             case '^' :
1444               if (getNextChar('='))
1445                 return TokenNameXOR_EQUAL;
1446               return TokenNameXOR;
1447             case '?' :
1448               if (getNextChar('>')) {
1449                 phpMode = false;
1450                 if (currentPosition == source.length) {
1451                   phpMode = true;
1452                   return TokenNameINLINE_HTML;
1453                 }
1454                 return getInlinedHTML(currentPosition - 2);
1455               }
1456               return TokenNameQUESTION;
1457             case ':' :
1458               if (getNextChar(':'))
1459                 return TokenNamePAAMAYIM_NEKUDOTAYIM;
1460               return TokenNameCOLON;
1461             case '@' :
1462               return TokenNameAT;
1463             case '\'' :
1464               consumeStringConstant();
1465               return TokenNameStringSingleQuote;
1466             case '"' :
1467               if (tokenizeStrings) {
1468                 consumeStringLiteral();
1469                 return TokenNameStringDoubleQuote;
1470               }
1471               return TokenNameEncapsedString2;
1472             case '`' :
1473               if (tokenizeStrings) {
1474                 consumeStringInterpolated();
1475                 return TokenNameStringInterpolated;
1476               }
1477               return TokenNameEncapsedString0;
1478             case '#' :
1479             case '/' :
1480               {
1481                 char startChar = currentCharacter;
1482                 if (getNextChar('=')) {
1483                   return TokenNameDIVIDE_EQUAL;
1484                 }
1485                 int test;
1486                 if ((startChar == '#') || (test = getNextChar('/', '*')) == 0) {
1487                   //line comment
1488                         this.lastCommentLinePosition = this.currentPosition;
1489                   int endPositionForLineComment = 0;
1490                   try { //get the next char
1491                     currentCharacter = source[currentPosition++];
1492                     //                    if (((currentCharacter = source[currentPosition++])
1493                     //                      == '\\')
1494                     //                      && (source[currentPosition] == 'u')) {
1495                     //                      //-------------unicode traitement ------------
1496                     //                      int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1497                     //                      currentPosition++;
1498                     //                      while (source[currentPosition] == 'u') {
1499                     //                        currentPosition++;
1500                     //                      }
1501                     //                      if ((c1 =
1502                     //                        Character.getNumericValue(source[currentPosition++]))
1503                     //                        > 15
1504                     //                        || c1 < 0
1505                     //                        || (c2 =
1506                     //                          Character.getNumericValue(source[currentPosition++]))
1507                     //                          > 15
1508                     //                        || c2 < 0
1509                     //                        || (c3 =
1510                     //                          Character.getNumericValue(source[currentPosition++]))
1511                     //                          > 15
1512                     //                        || c3 < 0
1513                     //                        || (c4 =
1514                     //                          Character.getNumericValue(source[currentPosition++]))
1515                     //                          > 15
1516                     //                        || c4 < 0) {
1517                     //                        throw new
1518                     // InvalidInputException(INVALID_UNICODE_ESCAPE);
1519                     //                      } else {
1520                     //                        currentCharacter =
1521                     //                          (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1522                     //                      }
1523                     //                    }
1524                     //handle the \\u case manually into comment
1525                     //                    if (currentCharacter == '\\') {
1526                     //                      if (source[currentPosition] == '\\')
1527                     //                        currentPosition++;
1528                     //                    } //jump over the \\
1529                     boolean isUnicode = false;
1530                     while (currentCharacter != '\r' && currentCharacter != '\n') {
1531                         this.lastCommentLinePosition = this.currentPosition;
1532                       if (currentCharacter == '?') {
1533                         if (getNextChar('>')) {
1534                           startPosition = currentPosition - 2;
1535                           phpMode = false;
1536                           return TokenNameINLINE_HTML;
1537                         }
1538                       }
1539                       //get the next char
1540                       isUnicode = false;
1541                       currentCharacter = source[currentPosition++];
1542                       //                      if (((currentCharacter = source[currentPosition++])
1543                       //                        == '\\')
1544                       //                        && (source[currentPosition] == 'u')) {
1545                       //                        isUnicode = true;
1546                       //                        //-------------unicode traitement ------------
1547                       //                        int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1548                       //                        currentPosition++;
1549                       //                        while (source[currentPosition] == 'u') {
1550                       //                          currentPosition++;
1551                       //                        }
1552                       //                        if ((c1 =
1553                       //                          Character.getNumericValue(source[currentPosition++]))
1554                       //                          > 15
1555                       //                          || c1 < 0
1556                       //                          || (c2 =
1557                       //                            Character.getNumericValue(
1558                       //                              source[currentPosition++]))
1559                       //                            > 15
1560                       //                          || c2 < 0
1561                       //                          || (c3 =
1562                       //                            Character.getNumericValue(
1563                       //                              source[currentPosition++]))
1564                       //                            > 15
1565                       //                          || c3 < 0
1566                       //                          || (c4 =
1567                       //                            Character.getNumericValue(
1568                       //                              source[currentPosition++]))
1569                       //                            > 15
1570                       //                          || c4 < 0) {
1571                       //                          throw new
1572                       // InvalidInputException(INVALID_UNICODE_ESCAPE);
1573                       //                        } else {
1574                       //                          currentCharacter =
1575                       //                            (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1576                       //                        }
1577                       //                      }
1578                       //handle the \\u case manually into comment
1579                       //                      if (currentCharacter == '\\') {
1580                       //                        if (source[currentPosition] == '\\')
1581                       //                          currentPosition++;
1582                       //                      } //jump over the \\
1583                     }
1584                     if (isUnicode) {
1585                       endPositionForLineComment = currentPosition - 6;
1586                     } else {
1587                       endPositionForLineComment = currentPosition - 1;
1588                     }
1589 //                    recordComment(false);
1590                     recordComment(TokenNameCOMMENT_LINE);
1591                     if (this.taskTags != null) checkTaskTag(this.startPosition, this.currentPosition);
1592                                         if ((currentCharacter == '\r')
1593                         || (currentCharacter == '\n')) {
1594                       checkNonExternalizeString();
1595                       if (recordLineSeparator) {
1596                         if (isUnicode) {
1597                           pushUnicodeLineSeparator();
1598                         } else {
1599                           pushLineSeparator();
1600                         }
1601                       } else {
1602                         currentLine = null;
1603                       }
1604                     }
1605                     if (tokenizeComments) {
1606                       if (!isUnicode) {
1607                         currentPosition = endPositionForLineComment;
1608                         // reset one character behind
1609                       }
1610                       return TokenNameCOMMENT_LINE;
1611                     }
1612                   } catch (IndexOutOfBoundsException e) { //an eof will them
1613                     // be generated
1614                     if (tokenizeComments) {
1615                       currentPosition--;
1616                       // reset one character behind
1617                       return TokenNameCOMMENT_LINE;
1618                     }
1619                   }
1620                   break;
1621                 }
1622                 if (test > 0) {
1623                   //traditional and annotation comment
1624                   boolean isJavadoc = false, star = false;
1625                   // consume next character
1626                   unicodeAsBackSlash = false;
1627                   currentCharacter = source[currentPosition++];
1628                   //                  if (((currentCharacter = source[currentPosition++]) ==
1629                   // '\\')
1630                   //                    && (source[currentPosition] == 'u')) {
1631                   //                    getNextUnicodeChar();
1632                   //                  } else {
1633                   //                    if (withoutUnicodePtr != 0) {
1634                   //                      withoutUnicodeBuffer[++withoutUnicodePtr] =
1635                   //                        currentCharacter;
1636                   //                    }
1637                   //                  }
1638                   if (currentCharacter == '*') {
1639                     isJavadoc = true;
1640                     star = true;
1641                   }
1642                   if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1643                     checkNonExternalizeString();
1644                     if (recordLineSeparator) {
1645                       pushLineSeparator();
1646                     } else {
1647                       currentLine = null;
1648                     }
1649                   }
1650                   try { //get the next char
1651                     currentCharacter = source[currentPosition++];
1652                     //                    if (((currentCharacter = source[currentPosition++])
1653                     //                      == '\\')
1654                     //                      && (source[currentPosition] == 'u')) {
1655                     //                      //-------------unicode traitement ------------
1656                     //                      getNextUnicodeChar();
1657                     //                    }
1658                     //handle the \\u case manually into comment
1659                     //                    if (currentCharacter == '\\') {
1660                     //                      if (source[currentPosition] == '\\')
1661                     //                        currentPosition++;
1662                     //                      //jump over the \\
1663                     //                    }
1664                     // empty comment is not a javadoc /**/
1665                     if (currentCharacter == '/') {
1666                       isJavadoc = false;
1667                     }
1668                     //loop until end of comment */
1669                     while ((currentCharacter != '/') || (!star)) {
1670                       if ((currentCharacter == '\r')
1671                           || (currentCharacter == '\n')) {
1672                         checkNonExternalizeString();
1673                         if (recordLineSeparator) {
1674                           pushLineSeparator();
1675                         } else {
1676                           currentLine = null;
1677                         }
1678                       }
1679                       star = currentCharacter == '*';
1680                       //get next char
1681                       currentCharacter = source[currentPosition++];
1682                       //                      if (((currentCharacter = source[currentPosition++])
1683                       //                        == '\\')
1684                       //                        && (source[currentPosition] == 'u')) {
1685                       //                        //-------------unicode traitement ------------
1686                       //                        getNextUnicodeChar();
1687                       //                      }
1688                       //handle the \\u case manually into comment
1689                       //                      if (currentCharacter == '\\') {
1690                       //                        if (source[currentPosition] == '\\')
1691                       //                          currentPosition++;
1692                       //                      } //jump over the \\
1693                     }
1694                     //recordComment(isJavadoc);
1695                     if (isJavadoc) {
1696                         recordComment(TokenNameCOMMENT_PHPDOC);
1697                     } else {
1698                         recordComment(TokenNameCOMMENT_BLOCK);
1699                     }
1700
1701                     if (tokenizeComments) {
1702                       if (isJavadoc)
1703                         return TokenNameCOMMENT_PHPDOC;
1704                       return TokenNameCOMMENT_BLOCK;
1705                     }
1706                   } catch (IndexOutOfBoundsException e) {
1707                     throw new InvalidInputException(UNTERMINATED_COMMENT);
1708                   }
1709                   break;
1710                 }
1711                 return TokenNameDIVIDE;
1712               }
1713             case '\u001a' :
1714               if (atEnd())
1715                 return TokenNameEOF;
1716               //the atEnd may not be <currentPosition == source.length> if
1717               // source is only some part of a real (external) stream
1718               throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1719             default :
1720               if (currentCharacter == '$') {
1721                 int oldPosition = currentPosition;
1722                 try {
1723                   currentCharacter = source[currentPosition++];
1724                   if (isPHPIdentifierStart(currentCharacter)) {
1725                     return scanIdentifierOrKeyword(true);
1726                   } else {
1727                     currentPosition = oldPosition;
1728                     return TokenNameDOLLAR;
1729                   }
1730                 } catch (IndexOutOfBoundsException e) {
1731                   currentPosition = oldPosition;
1732                   return TokenNameDOLLAR;
1733                 }
1734               }
1735               if (isPHPIdentifierStart(currentCharacter))
1736                 return scanIdentifierOrKeyword(false);
1737               if (Character.isDigit(currentCharacter))
1738                 return scanNumber(false);
1739               return TokenNameERROR;
1740           }
1741         }
1742       } //-----------------end switch while try--------------------
1743       catch (IndexOutOfBoundsException e) {
1744       }
1745     }
1746     return TokenNameEOF;
1747   }
1748
1749   private int getInlinedHTML(int start) throws InvalidInputException {
1750         int token = getInlinedHTMLToken(start);
1751         if (token == TokenNameINLINE_HTML) {
1752 //              Stack stack = new Stack();
1753 //              // scan html for errors
1754 //              Source inlinedHTMLSource = new Source(new String(source, startPosition, currentPosition - startPosition));
1755 //              int lastPHPEndPos=0;
1756 //              for (Iterator i=inlinedHTMLSource.getNextTagIterator(0); i.hasNext();) {
1757 //                  Tag tag=(Tag)i.next();
1758 //
1759 //                  if (tag instanceof StartTag) {
1760 //                      StartTag startTag=(StartTag)tag;
1761 //                    //  System.out.println("startTag: "+tag);
1762 //                      if (startTag.isServerTag()) {
1763 //                        // TODO : what to do with a server tag ?
1764 //                      } else {
1765 //                          // do whatever with HTML start tag
1766 //                          // use startTag.getElement() to find the element corresponding
1767 //                          // to this start tag which may be useful if you implement code
1768 //                          // folding etc
1769 //                              stack.push(startTag);
1770 //                      }
1771 //                  } else {
1772 //                      EndTag endTag=(EndTag)tag;
1773 //                      StartTag stag = (StartTag) stack.peek();
1774 ////                  System.out.println("endTag: "+tag);
1775 //                      // do whatever with HTML end tag.
1776 //                  }
1777 //              }
1778         }
1779         return token;
1780   }
1781   /**
1782    * @return @throws
1783    *         InvalidInputException
1784    */
1785   private int getInlinedHTMLToken(int start) throws InvalidInputException {
1786     //    int htmlPosition = start;
1787     if (currentPosition > source.length) {
1788       currentPosition = source.length;
1789       return TokenNameEOF;
1790     }
1791     startPosition = start;
1792     try {
1793       while (!phpMode) {
1794         currentCharacter = source[currentPosition++];
1795         if (currentCharacter == '<') {
1796           if (getNextChar('?')) {
1797             currentCharacter = source[currentPosition++];
1798             if ((currentCharacter == ' ')
1799                 || Character.isWhitespace(currentCharacter)) {
1800               // <?
1801               phpMode = true;
1802               return TokenNameINLINE_HTML;
1803             } else {
1804               boolean phpStart = (currentCharacter == 'P')
1805                   || (currentCharacter == 'p');
1806               if (phpStart) {
1807                 int test = getNextChar('H', 'h');
1808                 if (test >= 0) {
1809                   test = getNextChar('P', 'p');
1810                   if (test >= 0) {
1811                     // <?PHP <?php
1812                     phpMode = true;
1813                     return TokenNameINLINE_HTML;
1814                   }
1815                 }
1816               }
1817             }
1818           }
1819         }
1820         if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1821           if (recordLineSeparator) {
1822             pushLineSeparator();
1823           } else {
1824             currentLine = null;
1825           }
1826         }
1827       } //-----------------while--------------------
1828       phpMode = true;
1829       return TokenNameINLINE_HTML;
1830     } //-----------------try--------------------
1831     catch (IndexOutOfBoundsException e) {
1832       startPosition = start;
1833       currentPosition--;
1834     }
1835     phpMode = true;
1836     return TokenNameINLINE_HTML;
1837   }
1838   //  public final void getNextUnicodeChar()
1839   //    throws IndexOutOfBoundsException, InvalidInputException {
1840   //    //VOID
1841   //    //handle the case of unicode.
1842   //    //when a unicode appears then we must use a buffer that holds char
1843   // internal values
1844   //    //At the end of this method currentCharacter holds the new visited char
1845   //    //and currentPosition points right next after it
1846   //
1847   //    //ALL getNextChar.... ARE OPTIMIZED COPIES
1848   //
1849   //    int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
1850   //    currentPosition++;
1851   //    while (source[currentPosition] == 'u') {
1852   //      currentPosition++;
1853   //      unicodeSize++;
1854   //    }
1855   //
1856   //    if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
1857   //      || c1 < 0
1858   //      || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
1859   //      || c2 < 0
1860   //      || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
1861   //      || c3 < 0
1862   //      || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
1863   //      || c4 < 0) {
1864   //      throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1865   //    } else {
1866   //      currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1867   //      //need the unicode buffer
1868   //      if (withoutUnicodePtr == 0) {
1869   //        //buffer all the entries that have been left aside....
1870   //        withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
1871   //        System.arraycopy(
1872   //          source,
1873   //          startPosition,
1874   //          withoutUnicodeBuffer,
1875   //          1,
1876   //          withoutUnicodePtr);
1877   //      }
1878   //      //fill the buffer with the char
1879   //      withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1880   //    }
1881   //    unicodeAsBackSlash = currentCharacter == '\\';
1882   //  }
1883   /*
1884    * Tokenize a method body, assuming that curly brackets are properly
1885    * balanced.
1886    */
1887   public final void jumpOverMethodBody() {
1888     this.wasAcr = false;
1889     int found = 1;
1890     try {
1891       while (true) { //loop for jumping over comments
1892         // ---------Consume white space and handles startPosition---------
1893         boolean isWhiteSpace;
1894         do {
1895           startPosition = currentPosition;
1896           currentCharacter = source[currentPosition++];
1897           //          if (((currentCharacter = source[currentPosition++]) == '\\')
1898           //            && (source[currentPosition] == 'u')) {
1899           //            isWhiteSpace = jumpOverUnicodeWhiteSpace();
1900           //          } else {
1901           if (recordLineSeparator
1902               && ((currentCharacter == '\r') || (currentCharacter == '\n')))
1903             pushLineSeparator();
1904           isWhiteSpace = Character.isWhitespace(currentCharacter);
1905           //          }
1906         } while (isWhiteSpace);
1907         // -------consume token until } is found---------
1908         switch (currentCharacter) {
1909           case '{' :
1910             found++;
1911             break;
1912           case '}' :
1913             found--;
1914             if (found == 0)
1915               return;
1916             break;
1917           case '\'' :
1918             {
1919               boolean test;
1920               test = getNextChar('\\');
1921               if (test) {
1922                 try {
1923                   scanDoubleQuotedEscapeCharacter();
1924                 } catch (InvalidInputException ex) {
1925                 };
1926               } else {
1927                 //                try { // consume next character
1928                 unicodeAsBackSlash = false;
1929                 currentCharacter = source[currentPosition++];
1930                 //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1931                 //                    && (source[currentPosition] == 'u')) {
1932                 //                    getNextUnicodeChar();
1933                 //                  } else {
1934                 if (withoutUnicodePtr != 0) {
1935                   withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1936                 }
1937                 //                  }
1938                 //                } catch (InvalidInputException ex) {
1939                 //                };
1940               }
1941               getNextChar('\'');
1942               break;
1943             }
1944           case '"' :
1945             try {
1946               //              try { // consume next character
1947               unicodeAsBackSlash = false;
1948               currentCharacter = source[currentPosition++];
1949               //                if (((currentCharacter = source[currentPosition++]) == '\\')
1950               //                  && (source[currentPosition] == 'u')) {
1951               //                  getNextUnicodeChar();
1952               //                } else {
1953               if (withoutUnicodePtr != 0) {
1954                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1955               }
1956               //                }
1957               //              } catch (InvalidInputException ex) {
1958               //              };
1959               while (currentCharacter != '"') {
1960                 if (currentCharacter == '\r') {
1961                   if (source[currentPosition] == '\n')
1962                     currentPosition++;
1963                   break;
1964                   // the string cannot go further that the line
1965                 }
1966                 if (currentCharacter == '\n') {
1967                   break;
1968                   // the string cannot go further that the line
1969                 }
1970                 if (currentCharacter == '\\') {
1971                   try {
1972                     scanDoubleQuotedEscapeCharacter();
1973                   } catch (InvalidInputException ex) {
1974                   };
1975                 }
1976                 //                try { // consume next character
1977                 unicodeAsBackSlash = false;
1978                 currentCharacter = source[currentPosition++];
1979                 //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1980                 //                    && (source[currentPosition] == 'u')) {
1981                 //                    getNextUnicodeChar();
1982                 //                  } else {
1983                 if (withoutUnicodePtr != 0) {
1984                   withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1985                 }
1986                 //                  }
1987                 //                } catch (InvalidInputException ex) {
1988                 //                };
1989               }
1990             } catch (IndexOutOfBoundsException e) {
1991               return;
1992             }
1993             break;
1994           case '/' :
1995             {
1996               int test;
1997               if ((test = getNextChar('/', '*')) == 0) {
1998                 //line comment
1999                 try {
2000                   //get the next char
2001                   currentCharacter = source[currentPosition++];
2002                   //                  if (((currentCharacter = source[currentPosition++]) ==
2003                   // '\\')
2004                   //                    && (source[currentPosition] == 'u')) {
2005                   //                    //-------------unicode traitement ------------
2006                   //                    int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2007                   //                    currentPosition++;
2008                   //                    while (source[currentPosition] == 'u') {
2009                   //                      currentPosition++;
2010                   //                    }
2011                   //                    if ((c1 =
2012                   //                      Character.getNumericValue(source[currentPosition++]))
2013                   //                      > 15
2014                   //                      || c1 < 0
2015                   //                      || (c2 =
2016                   //                        Character.getNumericValue(source[currentPosition++]))
2017                   //                        > 15
2018                   //                      || c2 < 0
2019                   //                      || (c3 =
2020                   //                        Character.getNumericValue(source[currentPosition++]))
2021                   //                        > 15
2022                   //                      || c3 < 0
2023                   //                      || (c4 =
2024                   //                        Character.getNumericValue(source[currentPosition++]))
2025                   //                        > 15
2026                   //                      || c4 < 0) {
2027                   //                      //error don't care of the value
2028                   //                      currentCharacter = 'A';
2029                   //                    } //something different from \n and \r
2030                   //                    else {
2031                   //                      currentCharacter =
2032                   //                        (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2033                   //                    }
2034                   //                  }
2035                   while (currentCharacter != '\r' && currentCharacter != '\n') {
2036                     //get the next char
2037                     currentCharacter = source[currentPosition++];
2038                     //                    if (((currentCharacter = source[currentPosition++])
2039                     //                      == '\\')
2040                     //                      && (source[currentPosition] == 'u')) {
2041                     //                      //-------------unicode traitement ------------
2042                     //                      int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2043                     //                      currentPosition++;
2044                     //                      while (source[currentPosition] == 'u') {
2045                     //                        currentPosition++;
2046                     //                      }
2047                     //                      if ((c1 =
2048                     //                        Character.getNumericValue(source[currentPosition++]))
2049                     //                        > 15
2050                     //                        || c1 < 0
2051                     //                        || (c2 =
2052                     //                          Character.getNumericValue(source[currentPosition++]))
2053                     //                          > 15
2054                     //                        || c2 < 0
2055                     //                        || (c3 =
2056                     //                          Character.getNumericValue(source[currentPosition++]))
2057                     //                          > 15
2058                     //                        || c3 < 0
2059                     //                        || (c4 =
2060                     //                          Character.getNumericValue(source[currentPosition++]))
2061                     //                          > 15
2062                     //                        || c4 < 0) {
2063                     //                        //error don't care of the value
2064                     //                        currentCharacter = 'A';
2065                     //                      } //something different from \n and \r
2066                     //                      else {
2067                     //                        currentCharacter =
2068                     //                          (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2069                     //                      }
2070                     //                    }
2071                   }
2072                   if (recordLineSeparator
2073                       && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2074                     pushLineSeparator();
2075                 } catch (IndexOutOfBoundsException e) {
2076                 } //an eof will them be generated
2077                 break;
2078               }
2079               if (test > 0) {
2080                 //traditional and annotation comment
2081                 boolean star = false;
2082                 //                try { // consume next character
2083                 unicodeAsBackSlash = false;
2084                 currentCharacter = source[currentPosition++];
2085                 //                  if (((currentCharacter = source[currentPosition++]) == '\\')
2086                 //                    && (source[currentPosition] == 'u')) {
2087                 //                    getNextUnicodeChar();
2088                 //                  } else {
2089                 if (withoutUnicodePtr != 0) {
2090                   withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2091                 }
2092                 //                  };
2093                 //                } catch (InvalidInputException ex) {
2094                 //                };
2095                 if (currentCharacter == '*') {
2096                   star = true;
2097                 }
2098                 if (recordLineSeparator
2099                     && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2100                   pushLineSeparator();
2101                 try { //get the next char
2102                   currentCharacter = source[currentPosition++];
2103                   //                  if (((currentCharacter = source[currentPosition++]) ==
2104                   // '\\')
2105                   //                    && (source[currentPosition] == 'u')) {
2106                   //                    //-------------unicode traitement ------------
2107                   //                    int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2108                   //                    currentPosition++;
2109                   //                    while (source[currentPosition] == 'u') {
2110                   //                      currentPosition++;
2111                   //                    }
2112                   //                    if ((c1 =
2113                   //                      Character.getNumericValue(source[currentPosition++]))
2114                   //                      > 15
2115                   //                      || c1 < 0
2116                   //                      || (c2 =
2117                   //                        Character.getNumericValue(source[currentPosition++]))
2118                   //                        > 15
2119                   //                      || c2 < 0
2120                   //                      || (c3 =
2121                   //                        Character.getNumericValue(source[currentPosition++]))
2122                   //                        > 15
2123                   //                      || c3 < 0
2124                   //                      || (c4 =
2125                   //                        Character.getNumericValue(source[currentPosition++]))
2126                   //                        > 15
2127                   //                      || c4 < 0) {
2128                   //                      //error don't care of the value
2129                   //                      currentCharacter = 'A';
2130                   //                    } //something different from * and /
2131                   //                    else {
2132                   //                      currentCharacter =
2133                   //                        (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2134                   //                    }
2135                   //                  }
2136                   //loop until end of comment */
2137                   while ((currentCharacter != '/') || (!star)) {
2138                     if (recordLineSeparator
2139                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2140                       pushLineSeparator();
2141                     star = currentCharacter == '*';
2142                     //get next char
2143                     currentCharacter = source[currentPosition++];
2144                     //                    if (((currentCharacter = source[currentPosition++])
2145                     //                      == '\\')
2146                     //                      && (source[currentPosition] == 'u')) {
2147                     //                      //-------------unicode traitement ------------
2148                     //                      int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2149                     //                      currentPosition++;
2150                     //                      while (source[currentPosition] == 'u') {
2151                     //                        currentPosition++;
2152                     //                      }
2153                     //                      if ((c1 =
2154                     //                        Character.getNumericValue(source[currentPosition++]))
2155                     //                        > 15
2156                     //                        || c1 < 0
2157                     //                        || (c2 =
2158                     //                          Character.getNumericValue(source[currentPosition++]))
2159                     //                          > 15
2160                     //                        || c2 < 0
2161                     //                        || (c3 =
2162                     //                          Character.getNumericValue(source[currentPosition++]))
2163                     //                          > 15
2164                     //                        || c3 < 0
2165                     //                        || (c4 =
2166                     //                          Character.getNumericValue(source[currentPosition++]))
2167                     //                          > 15
2168                     //                        || c4 < 0) {
2169                     //                        //error don't care of the value
2170                     //                        currentCharacter = 'A';
2171                     //                      } //something different from * and /
2172                     //                      else {
2173                     //                        currentCharacter =
2174                     //                          (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2175                     //                      }
2176                     //                    }
2177                   }
2178                 } catch (IndexOutOfBoundsException e) {
2179                   return;
2180                 }
2181                 break;
2182               }
2183               break;
2184             }
2185           default :
2186             if (isPHPIdentifierStart(currentCharacter)
2187                 || currentCharacter == '$') {
2188               try {
2189                 scanIdentifierOrKeyword((currentCharacter == '$'));
2190               } catch (InvalidInputException ex) {
2191               };
2192               break;
2193             }
2194             if (Character.isDigit(currentCharacter)) {
2195               try {
2196                 scanNumber(false);
2197               } catch (InvalidInputException ex) {
2198               };
2199               break;
2200             }
2201         }
2202       }
2203       //-----------------end switch while try--------------------
2204     } catch (IndexOutOfBoundsException e) {
2205     } catch (InvalidInputException e) {
2206     }
2207     return;
2208   }
2209   //  public final boolean jumpOverUnicodeWhiteSpace()
2210   //    throws InvalidInputException {
2211   //    //BOOLEAN
2212   //    //handle the case of unicode. Jump over the next whiteSpace
2213   //    //making startPosition pointing on the next available char
2214   //    //On false, the currentCharacter is filled up with a potential
2215   //    //correct char
2216   //
2217   //    try {
2218   //      this.wasAcr = false;
2219   //      int c1, c2, c3, c4;
2220   //      int unicodeSize = 6;
2221   //      currentPosition++;
2222   //      while (source[currentPosition] == 'u') {
2223   //        currentPosition++;
2224   //        unicodeSize++;
2225   //      }
2226   //
2227   //      if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
2228   //        || c1 < 0)
2229   //        || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
2230   //          || c2 < 0)
2231   //        || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
2232   //          || c3 < 0)
2233   //        || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
2234   //          || c4 < 0)) {
2235   //        throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2236   //      }
2237   //
2238   //      currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2239   //      if (recordLineSeparator
2240   //        && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2241   //        pushLineSeparator();
2242   //      if (Character.isWhitespace(currentCharacter))
2243   //        return true;
2244   //
2245   //      //buffer the new char which is not a white space
2246   //      withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2247   //      //withoutUnicodePtr == 1 is true here
2248   //      return false;
2249   //    } catch (IndexOutOfBoundsException e) {
2250   //      throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2251   //    }
2252   //  }
2253   public final int[] getLineEnds() {
2254     //return a bounded copy of this.lineEnds
2255     int[] copy;
2256     System.arraycopy(lineEnds, 0, copy = new int[linePtr + 1], 0, linePtr + 1);
2257     return copy;
2258   }
2259   public char[] getSource() {
2260     return this.source;
2261   }
2262   final char[] optimizedCurrentTokenSource1() {
2263     //return always the same char[] build only once
2264     //optimization at no speed cost of 99.5 % of the singleCharIdentifier
2265     char charOne = source[startPosition];
2266     switch (charOne) {
2267       case 'a' :
2268         return charArray_a;
2269       case 'b' :
2270         return charArray_b;
2271       case 'c' :
2272         return charArray_c;
2273       case 'd' :
2274         return charArray_d;
2275       case 'e' :
2276         return charArray_e;
2277       case 'f' :
2278         return charArray_f;
2279       case 'g' :
2280         return charArray_g;
2281       case 'h' :
2282         return charArray_h;
2283       case 'i' :
2284         return charArray_i;
2285       case 'j' :
2286         return charArray_j;
2287       case 'k' :
2288         return charArray_k;
2289       case 'l' :
2290         return charArray_l;
2291       case 'm' :
2292         return charArray_m;
2293       case 'n' :
2294         return charArray_n;
2295       case 'o' :
2296         return charArray_o;
2297       case 'p' :
2298         return charArray_p;
2299       case 'q' :
2300         return charArray_q;
2301       case 'r' :
2302         return charArray_r;
2303       case 's' :
2304         return charArray_s;
2305       case 't' :
2306         return charArray_t;
2307       case 'u' :
2308         return charArray_u;
2309       case 'v' :
2310         return charArray_v;
2311       case 'w' :
2312         return charArray_w;
2313       case 'x' :
2314         return charArray_x;
2315       case 'y' :
2316         return charArray_y;
2317       case 'z' :
2318         return charArray_z;
2319       default :
2320         return new char[]{charOne};
2321     }
2322   }
2323   final char[] optimizedCurrentTokenSource2() {
2324     //try to return the same char[] build only once
2325     char c0, c1;
2326     int hash = (((c0 = source[startPosition]) << 6) + (c1 = source[startPosition + 1]))
2327         % TableSize;
2328     char[][] table = charArray_length[0][hash];
2329     int i = newEntry2;
2330     while (++i < InternalTableSize) {
2331       char[] charArray = table[i];
2332       if ((c0 == charArray[0]) && (c1 == charArray[1]))
2333         return charArray;
2334     }
2335     //---------other side---------
2336     i = -1;
2337     int max = newEntry2;
2338     while (++i <= max) {
2339       char[] charArray = table[i];
2340       if ((c0 == charArray[0]) && (c1 == charArray[1]))
2341         return charArray;
2342     }
2343     //--------add the entry-------
2344     if (++max >= InternalTableSize)
2345       max = 0;
2346     char[] r;
2347     table[max] = (r = new char[]{c0, c1});
2348     newEntry2 = max;
2349     return r;
2350   }
2351   final char[] optimizedCurrentTokenSource3() {
2352     //try to return the same char[] build only once
2353     char c0, c1, c2;
2354     int hash = (((c0 = source[startPosition]) << 12)
2355         + ((c1 = source[startPosition + 1]) << 6) + (c2 = source[startPosition + 2]))
2356         % TableSize;
2357     char[][] table = charArray_length[1][hash];
2358     int i = newEntry3;
2359     while (++i < InternalTableSize) {
2360       char[] charArray = table[i];
2361       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2362         return charArray;
2363     }
2364     //---------other side---------
2365     i = -1;
2366     int max = newEntry3;
2367     while (++i <= max) {
2368       char[] charArray = table[i];
2369       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2370         return charArray;
2371     }
2372     //--------add the entry-------
2373     if (++max >= InternalTableSize)
2374       max = 0;
2375     char[] r;
2376     table[max] = (r = new char[]{c0, c1, c2});
2377     newEntry3 = max;
2378     return r;
2379   }
2380   final char[] optimizedCurrentTokenSource4() {
2381     //try to return the same char[] build only once
2382     char c0, c1, c2, c3;
2383     long hash = ((((long) (c0 = source[startPosition])) << 18)
2384         + ((c1 = source[startPosition + 1]) << 12)
2385         + ((c2 = source[startPosition + 2]) << 6) + (c3 = source[startPosition + 3]))
2386         % TableSize;
2387     char[][] table = charArray_length[2][(int) hash];
2388     int i = newEntry4;
2389     while (++i < InternalTableSize) {
2390       char[] charArray = table[i];
2391       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])
2392           && (c3 == charArray[3]))
2393         return charArray;
2394     }
2395     //---------other side---------
2396     i = -1;
2397     int max = newEntry4;
2398     while (++i <= max) {
2399       char[] charArray = table[i];
2400       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])
2401           && (c3 == charArray[3]))
2402         return charArray;
2403     }
2404     //--------add the entry-------
2405     if (++max >= InternalTableSize)
2406       max = 0;
2407     char[] r;
2408     table[max] = (r = new char[]{c0, c1, c2, c3});
2409     newEntry4 = max;
2410     return r;
2411   }
2412   final char[] optimizedCurrentTokenSource5() {
2413     //try to return the same char[] build only once
2414     char c0, c1, c2, c3, c4;
2415     long hash = ((((long) (c0 = source[startPosition])) << 24)
2416         + (((long) (c1 = source[startPosition + 1])) << 18)
2417         + ((c2 = source[startPosition + 2]) << 12)
2418         + ((c3 = source[startPosition + 3]) << 6) + (c4 = source[startPosition + 4]))
2419         % TableSize;
2420     char[][] table = charArray_length[3][(int) hash];
2421     int i = newEntry5;
2422     while (++i < InternalTableSize) {
2423       char[] charArray = table[i];
2424       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])
2425           && (c3 == charArray[3]) && (c4 == charArray[4]))
2426         return charArray;
2427     }
2428     //---------other side---------
2429     i = -1;
2430     int max = newEntry5;
2431     while (++i <= max) {
2432       char[] charArray = table[i];
2433       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])
2434           && (c3 == charArray[3]) && (c4 == charArray[4]))
2435         return charArray;
2436     }
2437     //--------add the entry-------
2438     if (++max >= InternalTableSize)
2439       max = 0;
2440     char[] r;
2441     table[max] = (r = new char[]{c0, c1, c2, c3, c4});
2442     newEntry5 = max;
2443     return r;
2444   }
2445   final char[] optimizedCurrentTokenSource6() {
2446     //try to return the same char[] build only once
2447     char c0, c1, c2, c3, c4, c5;
2448     long hash = ((((long) (c0 = source[startPosition])) << 32)
2449         + (((long) (c1 = source[startPosition + 1])) << 24)
2450         + (((long) (c2 = source[startPosition + 2])) << 18)
2451         + ((c3 = source[startPosition + 3]) << 12)
2452         + ((c4 = source[startPosition + 4]) << 6) + (c5 = source[startPosition + 5]))
2453         % TableSize;
2454     char[][] table = charArray_length[4][(int) hash];
2455     int i = newEntry6;
2456     while (++i < InternalTableSize) {
2457       char[] charArray = table[i];
2458       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])
2459           && (c3 == charArray[3]) && (c4 == charArray[4])
2460           && (c5 == charArray[5]))
2461         return charArray;
2462     }
2463     //---------other side---------
2464     i = -1;
2465     int max = newEntry6;
2466     while (++i <= max) {
2467       char[] charArray = table[i];
2468       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2])
2469           && (c3 == charArray[3]) && (c4 == charArray[4])
2470           && (c5 == charArray[5]))
2471         return charArray;
2472     }
2473     //--------add the entry-------
2474     if (++max >= InternalTableSize)
2475       max = 0;
2476     char[] r;
2477     table[max] = (r = new char[]{c0, c1, c2, c3, c4, c5});
2478     newEntry6 = max;
2479     return r;
2480   }
2481   public final void pushLineSeparator() throws InvalidInputException {
2482     //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2483     final int INCREMENT = 250;
2484     if (this.checkNonExternalizedStringLiterals) {
2485       // reinitialize the current line for non externalize strings purpose
2486       currentLine = null;
2487     }
2488     //currentCharacter is at position currentPosition-1
2489     // cr 000D
2490     if (currentCharacter == '\r') {
2491       int separatorPos = currentPosition - 1;
2492       if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2493         return;
2494       //System.out.println("CR-" + separatorPos);
2495       try {
2496         lineEnds[++linePtr] = separatorPos;
2497       } catch (IndexOutOfBoundsException e) {
2498         //linePtr value is correct
2499         int oldLength = lineEnds.length;
2500         int[] old = lineEnds;
2501         lineEnds = new int[oldLength + INCREMENT];
2502         System.arraycopy(old, 0, lineEnds, 0, oldLength);
2503         lineEnds[linePtr] = separatorPos;
2504       }
2505       // look-ahead for merged cr+lf
2506       try {
2507         if (source[currentPosition] == '\n') {
2508           //System.out.println("look-ahead LF-" + currentPosition);
2509           lineEnds[linePtr] = currentPosition;
2510           currentPosition++;
2511           wasAcr = false;
2512         } else {
2513           wasAcr = true;
2514         }
2515       } catch (IndexOutOfBoundsException e) {
2516         wasAcr = true;
2517       }
2518     } else {
2519       // lf 000A
2520       if (currentCharacter == '\n') {
2521         //must merge eventual cr followed by lf
2522         if (wasAcr && (lineEnds[linePtr] == (currentPosition - 2))) {
2523           //System.out.println("merge LF-" + (currentPosition - 1));
2524           lineEnds[linePtr] = currentPosition - 1;
2525         } else {
2526           int separatorPos = currentPosition - 1;
2527           if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2528             return;
2529           // System.out.println("LF-" + separatorPos);
2530           try {
2531             lineEnds[++linePtr] = separatorPos;
2532           } catch (IndexOutOfBoundsException e) {
2533             //linePtr value is correct
2534             int oldLength = lineEnds.length;
2535             int[] old = lineEnds;
2536             lineEnds = new int[oldLength + INCREMENT];
2537             System.arraycopy(old, 0, lineEnds, 0, oldLength);
2538             lineEnds[linePtr] = separatorPos;
2539           }
2540         }
2541         wasAcr = false;
2542       }
2543     }
2544   }
2545   public final void pushUnicodeLineSeparator() {
2546     // isUnicode means that the \r or \n has been read as a unicode character
2547     //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2548     final int INCREMENT = 250;
2549     //currentCharacter is at position currentPosition-1
2550     if (this.checkNonExternalizedStringLiterals) {
2551       // reinitialize the current line for non externalize strings purpose
2552       currentLine = null;
2553     }
2554     // cr 000D
2555     if (currentCharacter == '\r') {
2556       int separatorPos = currentPosition - 6;
2557       if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2558         return;
2559       //System.out.println("CR-" + separatorPos);
2560       try {
2561         lineEnds[++linePtr] = separatorPos;
2562       } catch (IndexOutOfBoundsException e) {
2563         //linePtr value is correct
2564         int oldLength = lineEnds.length;
2565         int[] old = lineEnds;
2566         lineEnds = new int[oldLength + INCREMENT];
2567         System.arraycopy(old, 0, lineEnds, 0, oldLength);
2568         lineEnds[linePtr] = separatorPos;
2569       }
2570       // look-ahead for merged cr+lf
2571       if (source[currentPosition] == '\n') {
2572         //System.out.println("look-ahead LF-" + currentPosition);
2573         lineEnds[linePtr] = currentPosition;
2574         currentPosition++;
2575         wasAcr = false;
2576       } else {
2577         wasAcr = true;
2578       }
2579     } else {
2580       // lf 000A
2581       if (currentCharacter == '\n') {
2582         //must merge eventual cr followed by lf
2583         if (wasAcr && (lineEnds[linePtr] == (currentPosition - 7))) {
2584           //System.out.println("merge LF-" + (currentPosition - 1));
2585           lineEnds[linePtr] = currentPosition - 6;
2586         } else {
2587           int separatorPos = currentPosition - 6;
2588           if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2589             return;
2590           // System.out.println("LF-" + separatorPos);
2591           try {
2592             lineEnds[++linePtr] = separatorPos;
2593           } catch (IndexOutOfBoundsException e) {
2594             //linePtr value is correct
2595             int oldLength = lineEnds.length;
2596             int[] old = lineEnds;
2597             lineEnds = new int[oldLength + INCREMENT];
2598             System.arraycopy(old, 0, lineEnds, 0, oldLength);
2599             lineEnds[linePtr] = separatorPos;
2600           }
2601         }
2602         wasAcr = false;
2603       }
2604     }
2605   }
2606   public void recordComment(int token) {
2607         // compute position
2608         int stopPosition = this.currentPosition;
2609         switch (token) {
2610                 case TokenNameCOMMENT_LINE:
2611                         stopPosition = -this.lastCommentLinePosition;
2612                         break;
2613                 case TokenNameCOMMENT_BLOCK:
2614                         stopPosition = -this.currentPosition;
2615                         break;
2616         }
2617
2618         // a new comment is recorded
2619         int length = this.commentStops.length;
2620         if (++this.commentPtr >=  length) {
2621                 System.arraycopy(this.commentStops, 0, this.commentStops = new int[length + 30], 0, length);
2622                 //grows the positions buffers too
2623                 System.arraycopy(this.commentStarts, 0, this.commentStarts = new int[length + 30], 0, length);
2624         }
2625         this.commentStops[this.commentPtr] = stopPosition;
2626         this.commentStarts[this.commentPtr] = this.startPosition;
2627 }
2628 //  public final void recordComment(boolean isJavadoc) {
2629 //    // a new annotation comment is recorded
2630 //    try {
2631 //      commentStops[++commentPtr] = isJavadoc
2632 //          ? currentPosition
2633 //          : -currentPosition;
2634 //    } catch (IndexOutOfBoundsException e) {
2635 //      int oldStackLength = commentStops.length;
2636 //      int[] oldStack = commentStops;
2637 //      commentStops = new int[oldStackLength + 30];
2638 //      System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength);
2639 //      commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition;
2640 //      //grows the positions buffers too
2641 //      int[] old = commentStarts;
2642 //      commentStarts = new int[oldStackLength + 30];
2643 //      System.arraycopy(old, 0, commentStarts, 0, oldStackLength);
2644 //    }
2645 //    //the buffer is of a correct size here
2646 //    commentStarts[commentPtr] = startPosition;
2647 //  }
2648   public void resetTo(int begin, int end) {
2649     //reset the scanner to a given position where it may rescan again
2650     diet = false;
2651     initialPosition = startPosition = currentPosition = begin;
2652     eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
2653     commentPtr = -1; // reset comment stack
2654   }
2655   public final void scanSingleQuotedEscapeCharacter()
2656       throws InvalidInputException {
2657     // the string with "\\u" is a legal string of two chars \ and u
2658     //thus we use a direct access to the source (for regular cases).
2659     //    if (unicodeAsBackSlash) {
2660     //      // consume next character
2661     //      unicodeAsBackSlash = false;
2662     //      if (((currentCharacter = source[currentPosition++]) == '\\')
2663     //        && (source[currentPosition] == 'u')) {
2664     //        getNextUnicodeChar();
2665     //      } else {
2666     //        if (withoutUnicodePtr != 0) {
2667     //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2668     //        }
2669     //      }
2670     //    } else
2671     currentCharacter = source[currentPosition++];
2672     switch (currentCharacter) {
2673       case '\'' :
2674         currentCharacter = '\'';
2675         break;
2676       case '\\' :
2677         currentCharacter = '\\';
2678         break;
2679       default :
2680         currentCharacter = '\\';
2681         currentPosition--;
2682     }
2683   }
2684   public final void scanDoubleQuotedEscapeCharacter()
2685       throws InvalidInputException {
2686     // the string with "\\u" is a legal string of two chars \ and u
2687     //thus we use a direct access to the source (for regular cases).
2688     //    if (unicodeAsBackSlash) {
2689     //      // consume next character
2690     //      unicodeAsBackSlash = false;
2691     //      if (((currentCharacter = source[currentPosition++]) == '\\')
2692     //        && (source[currentPosition] == 'u')) {
2693     //        getNextUnicodeChar();
2694     //      } else {
2695     //        if (withoutUnicodePtr != 0) {
2696     //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2697     //        }
2698     //      }
2699     //    } else
2700     currentCharacter = source[currentPosition++];
2701     switch (currentCharacter) {
2702       //      case 'b' :
2703       //        currentCharacter = '\b';
2704       //        break;
2705       case 't' :
2706         currentCharacter = '\t';
2707         break;
2708       case 'n' :
2709         currentCharacter = '\n';
2710         break;
2711       //      case 'f' :
2712       //        currentCharacter = '\f';
2713       //        break;
2714       case 'r' :
2715         currentCharacter = '\r';
2716         break;
2717       case '\"' :
2718         currentCharacter = '\"';
2719         break;
2720       case '\'' :
2721         currentCharacter = '\'';
2722         break;
2723       case '\\' :
2724         currentCharacter = '\\';
2725         break;
2726       case '$' :
2727         currentCharacter = '$';
2728         break;
2729       default :
2730         // -----------octal escape--------------
2731         // OctalDigit
2732         // OctalDigit OctalDigit
2733         // ZeroToThree OctalDigit OctalDigit
2734         int number = Character.getNumericValue(currentCharacter);
2735         if (number >= 0 && number <= 7) {
2736           boolean zeroToThreeNot = number > 3;
2737           if (Character.isDigit(currentCharacter = source[currentPosition++])) {
2738             int digit = Character.getNumericValue(currentCharacter);
2739             if (digit >= 0 && digit <= 7) {
2740               number = (number * 8) + digit;
2741               if (Character
2742                   .isDigit(currentCharacter = source[currentPosition++])) {
2743                 if (zeroToThreeNot) { // has read \NotZeroToThree OctalDigit
2744                   // Digit --> ignore last character
2745                   currentPosition--;
2746                 } else {
2747                   digit = Character.getNumericValue(currentCharacter);
2748                   if (digit >= 0 && digit <= 7) {
2749                     // has read \ZeroToThree OctalDigit OctalDigit
2750                     number = (number * 8) + digit;
2751                   } else { // has read \ZeroToThree OctalDigit NonOctalDigit
2752                     // --> ignore last character
2753                     currentPosition--;
2754                   }
2755                 }
2756               } else { // has read \OctalDigit NonDigit--> ignore last
2757                 // character
2758                 currentPosition--;
2759               }
2760             } else { // has read \OctalDigit NonOctalDigit--> ignore last
2761               // character
2762               currentPosition--;
2763             }
2764           } else { // has read \OctalDigit --> ignore last character
2765             currentPosition--;
2766           }
2767           if (number > 255)
2768             throw new InvalidInputException(INVALID_ESCAPE);
2769           currentCharacter = (char) number;
2770         }
2771     //else
2772     //     throw new InvalidInputException(INVALID_ESCAPE);
2773     }
2774   }
2775   //  public int scanIdentifierOrKeyword() throws InvalidInputException {
2776   //    return scanIdentifierOrKeyword( false );
2777   //  }
2778   public int scanIdentifierOrKeyword(boolean isVariable)
2779       throws InvalidInputException {
2780     //test keywords
2781     //first dispatch on the first char.
2782     //then the length. If there are several
2783     //keywors with the same length AND the same first char, then do another
2784     //disptach on the second char :-)...cool....but fast !
2785     useAssertAsAnIndentifier = false;
2786     while (getNextCharAsJavaIdentifierPart()) {
2787     };
2788     if (isVariable) {
2789       //      if (new String(getCurrentTokenSource()).equals("$this")) {
2790       //        return TokenNamethis;
2791       //      }
2792       return TokenNameVariable;
2793     }
2794     int index, length;
2795     char[] data;
2796     char firstLetter;
2797     //    if (withoutUnicodePtr == 0)
2798     //quick test on length == 1 but not on length > 12 while most identifier
2799     //have a length which is <= 12...but there are lots of identifier with
2800     //only one char....
2801     //      {
2802     if ((length = currentPosition - startPosition) == 1)
2803       return TokenNameIdentifier;
2804     //  data = source;
2805     data = new char[length];
2806     index = startPosition;
2807     for (int i = 0; i < length; i++) {
2808       data[i] = Character.toLowerCase(source[index + i]);
2809     }
2810     index = 0;
2811     //    } else {
2812     //      if ((length = withoutUnicodePtr) == 1)
2813     //        return TokenNameIdentifier;
2814     //      // data = withoutUnicodeBuffer;
2815     //      data = new char[withoutUnicodeBuffer.length];
2816     //      for (int i = 0; i < withoutUnicodeBuffer.length; i++) {
2817     //        data[i] = Character.toLowerCase(withoutUnicodeBuffer[i]);
2818     //      }
2819     //      index = 1;
2820     //    }
2821     firstLetter = data[index];
2822     switch (firstLetter) {
2823       case '_' :
2824         switch (length) {
2825           case 8 :
2826             //__FILE__
2827             if ((data[++index] == '_') && (data[++index] == 'f')
2828                 && (data[++index] == 'i') && (data[++index] == 'l')
2829                 && (data[++index] == 'e') && (data[++index] == '_')
2830                 && (data[++index] == '_'))
2831               return TokenNameFILE;
2832             index = 0; //__LINE__
2833             if ((data[++index] == '_') && (data[++index] == 'l')
2834                 && (data[++index] == 'i') && (data[++index] == 'n')
2835                 && (data[++index] == 'e') && (data[++index] == '_')
2836                 && (data[++index] == '_'))
2837               return TokenNameLINE;
2838             break;
2839           case 9 :
2840             //__CLASS__
2841             if ((data[++index] == '_') && (data[++index] == 'c')
2842                 && (data[++index] == 'l') && (data[++index] == 'a')
2843                 && (data[++index] == 's') && (data[++index] == 's')
2844                 && (data[++index] == '_') && (data[++index] == '_'))
2845               return TokenNameCLASS_C;
2846             break;
2847           case 11 :
2848             //__METHOD__
2849             if ((data[++index] == '_') && (data[++index] == 'm')
2850                 && (data[++index] == 'e') && (data[++index] == 't')
2851                 && (data[++index] == 'h') && (data[++index] == 'o')
2852                 && (data[++index] == 'd') && (data[++index] == '_')
2853                 && (data[++index] == '_'))
2854               return TokenNameMETHOD_C;
2855             break;
2856           case 12 :
2857             //__FUNCTION__
2858             if ((data[++index] == '_') && (data[++index] == 'f')
2859                 && (data[++index] == 'u') && (data[++index] == 'n')
2860                 && (data[++index] == 'c') && (data[++index] == 't')
2861                 && (data[++index] == 'i') && (data[++index] == 'o')
2862                 && (data[++index] == 'n') && (data[++index] == '_')
2863                 && (data[++index] == '_'))
2864               return TokenNameFUNC_C;
2865             break;
2866         }
2867         return TokenNameIdentifier;
2868       case 'a' :
2869         // as and array abstract
2870         switch (length) {
2871           case 2 :
2872             //as
2873             if ((data[++index] == 's')) {
2874               return TokenNameas;
2875             } else {
2876               return TokenNameIdentifier;
2877             }
2878           case 3 :
2879             //and
2880             if ((data[++index] == 'n') && (data[++index] == 'd')) {
2881               return TokenNameand;
2882             } else {
2883               return TokenNameIdentifier;
2884             }
2885           case 5 :
2886             // array
2887             if ((data[++index] == 'r') && (data[++index] == 'r')
2888                 && (data[++index] == 'a') && (data[++index] == 'y'))
2889               return TokenNamearray;
2890             else
2891               return TokenNameIdentifier;
2892           case 8 :
2893             if ((data[++index] == 'b') && (data[++index] == 's')
2894                 && (data[++index] == 't') && (data[++index] == 'r')
2895                 && (data[++index] == 'a') && (data[++index] == 'c')
2896                 && (data[++index] == 't'))
2897               return TokenNameabstract;
2898             else
2899               return TokenNameIdentifier;
2900           default :
2901             return TokenNameIdentifier;
2902         }
2903       case 'b' :
2904         //break
2905         switch (length) {
2906           case 5 :
2907             if ((data[++index] == 'r') && (data[++index] == 'e')
2908                 && (data[++index] == 'a') && (data[++index] == 'k'))
2909               return TokenNamebreak;
2910             else
2911               return TokenNameIdentifier;
2912           default :
2913             return TokenNameIdentifier;
2914         }
2915       case 'c' :
2916         //case catch class clone const continue
2917         switch (length) {
2918           case 4 :
2919             if ((data[++index] == 'a') && (data[++index] == 's')
2920                 && (data[++index] == 'e'))
2921               return TokenNamecase;
2922             else
2923               return TokenNameIdentifier;
2924           case 5 :
2925             if ((data[++index] == 'a') && (data[++index] == 't')
2926                 && (data[++index] == 'c') && (data[++index] == 'h'))
2927               return TokenNamecatch;
2928             index = 0;
2929             if ((data[++index] == 'l') && (data[++index] == 'a')
2930                 && (data[++index] == 's') && (data[++index] == 's'))
2931               return TokenNameclass;
2932             index = 0;
2933             if ((data[++index] == 'l') && (data[++index] == 'o')
2934                 && (data[++index] == 'n') && (data[++index] == 'e'))
2935               return TokenNameclone;
2936             index = 0;
2937             if ((data[++index] == 'o') && (data[++index] == 'n')
2938                 && (data[++index] == 's') && (data[++index] == 't'))
2939               return TokenNameconst;
2940             else
2941               return TokenNameIdentifier;
2942           case 8 :
2943             if ((data[++index] == 'o') && (data[++index] == 'n')
2944                 && (data[++index] == 't') && (data[++index] == 'i')
2945                 && (data[++index] == 'n') && (data[++index] == 'u')
2946                 && (data[++index] == 'e'))
2947               return TokenNamecontinue;
2948             else
2949               return TokenNameIdentifier;
2950           default :
2951             return TokenNameIdentifier;
2952         }
2953       case 'd' :
2954         // declare default do die
2955         // TODO delete define ==> no keyword !
2956         switch (length) {
2957           case 2 :
2958             if ((data[++index] == 'o'))
2959               return TokenNamedo;
2960             else
2961               return TokenNameIdentifier;
2962           //          case 6 :
2963           //            if ((data[++index] == 'e')
2964           //              && (data[++index] == 'f')
2965           //              && (data[++index] == 'i')
2966           //              && (data[++index] == 'n')
2967           //              && (data[++index] == 'e'))
2968           //              return TokenNamedefine;
2969           //            else
2970           //              return TokenNameIdentifier;
2971           case 7 :
2972             if ((data[++index] == 'e') && (data[++index] == 'c')
2973                 && (data[++index] == 'l') && (data[++index] == 'a')
2974                 && (data[++index] == 'r') && (data[++index] == 'e'))
2975               return TokenNamedeclare;
2976             index = 0;
2977             if ((data[++index] == 'e') && (data[++index] == 'f')
2978                 && (data[++index] == 'a') && (data[++index] == 'u')
2979                 && (data[++index] == 'l') && (data[++index] == 't'))
2980               return TokenNamedefault;
2981             else
2982               return TokenNameIdentifier;
2983           default :
2984             return TokenNameIdentifier;
2985         }
2986       case 'e' :
2987         //echo else exit elseif extends eval
2988         switch (length) {
2989           case 4 :
2990             if ((data[++index] == 'c') && (data[++index] == 'h')
2991                 && (data[++index] == 'o'))
2992               return TokenNameecho;
2993             else if ((data[index] == 'l') && (data[++index] == 's')
2994                 && (data[++index] == 'e'))
2995               return TokenNameelse;
2996             else if ((data[index] == 'x') && (data[++index] == 'i')
2997                 && (data[++index] == 't'))
2998               return TokenNameexit;
2999             else if ((data[index] == 'v') && (data[++index] == 'a')
3000                 && (data[++index] == 'l'))
3001               return TokenNameeval;
3002             else
3003               return TokenNameIdentifier;
3004           case 5 :
3005             // endif empty
3006             if ((data[++index] == 'n') && (data[++index] == 'd')
3007                 && (data[++index] == 'i') && (data[++index] == 'f'))
3008               return TokenNameendif;
3009             if ((data[index] == 'm') && (data[++index] == 'p')
3010                 && (data[++index] == 't') && (data[++index] == 'y'))
3011               return TokenNameempty;
3012             else
3013               return TokenNameIdentifier;
3014           case 6 :
3015             // endfor
3016             if ((data[++index] == 'n') && (data[++index] == 'd')
3017                 && (data[++index] == 'f') && (data[++index] == 'o')
3018                 && (data[++index] == 'r'))
3019               return TokenNameendfor;
3020             else if ((data[index] == 'l') && (data[++index] == 's')
3021                 && (data[++index] == 'e') && (data[++index] == 'i')
3022                 && (data[++index] == 'f'))
3023               return TokenNameelseif;
3024             else
3025               return TokenNameIdentifier;
3026           case 7 :
3027             if ((data[++index] == 'x') && (data[++index] == 't')
3028                 && (data[++index] == 'e') && (data[++index] == 'n')
3029                 && (data[++index] == 'd') && (data[++index] == 's'))
3030               return TokenNameextends;
3031             else
3032               return TokenNameIdentifier;
3033           case 8 :
3034             // endwhile
3035             if ((data[++index] == 'n') && (data[++index] == 'd')
3036                 && (data[++index] == 'w') && (data[++index] == 'h')
3037                 && (data[++index] == 'i') && (data[++index] == 'l')
3038                 && (data[++index] == 'e'))
3039               return TokenNameendwhile;
3040             else
3041               return TokenNameIdentifier;
3042           case 9 :
3043             // endswitch
3044             if ((data[++index] == 'n') && (data[++index] == 'd')
3045                 && (data[++index] == 's') && (data[++index] == 'w')
3046                 && (data[++index] == 'i') && (data[++index] == 't')
3047                 && (data[++index] == 'c') && (data[++index] == 'h'))
3048               return TokenNameendswitch;
3049             else
3050               return TokenNameIdentifier;
3051           case 10 :
3052             // enddeclare
3053             if ((data[++index] == 'n') && (data[++index] == 'd')
3054                 && (data[++index] == 'd') && (data[++index] == 'e')
3055                 && (data[++index] == 'c') && (data[++index] == 'l')
3056                 && (data[++index] == 'a') && (data[++index] == 'r')
3057                 && (data[++index] == 'e'))
3058               return TokenNameendforeach;
3059             index = 0;
3060             if ((data[++index] == 'n') // endforeach
3061                 && (data[++index] == 'd') && (data[++index] == 'f')
3062                 && (data[++index] == 'o') && (data[++index] == 'r')
3063                 && (data[++index] == 'e') && (data[++index] == 'a')
3064                 && (data[++index] == 'c') && (data[++index] == 'h'))
3065               return TokenNameendforeach;
3066             else
3067               return TokenNameIdentifier;
3068           default :
3069             return TokenNameIdentifier;
3070         }
3071       case 'f' :
3072         //for false final function
3073         switch (length) {
3074           case 3 :
3075             if ((data[++index] == 'o') && (data[++index] == 'r'))
3076               return TokenNamefor;
3077             else
3078               return TokenNameIdentifier;
3079           case 5 :
3080             //            if ((data[++index] == 'a') && (data[++index] == 'l')
3081             //                && (data[++index] == 's') && (data[++index] == 'e'))
3082             //              return TokenNamefalse;
3083             if ((data[++index] == 'i') && (data[++index] == 'n')
3084                 && (data[++index] == 'a') && (data[++index] == 'l'))
3085               return TokenNamefinal;
3086             else
3087               return TokenNameIdentifier;
3088           case 7 :
3089             // foreach
3090             if ((data[++index] == 'o') && (data[++index] == 'r')
3091                 && (data[++index] == 'e') && (data[++index] == 'a')
3092                 && (data[++index] == 'c') && (data[++index] == 'h'))
3093               return TokenNameforeach;
3094             else
3095               return TokenNameIdentifier;
3096           case 8 :
3097             // function
3098             if ((data[++index] == 'u') && (data[++index] == 'n')
3099                 && (data[++index] == 'c') && (data[++index] == 't')
3100                 && (data[++index] == 'i') && (data[++index] == 'o')
3101                 && (data[++index] == 'n'))
3102               return TokenNamefunction;
3103             else
3104               return TokenNameIdentifier;
3105           default :
3106             return TokenNameIdentifier;
3107         }
3108       case 'g' :
3109         //global
3110         if (length == 6) {
3111           if ((data[++index] == 'l') && (data[++index] == 'o')
3112               && (data[++index] == 'b') && (data[++index] == 'a')
3113               && (data[++index] == 'l')) {
3114             return TokenNameglobal;
3115           }
3116         }
3117         return TokenNameIdentifier;
3118       case 'i' :
3119         //if int isset include include_once instanceof interface implements
3120         switch (length) {
3121           case 2 :
3122             if (data[++index] == 'f')
3123               return TokenNameif;
3124             else
3125               return TokenNameIdentifier;
3126           //          case 3 :
3127           //            if ((data[++index] == 'n') && (data[++index] == 't'))
3128           //              return TokenNameint;
3129           //            else
3130           //              return TokenNameIdentifier;
3131           case 5 :
3132             if ((data[++index] == 's') && (data[++index] == 's')
3133                 && (data[++index] == 'e') && (data[++index] == 't'))
3134               return TokenNameisset;
3135             else
3136               return TokenNameIdentifier;
3137           case 7 :
3138             if ((data[++index] == 'n') && (data[++index] == 'c')
3139                 && (data[++index] == 'l') && (data[++index] == 'u')
3140                 && (data[++index] == 'd') && (data[++index] == 'e'))
3141               return TokenNameinclude;
3142             else
3143               return TokenNameIdentifier;
3144           case 9 :
3145             // interface
3146             if ((data[++index] == 'n') && (data[++index] == 't')
3147                 && (data[++index] == 'e') && (data[++index] == 'r')
3148                 && (data[++index] == 'f') && (data[++index] == 'a')
3149                 && (data[++index] == 'c') && (data[++index] == 'e'))
3150               return TokenNameinterface;
3151             else
3152               return TokenNameIdentifier;
3153           case 10 :
3154             // instanceof
3155             if ((data[++index] == 'n') && (data[++index] == 's')
3156                 && (data[++index] == 't') && (data[++index] == 'a')
3157                 && (data[++index] == 'n') && (data[++index] == 'c')
3158                 && (data[++index] == 'e') && (data[++index] == 'o')
3159                 && (data[++index] == 'f'))
3160               return TokenNameinstanceof;
3161             if ((data[index] == 'm') && (data[++index] == 'p')
3162                 && (data[++index] == 'l') && (data[++index] == 'e')
3163                 && (data[++index] == 'm') && (data[++index] == 'e')
3164                 && (data[++index] == 'n') && (data[++index] == 't')
3165                 && (data[++index] == 's'))
3166               return TokenNameimplements;
3167             else
3168               return TokenNameIdentifier;
3169           case 12 :
3170             if ((data[++index] == 'n') && (data[++index] == 'c')
3171                 && (data[++index] == 'l') && (data[++index] == 'u')
3172                 && (data[++index] == 'd') && (data[++index] == 'e')
3173                 && (data[++index] == '_') && (data[++index] == 'o')
3174                 && (data[++index] == 'n') && (data[++index] == 'c')
3175                 && (data[++index] == 'e'))
3176               return TokenNameinclude_once;
3177             else
3178               return TokenNameIdentifier;
3179           default :
3180             return TokenNameIdentifier;
3181         }
3182       case 'l' :
3183         //list
3184         if (length == 4) {
3185           if ((data[++index] == 'i') && (data[++index] == 's')
3186               && (data[++index] == 't')) {
3187             return TokenNamelist;
3188           }
3189         }
3190         return TokenNameIdentifier;
3191       case 'n' :
3192         // new null
3193         switch (length) {
3194           case 3 :
3195             if ((data[++index] == 'e') && (data[++index] == 'w'))
3196               return TokenNamenew;
3197             else
3198               return TokenNameIdentifier;
3199           //          case 4 :
3200           //            if ((data[++index] == 'u') && (data[++index] == 'l')
3201           //                && (data[++index] == 'l'))
3202           //              return TokenNamenull;
3203           //            else
3204           //              return TokenNameIdentifier;
3205           default :
3206             return TokenNameIdentifier;
3207         }
3208       case 'o' :
3209         // or old_function
3210         if (length == 2) {
3211           if (data[++index] == 'r') {
3212             return TokenNameor;
3213           }
3214         }
3215         //        if (length == 12) {
3216         //          if ((data[++index] == 'l')
3217         //            && (data[++index] == 'd')
3218         //            && (data[++index] == '_')
3219         //            && (data[++index] == 'f')
3220         //            && (data[++index] == 'u')
3221         //            && (data[++index] == 'n')
3222         //            && (data[++index] == 'c')
3223         //            && (data[++index] == 't')
3224         //            && (data[++index] == 'i')
3225         //            && (data[++index] == 'o')
3226         //            && (data[++index] == 'n')) {
3227         //            return TokenNameold_function;
3228         //          }
3229         //        }
3230         return TokenNameIdentifier;
3231       case 'p' :
3232         // print public private protected
3233         switch (length) {
3234           case 5 :
3235             if ((data[++index] == 'r') && (data[++index] == 'i')
3236                 && (data[++index] == 'n') && (data[++index] == 't')) {
3237               return TokenNameprint;
3238             } else
3239               return TokenNameIdentifier;
3240           case 6 :
3241             if ((data[++index] == 'u') && (data[++index] == 'b')
3242                 && (data[++index] == 'l') && (data[++index] == 'i')
3243                 && (data[++index] == 'c')) {
3244               return TokenNamepublic;
3245             } else
3246               return TokenNameIdentifier;
3247           case 7 :
3248             if ((data[++index] == 'r') && (data[++index] == 'i')
3249                 && (data[++index] == 'v') && (data[++index] == 'a')
3250                 && (data[++index] == 't') && (data[++index] == 'e')) {
3251               return TokenNameprivate;
3252             } else
3253               return TokenNameIdentifier;
3254           case 9 :
3255             if ((data[++index] == 'r') && (data[++index] == 'o')
3256                 && (data[++index] == 't') && (data[++index] == 'e')
3257                 && (data[++index] == 'c') && (data[++index] == 't')
3258                 && (data[++index] == 'e') && (data[++index] == 'd')) {
3259               return TokenNameprotected;
3260             } else
3261               return TokenNameIdentifier;
3262         }
3263         return TokenNameIdentifier;
3264       case 'r' :
3265         //return require require_once
3266         if (length == 6) {
3267           if ((data[++index] == 'e') && (data[++index] == 't')
3268               && (data[++index] == 'u') && (data[++index] == 'r')
3269               && (data[++index] == 'n')) {
3270             return TokenNamereturn;
3271           }
3272         } else if (length == 7) {
3273           if ((data[++index] == 'e') && (data[++index] == 'q')
3274               && (data[++index] == 'u') && (data[++index] == 'i')
3275               && (data[++index] == 'r') && (data[++index] == 'e')) {
3276             return TokenNamerequire;
3277           }
3278         } else if (length == 12) {
3279           if ((data[++index] == 'e') && (data[++index] == 'q')
3280               && (data[++index] == 'u') && (data[++index] == 'i')
3281               && (data[++index] == 'r') && (data[++index] == 'e')
3282               && (data[++index] == '_') && (data[++index] == 'o')
3283               && (data[++index] == 'n') && (data[++index] == 'c')
3284               && (data[++index] == 'e')) {
3285             return TokenNamerequire_once;
3286           }
3287         } else
3288           return TokenNameIdentifier;
3289       case 's' :
3290         //static switch
3291         switch (length) {
3292           case 6 :
3293             if (data[++index] == 't')
3294               if ((data[++index] == 'a') && (data[++index] == 't')
3295                   && (data[++index] == 'i') && (data[++index] == 'c')) {
3296                 return TokenNamestatic;
3297               } else
3298                 return TokenNameIdentifier;
3299             else if ((data[index] == 'w') && (data[++index] == 'i')
3300                 && (data[++index] == 't') && (data[++index] == 'c')
3301                 && (data[++index] == 'h'))
3302               return TokenNameswitch;
3303             else
3304               return TokenNameIdentifier;
3305           default :
3306             return TokenNameIdentifier;
3307         }
3308       case 't' :
3309         // try true throw
3310         switch (length) {
3311           case 3 :
3312             if ((data[++index] == 'r') && (data[++index] == 'y'))
3313               return TokenNametry;
3314             else
3315               return TokenNameIdentifier;
3316           //          case 4 :
3317           //            if ((data[++index] == 'r') && (data[++index] == 'u')
3318           //                && (data[++index] == 'e'))
3319           //              return TokenNametrue;
3320           //            else
3321           //              return TokenNameIdentifier;
3322           case 5 :
3323             if ((data[++index] == 'h') && (data[++index] == 'r')
3324                 && (data[++index] == 'o') && (data[++index] == 'w'))
3325               return TokenNamethrow;
3326             else
3327               return TokenNameIdentifier;
3328           default :
3329             return TokenNameIdentifier;
3330         }
3331       case 'u' :
3332         //use unset
3333         switch (length) {
3334           case 3 :
3335             if ((data[++index] == 's') && (data[++index] == 'e'))
3336               return TokenNameuse;
3337             else
3338               return TokenNameIdentifier;
3339           case 5 :
3340             if ((data[++index] == 'n') && (data[++index] == 's')
3341                 && (data[++index] == 'e') && (data[++index] == 't'))
3342               return TokenNameunset;
3343             else
3344               return TokenNameIdentifier;
3345           default :
3346             return TokenNameIdentifier;
3347         }
3348       case 'v' :
3349         //var
3350         switch (length) {
3351           case 3 :
3352             if ((data[++index] == 'a') && (data[++index] == 'r'))
3353               return TokenNamevar;
3354             else
3355               return TokenNameIdentifier;
3356           default :
3357             return TokenNameIdentifier;
3358         }
3359       case 'w' :
3360         //while
3361         switch (length) {
3362           case 5 :
3363             if ((data[++index] == 'h') && (data[++index] == 'i')
3364                 && (data[++index] == 'l') && (data[++index] == 'e'))
3365               return TokenNamewhile;
3366             else
3367               return TokenNameIdentifier;
3368           //case 6:if ( (data[++index] =='i') && (data[++index]=='d') &&
3369           // (data[++index]=='e') && (data[++index]=='f')&&
3370           // (data[++index]=='p'))
3371           //return TokenNamewidefp ;
3372           //else
3373           //return TokenNameIdentifier;
3374           default :
3375             return TokenNameIdentifier;
3376         }
3377       case 'x' :
3378         //xor
3379         switch (length) {
3380           case 3 :
3381             if ((data[++index] == 'o') && (data[++index] == 'r'))
3382               return TokenNamexor;
3383             else
3384               return TokenNameIdentifier;
3385           default :
3386             return TokenNameIdentifier;
3387         }
3388       default :
3389         return TokenNameIdentifier;
3390     }
3391   }
3392   public int scanNumber(boolean dotPrefix) throws InvalidInputException {
3393     //when entering this method the currentCharacter is the firt
3394     //digit of the number , i.e. it may be preceeded by a . when
3395     //dotPrefix is true
3396     boolean floating = dotPrefix;
3397     if ((!dotPrefix) && (currentCharacter == '0')) {
3398       if (getNextChar('x', 'X') >= 0) { //----------hexa-----------------
3399         //force the first char of the hexa number do exist...
3400         // consume next character
3401         unicodeAsBackSlash = false;
3402         currentCharacter = source[currentPosition++];
3403         //        if (((currentCharacter = source[currentPosition++]) == '\\')
3404         //          && (source[currentPosition] == 'u')) {
3405         //          getNextUnicodeChar();
3406         //        } else {
3407         //          if (withoutUnicodePtr != 0) {
3408         //            withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3409         //          }
3410         //        }
3411         if (Character.digit(currentCharacter, 16) == -1)
3412           throw new InvalidInputException(INVALID_HEXA);
3413         //---end forcing--
3414         while (getNextCharAsDigit(16)) {
3415         };
3416         //        if (getNextChar('l', 'L') >= 0)
3417         //          return TokenNameLongLiteral;
3418         //        else
3419         return TokenNameIntegerLiteral;
3420       }
3421       //there is x or X in the number
3422       //potential octal ! ... some one may write 000099.0 ! thus 00100 <
3423       // 00078.0 is true !!!!! crazy language
3424       if (getNextCharAsDigit()) {
3425         //-------------potential octal-----------------
3426         while (getNextCharAsDigit()) {
3427         };
3428         //        if (getNextChar('l', 'L') >= 0) {
3429         //          return TokenNameLongLiteral;
3430         //        }
3431         //
3432         //        if (getNextChar('f', 'F') >= 0) {
3433         //          return TokenNameFloatingPointLiteral;
3434         //        }
3435         if (getNextChar('d', 'D') >= 0) {
3436           return TokenNameDoubleLiteral;
3437         } else { //make the distinction between octal and float ....
3438           if (getNextChar('.')) { //bingo ! ....
3439             while (getNextCharAsDigit()) {
3440             };
3441             if (getNextChar('e', 'E') >= 0) {
3442               // consume next character
3443               unicodeAsBackSlash = false;
3444               currentCharacter = source[currentPosition++];
3445               //              if (((currentCharacter = source[currentPosition++]) == '\\')
3446               //                && (source[currentPosition] == 'u')) {
3447               //                getNextUnicodeChar();
3448               //              } else {
3449               //                if (withoutUnicodePtr != 0) {
3450               //                  withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3451               //                }
3452               //              }
3453               if ((currentCharacter == '-') || (currentCharacter == '+')) {
3454                 // consume next character
3455                 unicodeAsBackSlash = false;
3456                 currentCharacter = source[currentPosition++];
3457                 //                if (((currentCharacter = source[currentPosition++]) == '\\')
3458                 //                  && (source[currentPosition] == 'u')) {
3459                 //                  getNextUnicodeChar();
3460                 //                } else {
3461                 //                  if (withoutUnicodePtr != 0) {
3462                 //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
3463                 //                      currentCharacter;
3464                 //                  }
3465                 //                }
3466               }
3467               if (!Character.isDigit(currentCharacter))
3468                 throw new InvalidInputException(INVALID_FLOAT);
3469               while (getNextCharAsDigit()) {
3470               };
3471             }
3472             //            if (getNextChar('f', 'F') >= 0)
3473             //              return TokenNameFloatingPointLiteral;
3474             getNextChar('d', 'D'); //jump over potential d or D
3475             return TokenNameDoubleLiteral;
3476           } else {
3477             return TokenNameIntegerLiteral;
3478           }
3479         }
3480       } else {
3481         /* carry on */
3482       }
3483     }
3484     while (getNextCharAsDigit()) {
3485     };
3486     //    if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3487     //      return TokenNameLongLiteral;
3488     if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty
3489       while (getNextCharAsDigit()) {
3490       };
3491       floating = true;
3492     }
3493     //if floating is true both exponant and suffix may be optional
3494     if (getNextChar('e', 'E') >= 0) {
3495       floating = true;
3496       // consume next character
3497       unicodeAsBackSlash = false;
3498       currentCharacter = source[currentPosition++];
3499       //      if (((currentCharacter = source[currentPosition++]) == '\\')
3500       //        && (source[currentPosition] == 'u')) {
3501       //        getNextUnicodeChar();
3502       //      } else {
3503       //        if (withoutUnicodePtr != 0) {
3504       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3505       //        }
3506       //      }
3507       if ((currentCharacter == '-') || (currentCharacter == '+')) { // consume
3508         // next
3509         // character
3510         unicodeAsBackSlash = false;
3511         currentCharacter = source[currentPosition++];
3512         //        if (((currentCharacter = source[currentPosition++]) == '\\')
3513         //          && (source[currentPosition] == 'u')) {
3514         //          getNextUnicodeChar();
3515         //        } else {
3516         //          if (withoutUnicodePtr != 0) {
3517         //            withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3518         //          }
3519         //        }
3520       }
3521       if (!Character.isDigit(currentCharacter))
3522         throw new InvalidInputException(INVALID_FLOAT);
3523       while (getNextCharAsDigit()) {
3524       };
3525     }
3526     if (getNextChar('d', 'D') >= 0)
3527       return TokenNameDoubleLiteral;
3528     //    if (getNextChar('f', 'F') >= 0)
3529     //      return TokenNameFloatingPointLiteral;
3530     //the long flag has been tested before
3531     return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral;
3532   }
3533   /**
3534    * Search the line number corresponding to a specific position
3535    *
3536    */
3537   public final int getLineNumber(int position) {
3538     if (lineEnds == null)
3539       return 1;
3540     int length = linePtr + 1;
3541     if (length == 0)
3542       return 1;
3543     int g = 0, d = length - 1;
3544     int m = 0;
3545     while (g <= d) {
3546       m = (g + d) / 2;
3547       if (position < lineEnds[m]) {
3548         d = m - 1;
3549       } else if (position > lineEnds[m]) {
3550         g = m + 1;
3551       } else {
3552         return m + 1;
3553       }
3554     }
3555     if (position < lineEnds[m]) {
3556       return m + 1;
3557     }
3558     return m + 2;
3559   }
3560   public void setPHPMode(boolean mode) {
3561     phpMode = mode;
3562   }
3563   public final void setSource(char[] source) {
3564     //the source-buffer is set to sourceString
3565     if (source == null) {
3566       this.source = new char[0];
3567     } else {
3568       this.source = source;
3569     }
3570     startPosition = -1;
3571     initialPosition = currentPosition = 0;
3572     containsAssertKeyword = false;
3573     withoutUnicodeBuffer = new char[this.source.length];
3574     encapsedStringStack = new Stack();
3575   }
3576   public String toString() {
3577     if (startPosition == source.length)
3578       return "EOF\n\n" + new String(source); //$NON-NLS-1$
3579     if (currentPosition > source.length)
3580       return "behind the EOF :-( ....\n\n" + new String(source); //$NON-NLS-1$
3581     char front[] = new char[startPosition];
3582     System.arraycopy(source, 0, front, 0, startPosition);
3583     int middleLength = (currentPosition - 1) - startPosition + 1;
3584     char middle[];
3585     if (middleLength > -1) {
3586       middle = new char[middleLength];
3587       System.arraycopy(source, startPosition, middle, 0, middleLength);
3588     } else {
3589       middle = new char[0];
3590     }
3591     char end[] = new char[source.length - (currentPosition - 1)];
3592     System.arraycopy(source, (currentPosition - 1) + 1, end, 0, source.length
3593         - (currentPosition - 1) - 1);
3594     return new String(front)
3595         + "\n===============================\nStarts here -->" //$NON-NLS-1$
3596         + new String(middle)
3597         + "<-- Ends here\n===============================\n" //$NON-NLS-1$
3598         + new String(end);
3599   }
3600   public final String toStringAction(int act) {
3601     switch (act) {
3602       case TokenNameERROR :
3603         return "ScannerError"; // + new String(getCurrentTokenSource()) + ")";
3604       // //$NON-NLS-1$
3605       case TokenNameINLINE_HTML :
3606         return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3607       case TokenNameIdentifier :
3608         return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3609       case TokenNameVariable :
3610         return "Variable(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3611       case TokenNameabstract :
3612         return "abstract"; //$NON-NLS-1$
3613       case TokenNameand :
3614         return "AND"; //$NON-NLS-1$
3615       case TokenNamearray :
3616         return "array"; //$NON-NLS-1$
3617       case TokenNameas :
3618         return "as"; //$NON-NLS-1$
3619       case TokenNamebreak :
3620         return "break"; //$NON-NLS-1$
3621       case TokenNamecase :
3622         return "case"; //$NON-NLS-1$
3623       case TokenNameclass :
3624         return "class"; //$NON-NLS-1$
3625       case TokenNamecatch :
3626         return "catch"; //$NON-NLS-1$
3627       case TokenNameclone :
3628         //$NON-NLS-1$
3629         return "clone";
3630       case TokenNameconst :
3631         //$NON-NLS-1$
3632         return "const";
3633       case TokenNamecontinue :
3634         return "continue"; //$NON-NLS-1$
3635       case TokenNamedefault :
3636         return "default"; //$NON-NLS-1$
3637       //      case TokenNamedefine :
3638       //        return "define"; //$NON-NLS-1$
3639       case TokenNamedo :
3640         return "do"; //$NON-NLS-1$
3641       case TokenNameecho :
3642         return "echo"; //$NON-NLS-1$
3643       case TokenNameelse :
3644         return "else"; //$NON-NLS-1$
3645       case TokenNameelseif :
3646         return "elseif"; //$NON-NLS-1$
3647       case TokenNameendfor :
3648         return "endfor"; //$NON-NLS-1$
3649       case TokenNameendforeach :
3650         return "endforeach"; //$NON-NLS-1$
3651       case TokenNameendif :
3652         return "endif"; //$NON-NLS-1$
3653       case TokenNameendswitch :
3654         return "endswitch"; //$NON-NLS-1$
3655       case TokenNameendwhile :
3656         return "endwhile"; //$NON-NLS-1$
3657       case TokenNameexit:
3658         return "exit";
3659       case TokenNameextends :
3660         return "extends"; //$NON-NLS-1$
3661       //      case TokenNamefalse :
3662       //        return "false"; //$NON-NLS-1$
3663       case TokenNamefinal :
3664         return "final"; //$NON-NLS-1$
3665       case TokenNamefor :
3666         return "for"; //$NON-NLS-1$
3667       case TokenNameforeach :
3668         return "foreach"; //$NON-NLS-1$
3669       case TokenNamefunction :
3670         return "function"; //$NON-NLS-1$
3671       case TokenNameglobal :
3672         return "global"; //$NON-NLS-1$
3673       case TokenNameif :
3674         return "if"; //$NON-NLS-1$
3675       case TokenNameimplements :
3676         return "implements"; //$NON-NLS-1$
3677       case TokenNameinclude :
3678         return "include"; //$NON-NLS-1$
3679       case TokenNameinclude_once :
3680         return "include_once"; //$NON-NLS-1$
3681       case TokenNameinstanceof :
3682         return "instanceof"; //$NON-NLS-1$
3683       case TokenNameinterface :
3684         return "interface"; //$NON-NLS-1$
3685       case TokenNameisset :
3686         return "isset"; //$NON-NLS-1$
3687       case TokenNamelist :
3688         return "list"; //$NON-NLS-1$
3689       case TokenNamenew :
3690         return "new"; //$NON-NLS-1$
3691       //      case TokenNamenull :
3692       //        return "null"; //$NON-NLS-1$
3693       case TokenNameor :
3694         return "OR"; //$NON-NLS-1$
3695       case TokenNameprint :
3696         return "print"; //$NON-NLS-1$
3697       case TokenNameprivate :
3698         return "private"; //$NON-NLS-1$
3699       case TokenNameprotected :
3700         return "protected"; //$NON-NLS-1$
3701       case TokenNamepublic :
3702         return "public"; //$NON-NLS-1$
3703       case TokenNamerequire :
3704         return "require"; //$NON-NLS-1$
3705       case TokenNamerequire_once :
3706         return "require_once"; //$NON-NLS-1$
3707       case TokenNamereturn :
3708         return "return"; //$NON-NLS-1$
3709       case TokenNamestatic :
3710         return "static"; //$NON-NLS-1$
3711       case TokenNameswitch :
3712         return "switch"; //$NON-NLS-1$
3713       //      case TokenNametrue :
3714       //        return "true"; //$NON-NLS-1$
3715       case TokenNameunset :
3716         return "unset"; //$NON-NLS-1$
3717       case TokenNamevar :
3718         return "var"; //$NON-NLS-1$
3719       case TokenNamewhile :
3720         return "while"; //$NON-NLS-1$
3721       case TokenNamexor :
3722         return "XOR"; //$NON-NLS-1$
3723       //      case TokenNamethis :
3724       //        return "$this"; //$NON-NLS-1$
3725       case TokenNameIntegerLiteral :
3726         return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3727       case TokenNameDoubleLiteral :
3728         return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3729       case TokenNameStringDoubleQuote :
3730         return "StringLiteral(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3731       case TokenNameStringSingleQuote :
3732         return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3733       case TokenNameStringInterpolated :
3734         return "StringInterpolated(" + new String(getCurrentTokenSource())
3735             + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3736       case TokenNameEncapsedString0 :
3737         return "`"; //$NON-NLS-1$
3738       case TokenNameEncapsedString1 :
3739         return "\'"; //$NON-NLS-1$
3740       case TokenNameEncapsedString2 :
3741         return "\""; //$NON-NLS-1$
3742       case TokenNameSTRING :
3743         return "STRING(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3744       case TokenNameHEREDOC :
3745         return "HEREDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3746       case TokenNamePLUS_PLUS :
3747         return "++"; //$NON-NLS-1$
3748       case TokenNameMINUS_MINUS :
3749         return "--"; //$NON-NLS-1$
3750       case TokenNameEQUAL_EQUAL :
3751         return "=="; //$NON-NLS-1$
3752       case TokenNameEQUAL_EQUAL_EQUAL :
3753         return "==="; //$NON-NLS-1$
3754       case TokenNameEQUAL_GREATER :
3755         return "=>"; //$NON-NLS-1$
3756       case TokenNameLESS_EQUAL :
3757         return "<="; //$NON-NLS-1$
3758       case TokenNameGREATER_EQUAL :
3759         return ">="; //$NON-NLS-1$
3760       case TokenNameNOT_EQUAL :
3761         return "!="; //$NON-NLS-1$
3762       case TokenNameNOT_EQUAL_EQUAL :
3763         return "!=="; //$NON-NLS-1$
3764       case TokenNameLEFT_SHIFT :
3765         return "<<"; //$NON-NLS-1$
3766       case TokenNameRIGHT_SHIFT :
3767         return ">>"; //$NON-NLS-1$
3768       case TokenNamePLUS_EQUAL :
3769         return "+="; //$NON-NLS-1$
3770       case TokenNameMINUS_EQUAL :
3771         return "-="; //$NON-NLS-1$
3772       case TokenNameMULTIPLY_EQUAL :
3773         return "*="; //$NON-NLS-1$
3774       case TokenNameDIVIDE_EQUAL :
3775         return "/="; //$NON-NLS-1$
3776       case TokenNameAND_EQUAL :
3777         return "&="; //$NON-NLS-1$
3778       case TokenNameOR_EQUAL :
3779         return "|="; //$NON-NLS-1$
3780       case TokenNameXOR_EQUAL :
3781         return "^="; //$NON-NLS-1$
3782       case TokenNameREMAINDER_EQUAL :
3783         return "%="; //$NON-NLS-1$
3784       case TokenNameDOT_EQUAL :
3785         return ".="; //$NON-NLS-1$
3786       case TokenNameLEFT_SHIFT_EQUAL :
3787         return "<<="; //$NON-NLS-1$
3788       case TokenNameRIGHT_SHIFT_EQUAL :
3789         return ">>="; //$NON-NLS-1$
3790       case TokenNameOR_OR :
3791         return "||"; //$NON-NLS-1$
3792       case TokenNameAND_AND :
3793         return "&&"; //$NON-NLS-1$
3794       case TokenNamePLUS :
3795         return "+"; //$NON-NLS-1$
3796       case TokenNameMINUS :
3797         return "-"; //$NON-NLS-1$
3798       case TokenNameMINUS_GREATER :
3799         return "->";
3800       case TokenNameNOT :
3801         return "!"; //$NON-NLS-1$
3802       case TokenNameREMAINDER :
3803         return "%"; //$NON-NLS-1$
3804       case TokenNameXOR :
3805         return "^"; //$NON-NLS-1$
3806       case TokenNameAND :
3807         return "&"; //$NON-NLS-1$
3808       case TokenNameMULTIPLY :
3809         return "*"; //$NON-NLS-1$
3810       case TokenNameOR :
3811         return "|"; //$NON-NLS-1$
3812       case TokenNameTWIDDLE :
3813         return "~"; //$NON-NLS-1$
3814       case TokenNameTWIDDLE_EQUAL :
3815         return "~="; //$NON-NLS-1$
3816       case TokenNameDIVIDE :
3817         return "/"; //$NON-NLS-1$
3818       case TokenNameGREATER :
3819         return ">"; //$NON-NLS-1$
3820       case TokenNameLESS :
3821         return "<"; //$NON-NLS-1$
3822       case TokenNameLPAREN :
3823         return "("; //$NON-NLS-1$
3824       case TokenNameRPAREN :
3825         return ")"; //$NON-NLS-1$
3826       case TokenNameLBRACE :
3827         return "{"; //$NON-NLS-1$
3828       case TokenNameRBRACE :
3829         return "}"; //$NON-NLS-1$
3830       case TokenNameLBRACKET :
3831         return "["; //$NON-NLS-1$
3832       case TokenNameRBRACKET :
3833         return "]"; //$NON-NLS-1$
3834       case TokenNameSEMICOLON :
3835         return ";"; //$NON-NLS-1$
3836       case TokenNameQUESTION :
3837         return "?"; //$NON-NLS-1$
3838       case TokenNameCOLON :
3839         return ":"; //$NON-NLS-1$
3840       case TokenNameCOMMA :
3841         return ","; //$NON-NLS-1$
3842       case TokenNameDOT :
3843         return "."; //$NON-NLS-1$
3844       case TokenNameEQUAL :
3845         return "="; //$NON-NLS-1$
3846       case TokenNameAT :
3847         return "@";
3848       case TokenNameDOLLAR :
3849         return "$";
3850       case TokenNameDOLLAR_LBRACE :
3851         return "${";
3852       case TokenNameEOF :
3853         return "EOF"; //$NON-NLS-1$
3854       case TokenNameWHITESPACE :
3855         return "WHITESPACE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3856       case TokenNameCOMMENT_LINE :
3857         return "COMMENT_LINE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3858       case TokenNameCOMMENT_BLOCK :
3859         return "COMMENT_BLOCK(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3860       case TokenNameCOMMENT_PHPDOC :
3861         return "COMMENT_PHPDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3862       //      case TokenNameHTML :
3863       //        return "HTML(" + new String(getCurrentTokenSource()) + ")";
3864       // //$NON-NLS-1$
3865       case TokenNameFILE :
3866         return "__FILE__"; //$NON-NLS-1$
3867       case TokenNameLINE :
3868         return "__LINE__"; //$NON-NLS-1$
3869       case TokenNameCLASS_C :
3870         return "__CLASS__"; //$NON-NLS-1$
3871       case TokenNameMETHOD_C :
3872         return "__METHOD__"; //$NON-NLS-1$
3873       case TokenNameFUNC_C :
3874         return "__FUNCTION__"; //$NON-NLS-1
3875       case TokenNameboolCAST :
3876         return "( bool )"; //$NON-NLS-1$
3877       case TokenNameintCAST :
3878         return "( int )"; //$NON-NLS-1$
3879       case TokenNamedoubleCAST :
3880         return "( double )"; //$NON-NLS-1$
3881       case TokenNameobjectCAST :
3882         return "( object )"; //$NON-NLS-1$
3883       case TokenNamestringCAST :
3884         return "( string )"; //$NON-NLS-1$
3885       default :
3886         return "not-a-token(" + (new Integer(act)) + ") "
3887             + new String(getCurrentTokenSource()); //$NON-NLS-1$
3888     }
3889   }
3890
3891   public Scanner() {
3892     this(false, false);
3893   }
3894   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
3895     this(tokenizeComments, tokenizeWhiteSpace, false);
3896   }
3897   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
3898       boolean checkNonExternalizedStringLiterals) {
3899     this(tokenizeComments, tokenizeWhiteSpace,
3900         checkNonExternalizedStringLiterals, false);
3901   }
3902   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
3903       boolean checkNonExternalizedStringLiterals, boolean assertMode) {
3904     this(tokenizeComments, tokenizeWhiteSpace,
3905         checkNonExternalizedStringLiterals, assertMode, false, null, null);
3906   }
3907   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
3908       boolean checkNonExternalizedStringLiterals, boolean assertMode,
3909       boolean tokenizeStrings,
3910       char[][] taskTags,
3911           char[][] taskPriorities) {
3912     this.eofPosition = Integer.MAX_VALUE;
3913     this.tokenizeComments = tokenizeComments;
3914     this.tokenizeWhiteSpace = tokenizeWhiteSpace;
3915     this.tokenizeStrings = tokenizeStrings;
3916     this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
3917     this.assertMode = assertMode;
3918     this.encapsedStringStack = null;
3919     this.taskTags = taskTags;
3920         this.taskPriorities = taskPriorities;
3921   }
3922   private void checkNonExternalizeString() throws InvalidInputException {
3923     if (currentLine == null)
3924       return;
3925     parseTags(currentLine);
3926   }
3927   private void parseTags(NLSLine line) throws InvalidInputException {
3928     String s = new String(getCurrentTokenSource());
3929     int pos = s.indexOf(TAG_PREFIX);
3930     int lineLength = line.size();
3931     while (pos != -1) {
3932       int start = pos + TAG_PREFIX_LENGTH;
3933       int end = s.indexOf(TAG_POSTFIX, start);
3934       String index = s.substring(start, end);
3935       int i = 0;
3936       try {
3937         i = Integer.parseInt(index) - 1;
3938         // Tags are one based not zero based.
3939       } catch (NumberFormatException e) {
3940         i = -1; // we don't want to consider this as a valid NLS tag
3941       }
3942       if (line.exists(i)) {
3943         line.set(i, null);
3944       }
3945       pos = s.indexOf(TAG_PREFIX, start);
3946     }
3947     this.nonNLSStrings = new StringLiteral[lineLength];
3948     int nonNLSCounter = 0;
3949     for (Iterator iterator = line.iterator(); iterator.hasNext();) {
3950       StringLiteral literal = (StringLiteral) iterator.next();
3951       if (literal != null) {
3952         this.nonNLSStrings[nonNLSCounter++] = literal;
3953       }
3954     }
3955     if (nonNLSCounter == 0) {
3956       this.nonNLSStrings = null;
3957       currentLine = null;
3958       return;
3959     }
3960     this.wasNonExternalizedStringLiteral = true;
3961     if (nonNLSCounter != lineLength) {
3962       System.arraycopy(this.nonNLSStrings, 0,
3963           (this.nonNLSStrings = new StringLiteral[nonNLSCounter]), 0,
3964           nonNLSCounter);
3965     }
3966     currentLine = null;
3967   }
3968   public final void scanEscapeCharacter() throws InvalidInputException {
3969     // the string with "\\u" is a legal string of two chars \ and u
3970     //thus we use a direct access to the source (for regular cases).
3971     if (unicodeAsBackSlash) {
3972       // consume next character
3973       unicodeAsBackSlash = false;
3974       //                        if (((currentCharacter = source[currentPosition++]) == '\\') &&
3975       // (source[currentPosition] == 'u')) {
3976       //                                getNextUnicodeChar();
3977       //                        } else {
3978       if (withoutUnicodePtr != 0) {
3979         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3980         //                              }
3981       }
3982     } else
3983       currentCharacter = source[currentPosition++];
3984     switch (currentCharacter) {
3985       case 'b' :
3986         currentCharacter = '\b';
3987         break;
3988       case 't' :
3989         currentCharacter = '\t';
3990         break;
3991       case 'n' :
3992         currentCharacter = '\n';
3993         break;
3994       case 'f' :
3995         currentCharacter = '\f';
3996         break;
3997       case 'r' :
3998         currentCharacter = '\r';
3999         break;
4000       case '\"' :
4001         currentCharacter = '\"';
4002         break;
4003       case '\'' :
4004         currentCharacter = '\'';
4005         break;
4006       case '\\' :
4007         currentCharacter = '\\';
4008         break;
4009       default :
4010         // -----------octal escape--------------
4011         // OctalDigit
4012         // OctalDigit OctalDigit
4013         // ZeroToThree OctalDigit OctalDigit
4014         int number = Character.getNumericValue(currentCharacter);
4015         if (number >= 0 && number <= 7) {
4016           boolean zeroToThreeNot = number > 3;
4017           if (Character.isDigit(currentCharacter = source[currentPosition++])) {
4018             int digit = Character.getNumericValue(currentCharacter);
4019             if (digit >= 0 && digit <= 7) {
4020               number = (number * 8) + digit;
4021               if (Character
4022                   .isDigit(currentCharacter = source[currentPosition++])) {
4023                 if (zeroToThreeNot) { // has read \NotZeroToThree OctalDigit
4024                   // Digit --> ignore last character
4025                   currentPosition--;
4026                 } else {
4027                   digit = Character.getNumericValue(currentCharacter);
4028                   if (digit >= 0 && digit <= 7) { // has read \ZeroToThree
4029                     // OctalDigit OctalDigit
4030                     number = (number * 8) + digit;
4031                   } else { // has read \ZeroToThree OctalDigit NonOctalDigit
4032                     // --> ignore last character
4033                     currentPosition--;
4034                   }
4035                 }
4036               } else { // has read \OctalDigit NonDigit--> ignore last
4037                 // character
4038                 currentPosition--;
4039               }
4040             } else { // has read \OctalDigit NonOctalDigit--> ignore last
4041               // character
4042               currentPosition--;
4043             }
4044           } else { // has read \OctalDigit --> ignore last character
4045             currentPosition--;
4046           }
4047           if (number > 255)
4048             throw new InvalidInputException(INVALID_ESCAPE);
4049           currentCharacter = (char) number;
4050         } else
4051           throw new InvalidInputException(INVALID_ESCAPE);
4052     }
4053   }
4054   // chech presence of task: tags
4055   public void checkTaskTag(int commentStart, int commentEnd) {
4056     // only look for newer task: tags
4057     if (this.foundTaskCount > 0
4058         && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
4059       return;
4060     }
4061     int foundTaskIndex = this.foundTaskCount;
4062     nextChar : for (int i = commentStart; i < commentEnd
4063         && i < this.eofPosition; i++) {
4064       char[] tag = null;
4065       char[] priority = null;
4066       // check for tag occurrence
4067       nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) {
4068         tag = this.taskTags[itag];
4069         priority = this.taskPriorities != null
4070             && itag < this.taskPriorities.length
4071             ? this.taskPriorities[itag]
4072             : null;
4073         int tagLength = tag.length;
4074         for (int t = 0; t < tagLength; t++) {
4075           if (this.source[i + t] != tag[t])
4076             continue nextTag;
4077         }
4078         if (this.foundTaskTags == null) {
4079           this.foundTaskTags = new char[5][];
4080           this.foundTaskMessages = new char[5][];
4081           this.foundTaskPriorities = new char[5][];
4082           this.foundTaskPositions = new int[5][];
4083         } else if (this.foundTaskCount == this.foundTaskTags.length) {
4084           System.arraycopy(this.foundTaskTags, 0,
4085               this.foundTaskTags = new char[this.foundTaskCount * 2][], 0,
4086               this.foundTaskCount);
4087           System.arraycopy(this.foundTaskMessages, 0,
4088               this.foundTaskMessages = new char[this.foundTaskCount * 2][], 0,
4089               this.foundTaskCount);
4090           System.arraycopy(this.foundTaskPriorities, 0,
4091               this.foundTaskPriorities = new char[this.foundTaskCount * 2][],
4092               0, this.foundTaskCount);
4093           System.arraycopy(this.foundTaskPositions, 0,
4094               this.foundTaskPositions = new int[this.foundTaskCount * 2][], 0,
4095               this.foundTaskCount);
4096         }
4097         this.foundTaskTags[this.foundTaskCount] = tag;
4098         this.foundTaskPriorities[this.foundTaskCount] = priority;
4099         this.foundTaskPositions[this.foundTaskCount] = new int[]{i,
4100             i + tagLength - 1};
4101         this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
4102         this.foundTaskCount++;
4103         i += tagLength - 1; // will be incremented when looping
4104       }
4105     }
4106     for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
4107       // retrieve message start and end positions
4108       int msgStart = this.foundTaskPositions[i][0]
4109           + this.foundTaskTags[i].length;
4110       int max_value = i + 1 < this.foundTaskCount
4111           ? this.foundTaskPositions[i + 1][0] - 1
4112           : commentEnd - 1;
4113       // at most beginning of next task
4114       if (max_value < msgStart)
4115         max_value = msgStart; // would only occur if tag is before EOF.
4116       int end = -1;
4117       char c;
4118       for (int j = msgStart; j < max_value; j++) {
4119         if ((c = this.source[j]) == '\n' || c == '\r') {
4120           end = j - 1;
4121           break;
4122         }
4123       }
4124       if (end == -1) {
4125         for (int j = max_value; j > msgStart; j--) {
4126           if ((c = this.source[j]) == '*') {
4127             end = j - 1;
4128             break;
4129           }
4130         }
4131         if (end == -1)
4132           end = max_value;
4133       }
4134       if (msgStart == end)
4135         continue; // empty
4136       // trim the message
4137       while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
4138         end--;
4139       while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
4140         msgStart++;
4141       // update the end position of the task
4142       this.foundTaskPositions[i][1] = end;
4143       // get the message source
4144       final int messageLength = end - msgStart + 1;
4145       char[] message = new char[messageLength];
4146       System.arraycopy(source, msgStart, message, 0, messageLength);
4147       this.foundTaskMessages[i] = message;
4148     }
4149   }
4150 }