net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2001, 2002 International Business Machines Corp. and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v0.5
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v05.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  ******************************************************************************/
  11 package net.sourceforge.phpdt.internal.compiler.parser;
  12
  13 import java.util.ArrayList;
  14 import java.util.Iterator;
  15 import java.util.List;
  16
  17 import net.sourceforge.phpdt.core.compiler.CharOperation;
  18 import net.sourceforge.phpdt.core.compiler.IScanner;
  19 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
  20 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
  21 import net.sourceforge.phpdt.internal.compiler.ast.StringLiteral;
  22
  23 public class Scanner implements IScanner, ITerminalSymbols {
  24
  25   /*
  26    * APIs ares - getNextToken() which return the current type of the token (this value is not memorized by the scanner) -
  27    * getCurrentTokenSource() which provides with the token "REAL" source (aka all unicode have been transformed into a correct
  28    * char) - sourceStart gives the position into the stream - currentPosition-1 gives the sourceEnd position into the stream
  29    */
  30
  31   // 1.4 feature
  32   private boolean assertMode;
  33   public boolean useAssertAsAnIndentifier = false;
  34   //flag indicating if processed source contains occurrences of keyword assert
  35   public boolean containsAssertKeyword = false;
  36
  37   public boolean recordLineSeparator;
  38   public boolean phpMode = false;
  39
  40   public char currentCharacter;
  41   public int startPosition;
  42   public int currentPosition;
  43   public int initialPosition, eofPosition;
  44   // after this position eof are generated instead of real token from the source
  45
  46   public boolean tokenizeComments;
  47   public boolean tokenizeWhiteSpace;
  48
  49   //source should be viewed as a window (aka a part)
  50   //of a entire very large stream
  51   public char source[];
  52
  53   //unicode support
  54   public char[] withoutUnicodeBuffer;
  55   public int withoutUnicodePtr;
  56   //when == 0 ==> no unicode in the current token
  57   public boolean unicodeAsBackSlash = false;
  58
  59   public boolean scanningFloatLiteral = false;
  60
  61   //support for /** comments
  62   //public char[][] comments = new char[10][];
  63   public int[] commentStops = new int[10];
  64   public int[] commentStarts = new int[10];
  65   public int commentPtr = -1; // no comment test with commentPtr value -1
  66
  67   //diet parsing support - jump over some method body when requested
  68   public boolean diet = false;
  69
  70   //support for the poor-line-debuggers ....
  71   //remember the position of the cr/lf
  72   public int[] lineEnds = new int[250];
  73   public int linePtr = -1;
  74   public boolean wasAcr = false;
  75
  76   public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
  77
  78   public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
  79   public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
  80   public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
  81   public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
  82   public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
  83   public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
  84   public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
  85
  86   public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
  87   public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
  88   public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
  89   public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
  90
  91   //----------------optimized identifier managment------------------
  92   static final char[] charArray_a = new char[] { 'a' },
  93     charArray_b = new char[] { 'b' },
  94     charArray_c = new char[] { 'c' },
  95     charArray_d = new char[] { 'd' },
  96     charArray_e = new char[] { 'e' },
  97     charArray_f = new char[] { 'f' },
  98     charArray_g = new char[] { 'g' },
  99     charArray_h = new char[] { 'h' },
 100     charArray_i = new char[] { 'i' },
 101     charArray_j = new char[] { 'j' },
 102     charArray_k = new char[] { 'k' },
 103     charArray_l = new char[] { 'l' },
 104     charArray_m = new char[] { 'm' },
 105     charArray_n = new char[] { 'n' },
 106     charArray_o = new char[] { 'o' },
 107     charArray_p = new char[] { 'p' },
 108     charArray_q = new char[] { 'q' },
 109     charArray_r = new char[] { 'r' },
 110     charArray_s = new char[] { 's' },
 111     charArray_t = new char[] { 't' },
 112     charArray_u = new char[] { 'u' },
 113     charArray_v = new char[] { 'v' },
 114     charArray_w = new char[] { 'w' },
 115     charArray_x = new char[] { 'x' },
 116     charArray_y = new char[] { 'y' },
 117     charArray_z = new char[] { 'z' };
 118
 119   static final char[] initCharArray = new char[] { '\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000' };
 120   static final int TableSize = 30, InternalTableSize = 6;
 121   //30*6 = 180 entries
 122   public static final int OptimizedLength = 6;
 123   public /* static */
 124   final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][];
 125   // support for detecting non-externalized string literals
 126   int currentLineNr = -1;
 127   int previousLineNr = -1;
 128   NLSLine currentLine = null;
 129   List lines = new ArrayList();
 130   public static final String TAG_PREFIX = "//$NON-NLS-"; //$NON-NLS-1$
 131   public static final int TAG_PREFIX_LENGTH = TAG_PREFIX.length();
 132   public static final String TAG_POSTFIX = "$"; //$NON-NLS-1$
 133   public static final int TAG_POSTFIX_LENGTH = TAG_POSTFIX.length();
 134   public StringLiteral[] nonNLSStrings = null;
 135   public boolean checkNonExternalizedStringLiterals = true;
 136   public boolean wasNonExternalizedStringLiteral = false;
 137
 138   /* static */ {
 139     for (int i = 0; i < 6; i++) {
 140       for (int j = 0; j < TableSize; j++) {
 141         for (int k = 0; k < InternalTableSize; k++) {
 142           charArray_length[i][j][k] = initCharArray;
 143         }
 144       }
 145     }
 146   }
 147   static int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0, newEntry5 = 0, newEntry6 = 0;
 148
 149   public static final int RoundBracket = 0;
 150   public static final int SquareBracket = 1;
 151   public static final int CurlyBracket = 2;
 152   public static final int BracketKinds = 3;
 153
 154   // task tag support
 155   public char[][] foundTaskTags = null;
 156   public char[][] foundTaskMessages;
 157   public char[][] foundTaskPriorities = null;
 158   public int[][] foundTaskPositions;
 159   public int foundTaskCount = 0;
 160   public char[][] taskTags = null;
 161   public char[][] taskPriorities = null;
 162
 163   public static final boolean DEBUG = true;
 164
 165   public Scanner() {
 166     this(false, false);
 167   }
 168   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
 169     this(tokenizeComments, tokenizeWhiteSpace, false);
 170   }
 171
 172   /**
 173    * Determines if the specified character is permissible as the first character in a PHP identifier
 174    */
 175   public static boolean isPHPIdentifierStart(char ch) {
 176     return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
 177   }
 178
 179   /**
 180    * Determines if the specified character may be part of a PHP identifier as other than the first character
 181    */
 182   public static boolean isPHPIdentifierPart(char ch) {
 183     return Character.isLetterOrDigit(ch) || (ch == '_') || (0x7F <= ch && ch <= 0xFF);
 184   }
 185
 186   public final boolean atEnd() {
 187     // This code is not relevant if source is
 188     // Only a part of the real stream input
 189
 190     return source.length == currentPosition;
 191   }
 192   public char[] getCurrentIdentifierSource() {
 193     //return the token REAL source (aka unicodes are precomputed)
 194
 195     char[] result;
 196     //    if (withoutUnicodePtr != 0)
 197     //      //0 is used as a fast test flag so the real first char is in position 1
 198     //      System.arraycopy(
 199     //        withoutUnicodeBuffer,
 200     //        1,
 201     //        result = new char[withoutUnicodePtr],
 202     //        0,
 203     //        withoutUnicodePtr);
 204     //    else {
 205     int length = currentPosition - startPosition;
 206     switch (length) { // see OptimizedLength
 207       case 1 :
 208         return optimizedCurrentTokenSource1();
 209       case 2 :
 210         return optimizedCurrentTokenSource2();
 211       case 3 :
 212         return optimizedCurrentTokenSource3();
 213       case 4 :
 214         return optimizedCurrentTokenSource4();
 215       case 5 :
 216         return optimizedCurrentTokenSource5();
 217       case 6 :
 218         return optimizedCurrentTokenSource6();
 219     }
 220     //no optimization
 221     System.arraycopy(source, startPosition, result = new char[length], 0, length);
 222     //   }
 223     return result;
 224   }
 225   public int getCurrentTokenEndPosition() {
 226     return this.currentPosition - 1;
 227   }
 228
 229   public final char[] getCurrentTokenSource() {
 230     // Return the token REAL source (aka unicodes are precomputed)
 231
 232     char[] result;
 233     //    if (withoutUnicodePtr != 0)
 234     //      // 0 is used as a fast test flag so the real first char is in position 1
 235     //      System.arraycopy(
 236     //        withoutUnicodeBuffer,
 237     //        1,
 238     //        result = new char[withoutUnicodePtr],
 239     //        0,
 240     //        withoutUnicodePtr);
 241     //    else {
 242     int length;
 243     System.arraycopy(source, startPosition, result = new char[length = currentPosition - startPosition], 0, length);
 244     //    }
 245     return result;
 246   }
 247
 248   public final char[] getCurrentTokenSource(int startPos) {
 249     // Return the token REAL source (aka unicodes are precomputed)
 250
 251     char[] result;
 252     //    if (withoutUnicodePtr != 0)
 253     //      // 0 is used as a fast test flag so the real first char is in position 1
 254     //      System.arraycopy(
 255     //        withoutUnicodeBuffer,
 256     //        1,
 257     //        result = new char[withoutUnicodePtr],
 258     //        0,
 259     //        withoutUnicodePtr);
 260     //    else {
 261     int length;
 262     System.arraycopy(source, startPos, result = new char[length = currentPosition - startPos], 0, length);
 263     //  }
 264     return result;
 265   }
 266
 267   public final char[] getCurrentTokenSourceString() {
 268     //return the token REAL source (aka unicodes are precomputed).
 269     //REMOVE the two " that are at the beginning and the end.
 270
 271     char[] result;
 272     if (withoutUnicodePtr != 0)
 273       //0 is used as a fast test flag so the real first char is in position 1
 274       System.arraycopy(withoutUnicodeBuffer, 2,
 275       //2 is 1 (real start) + 1 (to jump over the ")
 276       result = new char[withoutUnicodePtr - 2], 0, withoutUnicodePtr - 2);
 277     else {
 278       int length;
 279       System.arraycopy(source, startPosition + 1, result = new char[length = currentPosition - startPosition - 2], 0, length);
 280     }
 281     return result;
 282   }
 283   public int getCurrentTokenStartPosition() {
 284     return this.startPosition;
 285   }
 286
 287   public final char[] getCurrentStringLiteralSource() {
 288     // Return the token REAL source (aka unicodes are precomputed)
 289
 290     char[] result;
 291
 292     int length;
 293     System.arraycopy(source, startPosition + 1, result = new char[length = currentPosition - startPosition - 2], 0, length);
 294     //    }
 295     return result;
 296   }
 297
 298   /*
 299    * Search the source position corresponding to the end of a given line number
 300    *
 301    * Line numbers are 1-based, and relative to the scanner initialPosition. Character positions are 0-based.
 302    *
 303    * In case the given line number is inconsistent, answers -1.
 304    */
 305   public final int getLineEnd(int lineNumber) {
 306
 307     if (lineEnds == null)
 308       return -1;
 309     if (lineNumber >= lineEnds.length)
 310       return -1;
 311     if (lineNumber <= 0)
 312       return -1;
 313
 314     if (lineNumber == lineEnds.length - 1)
 315       return eofPosition;
 316     return lineEnds[lineNumber - 1];
 317     // next line start one character behind the lineEnd of the previous line
 318   }
 319   /**
 320    * Search the source position corresponding to the beginning of a given line number
 321    *
 322    * Line numbers are 1-based, and relative to the scanner initialPosition. Character positions are 0-based.
 323    *
 324    * e.g. getLineStart(1) --> 0 i.e. first line starts at character 0.
 325    *
 326    * In case the given line number is inconsistent, answers -1.
 327    */
 328   public final int getLineStart(int lineNumber) {
 329
 330     if (lineEnds == null)
 331       return -1;
 332     if (lineNumber >= lineEnds.length)
 333       return -1;
 334     if (lineNumber <= 0)
 335       return -1;
 336
 337     if (lineNumber == 1)
 338       return initialPosition;
 339     return lineEnds[lineNumber - 2] + 1;
 340     // next line start one character behind the lineEnd of the previous line
 341   }
 342   public final boolean getNextChar(char testedChar) {
 343     //BOOLEAN
 344     //handle the case of unicode.
 345     //when a unicode appears then we must use a buffer that holds char internal values
 346     //At the end of this method currentCharacter holds the new visited char
 347     //and currentPosition points right next after it
 348     //Both previous lines are true if the currentCharacter is == to the testedChar
 349     //On false, no side effect has occured.
 350
 351     //ALL getNextChar.... ARE OPTIMIZED COPIES
 352
 353     int temp = currentPosition;
 354     try {
 355       currentCharacter = source[currentPosition++];
 356       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 357       //        && (source[currentPosition] == 'u')) {
 358       //        //-------------unicode traitement ------------
 359       //        int c1, c2, c3, c4;
 360       //        int unicodeSize = 6;
 361       //        currentPosition++;
 362       //        while (source[currentPosition] == 'u') {
 363       //          currentPosition++;
 364       //          unicodeSize++;
 365       //        }
 366       //
 367       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 368       //          || c1 < 0)
 369       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 370       //            || c2 < 0)
 371       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 372       //            || c3 < 0)
 373       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 374       //            || c4 < 0)) {
 375       //          currentPosition = temp;
 376       //          return false;
 377       //        }
 378       //
 379       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 380       //        if (currentCharacter != testedChar) {
 381       //          currentPosition = temp;
 382       //          return false;
 383       //        }
 384       //        unicodeAsBackSlash = currentCharacter == '\\';
 385       //
 386       //        //need the unicode buffer
 387       //        if (withoutUnicodePtr == 0) {
 388       //          //buffer all the entries that have been left aside....
 389       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 390       //          System.arraycopy(
 391       //            source,
 392       //            startPosition,
 393       //            withoutUnicodeBuffer,
 394       //            1,
 395       //            withoutUnicodePtr);
 396       //        }
 397       //        //fill the buffer with the char
 398       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 399       //        return true;
 400       //
 401       //      } //-------------end unicode traitement--------------
 402       //      else {
 403       if (currentCharacter != testedChar) {
 404         currentPosition = temp;
 405         return false;
 406       }
 407       unicodeAsBackSlash = false;
 408       //        if (withoutUnicodePtr != 0)
 409       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 410       return true;
 411       //      }
 412     } catch (IndexOutOfBoundsException e) {
 413       unicodeAsBackSlash = false;
 414       currentPosition = temp;
 415       return false;
 416     }
 417   }
 418   public final int getNextChar(char testedChar1, char testedChar2) {
 419     //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
 420     //test can be done with (x==0) for the first and (x>0) for the second
 421     //handle the case of unicode.
 422     //when a unicode appears then we must use a buffer that holds char internal values
 423     //At the end of this method currentCharacter holds the new visited char
 424     //and currentPosition points right next after it
 425     //Both previous lines are true if the currentCharacter is == to the testedChar1/2
 426     //On false, no side effect has occured.
 427
 428     //ALL getNextChar.... ARE OPTIMIZED COPIES
 429
 430     int temp = currentPosition;
 431     try {
 432       int result;
 433       currentCharacter = source[currentPosition++];
 434       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 435       //        && (source[currentPosition] == 'u')) {
 436       //        //-------------unicode traitement ------------
 437       //        int c1, c2, c3, c4;
 438       //        int unicodeSize = 6;
 439       //        currentPosition++;
 440       //        while (source[currentPosition] == 'u') {
 441       //          currentPosition++;
 442       //          unicodeSize++;
 443       //        }
 444       //
 445       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 446       //          || c1 < 0)
 447       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 448       //            || c2 < 0)
 449       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 450       //            || c3 < 0)
 451       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 452       //            || c4 < 0)) {
 453       //          currentPosition = temp;
 454       //          return 2;
 455       //        }
 456       //
 457       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 458       //        if (currentCharacter == testedChar1)
 459       //          result = 0;
 460       //        else if (currentCharacter == testedChar2)
 461       //          result = 1;
 462       //        else {
 463       //          currentPosition = temp;
 464       //          return -1;
 465       //        }
 466       //
 467       //        //need the unicode buffer
 468       //        if (withoutUnicodePtr == 0) {
 469       //          //buffer all the entries that have been left aside....
 470       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 471       //          System.arraycopy(
 472       //            source,
 473       //            startPosition,
 474       //            withoutUnicodeBuffer,
 475       //            1,
 476       //            withoutUnicodePtr);
 477       //        }
 478       //        //fill the buffer with the char
 479       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 480       //        return result;
 481       //      } //-------------end unicode traitement--------------
 482       //      else {
 483       if (currentCharacter == testedChar1)
 484         result = 0;
 485       else if (currentCharacter == testedChar2)
 486         result = 1;
 487       else {
 488         currentPosition = temp;
 489         return -1;
 490       }
 491
 492       //        if (withoutUnicodePtr != 0)
 493       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 494       return result;
 495       //     }
 496     } catch (IndexOutOfBoundsException e) {
 497       currentPosition = temp;
 498       return -1;
 499     }
 500   }
 501   public final boolean getNextCharAsDigit() {
 502     //BOOLEAN
 503     //handle the case of unicode.
 504     //when a unicode appears then we must use a buffer that holds char internal values
 505     //At the end of this method currentCharacter holds the new visited char
 506     //and currentPosition points right next after it
 507     //Both previous lines are true if the currentCharacter is a digit
 508     //On false, no side effect has occured.
 509
 510     //ALL getNextChar.... ARE OPTIMIZED COPIES
 511
 512     int temp = currentPosition;
 513     try {
 514       currentCharacter = source[currentPosition++];
 515       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 516       //        && (source[currentPosition] == 'u')) {
 517       //        //-------------unicode traitement ------------
 518       //        int c1, c2, c3, c4;
 519       //        int unicodeSize = 6;
 520       //        currentPosition++;
 521       //        while (source[currentPosition] == 'u') {
 522       //          currentPosition++;
 523       //          unicodeSize++;
 524       //        }
 525       //
 526       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 527       //          || c1 < 0)
 528       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 529       //            || c2 < 0)
 530       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 531       //            || c3 < 0)
 532       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 533       //            || c4 < 0)) {
 534       //          currentPosition = temp;
 535       //          return false;
 536       //        }
 537       //
 538       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 539       //        if (!Character.isDigit(currentCharacter)) {
 540       //          currentPosition = temp;
 541       //          return false;
 542       //        }
 543       //
 544       //        //need the unicode buffer
 545       //        if (withoutUnicodePtr == 0) {
 546       //          //buffer all the entries that have been left aside....
 547       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 548       //          System.arraycopy(
 549       //            source,
 550       //            startPosition,
 551       //            withoutUnicodeBuffer,
 552       //            1,
 553       //            withoutUnicodePtr);
 554       //        }
 555       //        //fill the buffer with the char
 556       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 557       //        return true;
 558       //      } //-------------end unicode traitement--------------
 559       //      else {
 560       if (!Character.isDigit(currentCharacter)) {
 561         currentPosition = temp;
 562         return false;
 563       }
 564       //        if (withoutUnicodePtr != 0)
 565       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 566       return true;
 567       //      }
 568     } catch (IndexOutOfBoundsException e) {
 569       currentPosition = temp;
 570       return false;
 571     }
 572   }
 573   public final boolean getNextCharAsDigit(int radix) {
 574     //BOOLEAN
 575     //handle the case of unicode.
 576     //when a unicode appears then we must use a buffer that holds char internal values
 577     //At the end of this method currentCharacter holds the new visited char
 578     //and currentPosition points right next after it
 579     //Both previous lines are true if the currentCharacter is a digit base on radix
 580     //On false, no side effect has occured.
 581
 582     //ALL getNextChar.... ARE OPTIMIZED COPIES
 583
 584     int temp = currentPosition;
 585     try {
 586       currentCharacter = source[currentPosition++];
 587       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 588       //        && (source[currentPosition] == 'u')) {
 589       //        //-------------unicode traitement ------------
 590       //        int c1, c2, c3, c4;
 591       //        int unicodeSize = 6;
 592       //        currentPosition++;
 593       //        while (source[currentPosition] == 'u') {
 594       //          currentPosition++;
 595       //          unicodeSize++;
 596       //        }
 597       //
 598       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 599       //          || c1 < 0)
 600       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 601       //            || c2 < 0)
 602       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 603       //            || c3 < 0)
 604       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 605       //            || c4 < 0)) {
 606       //          currentPosition = temp;
 607       //          return false;
 608       //        }
 609       //
 610       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 611       //        if (Character.digit(currentCharacter, radix) == -1) {
 612       //          currentPosition = temp;
 613       //          return false;
 614       //        }
 615       //
 616       //        //need the unicode buffer
 617       //        if (withoutUnicodePtr == 0) {
 618       //          //buffer all the entries that have been left aside....
 619       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 620       //          System.arraycopy(
 621       //            source,
 622       //            startPosition,
 623       //            withoutUnicodeBuffer,
 624       //            1,
 625       //            withoutUnicodePtr);
 626       //        }
 627       //        //fill the buffer with the char
 628       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 629       //        return true;
 630       //      } //-------------end unicode traitement--------------
 631       //      else {
 632       if (Character.digit(currentCharacter, radix) == -1) {
 633         currentPosition = temp;
 634         return false;
 635       }
 636       //        if (withoutUnicodePtr != 0)
 637       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 638       return true;
 639       //      }
 640     } catch (IndexOutOfBoundsException e) {
 641       currentPosition = temp;
 642       return false;
 643     }
 644   }
 645   public boolean getNextCharAsJavaIdentifierPart() {
 646     //BOOLEAN
 647     //handle the case of unicode.
 648     //when a unicode appears then we must use a buffer that holds char internal values
 649     //At the end of this method currentCharacter holds the new visited char
 650     //and currentPosition points right next after it
 651     //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
 652     //On false, no side effect has occured.
 653
 654     //ALL getNextChar.... ARE OPTIMIZED COPIES
 655
 656     int temp = currentPosition;
 657     try {
 658       currentCharacter = source[currentPosition++];
 659       //      if (((currentCharacter = source[currentPosition++]) == '\\')
 660       //        && (source[currentPosition] == 'u')) {
 661       //        //-------------unicode traitement ------------
 662       //        int c1, c2, c3, c4;
 663       //        int unicodeSize = 6;
 664       //        currentPosition++;
 665       //        while (source[currentPosition] == 'u') {
 666       //          currentPosition++;
 667       //          unicodeSize++;
 668       //        }
 669       //
 670       //        if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
 671       //          || c1 < 0)
 672       //          || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
 673       //            || c2 < 0)
 674       //          || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
 675       //            || c3 < 0)
 676       //          || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
 677       //            || c4 < 0)) {
 678       //          currentPosition = temp;
 679       //          return false;
 680       //        }
 681       //
 682       //        currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 683       //        if (!isPHPIdentifierPart(currentCharacter)) {
 684       //          currentPosition = temp;
 685       //          return false;
 686       //        }
 687       //
 688       //        //need the unicode buffer
 689       //        if (withoutUnicodePtr == 0) {
 690       //          //buffer all the entries that have been left aside....
 691       //          withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
 692       //          System.arraycopy(
 693       //            source,
 694       //            startPosition,
 695       //            withoutUnicodeBuffer,
 696       //            1,
 697       //            withoutUnicodePtr);
 698       //        }
 699       //        //fill the buffer with the char
 700       //        withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 701       //        return true;
 702       //      } //-------------end unicode traitement--------------
 703       //      else {
 704       if (!isPHPIdentifierPart(currentCharacter)) {
 705         currentPosition = temp;
 706         return false;
 707       }
 708
 709       //        if (withoutUnicodePtr != 0)
 710       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 711       return true;
 712       //      }
 713     } catch (IndexOutOfBoundsException e) {
 714       currentPosition = temp;
 715       return false;
 716     }
 717   }
 718
 719   public int getNextToken() throws InvalidInputException {
 720     int htmlPosition = currentPosition;
 721     try {
 722       while (!phpMode) {
 723         currentCharacter = source[currentPosition++];
 724         if (currentCharacter == '<') {
 725           if (getNextChar('?')) {
 726             currentCharacter = source[currentPosition++];
 727             if ((currentCharacter == ' ') || Character.isWhitespace(currentCharacter)) {
 728               // <?
 729               startPosition = currentPosition;
 730               phpMode = true;
 731               if (tokenizeWhiteSpace) {
 732                 // && (whiteStart != currentPosition - 1)) {
 733                 // reposition scanner in case we are interested by spaces as tokens
 734                 startPosition = htmlPosition;
 735                 return TokenNameHTML;
 736               }
 737             } else {
 738               boolean phpStart = (currentCharacter == 'P') || (currentCharacter == 'p');
 739               if (phpStart) {
 740                 int test = getNextChar('H', 'h');
 741                 if (test >= 0) {
 742                   test = getNextChar('P', 'p');
 743                   if (test >= 0) {
 744                     // <?PHP <?php
 745                     startPosition = currentPosition;
 746                     phpMode = true;
 747
 748                     if (tokenizeWhiteSpace) {
 749                       // && (whiteStart != currentPosition - 1)) {
 750                       // reposition scanner in case we are interested by spaces as tokens
 751                       startPosition = htmlPosition;
 752                       return TokenNameHTML;
 753                     }
 754                   }
 755                 }
 756               }
 757             }
 758           }
 759         }
 760
 761         if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
 762           if (recordLineSeparator) {
 763             pushLineSeparator();
 764           } else {
 765             currentLine = null;
 766           }
 767         }
 768       }
 769     } //-----------------end switch while try--------------------
 770     catch (IndexOutOfBoundsException e) {
 771       if (tokenizeWhiteSpace) {
 772         // && (whiteStart != currentPosition - 1)) {
 773         // reposition scanner in case we are interested by spaces as tokens
 774         startPosition = htmlPosition;
 775       }
 776       return TokenNameEOF;
 777     }
 778
 779     if (phpMode) {
 780       this.wasAcr = false;
 781       if (diet) {
 782         jumpOverMethodBody();
 783         diet = false;
 784         return currentPosition > source.length ? TokenNameEOF : TokenNameRBRACE;
 785       }
 786       try {
 787         while (true) { //loop for jumping over comments
 788           withoutUnicodePtr = 0;
 789           //start with a new token (even comment written with unicode )
 790
 791           // ---------Consume white space and handles startPosition---------
 792           int whiteStart = currentPosition;
 793           boolean isWhiteSpace;
 794           do {
 795             startPosition = currentPosition;
 796             currentCharacter = source[currentPosition++];
 797             //            if (((currentCharacter = source[currentPosition++]) == '\\')
 798             //              && (source[currentPosition] == 'u')) {
 799             //              isWhiteSpace = jumpOverUnicodeWhiteSpace();
 800             //            } else {
 801             if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
 802               checkNonExternalizeString();
 803               if (recordLineSeparator) {
 804                 pushLineSeparator();
 805               } else {
 806                 currentLine = null;
 807               }
 808             }
 809             isWhiteSpace = (currentCharacter == ' ') || Character.isWhitespace(currentCharacter);
 810             //            }
 811           } while (isWhiteSpace);
 812           if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
 813             // reposition scanner in case we are interested by spaces as tokens
 814             currentPosition--;
 815             startPosition = whiteStart;
 816             return TokenNameWHITESPACE;
 817           }
 818           //little trick to get out in the middle of a source compuation
 819           if (currentPosition > eofPosition)
 820             return TokenNameEOF;
 821
 822           // ---------Identify the next token-------------
 823
 824           switch (currentCharacter) {
 825             case '(' :
 826               return TokenNameLPAREN;
 827             case ')' :
 828               return TokenNameRPAREN;
 829             case '{' :
 830               return TokenNameLBRACE;
 831             case '}' :
 832               return TokenNameRBRACE;
 833             case '[' :
 834               return TokenNameLBRACKET;
 835             case ']' :
 836               return TokenNameRBRACKET;
 837             case ';' :
 838               return TokenNameSEMICOLON;
 839             case ',' :
 840               return TokenNameCOMMA;
 841
 842             case '.' :
 843               if (getNextCharAsDigit())
 844                 return scanNumber(true);
 845               return TokenNameDOT;
 846             case '+' :
 847               {
 848                 int test;
 849                 if ((test = getNextChar('+', '=')) == 0)
 850                   return TokenNamePLUS_PLUS;
 851                 if (test > 0)
 852                   return TokenNamePLUS_EQUAL;
 853                 return TokenNamePLUS;
 854               }
 855             case '-' :
 856               {
 857                 int test;
 858                 if ((test = getNextChar('-', '=')) == 0)
 859                   return TokenNameMINUS_MINUS;
 860                 if (test > 0)
 861                   return TokenNameMINUS_EQUAL;
 862                 if (getNextChar('>'))
 863                   return TokenNameMINUS_GREATER;
 864
 865                 return TokenNameMINUS;
 866               }
 867             case '~' :
 868               if (getNextChar('='))
 869                 return TokenNameTWIDDLE_EQUAL;
 870               return TokenNameTWIDDLE;
 871             case '!' :
 872               if (getNextChar('=')) {
 873                 if (getNextChar('=')) {
 874                   return TokenNameNOT_EQUAL_EQUAL;
 875                 }
 876                 return TokenNameNOT_EQUAL;
 877               }
 878               return TokenNameNOT;
 879             case '*' :
 880               if (getNextChar('='))
 881                 return TokenNameMULTIPLY_EQUAL;
 882               return TokenNameMULTIPLY;
 883             case '%' :
 884               if (getNextChar('='))
 885                 return TokenNameREMAINDER_EQUAL;
 886               return TokenNameREMAINDER;
 887             case '<' :
 888               {
 889                 int test;
 890                 if ((test = getNextChar('=', '<')) == 0)
 891                   return TokenNameLESS_EQUAL;
 892                 if (test > 0) {
 893                   if (getNextChar('='))
 894                     return TokenNameLEFT_SHIFT_EQUAL;
 895                   if (getNextChar('<')) {
 896                     int heredocStart = currentPosition;
 897                     int heredocLength = 0;
 898                     currentCharacter = source[currentPosition++];
 899                     if (isPHPIdentifierStart(currentCharacter)) {
 900                       currentCharacter = source[currentPosition++];
 901                     } else {
 902                       return TokenNameERROR;
 903                     }
 904                     while (isPHPIdentifierPart(currentCharacter)) {
 905                       currentCharacter = source[currentPosition++];
 906                     }
 907
 908                     heredocLength = currentPosition - heredocStart - 1;
 909
 910                     // heredoc end-tag determination
 911                     boolean endTag = true;
 912                     char ch;
 913                     do {
 914                       ch = source[currentPosition++];
 915                       if (ch == '\r' || ch == '\n') {
 916                         if (recordLineSeparator) {
 917                           pushLineSeparator();
 918                         } else {
 919                           currentLine = null;
 920                         }
 921                         for (int i = 0; i < heredocLength; i++) {
 922                           if (source[currentPosition + i] != source[heredocStart + i]) {
 923                             endTag = false;
 924                             break;
 925                           }
 926                         }
 927                         if (endTag) {
 928                           currentPosition += heredocLength - 1;
 929                           currentCharacter = source[currentPosition++];
 930                           break; // do...while loop
 931                         } else {
 932                           endTag = true;
 933                         }
 934                       }
 935
 936                     } while (true);
 937
 938                     return TokenNameHEREDOC;
 939                   }
 940                   return TokenNameLEFT_SHIFT;
 941                 }
 942                 return TokenNameLESS;
 943               }
 944             case '>' :
 945               {
 946                 int test;
 947                 if ((test = getNextChar('=', '>')) == 0)
 948                   return TokenNameGREATER_EQUAL;
 949                 if (test > 0) {
 950                   if ((test = getNextChar('=', '>')) == 0)
 951                     return TokenNameRIGHT_SHIFT_EQUAL;
 952                   return TokenNameRIGHT_SHIFT;
 953                 }
 954                 return TokenNameGREATER;
 955               }
 956             case '=' :
 957               if (getNextChar('=')) {
 958                 if (getNextChar('=')) {
 959                   return TokenNameEQUAL_EQUAL_EQUAL;
 960                 }
 961                 return TokenNameEQUAL_EQUAL;
 962               }
 963               if (getNextChar('>'))
 964                 return TokenNameEQUAL_GREATER;
 965               return TokenNameEQUAL;
 966             case '&' :
 967               {
 968                 int test;
 969                 if ((test = getNextChar('&', '=')) == 0)
 970                   return TokenNameAND_AND;
 971                 if (test > 0)
 972                   return TokenNameAND_EQUAL;
 973                 return TokenNameAND;
 974               }
 975             case '|' :
 976               {
 977                 int test;
 978                 if ((test = getNextChar('|', '=')) == 0)
 979                   return TokenNameOR_OR;
 980                 if (test > 0)
 981                   return TokenNameOR_EQUAL;
 982                 return TokenNameOR;
 983               }
 984             case '^' :
 985               if (getNextChar('='))
 986                 return TokenNameXOR_EQUAL;
 987               return TokenNameXOR;
 988             case '?' :
 989               if (getNextChar('>')) {
 990                 phpMode = false;
 991                 return TokenNameStopPHP;
 992               }
 993               return TokenNameQUESTION;
 994             case ':' :
 995               if (getNextChar(':'))
 996                 return TokenNameCOLON_COLON;
 997               return TokenNameCOLON;
 998             case '@' :
 999               return TokenNameAT;
1000               //                                        case '\'' :
1001               //                                                {
1002               //                                                        int test;
1003               //                                                        if ((test = getNextChar('\n', '\r')) == 0) {
1004               //                                                                throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1005               //                                                        }
1006               //                                                        if (test > 0) {
1007               //                                                                // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1008               //                                                                for (int lookAhead = 0;
1009               //                                                                        lookAhead < 3;
1010               //                                                                        lookAhead++) {
1011               //                                                                        if (currentPosition + lookAhead
1012               //                                                                                == source.length)
1013               //                                                                                break;
1014               //                                                                        if (source[currentPosition + lookAhead]
1015               //                                                                                == '\n')
1016               //                                                                                break;
1017               //                                                                        if (source[currentPosition + lookAhead]
1018               //                                                                                == '\'') {
1019               //                                                                                currentPosition += lookAhead + 1;
1020               //                                                                                break;
1021               //                                                                        }
1022               //                                                                }
1023               //                                                                throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1024               //                                                        }
1025               //                                                }
1026               //                                                if (getNextChar('\'')) {
1027               //                                                        // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1028               //                                                        for (int lookAhead = 0;
1029               //                                                                lookAhead < 3;
1030               //                                                                lookAhead++) {
1031               //                                                                if (currentPosition + lookAhead
1032               //                                                                        == source.length)
1033               //                                                                        break;
1034               //                                                                if (source[currentPosition + lookAhead]
1035               //                                                                        == '\n')
1036               //                                                                        break;
1037               //                                                                if (source[currentPosition + lookAhead]
1038               //                                                                        == '\'') {
1039               //                                                                        currentPosition += lookAhead + 1;
1040               //                                                                        break;
1041               //                                                                }
1042               //                                                        }
1043               //                                                        throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1044               //                                                }
1045               //                                                if (getNextChar('\\'))
1046               //                                                        scanEscapeCharacter();
1047               //                                                else { // consume next character
1048               //                                                        unicodeAsBackSlash = false;
1049               //                                                        if (((currentCharacter = source[currentPosition++])
1050               //                                                                == '\\')
1051               //                                                                && (source[currentPosition] == 'u')) {
1052               //                                                                getNextUnicodeChar();
1053               //                                                        } else {
1054               //                                                                if (withoutUnicodePtr != 0) {
1055               //                                                                        withoutUnicodeBuffer[++withoutUnicodePtr] =
1056               //                                                                                currentCharacter;
1057               //                                                                }
1058               //                                                        }
1059               //                                                }
1060               //                                                // if (getNextChar('\''))
1061               //                                                // return TokenNameCharacterLiteral;
1062               //                                                // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1063               //                                                for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
1064               //                                                        if (currentPosition + lookAhead == source.length)
1065               //                                                                break;
1066               //                                                        if (source[currentPosition + lookAhead] == '\n')
1067               //                                                                break;
1068               //                                                        if (source[currentPosition + lookAhead] == '\'') {
1069               //                                                                currentPosition += lookAhead + 1;
1070               //                                                                break;
1071               //                                                        }
1072               //                                                }
1073               //                                                throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
1074             case '\'' :
1075               try {
1076                 // consume next character
1077                 unicodeAsBackSlash = false;
1078                 currentCharacter = source[currentPosition++];
1079                 //                if (((currentCharacter = source[currentPosition++]) == '\\')
1080                 //                  && (source[currentPosition] == 'u')) {
1081                 //                  getNextUnicodeChar();
1082                 //                } else {
1083                 //                  if (withoutUnicodePtr != 0) {
1084                 //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
1085                 //                      currentCharacter;
1086                 //                  }
1087                 //                }
1088
1089                 while (currentCharacter != '\'') {
1090
1091                   /** ** in PHP \r and \n are valid in string literals *** */
1092                   //                  if ((currentCharacter == '\n')
1093                   //                    || (currentCharacter == '\r')) {
1094                   //                    // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1095                   //                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1096                   //                      if (currentPosition + lookAhead == source.length)
1097                   //                        break;
1098                   //                      if (source[currentPosition + lookAhead] == '\n')
1099                   //                        break;
1100                   //                      if (source[currentPosition + lookAhead] == '\"') {
1101                   //                        currentPosition += lookAhead + 1;
1102                   //                        break;
1103                   //                      }
1104                   //                    }
1105                   //                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1106                   //                  }
1107                   if (currentCharacter == '\\') {
1108                     int escapeSize = currentPosition;
1109                     boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1110                     //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
1111                     scanSingleQuotedEscapeCharacter();
1112                     escapeSize = currentPosition - escapeSize;
1113                     if (withoutUnicodePtr == 0) {
1114                       //buffer all the entries that have been left aside....
1115                       withoutUnicodePtr = currentPosition - escapeSize - 1 - startPosition;
1116                       System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1, withoutUnicodePtr);
1117                       withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1118                     } else { //overwrite the / in the buffer
1119                       withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1120                       if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
1121                         withoutUnicodePtr--;
1122                       }
1123                     }
1124                   }
1125                   // consume next character
1126                   unicodeAsBackSlash = false;
1127                   currentCharacter = source[currentPosition++];
1128                   //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1129                   //                    && (source[currentPosition] == 'u')) {
1130                   //                    getNextUnicodeChar();
1131                   //                  } else {
1132                   if (withoutUnicodePtr != 0) {
1133                     withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1134                   }
1135                   //                  }
1136
1137                 }
1138               } catch (IndexOutOfBoundsException e) {
1139                 throw new InvalidInputException(UNTERMINATED_STRING);
1140               } catch (InvalidInputException e) {
1141                 if (e.getMessage().equals(INVALID_ESCAPE)) {
1142                   // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1143                   for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1144                     if (currentPosition + lookAhead == source.length)
1145                       break;
1146                     if (source[currentPosition + lookAhead] == '\n')
1147                       break;
1148                     if (source[currentPosition + lookAhead] == '\'') {
1149                       currentPosition += lookAhead + 1;
1150                       break;
1151                     }
1152                   }
1153
1154                 }
1155                 throw e; // rethrow
1156               }
1157               if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags //$NON-NLS-?$ where ? is an int.
1158                 if (currentLine == null) {
1159                   currentLine = new NLSLine();
1160                   lines.add(currentLine);
1161                 }
1162                 currentLine.add(new StringLiteral(getCurrentTokenSourceString(), startPosition, currentPosition - 1));
1163               }
1164               return TokenNameStringConstant;
1165             case '"' :
1166               try {
1167                 // consume next character
1168                 unicodeAsBackSlash = false;
1169                 currentCharacter = source[currentPosition++];
1170                 //                if (((currentCharacter = source[currentPosition++]) == '\\')
1171                 //                  && (source[currentPosition] == 'u')) {
1172                 //                  getNextUnicodeChar();
1173                 //                } else {
1174                 //                  if (withoutUnicodePtr != 0) {
1175                 //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
1176                 //                      currentCharacter;
1177                 //                  }
1178                 //                }
1179
1180                 while (currentCharacter != '"') {
1181
1182                   /** ** in PHP \r and \n are valid in string literals *** */
1183                   //                  if ((currentCharacter == '\n')
1184                   //                    || (currentCharacter == '\r')) {
1185                   //                    // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1186                   //                    for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1187                   //                      if (currentPosition + lookAhead == source.length)
1188                   //                        break;
1189                   //                      if (source[currentPosition + lookAhead] == '\n')
1190                   //                        break;
1191                   //                      if (source[currentPosition + lookAhead] == '\"') {
1192                   //                        currentPosition += lookAhead + 1;
1193                   //                        break;
1194                   //                      }
1195                   //                    }
1196                   //                    throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1197                   //                  }
1198                   if (currentCharacter == '\\') {
1199                     int escapeSize = currentPosition;
1200                     boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1201                     //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
1202                     scanDoubleQuotedEscapeCharacter();
1203                     escapeSize = currentPosition - escapeSize;
1204                     if (withoutUnicodePtr == 0) {
1205                       //buffer all the entries that have been left aside....
1206                       withoutUnicodePtr = currentPosition - escapeSize - 1 - startPosition;
1207                       System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1, withoutUnicodePtr);
1208                       withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1209                     } else { //overwrite the / in the buffer
1210                       withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1211                       if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
1212                         withoutUnicodePtr--;
1213                       }
1214                     }
1215                   }
1216                   // consume next character
1217                   unicodeAsBackSlash = false;
1218                   currentCharacter = source[currentPosition++];
1219                   //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1220                   //                    && (source[currentPosition] == 'u')) {
1221                   //                    getNextUnicodeChar();
1222                   //                  } else {
1223                   if (withoutUnicodePtr != 0) {
1224                     withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1225                   }
1226                   //                  }
1227
1228                 }
1229               } catch (IndexOutOfBoundsException e) {
1230                 throw new InvalidInputException(UNTERMINATED_STRING);
1231               } catch (InvalidInputException e) {
1232                 if (e.getMessage().equals(INVALID_ESCAPE)) {
1233                   // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1234                   for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1235                     if (currentPosition + lookAhead == source.length)
1236                       break;
1237                     if (source[currentPosition + lookAhead] == '\n')
1238                       break;
1239                     if (source[currentPosition + lookAhead] == '\"') {
1240                       currentPosition += lookAhead + 1;
1241                       break;
1242                     }
1243                   }
1244
1245                 }
1246                 throw e; // rethrow
1247               }
1248               if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags //$NON-NLS-?$ where ? is an int.
1249                 if (currentLine == null) {
1250                   currentLine = new NLSLine();
1251                   lines.add(currentLine);
1252                 }
1253                 currentLine.add(new StringLiteral(getCurrentTokenSourceString(), startPosition, currentPosition - 1));
1254               }
1255               return TokenNameStringLiteral;
1256             case '`' :
1257               try {
1258                 // consume next character
1259                 unicodeAsBackSlash = false;
1260                 currentCharacter = source[currentPosition++];
1261                 //                if (((currentCharacter = source[currentPosition++]) == '\\')
1262                 //                  && (source[currentPosition] == 'u')) {
1263                 //                  getNextUnicodeChar();
1264                 //                } else {
1265                 //                  if (withoutUnicodePtr != 0) {
1266                 //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
1267                 //                      currentCharacter;
1268                 //                  }
1269                 //                }
1270
1271                 while (currentCharacter != '`') {
1272
1273                   /** ** in PHP \r and \n are valid in string literals *** */
1274                   //                if ((currentCharacter == '\n')
1275                   //                  || (currentCharacter == '\r')) {
1276                   //                  // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1277                   //                  for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1278                   //                    if (currentPosition + lookAhead == source.length)
1279                   //                      break;
1280                   //                    if (source[currentPosition + lookAhead] == '\n')
1281                   //                      break;
1282                   //                    if (source[currentPosition + lookAhead] == '\"') {
1283                   //                      currentPosition += lookAhead + 1;
1284                   //                      break;
1285                   //                    }
1286                   //                  }
1287                   //                  throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1288                   //                }
1289                   if (currentCharacter == '\\') {
1290                     int escapeSize = currentPosition;
1291                     boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1292                     //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
1293                     scanDoubleQuotedEscapeCharacter();
1294                     escapeSize = currentPosition - escapeSize;
1295                     if (withoutUnicodePtr == 0) {
1296                       //buffer all the entries that have been left aside....
1297                       withoutUnicodePtr = currentPosition - escapeSize - 1 - startPosition;
1298                       System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1, withoutUnicodePtr);
1299                       withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1300                     } else { //overwrite the / in the buffer
1301                       withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1302                       if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
1303                         withoutUnicodePtr--;
1304                       }
1305                     }
1306                   }
1307                   // consume next character
1308                   unicodeAsBackSlash = false;
1309                   currentCharacter = source[currentPosition++];
1310                   //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1311                   //                    && (source[currentPosition] == 'u')) {
1312                   //                    getNextUnicodeChar();
1313                   //                  } else {
1314                   if (withoutUnicodePtr != 0) {
1315                     withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1316                   }
1317                   //                  }
1318
1319                 }
1320               } catch (IndexOutOfBoundsException e) {
1321                 throw new InvalidInputException(UNTERMINATED_STRING);
1322               } catch (InvalidInputException e) {
1323                 if (e.getMessage().equals(INVALID_ESCAPE)) {
1324                   // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
1325                   for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1326                     if (currentPosition + lookAhead == source.length)
1327                       break;
1328                     if (source[currentPosition + lookAhead] == '\n')
1329                       break;
1330                     if (source[currentPosition + lookAhead] == '`') {
1331                       currentPosition += lookAhead + 1;
1332                       break;
1333                     }
1334                   }
1335
1336                 }
1337                 throw e; // rethrow
1338               }
1339               if (checkNonExternalizedStringLiterals) { // check for presence of NLS tags //$NON-NLS-?$ where ? is an int.
1340                 if (currentLine == null) {
1341                   currentLine = new NLSLine();
1342                   lines.add(currentLine);
1343                 }
1344                 currentLine.add(new StringLiteral(getCurrentTokenSourceString(), startPosition, currentPosition - 1));
1345               }
1346               return TokenNameStringInterpolated;
1347             case '#' :
1348             case '/' :
1349               {
1350                 int test;
1351                 if ((currentCharacter == '#') || (test = getNextChar('/', '*')) == 0) {
1352                   //line comment
1353                   int endPositionForLineComment = 0;
1354                   try { //get the next char
1355                     currentCharacter = source[currentPosition++];
1356                     //                    if (((currentCharacter = source[currentPosition++])
1357                     //                      == '\\')
1358                     //                      && (source[currentPosition] == 'u')) {
1359                     //                      //-------------unicode traitement ------------
1360                     //                      int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1361                     //                      currentPosition++;
1362                     //                      while (source[currentPosition] == 'u') {
1363                     //                        currentPosition++;
1364                     //                      }
1365                     //                      if ((c1 =
1366                     //                        Character.getNumericValue(source[currentPosition++]))
1367                     //                        > 15
1368                     //                        || c1 < 0
1369                     //                        || (c2 =
1370                     //                          Character.getNumericValue(source[currentPosition++]))
1371                     //                          > 15
1372                     //                        || c2 < 0
1373                     //                        || (c3 =
1374                     //                          Character.getNumericValue(source[currentPosition++]))
1375                     //                          > 15
1376                     //                        || c3 < 0
1377                     //                        || (c4 =
1378                     //                          Character.getNumericValue(source[currentPosition++]))
1379                     //                          > 15
1380                     //                        || c4 < 0) {
1381                     //                        throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1382                     //                      } else {
1383                     //                        currentCharacter =
1384                     //                          (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1385                     //                      }
1386                     //                    }
1387
1388                     //handle the \\u case manually into comment
1389                     //                    if (currentCharacter == '\\') {
1390                     //                      if (source[currentPosition] == '\\')
1391                     //                        currentPosition++;
1392                     //                    } //jump over the \\
1393                     boolean isUnicode = false;
1394                     while (currentCharacter != '\r' && currentCharacter != '\n') {
1395                       if (currentCharacter == '?') {
1396                         if (getNextChar('>')) {
1397                           startPosition = currentPosition - 2;
1398                           phpMode = false;
1399                           return TokenNameStopPHP;
1400                         }
1401                       }
1402
1403                       //get the next char
1404                       isUnicode = false;
1405                       currentCharacter = source[currentPosition++];
1406                       //                      if (((currentCharacter = source[currentPosition++])
1407                       //                        == '\\')
1408                       //                        && (source[currentPosition] == 'u')) {
1409                       //                        isUnicode = true;
1410                       //                        //-------------unicode traitement ------------
1411                       //                        int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1412                       //                        currentPosition++;
1413                       //                        while (source[currentPosition] == 'u') {
1414                       //                          currentPosition++;
1415                       //                        }
1416                       //                        if ((c1 =
1417                       //                          Character.getNumericValue(source[currentPosition++]))
1418                       //                          > 15
1419                       //                          || c1 < 0
1420                       //                          || (c2 =
1421                       //                            Character.getNumericValue(
1422                       //                              source[currentPosition++]))
1423                       //                            > 15
1424                       //                          || c2 < 0
1425                       //                          || (c3 =
1426                       //                            Character.getNumericValue(
1427                       //                              source[currentPosition++]))
1428                       //                            > 15
1429                       //                          || c3 < 0
1430                       //                          || (c4 =
1431                       //                            Character.getNumericValue(
1432                       //                              source[currentPosition++]))
1433                       //                            > 15
1434                       //                          || c4 < 0) {
1435                       //                          throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1436                       //                        } else {
1437                       //                          currentCharacter =
1438                       //                            (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1439                       //                        }
1440                       //                      }
1441                       //handle the \\u case manually into comment
1442                       //                      if (currentCharacter == '\\') {
1443                       //                        if (source[currentPosition] == '\\')
1444                       //                          currentPosition++;
1445                       //                      } //jump over the \\
1446                     }
1447                     if (isUnicode) {
1448                       endPositionForLineComment = currentPosition - 6;
1449                     } else {
1450                       endPositionForLineComment = currentPosition - 1;
1451                     }
1452                     recordComment(false);
1453                     if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1454                       checkNonExternalizeString();
1455                       if (recordLineSeparator) {
1456                         if (isUnicode) {
1457                           pushUnicodeLineSeparator();
1458                         } else {
1459                           pushLineSeparator();
1460                         }
1461                       } else {
1462                         currentLine = null;
1463                       }
1464                     }
1465                     if (tokenizeComments) {
1466                       if (!isUnicode) {
1467                         currentPosition = endPositionForLineComment;
1468                         // reset one character behind
1469                       }
1470                       return TokenNameCOMMENT_LINE;
1471                     }
1472                   } catch (IndexOutOfBoundsException e) { //an eof will them be generated
1473                     if (tokenizeComments) {
1474                       currentPosition--;
1475                       // reset one character behind
1476                       return TokenNameCOMMENT_LINE;
1477                     }
1478                   }
1479                   break;
1480                 }
1481                 if (test > 0) {
1482                   //traditional and annotation comment
1483                   boolean isJavadoc = false, star = false;
1484                   // consume next character
1485                   unicodeAsBackSlash = false;
1486                   currentCharacter = source[currentPosition++];
1487                   //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1488                   //                    && (source[currentPosition] == 'u')) {
1489                   //                    getNextUnicodeChar();
1490                   //                  } else {
1491                   //                    if (withoutUnicodePtr != 0) {
1492                   //                      withoutUnicodeBuffer[++withoutUnicodePtr] =
1493                   //                        currentCharacter;
1494                   //                    }
1495                   //                  }
1496
1497                   if (currentCharacter == '*') {
1498                     isJavadoc = true;
1499                     star = true;
1500                   }
1501                   if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1502                     checkNonExternalizeString();
1503                     if (recordLineSeparator) {
1504                       pushLineSeparator();
1505                     } else {
1506                       currentLine = null;
1507                     }
1508                   }
1509                   try { //get the next char
1510                     currentCharacter = source[currentPosition++];
1511                     //                    if (((currentCharacter = source[currentPosition++])
1512                     //                      == '\\')
1513                     //                      && (source[currentPosition] == 'u')) {
1514                     //                      //-------------unicode traitement ------------
1515                     //                      getNextUnicodeChar();
1516                     //                    }
1517                     //handle the \\u case manually into comment
1518                     //                    if (currentCharacter == '\\') {
1519                     //                      if (source[currentPosition] == '\\')
1520                     //                        currentPosition++;
1521                     //                      //jump over the \\
1522                     //                    }
1523                     // empty comment is not a javadoc /**/
1524                     if (currentCharacter == '/') {
1525                       isJavadoc = false;
1526                     }
1527                     //loop until end of comment */
1528                     while ((currentCharacter != '/') || (!star)) {
1529                       if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
1530                         checkNonExternalizeString();
1531                         if (recordLineSeparator) {
1532                           pushLineSeparator();
1533                         } else {
1534                           currentLine = null;
1535                         }
1536                       }
1537                       star = currentCharacter == '*';
1538                       //get next char
1539                       currentCharacter = source[currentPosition++];
1540                       //                      if (((currentCharacter = source[currentPosition++])
1541                       //                        == '\\')
1542                       //                        && (source[currentPosition] == 'u')) {
1543                       //                        //-------------unicode traitement ------------
1544                       //                        getNextUnicodeChar();
1545                       //                      }
1546                       //handle the \\u case manually into comment
1547                       //                      if (currentCharacter == '\\') {
1548                       //                        if (source[currentPosition] == '\\')
1549                       //                          currentPosition++;
1550                       //                      } //jump over the \\
1551                     }
1552                     recordComment(isJavadoc);
1553                     if (tokenizeComments) {
1554                       if (isJavadoc)
1555                         return TokenNameCOMMENT_PHPDOC;
1556                       return TokenNameCOMMENT_BLOCK;
1557                     }
1558                   } catch (IndexOutOfBoundsException e) {
1559                     throw new InvalidInputException(UNTERMINATED_COMMENT);
1560                   }
1561                   break;
1562                 }
1563                 if (getNextChar('='))
1564                   return TokenNameDIVIDE_EQUAL;
1565                 return TokenNameDIVIDE;
1566               }
1567             case '\u001a' :
1568               if (atEnd())
1569                 return TokenNameEOF;
1570               //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
1571               throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1572
1573             default :
1574               if (currentCharacter == '$') {
1575                 while ((currentCharacter = source[currentPosition++]) == '$') {
1576                 }
1577                 if (currentCharacter == '{')
1578                   return TokenNameDOLLAR_LBRACE;
1579                 if (isPHPIdentifierStart(currentCharacter))
1580                   return scanIdentifierOrKeyword(true);
1581                 return TokenNameERROR;
1582               }
1583               if (isPHPIdentifierStart(currentCharacter))
1584                 return scanIdentifierOrKeyword(false);
1585               if (Character.isDigit(currentCharacter))
1586                 return scanNumber(false);
1587               return TokenNameERROR;
1588           }
1589         }
1590       } //-----------------end switch while try--------------------
1591       catch (IndexOutOfBoundsException e) {
1592       }
1593     }
1594     return TokenNameEOF;
1595   }
1596
1597   //  public final void getNextUnicodeChar()
1598   //    throws IndexOutOfBoundsException, InvalidInputException {
1599   //    //VOID
1600   //    //handle the case of unicode.
1601   //    //when a unicode appears then we must use a buffer that holds char internal values
1602   //    //At the end of this method currentCharacter holds the new visited char
1603   //    //and currentPosition points right next after it
1604   //
1605   //    //ALL getNextChar.... ARE OPTIMIZED COPIES
1606   //
1607   //    int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
1608   //    currentPosition++;
1609   //    while (source[currentPosition] == 'u') {
1610   //      currentPosition++;
1611   //      unicodeSize++;
1612   //    }
1613   //
1614   //    if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
1615   //      || c1 < 0
1616   //      || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
1617   //      || c2 < 0
1618   //      || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
1619   //      || c3 < 0
1620   //      || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
1621   //      || c4 < 0) {
1622   //      throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1623   //    } else {
1624   //      currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1625   //      //need the unicode buffer
1626   //      if (withoutUnicodePtr == 0) {
1627   //        //buffer all the entries that have been left aside....
1628   //        withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
1629   //        System.arraycopy(
1630   //          source,
1631   //          startPosition,
1632   //          withoutUnicodeBuffer,
1633   //          1,
1634   //          withoutUnicodePtr);
1635   //      }
1636   //      //fill the buffer with the char
1637   //      withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1638   //    }
1639   //    unicodeAsBackSlash = currentCharacter == '\\';
1640   //  }
1641   /*
1642    * Tokenize a method body, assuming that curly brackets are properly balanced.
1643    */
1644   public final void jumpOverMethodBody() {
1645
1646     this.wasAcr = false;
1647     int found = 1;
1648     try {
1649       while (true) { //loop for jumping over comments
1650         // ---------Consume white space and handles startPosition---------
1651         boolean isWhiteSpace;
1652         do {
1653           startPosition = currentPosition;
1654           currentCharacter = source[currentPosition++];
1655           //          if (((currentCharacter = source[currentPosition++]) == '\\')
1656           //            && (source[currentPosition] == 'u')) {
1657           //            isWhiteSpace = jumpOverUnicodeWhiteSpace();
1658           //          } else {
1659           if (recordLineSeparator && ((currentCharacter == '\r') || (currentCharacter == '\n')))
1660             pushLineSeparator();
1661           isWhiteSpace = Character.isWhitespace(currentCharacter);
1662           //          }
1663         } while (isWhiteSpace);
1664
1665         // -------consume token until } is found---------
1666         switch (currentCharacter) {
1667           case '{' :
1668             found++;
1669             break;
1670           case '}' :
1671             found--;
1672             if (found == 0)
1673               return;
1674             break;
1675           case '\'' :
1676             {
1677               boolean test;
1678               test = getNextChar('\\');
1679               if (test) {
1680                 try {
1681                   scanDoubleQuotedEscapeCharacter();
1682                 } catch (InvalidInputException ex) {
1683                 };
1684               } else {
1685                 //                try { // consume next character
1686                 unicodeAsBackSlash = false;
1687                 currentCharacter = source[currentPosition++];
1688                 //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1689                 //                    && (source[currentPosition] == 'u')) {
1690                 //                    getNextUnicodeChar();
1691                 //                  } else {
1692                 if (withoutUnicodePtr != 0) {
1693                   withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1694                 }
1695                 //                  }
1696                 //                } catch (InvalidInputException ex) {
1697                 //                };
1698               }
1699               getNextChar('\'');
1700               break;
1701             }
1702           case '"' :
1703             try {
1704               //              try { // consume next character
1705               unicodeAsBackSlash = false;
1706               currentCharacter = source[currentPosition++];
1707               //                if (((currentCharacter = source[currentPosition++]) == '\\')
1708               //                  && (source[currentPosition] == 'u')) {
1709               //                  getNextUnicodeChar();
1710               //                } else {
1711               if (withoutUnicodePtr != 0) {
1712                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1713               }
1714               //                }
1715               //              } catch (InvalidInputException ex) {
1716               //              };
1717               while (currentCharacter != '"') {
1718                 if (currentCharacter == '\r') {
1719                   if (source[currentPosition] == '\n')
1720                     currentPosition++;
1721                   break;
1722                   // the string cannot go further that the line
1723                 }
1724                 if (currentCharacter == '\n') {
1725                   break;
1726                   // the string cannot go further that the line
1727                 }
1728                 if (currentCharacter == '\\') {
1729                   try {
1730                     scanDoubleQuotedEscapeCharacter();
1731                   } catch (InvalidInputException ex) {
1732                   };
1733                 }
1734                 //                try { // consume next character
1735                 unicodeAsBackSlash = false;
1736                 currentCharacter = source[currentPosition++];
1737                 //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1738                 //                    && (source[currentPosition] == 'u')) {
1739                 //                    getNextUnicodeChar();
1740                 //                  } else {
1741                 if (withoutUnicodePtr != 0) {
1742                   withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1743                 }
1744                 //                  }
1745                 //                } catch (InvalidInputException ex) {
1746                 //                };
1747               }
1748             } catch (IndexOutOfBoundsException e) {
1749               return;
1750             }
1751             break;
1752           case '/' :
1753             {
1754               int test;
1755               if ((test = getNextChar('/', '*')) == 0) {
1756                 //line comment
1757                 try {
1758                   //get the next char
1759                   currentCharacter = source[currentPosition++];
1760                   //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1761                   //                    && (source[currentPosition] == 'u')) {
1762                   //                    //-------------unicode traitement ------------
1763                   //                    int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1764                   //                    currentPosition++;
1765                   //                    while (source[currentPosition] == 'u') {
1766                   //                      currentPosition++;
1767                   //                    }
1768                   //                    if ((c1 =
1769                   //                      Character.getNumericValue(source[currentPosition++]))
1770                   //                      > 15
1771                   //                      || c1 < 0
1772                   //                      || (c2 =
1773                   //                        Character.getNumericValue(source[currentPosition++]))
1774                   //                        > 15
1775                   //                      || c2 < 0
1776                   //                      || (c3 =
1777                   //                        Character.getNumericValue(source[currentPosition++]))
1778                   //                        > 15
1779                   //                      || c3 < 0
1780                   //                      || (c4 =
1781                   //                        Character.getNumericValue(source[currentPosition++]))
1782                   //                        > 15
1783                   //                      || c4 < 0) {
1784                   //                      //error don't care of the value
1785                   //                      currentCharacter = 'A';
1786                   //                    } //something different from \n and \r
1787                   //                    else {
1788                   //                      currentCharacter =
1789                   //                        (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1790                   //                    }
1791                   //                  }
1792
1793                   while (currentCharacter != '\r' && currentCharacter != '\n') {
1794                     //get the next char
1795                     currentCharacter = source[currentPosition++];
1796                     //                    if (((currentCharacter = source[currentPosition++])
1797                     //                      == '\\')
1798                     //                      && (source[currentPosition] == 'u')) {
1799                     //                      //-------------unicode traitement ------------
1800                     //                      int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1801                     //                      currentPosition++;
1802                     //                      while (source[currentPosition] == 'u') {
1803                     //                        currentPosition++;
1804                     //                      }
1805                     //                      if ((c1 =
1806                     //                        Character.getNumericValue(source[currentPosition++]))
1807                     //                        > 15
1808                     //                        || c1 < 0
1809                     //                        || (c2 =
1810                     //                          Character.getNumericValue(source[currentPosition++]))
1811                     //                          > 15
1812                     //                        || c2 < 0
1813                     //                        || (c3 =
1814                     //                          Character.getNumericValue(source[currentPosition++]))
1815                     //                          > 15
1816                     //                        || c3 < 0
1817                     //                        || (c4 =
1818                     //                          Character.getNumericValue(source[currentPosition++]))
1819                     //                          > 15
1820                     //                        || c4 < 0) {
1821                     //                        //error don't care of the value
1822                     //                        currentCharacter = 'A';
1823                     //                      } //something different from \n and \r
1824                     //                      else {
1825                     //                        currentCharacter =
1826                     //                          (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1827                     //                      }
1828                     //                    }
1829                   }
1830                   if (recordLineSeparator && ((currentCharacter == '\r') || (currentCharacter == '\n')))
1831                     pushLineSeparator();
1832                 } catch (IndexOutOfBoundsException e) {
1833                 } //an eof will them be generated
1834                 break;
1835               }
1836               if (test > 0) {
1837                 //traditional and annotation comment
1838                 boolean star = false;
1839                 //                try { // consume next character
1840                 unicodeAsBackSlash = false;
1841                 currentCharacter = source[currentPosition++];
1842                 //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1843                 //                    && (source[currentPosition] == 'u')) {
1844                 //                    getNextUnicodeChar();
1845                 //                  } else {
1846                 if (withoutUnicodePtr != 0) {
1847                   withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1848                 }
1849                 //                  };
1850                 //                } catch (InvalidInputException ex) {
1851                 //                };
1852                 if (currentCharacter == '*') {
1853                   star = true;
1854                 }
1855                 if (recordLineSeparator && ((currentCharacter == '\r') || (currentCharacter == '\n')))
1856                   pushLineSeparator();
1857                 try { //get the next char
1858                   currentCharacter = source[currentPosition++];
1859                   //                  if (((currentCharacter = source[currentPosition++]) == '\\')
1860                   //                    && (source[currentPosition] == 'u')) {
1861                   //                    //-------------unicode traitement ------------
1862                   //                    int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1863                   //                    currentPosition++;
1864                   //                    while (source[currentPosition] == 'u') {
1865                   //                      currentPosition++;
1866                   //                    }
1867                   //                    if ((c1 =
1868                   //                      Character.getNumericValue(source[currentPosition++]))
1869                   //                      > 15
1870                   //                      || c1 < 0
1871                   //                      || (c2 =
1872                   //                        Character.getNumericValue(source[currentPosition++]))
1873                   //                        > 15
1874                   //                      || c2 < 0
1875                   //                      || (c3 =
1876                   //                        Character.getNumericValue(source[currentPosition++]))
1877                   //                        > 15
1878                   //                      || c3 < 0
1879                   //                      || (c4 =
1880                   //                        Character.getNumericValue(source[currentPosition++]))
1881                   //                        > 15
1882                   //                      || c4 < 0) {
1883                   //                      //error don't care of the value
1884                   //                      currentCharacter = 'A';
1885                   //                    } //something different from * and /
1886                   //                    else {
1887                   //                      currentCharacter =
1888                   //                        (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1889                   //                    }
1890                   //                  }
1891                   //loop until end of comment */
1892                   while ((currentCharacter != '/') || (!star)) {
1893                     if (recordLineSeparator && ((currentCharacter == '\r') || (currentCharacter == '\n')))
1894                       pushLineSeparator();
1895                     star = currentCharacter == '*';
1896                     //get next char
1897                     currentCharacter = source[currentPosition++];
1898                     //                    if (((currentCharacter = source[currentPosition++])
1899                     //                      == '\\')
1900                     //                      && (source[currentPosition] == 'u')) {
1901                     //                      //-------------unicode traitement ------------
1902                     //                      int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1903                     //                      currentPosition++;
1904                     //                      while (source[currentPosition] == 'u') {
1905                     //                        currentPosition++;
1906                     //                      }
1907                     //                      if ((c1 =
1908                     //                        Character.getNumericValue(source[currentPosition++]))
1909                     //                        > 15
1910                     //                        || c1 < 0
1911                     //                        || (c2 =
1912                     //                          Character.getNumericValue(source[currentPosition++]))
1913                     //                          > 15
1914                     //                        || c2 < 0
1915                     //                        || (c3 =
1916                     //                          Character.getNumericValue(source[currentPosition++]))
1917                     //                          > 15
1918                     //                        || c3 < 0
1919                     //                        || (c4 =
1920                     //                          Character.getNumericValue(source[currentPosition++]))
1921                     //                          > 15
1922                     //                        || c4 < 0) {
1923                     //                        //error don't care of the value
1924                     //                        currentCharacter = 'A';
1925                     //                      } //something different from * and /
1926                     //                      else {
1927                     //                        currentCharacter =
1928                     //                          (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1929                     //                      }
1930                     //                    }
1931                   }
1932                 } catch (IndexOutOfBoundsException e) {
1933                   return;
1934                 }
1935                 break;
1936               }
1937               break;
1938             }
1939
1940           default :
1941             if (isPHPIdentifierStart(currentCharacter) || currentCharacter == '$') {
1942               try {
1943                 scanIdentifierOrKeyword((currentCharacter == '$'));
1944               } catch (InvalidInputException ex) {
1945               };
1946               break;
1947             }
1948             if (Character.isDigit(currentCharacter)) {
1949               try {
1950                 scanNumber(false);
1951               } catch (InvalidInputException ex) {
1952               };
1953               break;
1954             }
1955         }
1956       }
1957       //-----------------end switch while try--------------------
1958     } catch (IndexOutOfBoundsException e) {
1959     } catch (InvalidInputException e) {
1960     }
1961     return;
1962   }
1963   //  public final boolean jumpOverUnicodeWhiteSpace()
1964   //    throws InvalidInputException {
1965   //    //BOOLEAN
1966   //    //handle the case of unicode. Jump over the next whiteSpace
1967   //    //making startPosition pointing on the next available char
1968   //    //On false, the currentCharacter is filled up with a potential
1969   //    //correct char
1970   //
1971   //    try {
1972   //      this.wasAcr = false;
1973   //      int c1, c2, c3, c4;
1974   //      int unicodeSize = 6;
1975   //      currentPosition++;
1976   //      while (source[currentPosition] == 'u') {
1977   //        currentPosition++;
1978   //        unicodeSize++;
1979   //      }
1980   //
1981   //      if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
1982   //        || c1 < 0)
1983   //        || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
1984   //          || c2 < 0)
1985   //        || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
1986   //          || c3 < 0)
1987   //        || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
1988   //          || c4 < 0)) {
1989   //        throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
1990   //      }
1991   //
1992   //      currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1993   //      if (recordLineSeparator
1994   //        && ((currentCharacter == '\r') || (currentCharacter == '\n')))
1995   //        pushLineSeparator();
1996   //      if (Character.isWhitespace(currentCharacter))
1997   //        return true;
1998   //
1999   //      //buffer the new char which is not a white space
2000   //      withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2001   //      //withoutUnicodePtr == 1 is true here
2002   //      return false;
2003   //    } catch (IndexOutOfBoundsException e) {
2004   //      throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2005   //    }
2006   //  }
2007   public final int[] getLineEnds() {
2008     //return a bounded copy of this.lineEnds
2009
2010     int[] copy;
2011     System.arraycopy(lineEnds, 0, copy = new int[linePtr + 1], 0, linePtr + 1);
2012     return copy;
2013   }
2014
2015   public char[] getSource() {
2016     return this.source;
2017   }
2018   final char[] optimizedCurrentTokenSource1() {
2019     //return always the same char[] build only once
2020
2021     //optimization at no speed cost of 99.5 % of the singleCharIdentifier
2022     char charOne = source[startPosition];
2023     switch (charOne) {
2024       case 'a' :
2025         return charArray_a;
2026       case 'b' :
2027         return charArray_b;
2028       case 'c' :
2029         return charArray_c;
2030       case 'd' :
2031         return charArray_d;
2032       case 'e' :
2033         return charArray_e;
2034       case 'f' :
2035         return charArray_f;
2036       case 'g' :
2037         return charArray_g;
2038       case 'h' :
2039         return charArray_h;
2040       case 'i' :
2041         return charArray_i;
2042       case 'j' :
2043         return charArray_j;
2044       case 'k' :
2045         return charArray_k;
2046       case 'l' :
2047         return charArray_l;
2048       case 'm' :
2049         return charArray_m;
2050       case 'n' :
2051         return charArray_n;
2052       case 'o' :
2053         return charArray_o;
2054       case 'p' :
2055         return charArray_p;
2056       case 'q' :
2057         return charArray_q;
2058       case 'r' :
2059         return charArray_r;
2060       case 's' :
2061         return charArray_s;
2062       case 't' :
2063         return charArray_t;
2064       case 'u' :
2065         return charArray_u;
2066       case 'v' :
2067         return charArray_v;
2068       case 'w' :
2069         return charArray_w;
2070       case 'x' :
2071         return charArray_x;
2072       case 'y' :
2073         return charArray_y;
2074       case 'z' :
2075         return charArray_z;
2076       default :
2077         return new char[] { charOne };
2078     }
2079   }
2080
2081   final char[] optimizedCurrentTokenSource2() {
2082     //try to return the same char[] build only once
2083
2084     char c0, c1;
2085     int hash = (((c0 = source[startPosition]) << 6) + (c1 = source[startPosition + 1])) % TableSize;
2086     char[][] table = charArray_length[0][hash];
2087     int i = newEntry2;
2088     while (++i < InternalTableSize) {
2089       char[] charArray = table[i];
2090       if ((c0 == charArray[0]) && (c1 == charArray[1]))
2091         return charArray;
2092     }
2093     //---------other side---------
2094     i = -1;
2095     int max = newEntry2;
2096     while (++i <= max) {
2097       char[] charArray = table[i];
2098       if ((c0 == charArray[0]) && (c1 == charArray[1]))
2099         return charArray;
2100     }
2101     //--------add the entry-------
2102     if (++max >= InternalTableSize)
2103       max = 0;
2104     char[] r;
2105     table[max] = (r = new char[] { c0, c1 });
2106     newEntry2 = max;
2107     return r;
2108   }
2109
2110   final char[] optimizedCurrentTokenSource3() {
2111     //try to return the same char[] build only once
2112
2113     char c0, c1, c2;
2114     int hash =
2115       (((c0 = source[startPosition]) << 12) + ((c1 = source[startPosition + 1]) << 6) + (c2 = source[startPosition + 2]))
2116         % TableSize;
2117     char[][] table = charArray_length[1][hash];
2118     int i = newEntry3;
2119     while (++i < InternalTableSize) {
2120       char[] charArray = table[i];
2121       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2122         return charArray;
2123     }
2124     //---------other side---------
2125     i = -1;
2126     int max = newEntry3;
2127     while (++i <= max) {
2128       char[] charArray = table[i];
2129       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]))
2130         return charArray;
2131     }
2132     //--------add the entry-------
2133     if (++max >= InternalTableSize)
2134       max = 0;
2135     char[] r;
2136     table[max] = (r = new char[] { c0, c1, c2 });
2137     newEntry3 = max;
2138     return r;
2139   }
2140
2141   final char[] optimizedCurrentTokenSource4() {
2142     //try to return the same char[] build only once
2143
2144     char c0, c1, c2, c3;
2145     long hash =
2146       ((((long) (c0 = source[startPosition])) << 18)
2147         + ((c1 = source[startPosition + 1]) << 12)
2148         + ((c2 = source[startPosition + 2]) << 6)
2149         + (c3 = source[startPosition + 3]))
2150         % TableSize;
2151     char[][] table = charArray_length[2][(int) hash];
2152     int i = newEntry4;
2153     while (++i < InternalTableSize) {
2154       char[] charArray = table[i];
2155       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]))
2156         return charArray;
2157     }
2158     //---------other side---------
2159     i = -1;
2160     int max = newEntry4;
2161     while (++i <= max) {
2162       char[] charArray = table[i];
2163       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]))
2164         return charArray;
2165     }
2166     //--------add the entry-------
2167     if (++max >= InternalTableSize)
2168       max = 0;
2169     char[] r;
2170     table[max] = (r = new char[] { c0, c1, c2, c3 });
2171     newEntry4 = max;
2172     return r;
2173
2174   }
2175
2176   final char[] optimizedCurrentTokenSource5() {
2177     //try to return the same char[] build only once
2178
2179     char c0, c1, c2, c3, c4;
2180     long hash =
2181       ((((long) (c0 = source[startPosition])) << 24)
2182         + (((long) (c1 = source[startPosition + 1])) << 18)
2183         + ((c2 = source[startPosition + 2]) << 12)
2184         + ((c3 = source[startPosition + 3]) << 6)
2185         + (c4 = source[startPosition + 4]))
2186         % TableSize;
2187     char[][] table = charArray_length[3][(int) hash];
2188     int i = newEntry5;
2189     while (++i < InternalTableSize) {
2190       char[] charArray = table[i];
2191       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) && (c4 == charArray[4]))
2192         return charArray;
2193     }
2194     //---------other side---------
2195     i = -1;
2196     int max = newEntry5;
2197     while (++i <= max) {
2198       char[] charArray = table[i];
2199       if ((c0 == charArray[0]) && (c1 == charArray[1]) && (c2 == charArray[2]) && (c3 == charArray[3]) && (c4 == charArray[4]))
2200         return charArray;
2201     }
2202     //--------add the entry-------
2203     if (++max >= InternalTableSize)
2204       max = 0;
2205     char[] r;
2206     table[max] = (r = new char[] { c0, c1, c2, c3, c4 });
2207     newEntry5 = max;
2208     return r;
2209
2210   }
2211
2212   final char[] optimizedCurrentTokenSource6() {
2213     //try to return the same char[] build only once
2214
2215     char c0, c1, c2, c3, c4, c5;
2216     long hash =
2217       ((((long) (c0 = source[startPosition])) << 32)
2218         + (((long) (c1 = source[startPosition + 1])) << 24)
2219         + (((long) (c2 = source[startPosition + 2])) << 18)
2220         + ((c3 = source[startPosition + 3]) << 12)
2221         + ((c4 = source[startPosition + 4]) << 6)
2222         + (c5 = source[startPosition + 5]))
2223         % TableSize;
2224     char[][] table = charArray_length[4][(int) hash];
2225     int i = newEntry6;
2226     while (++i < InternalTableSize) {
2227       char[] charArray = table[i];
2228       if ((c0 == charArray[0])
2229         && (c1 == charArray[1])
2230         && (c2 == charArray[2])
2231         && (c3 == charArray[3])
2232         && (c4 == charArray[4])
2233         && (c5 == charArray[5]))
2234         return charArray;
2235     }
2236     //---------other side---------
2237     i = -1;
2238     int max = newEntry6;
2239     while (++i <= max) {
2240       char[] charArray = table[i];
2241       if ((c0 == charArray[0])
2242         && (c1 == charArray[1])
2243         && (c2 == charArray[2])
2244         && (c3 == charArray[3])
2245         && (c4 == charArray[4])
2246         && (c5 == charArray[5]))
2247         return charArray;
2248     }
2249     //--------add the entry-------
2250     if (++max >= InternalTableSize)
2251       max = 0;
2252     char[] r;
2253     table[max] = (r = new char[] { c0, c1, c2, c3, c4, c5 });
2254     newEntry6 = max;
2255     return r;
2256   }
2257
2258   public final void pushLineSeparator() throws InvalidInputException {
2259     //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2260     final int INCREMENT = 250;
2261
2262     if (this.checkNonExternalizedStringLiterals) {
2263       // reinitialize the current line for non externalize strings purpose
2264       currentLine = null;
2265     }
2266     //currentCharacter is at position currentPosition-1
2267
2268     // cr 000D
2269     if (currentCharacter == '\r') {
2270       int separatorPos = currentPosition - 1;
2271       if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2272         return;
2273       //System.out.println("CR-" + separatorPos);
2274       try {
2275         lineEnds[++linePtr] = separatorPos;
2276       } catch (IndexOutOfBoundsException e) {
2277         //linePtr value is correct
2278         int oldLength = lineEnds.length;
2279         int[] old = lineEnds;
2280         lineEnds = new int[oldLength + INCREMENT];
2281         System.arraycopy(old, 0, lineEnds, 0, oldLength);
2282         lineEnds[linePtr] = separatorPos;
2283       }
2284       // look-ahead for merged cr+lf
2285       try {
2286         if (source[currentPosition] == '\n') {
2287           //System.out.println("look-ahead LF-" + currentPosition);
2288           lineEnds[linePtr] = currentPosition;
2289           currentPosition++;
2290           wasAcr = false;
2291         } else {
2292           wasAcr = true;
2293         }
2294       } catch (IndexOutOfBoundsException e) {
2295         wasAcr = true;
2296       }
2297     } else {
2298       // lf 000A
2299       if (currentCharacter == '\n') {
2300         //must merge eventual cr followed by lf
2301         if (wasAcr && (lineEnds[linePtr] == (currentPosition - 2))) {
2302           //System.out.println("merge LF-" + (currentPosition - 1));
2303           lineEnds[linePtr] = currentPosition - 1;
2304         } else {
2305           int separatorPos = currentPosition - 1;
2306           if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2307             return;
2308           // System.out.println("LF-" + separatorPos);
2309           try {
2310             lineEnds[++linePtr] = separatorPos;
2311           } catch (IndexOutOfBoundsException e) {
2312             //linePtr value is correct
2313             int oldLength = lineEnds.length;
2314             int[] old = lineEnds;
2315             lineEnds = new int[oldLength + INCREMENT];
2316             System.arraycopy(old, 0, lineEnds, 0, oldLength);
2317             lineEnds[linePtr] = separatorPos;
2318           }
2319         }
2320         wasAcr = false;
2321       }
2322     }
2323   }
2324   public final void pushUnicodeLineSeparator() {
2325     // isUnicode means that the \r or \n has been read as a unicode character
2326
2327     //see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2328
2329     final int INCREMENT = 250;
2330     //currentCharacter is at position currentPosition-1
2331
2332     if (this.checkNonExternalizedStringLiterals) {
2333       // reinitialize the current line for non externalize strings purpose
2334       currentLine = null;
2335     }
2336
2337     // cr 000D
2338     if (currentCharacter == '\r') {
2339       int separatorPos = currentPosition - 6;
2340       if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2341         return;
2342       //System.out.println("CR-" + separatorPos);
2343       try {
2344         lineEnds[++linePtr] = separatorPos;
2345       } catch (IndexOutOfBoundsException e) {
2346         //linePtr value is correct
2347         int oldLength = lineEnds.length;
2348         int[] old = lineEnds;
2349         lineEnds = new int[oldLength + INCREMENT];
2350         System.arraycopy(old, 0, lineEnds, 0, oldLength);
2351         lineEnds[linePtr] = separatorPos;
2352       }
2353       // look-ahead for merged cr+lf
2354       if (source[currentPosition] == '\n') {
2355         //System.out.println("look-ahead LF-" + currentPosition);
2356         lineEnds[linePtr] = currentPosition;
2357         currentPosition++;
2358         wasAcr = false;
2359       } else {
2360         wasAcr = true;
2361       }
2362     } else {
2363       // lf 000A
2364       if (currentCharacter == '\n') {
2365         //must merge eventual cr followed by lf
2366         if (wasAcr && (lineEnds[linePtr] == (currentPosition - 7))) {
2367           //System.out.println("merge LF-" + (currentPosition - 1));
2368           lineEnds[linePtr] = currentPosition - 6;
2369         } else {
2370           int separatorPos = currentPosition - 6;
2371           if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2372             return;
2373           // System.out.println("LF-" + separatorPos);
2374           try {
2375             lineEnds[++linePtr] = separatorPos;
2376           } catch (IndexOutOfBoundsException e) {
2377             //linePtr value is correct
2378             int oldLength = lineEnds.length;
2379             int[] old = lineEnds;
2380             lineEnds = new int[oldLength + INCREMENT];
2381             System.arraycopy(old, 0, lineEnds, 0, oldLength);
2382             lineEnds[linePtr] = separatorPos;
2383           }
2384         }
2385         wasAcr = false;
2386       }
2387     }
2388   }
2389   public final void recordComment(boolean isJavadoc) {
2390
2391     // a new annotation comment is recorded
2392     try {
2393       commentStops[++commentPtr] = isJavadoc ? currentPosition : -currentPosition;
2394     } catch (IndexOutOfBoundsException e) {
2395       int oldStackLength = commentStops.length;
2396       int[] oldStack = commentStops;
2397       commentStops = new int[oldStackLength + 30];
2398       System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength);
2399       commentStops[commentPtr] = isJavadoc ? currentPosition : -currentPosition;
2400       //grows the positions buffers too
2401       int[] old = commentStarts;
2402       commentStarts = new int[oldStackLength + 30];
2403       System.arraycopy(old, 0, commentStarts, 0, oldStackLength);
2404     }
2405
2406     //the buffer is of a correct size here
2407     commentStarts[commentPtr] = startPosition;
2408   }
2409   public void resetTo(int begin, int end) {
2410     //reset the scanner to a given position where it may rescan again
2411
2412     diet = false;
2413     initialPosition = startPosition = currentPosition = begin;
2414     eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
2415     commentPtr = -1; // reset comment stack
2416   }
2417
2418   public final void scanSingleQuotedEscapeCharacter() throws InvalidInputException {
2419     // the string with "\\u" is a legal string of two chars \ and u
2420     //thus we use a direct access to the source (for regular cases).
2421
2422     //    if (unicodeAsBackSlash) {
2423     //      // consume next character
2424     //      unicodeAsBackSlash = false;
2425     //      if (((currentCharacter = source[currentPosition++]) == '\\')
2426     //        && (source[currentPosition] == 'u')) {
2427     //        getNextUnicodeChar();
2428     //      } else {
2429     //        if (withoutUnicodePtr != 0) {
2430     //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2431     //        }
2432     //      }
2433     //    } else
2434     currentCharacter = source[currentPosition++];
2435     switch (currentCharacter) {
2436       case '\'' :
2437         currentCharacter = '\'';
2438         break;
2439       case '\\' :
2440         currentCharacter = '\\';
2441         break;
2442       default :
2443         currentCharacter = '\\';
2444         currentPosition--;
2445     }
2446   }
2447
2448   public final void scanDoubleQuotedEscapeCharacter() throws InvalidInputException {
2449     // the string with "\\u" is a legal string of two chars \ and u
2450     //thus we use a direct access to the source (for regular cases).
2451
2452     //    if (unicodeAsBackSlash) {
2453     //      // consume next character
2454     //      unicodeAsBackSlash = false;
2455     //      if (((currentCharacter = source[currentPosition++]) == '\\')
2456     //        && (source[currentPosition] == 'u')) {
2457     //        getNextUnicodeChar();
2458     //      } else {
2459     //        if (withoutUnicodePtr != 0) {
2460     //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2461     //        }
2462     //      }
2463     //    } else
2464     currentCharacter = source[currentPosition++];
2465     switch (currentCharacter) {
2466       //      case 'b' :
2467       //        currentCharacter = '\b';
2468       //        break;
2469       case 't' :
2470         currentCharacter = '\t';
2471         break;
2472       case 'n' :
2473         currentCharacter = '\n';
2474         break;
2475         //      case 'f' :
2476         //        currentCharacter = '\f';
2477         //        break;
2478       case 'r' :
2479         currentCharacter = '\r';
2480         break;
2481       case '\"' :
2482         currentCharacter = '\"';
2483         break;
2484       case '\'' :
2485         currentCharacter = '\'';
2486         break;
2487       case '\\' :
2488         currentCharacter = '\\';
2489         break;
2490       case '$' :
2491         currentCharacter = '$';
2492         break;
2493       default :
2494         // -----------octal escape--------------
2495         // OctalDigit
2496         // OctalDigit OctalDigit
2497         // ZeroToThree OctalDigit OctalDigit
2498
2499         int number = Character.getNumericValue(currentCharacter);
2500         if (number >= 0 && number <= 7) {
2501           boolean zeroToThreeNot = number > 3;
2502           if (Character.isDigit(currentCharacter = source[currentPosition++])) {
2503             int digit = Character.getNumericValue(currentCharacter);
2504             if (digit >= 0 && digit <= 7) {
2505               number = (number * 8) + digit;
2506               if (Character.isDigit(currentCharacter = source[currentPosition++])) {
2507                 if (zeroToThreeNot) { // has read \NotZeroToThree OctalDigit Digit --> ignore last character
2508                   currentPosition--;
2509                 } else {
2510                   digit = Character.getNumericValue(currentCharacter);
2511                   if (digit >= 0 && digit <= 7) {
2512                     // has read \ZeroToThree OctalDigit OctalDigit
2513                     number = (number * 8) + digit;
2514                   } else { // has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
2515                     currentPosition--;
2516                   }
2517                 }
2518               } else { // has read \OctalDigit NonDigit--> ignore last character
2519                 currentPosition--;
2520               }
2521             } else { // has read \OctalDigit NonOctalDigit--> ignore last character
2522               currentPosition--;
2523             }
2524           } else { // has read \OctalDigit --> ignore last character
2525             currentPosition--;
2526           }
2527           if (number > 255)
2528             throw new InvalidInputException(INVALID_ESCAPE);
2529           currentCharacter = (char) number;
2530         }
2531         //else
2532         //     throw new InvalidInputException(INVALID_ESCAPE);
2533     }
2534   }
2535
2536   //  public int scanIdentifierOrKeyword() throws InvalidInputException {
2537   //    return scanIdentifierOrKeyword( false );
2538   //  }
2539
2540   public int scanIdentifierOrKeyword(boolean isVariable) throws InvalidInputException {
2541     //test keywords
2542
2543     //first dispatch on the first char.
2544     //then the length. If there are several
2545     //keywors with the same length AND the same first char, then do another
2546     //disptach on the second char :-)...cool....but fast !
2547
2548     useAssertAsAnIndentifier = false;
2549
2550     while (getNextCharAsJavaIdentifierPart()) {
2551     };
2552
2553     if (isVariable) {
2554       if (new String(getCurrentTokenSource()).equals("$this")) {
2555         return TokenNamethis;
2556       }
2557       return TokenNameVariable;
2558     }
2559     int index, length;
2560     char[] data;
2561     char firstLetter;
2562     //    if (withoutUnicodePtr == 0)
2563
2564     //quick test on length == 1 but not on length > 12 while most identifier
2565     //have a length which is <= 12...but there are lots of identifier with
2566     //only one char....
2567
2568     //      {
2569     if ((length = currentPosition - startPosition) == 1)
2570       return TokenNameIdentifier;
2571     //  data = source;
2572     data = new char[length];
2573     index = startPosition;
2574     for (int i = 0; i < length; i++) {
2575       data[i] = Character.toLowerCase(source[index + i]);
2576     }
2577     index = 0;
2578     //    } else {
2579     //      if ((length = withoutUnicodePtr) == 1)
2580     //        return TokenNameIdentifier;
2581     //      // data = withoutUnicodeBuffer;
2582     //      data = new char[withoutUnicodeBuffer.length];
2583     //      for (int i = 0; i < withoutUnicodeBuffer.length; i++) {
2584     //        data[i] = Character.toLowerCase(withoutUnicodeBuffer[i]);
2585     //      }
2586     //      index = 1;
2587     //    }
2588
2589     firstLetter = data[index];
2590     switch (firstLetter) {
2591
2592       case 'a' : // as and array abstract
2593         switch (length) {
2594           case 2 : //as
2595             if ((data[++index] == 's')) {
2596               return TokenNameas;
2597             } else {
2598               return TokenNameIdentifier;
2599             }
2600           case 3 : //and
2601             if ((data[++index] == 'n') && (data[++index] == 'd')) {
2602               return TokenNameAND;
2603             } else {
2604               return TokenNameIdentifier;
2605             }
2606           case 5 : // array
2607             if ((data[++index] == 'r') && (data[++index] == 'r') && (data[++index] == 'a') && (data[++index] == 'y'))
2608               return TokenNamearray;
2609             else
2610               return TokenNameIdentifier;
2611           case 8 :
2612             if ((data[++index] == 'b')
2613               && (data[++index] == 's')
2614               && (data[++index] == 't')
2615               && (data[++index] == 'r')
2616               && (data[++index] == 'a')
2617               && (data[++index] == 'c')
2618               && (data[++index] == 't'))
2619               return TokenNameabstract;
2620             else
2621               return TokenNameIdentifier;
2622           default :
2623             return TokenNameIdentifier;
2624         }
2625       case 'b' : //break
2626         switch (length) {
2627           case 5 :
2628             if ((data[++index] == 'r') && (data[++index] == 'e') && (data[++index] == 'a') && (data[++index] == 'k'))
2629               return TokenNamebreak;
2630             else
2631               return TokenNameIdentifier;
2632           default :
2633             return TokenNameIdentifier;
2634         }
2635
2636       case 'c' : //case catch class const continue
2637         switch (length) {
2638           case 4 :
2639             if ((data[++index] == 'a') && (data[++index] == 's') && (data[++index] == 'e'))
2640               return TokenNamecase;
2641             else
2642               return TokenNameIdentifier;
2643           case 5 :
2644             if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'c') && (data[++index] == 'h'))
2645               return TokenNamecatch;
2646             if ((data[index] == 'l') && (data[++index] == 'a') && (data[++index] == 's') && (data[++index] == 's'))
2647               return TokenNameclass;
2648             if ((data[index] == 'o') && (data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 't'))
2649               return TokenNameconst;
2650             else
2651               return TokenNameIdentifier;
2652           case 8 :
2653             if ((data[++index] == 'o')
2654               && (data[++index] == 'n')
2655               && (data[++index] == 't')
2656               && (data[++index] == 'i')
2657               && (data[++index] == 'n')
2658               && (data[++index] == 'u')
2659               && (data[++index] == 'e'))
2660               return TokenNamecontinue;
2661             else
2662               return TokenNameIdentifier;
2663           default :
2664             return TokenNameIdentifier;
2665         }
2666
2667       case 'd' : //define declare default do die
2668         switch (length) {
2669           case 2 :
2670             if ((data[++index] == 'o'))
2671               return TokenNamedo;
2672             else
2673               return TokenNameIdentifier;
2674           case 3 :
2675             if ((data[++index] == 'i') && (data[++index] == 'e'))
2676               return TokenNamedie;
2677             else
2678               return TokenNameIdentifier;
2679           case 6 :
2680             if ((data[++index] == 'e')
2681               && (data[++index] == 'f')
2682               && (data[++index] == 'i')
2683               && (data[++index] == 'n')
2684               && (data[++index] == 'e'))
2685               return TokenNamedefine;
2686             else
2687               return TokenNameIdentifier;
2688           case 7 :
2689             if ((data[++index] == 'e')
2690               && (data[++index] == 'c')
2691               && (data[++index] == 'l')
2692               && (data[++index] == 'a')
2693               && (data[++index] == 'r')
2694               && (data[++index] == 'e'))
2695               return TokenNamedeclare;
2696             index = 0;
2697             if ((data[++index] == 'e')
2698               && (data[++index] == 'f')
2699               && (data[++index] == 'a')
2700               && (data[++index] == 'u')
2701               && (data[++index] == 'l')
2702               && (data[++index] == 't'))
2703               return TokenNamedefault;
2704             else
2705               return TokenNameIdentifier;
2706           default :
2707             return TokenNameIdentifier;
2708         }
2709       case 'e' : //echo else exit elseif extends eval
2710         switch (length) {
2711           case 4 :
2712             if ((data[++index] == 'c') && (data[++index] == 'h') && (data[++index] == 'o'))
2713               return TokenNameecho;
2714             else if ((data[index] == 'l') && (data[++index] == 's') && (data[++index] == 'e'))
2715               return TokenNameelse;
2716             else if ((data[index] == 'x') && (data[++index] == 'i') && (data[++index] == 't'))
2717               return TokenNameexit;
2718             else if ((data[index] == 'v') && (data[++index] == 'a') && (data[++index] == 'l'))
2719               return TokenNameeval;
2720             else
2721               return TokenNameIdentifier;
2722           case 5 : // endif empty
2723             if ((data[++index] == 'n') && (data[++index] == 'd') && (data[++index] == 'i') && (data[++index] == 'f'))
2724               return TokenNameendif;
2725             if ((data[index] == 'm') && (data[++index] == 'p') && (data[++index] == 't') && (data[++index] == 'y'))
2726               return TokenNameempty;
2727             else
2728               return TokenNameIdentifier;
2729           case 6 : // endfor
2730             if ((data[++index] == 'n')
2731               && (data[++index] == 'd')
2732               && (data[++index] == 'f')
2733               && (data[++index] == 'o')
2734               && (data[++index] == 'r'))
2735               return TokenNameendfor;
2736             else if (
2737               (data[index] == 'l')
2738                 && (data[++index] == 's')
2739                 && (data[++index] == 'e')
2740                 && (data[++index] == 'i')
2741                 && (data[++index] == 'f'))
2742               return TokenNameelseif;
2743             else
2744               return TokenNameIdentifier;
2745           case 7 :
2746             if ((data[++index] == 'x')
2747               && (data[++index] == 't')
2748               && (data[++index] == 'e')
2749               && (data[++index] == 'n')
2750               && (data[++index] == 'd')
2751               && (data[++index] == 's'))
2752               return TokenNameextends;
2753             else
2754               return TokenNameIdentifier;
2755           case 8 : // endwhile
2756             if ((data[++index] == 'n')
2757               && (data[++index] == 'd')
2758               && (data[++index] == 'w')
2759               && (data[++index] == 'h')
2760               && (data[++index] == 'i')
2761               && (data[++index] == 'l')
2762               && (data[++index] == 'e'))
2763               return TokenNameendwhile;
2764             else
2765               return TokenNameIdentifier;
2766           case 9 : // endswitch
2767             if ((data[++index] == 'n')
2768               && (data[++index] == 'd')
2769               && (data[++index] == 's')
2770               && (data[++index] == 'w')
2771               && (data[++index] == 'i')
2772               && (data[++index] == 't')
2773               && (data[++index] == 'c')
2774               && (data[++index] == 'h'))
2775               return TokenNameendswitch;
2776             else
2777               return TokenNameIdentifier;
2778           case 10 : // enddeclare
2779             if ((data[++index] == 'n')
2780               && (data[++index] == 'd')
2781               && (data[++index] == 'd')
2782               && (data[++index] == 'e')
2783               && (data[++index] == 'c')
2784               && (data[++index] == 'l')
2785               && (data[++index] == 'a')
2786               && (data[++index] == 'r')
2787               && (data[++index] == 'e'))
2788               return TokenNameendforeach;
2789             index = 0;
2790             if ((data[++index] == 'n') // endforeach
2791               && (data[++index] == 'd')
2792               && (data[++index] == 'f')
2793               && (data[++index] == 'o')
2794               && (data[++index] == 'r')
2795               && (data[++index] == 'e')
2796               && (data[++index] == 'a')
2797               && (data[++index] == 'c')
2798               && (data[++index] == 'h'))
2799               return TokenNameendforeach;
2800             else
2801               return TokenNameIdentifier;
2802
2803           default :
2804             return TokenNameIdentifier;
2805         }
2806
2807       case 'f' : //for false final function
2808         switch (length) {
2809           case 3 :
2810             if ((data[++index] == 'o') && (data[++index] == 'r'))
2811               return TokenNamefor;
2812             else
2813               return TokenNameIdentifier;
2814           case 5 :
2815             if ((data[++index] == 'a') && (data[++index] == 'l') && (data[++index] == 's') && (data[++index] == 'e'))
2816               return TokenNamefalse;
2817             if ((data[index] == 'i') && (data[++index] == 'n') && (data[++index] == 'a') && (data[++index] == 'l'))
2818               return TokenNamefinal;
2819             else
2820               return TokenNameIdentifier;
2821           case 7 : // foreach
2822             if ((data[++index] == 'o')
2823               && (data[++index] == 'r')
2824               && (data[++index] == 'e')
2825               && (data[++index] == 'a')
2826               && (data[++index] == 'c')
2827               && (data[++index] == 'h'))
2828               return TokenNameforeach;
2829             else
2830               return TokenNameIdentifier;
2831           case 8 : // function
2832             if ((data[++index] == 'u')
2833               && (data[++index] == 'n')
2834               && (data[++index] == 'c')
2835               && (data[++index] == 't')
2836               && (data[++index] == 'i')
2837               && (data[++index] == 'o')
2838               && (data[++index] == 'n'))
2839               return TokenNamefunction;
2840             else
2841               return TokenNameIdentifier;
2842           default :
2843             return TokenNameIdentifier;
2844         }
2845       case 'g' : //global
2846         if (length == 6) {
2847           if ((data[++index] == 'l')
2848             && (data[++index] == 'o')
2849             && (data[++index] == 'b')
2850             && (data[++index] == 'a')
2851             && (data[++index] == 'l')) {
2852             return TokenNameglobal;
2853           }
2854         }
2855         return TokenNameIdentifier;
2856
2857       case 'i' : //if int isset include include_once instanceof interface implements
2858         switch (length) {
2859           case 2 :
2860             if (data[++index] == 'f')
2861               return TokenNameif;
2862             else
2863               return TokenNameIdentifier;
2864             //          case 3 :
2865             //            if ((data[++index] == 'n') && (data[++index] == 't'))
2866             //              return TokenNameint;
2867             //            else
2868             //              return TokenNameIdentifier;
2869           case 5 :
2870             if ((data[++index] == 's') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 't'))
2871               return TokenNameisset;
2872             else
2873               return TokenNameIdentifier;
2874           case 7 :
2875             if ((data[++index] == 'n')
2876               && (data[++index] == 'c')
2877               && (data[++index] == 'l')
2878               && (data[++index] == 'u')
2879               && (data[++index] == 'd')
2880               && (data[++index] == 'e'))
2881               return TokenNameinclude;
2882             else
2883               return TokenNameIdentifier;
2884           case 9 : // interface
2885             if ((data[++index] == 'n')
2886               && (data[++index] == 't')
2887               && (data[++index] == 'e')
2888               && (data[++index] == 'r')
2889               && (data[++index] == 'f')
2890               && (data[++index] == 'a')
2891               && (data[++index] == 'c')
2892               && (data[++index] == 'e'))
2893               return TokenNameinterface;
2894             else
2895               return TokenNameIdentifier;
2896           case 10 : // instanceof
2897             if ((data[++index] == 'n')
2898               && (data[++index] == 's')
2899               && (data[++index] == 't')
2900               && (data[++index] == 'a')
2901               && (data[++index] == 'n')
2902               && (data[++index] == 'c')
2903               && (data[++index] == 'e')
2904               && (data[++index] == 'o')
2905               && (data[++index] == 'f'))
2906               return TokenNameinstanceof;
2907             if ((data[index] == 'm')
2908               && (data[++index] == 'p')
2909               && (data[++index] == 'l')
2910               && (data[++index] == 'e')
2911               && (data[++index] == 'm')
2912               && (data[++index] == 'e')
2913               && (data[++index] == 'n')
2914               && (data[++index] == 't')
2915               && (data[++index] == 's'))
2916               return TokenNameimplements;
2917             else
2918               return TokenNameIdentifier;
2919           case 12 :
2920             if ((data[++index] == 'n')
2921               && (data[++index] == 'c')
2922               && (data[++index] == 'l')
2923               && (data[++index] == 'u')
2924               && (data[++index] == 'd')
2925               && (data[++index] == 'e')
2926               && (data[++index] == '_')
2927               && (data[++index] == 'o')
2928               && (data[++index] == 'n')
2929               && (data[++index] == 'c')
2930               && (data[++index] == 'e'))
2931               return TokenNameinclude_once;
2932             else
2933               return TokenNameIdentifier;
2934           default :
2935             return TokenNameIdentifier;
2936         }
2937
2938       case 'l' : //list
2939         if (length == 4) {
2940           if ((data[++index] == 'i') && (data[++index] == 's') && (data[++index] == 't')) {
2941             return TokenNamelist;
2942           }
2943         }
2944         return TokenNameIdentifier;
2945
2946       case 'n' : // new null
2947         switch (length) {
2948           case 3 :
2949             if ((data[++index] == 'e') && (data[++index] == 'w'))
2950               return TokenNamenew;
2951             else
2952               return TokenNameIdentifier;
2953           case 4 :
2954             if ((data[++index] == 'u') && (data[++index] == 'l') && (data[++index] == 'l'))
2955               return TokenNamenull;
2956             else
2957               return TokenNameIdentifier;
2958
2959           default :
2960             return TokenNameIdentifier;
2961         }
2962       case 'o' : // or old_function
2963         if (length == 2) {
2964           if (data[++index] == 'r') {
2965             return TokenNameOR;
2966           }
2967         }
2968         //        if (length == 12) {
2969         //          if ((data[++index] == 'l')
2970         //            && (data[++index] == 'd')
2971         //            && (data[++index] == '_')
2972         //            && (data[++index] == 'f')
2973         //            && (data[++index] == 'u')
2974         //            && (data[++index] == 'n')
2975         //            && (data[++index] == 'c')
2976         //            && (data[++index] == 't')
2977         //            && (data[++index] == 'i')
2978         //            && (data[++index] == 'o')
2979         //            && (data[++index] == 'n')) {
2980         //            return TokenNameold_function;
2981         //          }
2982         //        }
2983         return TokenNameIdentifier;
2984
2985       case 'p' : // print public private protected
2986         switch (length) {
2987           case 5 :
2988             if ((data[++index] == 'r') && (data[++index] == 'i') && (data[++index] == 'n') && (data[++index] == 't')) {
2989               return TokenNameprint;
2990             } else
2991               return TokenNameIdentifier;
2992           case 6 :
2993             if ((data[++index] == 'u')
2994               && (data[++index] == 'b')
2995               && (data[++index] == 'l')
2996               && (data[++index] == 'i')
2997               && (data[++index] == 'c')) {
2998               return TokenNamepublic;
2999             } else
3000               return TokenNameIdentifier;
3001           case 7 :
3002             if ((data[++index] == 'r')
3003               && (data[++index] == 'i')
3004               && (data[++index] == 'v')
3005               && (data[++index] == 'a')
3006               && (data[++index] == 't')
3007               && (data[++index] == 'e')) {
3008               return TokenNameprivate;
3009             } else
3010               return TokenNameIdentifier;
3011           case 9 :
3012             if ((data[++index] == 'r')
3013               && (data[++index] == 'o')
3014               && (data[++index] == 't')
3015               && (data[++index] == 'e')
3016               && (data[++index] == 'c')
3017               && (data[++index] == 't')
3018               && (data[++index] == 'e')
3019               && (data[++index] == 'd')) {
3020               return TokenNameprotected;
3021             } else
3022               return TokenNameIdentifier;
3023         }
3024         return TokenNameIdentifier;
3025       case 'r' : //return require require_once
3026         if (length == 6) {
3027           if ((data[++index] == 'e')
3028             && (data[++index] == 't')
3029             && (data[++index] == 'u')
3030             && (data[++index] == 'r')
3031             && (data[++index] == 'n')) {
3032             return TokenNamereturn;
3033           }
3034         } else if (length == 7) {
3035           if ((data[++index] == 'e')
3036             && (data[++index] == 'q')
3037             && (data[++index] == 'u')
3038             && (data[++index] == 'i')
3039             && (data[++index] == 'r')
3040             && (data[++index] == 'e')) {
3041             return TokenNamerequire;
3042           }
3043         } else if (length == 12) {
3044           if ((data[++index] == 'e')
3045             && (data[++index] == 'q')
3046             && (data[++index] == 'u')
3047             && (data[++index] == 'i')
3048             && (data[++index] == 'r')
3049             && (data[++index] == 'e')
3050             && (data[++index] == '_')
3051             && (data[++index] == 'o')
3052             && (data[++index] == 'n')
3053             && (data[++index] == 'c')
3054             && (data[++index] == 'e')) {
3055             return TokenNamerequire_once;
3056           }
3057         } else
3058           return TokenNameIdentifier;
3059
3060       case 's' : //static switch
3061         switch (length) {
3062           case 6 :
3063             if (data[++index] == 't')
3064               if ((data[++index] == 'a') && (data[++index] == 't') && (data[++index] == 'i') && (data[++index] == 'c')) {
3065                 return TokenNamestatic;
3066               } else
3067                 return TokenNameIdentifier;
3068             else if (
3069               (data[index] == 'w')
3070                 && (data[++index] == 'i')
3071                 && (data[++index] == 't')
3072                 && (data[++index] == 'c')
3073                 && (data[++index] == 'h'))
3074               return TokenNameswitch;
3075             else
3076               return TokenNameIdentifier;
3077           default :
3078             return TokenNameIdentifier;
3079         }
3080
3081       case 't' : // try true throw
3082         switch (length) {
3083           case 3 :
3084             if ((data[++index] == 'r') && (data[++index] == 'y'))
3085               return TokenNametry;
3086             else
3087               return TokenNameIdentifier;
3088           case 4 :
3089             if ((data[++index] == 'r') && (data[++index] == 'u') && (data[++index] == 'e'))
3090               return TokenNametrue;
3091             else
3092               return TokenNameIdentifier;
3093           case 5 :
3094             if ((data[++index] == 'h') && (data[++index] == 'r') && (data[++index] == 'o') && (data[++index] == 'w'))
3095               return TokenNamethrow;
3096             else
3097               return TokenNameIdentifier;
3098
3099           default :
3100             return TokenNameIdentifier;
3101         }
3102       case 'u' : //use unset
3103         switch (length) {
3104           case 3 :
3105             if ((data[++index] == 's') && (data[++index] == 'e'))
3106               return TokenNameuse;
3107             else
3108               return TokenNameIdentifier;
3109           case 5 :
3110             if ((data[++index] == 'n') && (data[++index] == 's') && (data[++index] == 'e') && (data[++index] == 't'))
3111               return TokenNameunset;
3112             else
3113               return TokenNameIdentifier;
3114           default :
3115             return TokenNameIdentifier;
3116         }
3117       case 'v' : //var
3118         switch (length) {
3119           case 3 :
3120             if ((data[++index] == 'a') && (data[++index] == 'r'))
3121               return TokenNamevar;
3122             else
3123               return TokenNameIdentifier;
3124
3125           default :
3126             return TokenNameIdentifier;
3127         }
3128
3129       case 'w' : //while
3130         switch (length) {
3131           case 5 :
3132             if ((data[++index] == 'h') && (data[++index] == 'i') && (data[++index] == 'l') && (data[++index] == 'e'))
3133               return TokenNamewhile;
3134             else
3135               return TokenNameIdentifier;
3136             //case 6:if ( (data[++index] =='i') && (data[++index]=='d') && (data[++index]=='e') && (data[++index]=='f')&&
3137             // (data[++index]=='p'))
3138             //return TokenNamewidefp ;
3139             //else
3140             //return TokenNameIdentifier;
3141           default :
3142             return TokenNameIdentifier;
3143         }
3144
3145       case 'x' : //xor
3146         switch (length) {
3147           case 3 :
3148             if ((data[++index] == 'o') && (data[++index] == 'r'))
3149               return TokenNameXOR;
3150             else
3151               return TokenNameIdentifier;
3152
3153           default :
3154             return TokenNameIdentifier;
3155         }
3156       default :
3157         return TokenNameIdentifier;
3158     }
3159   }
3160   public int scanNumber(boolean dotPrefix) throws InvalidInputException {
3161
3162     //when entering this method the currentCharacter is the firt
3163     //digit of the number , i.e. it may be preceeded by a . when
3164     //dotPrefix is true
3165
3166     boolean floating = dotPrefix;
3167     if ((!dotPrefix) && (currentCharacter == '0')) {
3168       if (getNextChar('x', 'X') >= 0) { //----------hexa-----------------
3169         //force the first char of the hexa number do exist...
3170         // consume next character
3171         unicodeAsBackSlash = false;
3172         currentCharacter = source[currentPosition++];
3173         //        if (((currentCharacter = source[currentPosition++]) == '\\')
3174         //          && (source[currentPosition] == 'u')) {
3175         //          getNextUnicodeChar();
3176         //        } else {
3177         //          if (withoutUnicodePtr != 0) {
3178         //            withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3179         //          }
3180         //        }
3181         if (Character.digit(currentCharacter, 16) == -1)
3182           throw new InvalidInputException(INVALID_HEXA);
3183         //---end forcing--
3184         while (getNextCharAsDigit(16)) {
3185         };
3186         //        if (getNextChar('l', 'L') >= 0)
3187         //          return TokenNameLongLiteral;
3188         //        else
3189         return TokenNameIntegerLiteral;
3190       }
3191
3192       //there is x or X in the number
3193       //potential octal ! ... some one may write 000099.0 ! thus 00100 < 00078.0 is true !!!!! crazy language
3194       if (getNextCharAsDigit()) {
3195         //-------------potential octal-----------------
3196         while (getNextCharAsDigit()) {
3197         };
3198
3199         //        if (getNextChar('l', 'L') >= 0) {
3200         //          return TokenNameLongLiteral;
3201         //        }
3202         //
3203         //        if (getNextChar('f', 'F') >= 0) {
3204         //          return TokenNameFloatingPointLiteral;
3205         //        }
3206
3207         if (getNextChar('d', 'D') >= 0) {
3208           return TokenNameDoubleLiteral;
3209         } else { //make the distinction between octal and float ....
3210           if (getNextChar('.')) { //bingo ! ....
3211             while (getNextCharAsDigit()) {
3212             };
3213             if (getNextChar('e', 'E') >= 0) {
3214               // consume next character
3215               unicodeAsBackSlash = false;
3216               currentCharacter = source[currentPosition++];
3217               //              if (((currentCharacter = source[currentPosition++]) == '\\')
3218               //                && (source[currentPosition] == 'u')) {
3219               //                getNextUnicodeChar();
3220               //              } else {
3221               //                if (withoutUnicodePtr != 0) {
3222               //                  withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3223               //                }
3224               //              }
3225
3226               if ((currentCharacter == '-') || (currentCharacter == '+')) {
3227                 // consume next character
3228                 unicodeAsBackSlash = false;
3229                 currentCharacter = source[currentPosition++];
3230                 //                if (((currentCharacter = source[currentPosition++]) == '\\')
3231                 //                  && (source[currentPosition] == 'u')) {
3232                 //                  getNextUnicodeChar();
3233                 //                } else {
3234                 //                  if (withoutUnicodePtr != 0) {
3235                 //                    withoutUnicodeBuffer[++withoutUnicodePtr] =
3236                 //                      currentCharacter;
3237                 //                  }
3238                 //                }
3239               }
3240               if (!Character.isDigit(currentCharacter))
3241                 throw new InvalidInputException(INVALID_FLOAT);
3242               while (getNextCharAsDigit()) {
3243               };
3244             }
3245             //            if (getNextChar('f', 'F') >= 0)
3246             //              return TokenNameFloatingPointLiteral;
3247             getNextChar('d', 'D'); //jump over potential d or D
3248             return TokenNameDoubleLiteral;
3249           } else {
3250             return TokenNameIntegerLiteral;
3251           }
3252         }
3253       } else {
3254         /* carry on */
3255       }
3256     }
3257
3258     while (getNextCharAsDigit()) {
3259     };
3260
3261     //    if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3262     //      return TokenNameLongLiteral;
3263
3264     if ((!dotPrefix) && (getNextChar('.'))) { //decimal part that can be empty
3265       while (getNextCharAsDigit()) {
3266       };
3267       floating = true;
3268     }
3269
3270     //if floating is true both exponant and suffix may be optional
3271
3272     if (getNextChar('e', 'E') >= 0) {
3273       floating = true;
3274       // consume next character
3275       unicodeAsBackSlash = false;
3276       currentCharacter = source[currentPosition++];
3277       //      if (((currentCharacter = source[currentPosition++]) == '\\')
3278       //        && (source[currentPosition] == 'u')) {
3279       //        getNextUnicodeChar();
3280       //      } else {
3281       //        if (withoutUnicodePtr != 0) {
3282       //          withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3283       //        }
3284       //      }
3285
3286       if ((currentCharacter == '-') || (currentCharacter == '+')) { // consume next character
3287         unicodeAsBackSlash = false;
3288         currentCharacter = source[currentPosition++];
3289         //        if (((currentCharacter = source[currentPosition++]) == '\\')
3290         //          && (source[currentPosition] == 'u')) {
3291         //          getNextUnicodeChar();
3292         //        } else {
3293         //          if (withoutUnicodePtr != 0) {
3294         //            withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3295         //          }
3296         //        }
3297       }
3298       if (!Character.isDigit(currentCharacter))
3299         throw new InvalidInputException(INVALID_FLOAT);
3300       while (getNextCharAsDigit()) {
3301       };
3302     }
3303
3304     if (getNextChar('d', 'D') >= 0)
3305       return TokenNameDoubleLiteral;
3306     //    if (getNextChar('f', 'F') >= 0)
3307     //      return TokenNameFloatingPointLiteral;
3308
3309     //the long flag has been tested before
3310
3311     return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral;
3312   }
3313   /**
3314    * Search the line number corresponding to a specific position
3315    *
3316    */
3317   public final int getLineNumber(int position) {
3318
3319     if (lineEnds == null)
3320       return 1;
3321     int length = linePtr + 1;
3322     if (length == 0)
3323       return 1;
3324     int g = 0, d = length - 1;
3325     int m = 0;
3326     while (g <= d) {
3327       m = (g + d) / 2;
3328       if (position < lineEnds[m]) {
3329         d = m - 1;
3330       } else if (position > lineEnds[m]) {
3331         g = m + 1;
3332       } else {
3333         return m + 1;
3334       }
3335     }
3336     if (position < lineEnds[m]) {
3337       return m + 1;
3338     }
3339     return m + 2;
3340   }
3341
3342   public void setPHPMode(boolean mode) {
3343     phpMode = mode;
3344   }
3345
3346   public final void setSource(char[] source) {
3347     //the source-buffer is set to sourceString
3348
3349     if (source == null) {
3350       this.source = new char[0];
3351     } else {
3352       this.source = source;
3353     }
3354     startPosition = -1;
3355     initialPosition = currentPosition = 0;
3356     containsAssertKeyword = false;
3357     withoutUnicodeBuffer = new char[this.source.length];
3358
3359   }
3360
3361   public String toString() {
3362     if (startPosition == source.length)
3363       return "EOF\n\n" + new String(source); //$NON-NLS-1$
3364     if (currentPosition > source.length)
3365       return "behind the EOF :-( ....\n\n" + new String(source); //$NON-NLS-1$
3366
3367     char front[] = new char[startPosition];
3368     System.arraycopy(source, 0, front, 0, startPosition);
3369
3370     int middleLength = (currentPosition - 1) - startPosition + 1;
3371     char middle[];
3372     if (middleLength > -1) {
3373       middle = new char[middleLength];
3374       System.arraycopy(source, startPosition, middle, 0, middleLength);
3375     } else {
3376       middle = new char[0];
3377     }
3378
3379     char end[] = new char[source.length - (currentPosition - 1)];
3380     System.arraycopy(source, (currentPosition - 1) + 1, end, 0, source.length - (currentPosition - 1) - 1);
3381
3382     return new String(front) + "\n===============================\nStarts here -->" //$NON-NLS-1$
3383     + new String(middle) + "<-- Ends here\n===============================\n" //$NON-NLS-1$
3384     + new String(end);
3385   }
3386   public final String toStringAction(int act) {
3387
3388     switch (act) {
3389       case TokenNameERROR :
3390         return "ScannerError"; // + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3391       case TokenNameStopPHP :
3392         return "StopPHP(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3393       case TokenNameIdentifier :
3394         return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3395       case TokenNameVariable :
3396         return "Variable(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3397       case TokenNameas :
3398         return "as"; //$NON-NLS-1$
3399       case TokenNamebreak :
3400         return "break"; //$NON-NLS-1$
3401       case TokenNamecase :
3402         return "case"; //$NON-NLS-1$
3403       case TokenNameclass :
3404         return "class"; //$NON-NLS-1$
3405       case TokenNamecontinue :
3406         return "continue"; //$NON-NLS-1$
3407       case TokenNamedefault :
3408         return "default"; //$NON-NLS-1$
3409       case TokenNamedefine :
3410         return "define"; //$NON-NLS-1$
3411       case TokenNamedo :
3412         return "do"; //$NON-NLS-1$
3413       case TokenNameecho :
3414         return "echo"; //$NON-NLS-1$
3415       case TokenNameelse :
3416         return "else"; //$NON-NLS-1$
3417       case TokenNameelseif :
3418         return "elseif"; //$NON-NLS-1$
3419       case TokenNameendfor :
3420         return "endfor"; //$NON-NLS-1$
3421       case TokenNameendforeach :
3422         return "endforeach"; //$NON-NLS-1$
3423       case TokenNameendif :
3424         return "endif"; //$NON-NLS-1$
3425       case TokenNameendswitch :
3426         return "endswitch"; //$NON-NLS-1$
3427       case TokenNameendwhile :
3428         return "endwhile"; //$NON-NLS-1$
3429       case TokenNameextends :
3430         return "extends"; //$NON-NLS-1$
3431       case TokenNamefalse :
3432         return "false"; //$NON-NLS-1$
3433       case TokenNamefor :
3434         return "for"; //$NON-NLS-1$
3435       case TokenNameforeach :
3436         return "foreach"; //$NON-NLS-1$
3437       case TokenNamefunction :
3438         return "function"; //$NON-NLS-1$
3439       case TokenNameglobal :
3440         return "global"; //$NON-NLS-1$
3441       case TokenNameif :
3442         return "if"; //$NON-NLS-1$
3443       case TokenNameinclude :
3444         return "include"; //$NON-NLS-1$
3445       case TokenNameinclude_once :
3446         return "include_once"; //$NON-NLS-1$
3447       case TokenNamelist :
3448         return "list"; //$NON-NLS-1$
3449       case TokenNamenew :
3450         return "new"; //$NON-NLS-1$
3451       case TokenNamenull :
3452         return "null"; //$NON-NLS-1$
3453       case TokenNameprint :
3454         return "print"; //$NON-NLS-1$
3455       case TokenNamerequire :
3456         return "require"; //$NON-NLS-1$
3457       case TokenNamerequire_once :
3458         return "require_once"; //$NON-NLS-1$
3459       case TokenNamereturn :
3460         return "return"; //$NON-NLS-1$
3461       case TokenNamestatic :
3462         return "static"; //$NON-NLS-1$
3463       case TokenNameswitch :
3464         return "switch"; //$NON-NLS-1$
3465       case TokenNametrue :
3466         return "true"; //$NON-NLS-1$
3467       case TokenNamevar :
3468         return "var"; //$NON-NLS-1$
3469       case TokenNamewhile :
3470         return "while"; //$NON-NLS-1$
3471       case TokenNamethis :
3472         return "$this"; //$NON-NLS-1$
3473       case TokenNameIntegerLiteral :
3474         return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3475       case TokenNameDoubleLiteral :
3476         return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3477       case TokenNameStringLiteral :
3478         return "String(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3479       case TokenNameStringConstant :
3480         return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3481       case TokenNameStringInterpolated :
3482         return "StringInterpolated(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3483       case TokenNameHEREDOC :
3484         return "HEREDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3485
3486       case TokenNamePLUS_PLUS :
3487         return "++"; //$NON-NLS-1$
3488       case TokenNameMINUS_MINUS :
3489         return "--"; //$NON-NLS-1$
3490       case TokenNameEQUAL_EQUAL :
3491         return "=="; //$NON-NLS-1$
3492       case TokenNameEQUAL_EQUAL_EQUAL :
3493         return "==="; //$NON-NLS-1$
3494       case TokenNameEQUAL_GREATER :
3495         return "=>"; //$NON-NLS-1$
3496       case TokenNameLESS_EQUAL :
3497         return "<="; //$NON-NLS-1$
3498       case TokenNameGREATER_EQUAL :
3499         return ">="; //$NON-NLS-1$
3500       case TokenNameNOT_EQUAL :
3501         return "!="; //$NON-NLS-1$
3502       case TokenNameNOT_EQUAL_EQUAL :
3503         return "!=="; //$NON-NLS-1$
3504       case TokenNameLEFT_SHIFT :
3505         return "<<"; //$NON-NLS-1$
3506       case TokenNameRIGHT_SHIFT :
3507         return ">>"; //$NON-NLS-1$
3508       case TokenNamePLUS_EQUAL :
3509         return "+="; //$NON-NLS-1$
3510       case TokenNameMINUS_EQUAL :
3511         return "-="; //$NON-NLS-1$
3512       case TokenNameMULTIPLY_EQUAL :
3513         return "*="; //$NON-NLS-1$
3514       case TokenNameDIVIDE_EQUAL :
3515         return "/="; //$NON-NLS-1$
3516       case TokenNameAND_EQUAL :
3517         return "&="; //$NON-NLS-1$
3518       case TokenNameOR_EQUAL :
3519         return "|="; //$NON-NLS-1$
3520       case TokenNameXOR_EQUAL :
3521         return "^="; //$NON-NLS-1$
3522       case TokenNameREMAINDER_EQUAL :
3523         return "%="; //$NON-NLS-1$
3524       case TokenNameLEFT_SHIFT_EQUAL :
3525         return "<<="; //$NON-NLS-1$
3526       case TokenNameRIGHT_SHIFT_EQUAL :
3527         return ">>="; //$NON-NLS-1$
3528       case TokenNameOR_OR :
3529         return "||"; //$NON-NLS-1$
3530       case TokenNameAND_AND :
3531         return "&&"; //$NON-NLS-1$
3532       case TokenNamePLUS :
3533         return "+"; //$NON-NLS-1$
3534       case TokenNameMINUS :
3535         return "-"; //$NON-NLS-1$
3536       case TokenNameMINUS_GREATER :
3537         return "->";
3538       case TokenNameNOT :
3539         return "!"; //$NON-NLS-1$
3540       case TokenNameREMAINDER :
3541         return "%"; //$NON-NLS-1$
3542       case TokenNameXOR :
3543         return "^"; //$NON-NLS-1$
3544       case TokenNameAND :
3545         return "&"; //$NON-NLS-1$
3546       case TokenNameMULTIPLY :
3547         return "*"; //$NON-NLS-1$
3548       case TokenNameOR :
3549         return "|"; //$NON-NLS-1$
3550       case TokenNameTWIDDLE :
3551         return "~"; //$NON-NLS-1$
3552       case TokenNameTWIDDLE_EQUAL :
3553         return "~="; //$NON-NLS-1$
3554       case TokenNameDIVIDE :
3555         return "/"; //$NON-NLS-1$
3556       case TokenNameGREATER :
3557         return ">"; //$NON-NLS-1$
3558       case TokenNameLESS :
3559         return "<"; //$NON-NLS-1$
3560       case TokenNameLPAREN :
3561         return "("; //$NON-NLS-1$
3562       case TokenNameRPAREN :
3563         return ")"; //$NON-NLS-1$
3564       case TokenNameLBRACE :
3565         return "{"; //$NON-NLS-1$
3566       case TokenNameRBRACE :
3567         return "}"; //$NON-NLS-1$
3568       case TokenNameLBRACKET :
3569         return "["; //$NON-NLS-1$
3570       case TokenNameRBRACKET :
3571         return "]"; //$NON-NLS-1$
3572       case TokenNameSEMICOLON :
3573         return ";"; //$NON-NLS-1$
3574       case TokenNameQUESTION :
3575         return "?"; //$NON-NLS-1$
3576       case TokenNameCOLON :
3577         return ":"; //$NON-NLS-1$
3578       case TokenNameCOMMA :
3579         return ","; //$NON-NLS-1$
3580       case TokenNameDOT :
3581         return "."; //$NON-NLS-1$
3582       case TokenNameEQUAL :
3583         return "="; //$NON-NLS-1$
3584       case TokenNameAT :
3585         return "@";
3586       case TokenNameDOLLAR_LBRACE :
3587         return "${";
3588       case TokenNameEOF :
3589         return "EOF"; //$NON-NLS-1$
3590       case TokenNameWHITESPACE :
3591         return "WHITESPACE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3592       case TokenNameCOMMENT_LINE :
3593         return "COMMENT_LINE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3594       case TokenNameCOMMENT_BLOCK :
3595         return "COMMENT_BLOCK(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3596       case TokenNameCOMMENT_PHPDOC :
3597         return "COMMENT_PHPDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3598       case TokenNameHTML :
3599         return "HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
3600       default :
3601         return "not-a-token(" + (new Integer(act)) + ") " + new String(getCurrentTokenSource()); //$NON-NLS-1$
3602     }
3603   }
3604
3605   public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace, boolean checkNonExternalizedStringLiterals) {
3606     this(tokenizeComments, tokenizeWhiteSpace, checkNonExternalizedStringLiterals, false);
3607   }
3608
3609   public Scanner(
3610     boolean tokenizeComments,
3611     boolean tokenizeWhiteSpace,
3612     boolean checkNonExternalizedStringLiterals,
3613     boolean assertMode) {
3614     this.eofPosition = Integer.MAX_VALUE;
3615     this.tokenizeComments = tokenizeComments;
3616     this.tokenizeWhiteSpace = tokenizeWhiteSpace;
3617     this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
3618     this.assertMode = assertMode;
3619   }
3620
3621   private void checkNonExternalizeString() throws InvalidInputException {
3622     if (currentLine == null)
3623       return;
3624     parseTags(currentLine);
3625   }
3626
3627   private void parseTags(NLSLine line) throws InvalidInputException {
3628     String s = new String(getCurrentTokenSource());
3629     int pos = s.indexOf(TAG_PREFIX);
3630     int lineLength = line.size();
3631     while (pos != -1) {
3632       int start = pos + TAG_PREFIX_LENGTH;
3633       int end = s.indexOf(TAG_POSTFIX, start);
3634       String index = s.substring(start, end);
3635       int i = 0;
3636       try {
3637         i = Integer.parseInt(index) - 1;
3638         // Tags are one based not zero based.
3639       } catch (NumberFormatException e) {
3640         i = -1; // we don't want to consider this as a valid NLS tag
3641       }
3642       if (line.exists(i)) {
3643         line.set(i, null);
3644       }
3645       pos = s.indexOf(TAG_PREFIX, start);
3646     }
3647
3648     this.nonNLSStrings = new StringLiteral[lineLength];
3649     int nonNLSCounter = 0;
3650     for (Iterator iterator = line.iterator(); iterator.hasNext();) {
3651       StringLiteral literal = (StringLiteral) iterator.next();
3652       if (literal != null) {
3653         this.nonNLSStrings[nonNLSCounter++] = literal;
3654       }
3655     }
3656     if (nonNLSCounter == 0) {
3657       this.nonNLSStrings = null;
3658       currentLine = null;
3659       return;
3660     }
3661     this.wasNonExternalizedStringLiteral = true;
3662     if (nonNLSCounter != lineLength) {
3663       System.arraycopy(this.nonNLSStrings, 0, (this.nonNLSStrings = new StringLiteral[nonNLSCounter]), 0, nonNLSCounter);
3664     }
3665     currentLine = null;
3666   }
3667
3668   public final void scanEscapeCharacter() throws InvalidInputException {
3669     // the string with "\\u" is a legal string of two chars \ and u
3670     //thus we use a direct access to the source (for regular cases).
3671
3672     if (unicodeAsBackSlash) {
3673       // consume next character
3674       unicodeAsBackSlash = false;
3675       //                        if (((currentCharacter = source[currentPosition++]) == '\\') && (source[currentPosition] == 'u')) {
3676       //                                getNextUnicodeChar();
3677       //                        } else {
3678       if (withoutUnicodePtr != 0) {
3679         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3680         //                              }
3681       }
3682     } else
3683       currentCharacter = source[currentPosition++];
3684     switch (currentCharacter) {
3685       case 'b' :
3686         currentCharacter = '\b';
3687         break;
3688       case 't' :
3689         currentCharacter = '\t';
3690         break;
3691       case 'n' :
3692         currentCharacter = '\n';
3693         break;
3694       case 'f' :
3695         currentCharacter = '\f';
3696         break;
3697       case 'r' :
3698         currentCharacter = '\r';
3699         break;
3700       case '\"' :
3701         currentCharacter = '\"';
3702         break;
3703       case '\'' :
3704         currentCharacter = '\'';
3705         break;
3706       case '\\' :
3707         currentCharacter = '\\';
3708         break;
3709       default :
3710         // -----------octal escape--------------
3711         // OctalDigit
3712         // OctalDigit OctalDigit
3713         // ZeroToThree OctalDigit OctalDigit
3714
3715         int number = Character.getNumericValue(currentCharacter);
3716         if (number >= 0 && number <= 7) {
3717           boolean zeroToThreeNot = number > 3;
3718           if (Character.isDigit(currentCharacter = source[currentPosition++])) {
3719             int digit = Character.getNumericValue(currentCharacter);
3720             if (digit >= 0 && digit <= 7) {
3721               number = (number * 8) + digit;
3722               if (Character.isDigit(currentCharacter = source[currentPosition++])) {
3723                 if (zeroToThreeNot) { // has read \NotZeroToThree OctalDigit Digit --> ignore last character
3724                   currentPosition--;
3725                 } else {
3726                   digit = Character.getNumericValue(currentCharacter);
3727                   if (digit >= 0 && digit <= 7) { // has read \ZeroToThree OctalDigit OctalDigit
3728                     number = (number * 8) + digit;
3729                   } else { // has read \ZeroToThree OctalDigit NonOctalDigit --> ignore last character
3730                     currentPosition--;
3731                   }
3732                 }
3733               } else { // has read \OctalDigit NonDigit--> ignore last character
3734                 currentPosition--;
3735               }
3736             } else { // has read \OctalDigit NonOctalDigit--> ignore last character
3737               currentPosition--;
3738             }
3739           } else { // has read \OctalDigit --> ignore last character
3740             currentPosition--;
3741           }
3742           if (number > 255)
3743             throw new InvalidInputException(INVALID_ESCAPE);
3744           currentCharacter = (char) number;
3745         } else
3746           throw new InvalidInputException(INVALID_ESCAPE);
3747     }
3748   }
3749
3750   // chech presence of task: tags
3751   public void checkTaskTag(int commentStart, int commentEnd) {
3752
3753     // only look for newer task: tags
3754     if (this.foundTaskCount > 0 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
3755       return;
3756     }
3757     int foundTaskIndex = this.foundTaskCount;
3758     nextChar : for (int i = commentStart; i < commentEnd && i < this.eofPosition; i++) {
3759
3760       char[] tag = null;
3761       char[] priority = null;
3762
3763       // check for tag occurrence
3764       nextTag : for (int itag = 0; itag < this.taskTags.length; itag++) {
3765         tag = this.taskTags[itag];
3766         priority = this.taskPriorities != null && itag < this.taskPriorities.length ? this.taskPriorities[itag] : null;
3767         int tagLength = tag.length;
3768         for (int t = 0; t < tagLength; t++) {
3769           if (this.source[i + t] != tag[t])
3770             continue nextTag;
3771         }
3772
3773         if (this.foundTaskTags == null) {
3774           this.foundTaskTags = new char[5][];
3775           this.foundTaskMessages = new char[5][];
3776           this.foundTaskPriorities = new char[5][];
3777           this.foundTaskPositions = new int[5][];
3778         } else if (this.foundTaskCount == this.foundTaskTags.length) {
3779           System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
3780           System.arraycopy(
3781             this.foundTaskMessages,
3782             0,
3783             this.foundTaskMessages = new char[this.foundTaskCount * 2][],
3784             0,
3785             this.foundTaskCount);
3786           System.arraycopy(
3787             this.foundTaskPriorities,
3788             0,
3789             this.foundTaskPriorities = new char[this.foundTaskCount * 2][],
3790             0,
3791             this.foundTaskCount);
3792           System.arraycopy(
3793             this.foundTaskPositions,
3794             0,
3795             this.foundTaskPositions = new int[this.foundTaskCount * 2][],
3796             0,
3797             this.foundTaskCount);
3798         }
3799         this.foundTaskTags[this.foundTaskCount] = tag;
3800         this.foundTaskPriorities[this.foundTaskCount] = priority;
3801         this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i + tagLength - 1 };
3802         this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
3803         this.foundTaskCount++;
3804
3805         i += tagLength - 1; // will be incremented when looping
3806       }
3807     }
3808
3809     for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
3810       // retrieve message start and end positions
3811       int msgStart = this.foundTaskPositions[i][0] + this.foundTaskTags[i].length;
3812       int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1 : commentEnd - 1;
3813       // at most beginning of next task
3814       if (max_value < msgStart)
3815         max_value = msgStart; // would only occur if tag is before EOF.
3816       int end = -1;
3817       char c;
3818
3819       for (int j = msgStart; j < max_value; j++) {
3820         if ((c = this.source[j]) == '\n' || c == '\r') {
3821           end = j - 1;
3822           break;
3823         }
3824       }
3825
3826       if (end == -1) {
3827         for (int j = max_value; j > msgStart; j--) {
3828           if ((c = this.source[j]) == '*') {
3829             end = j - 1;
3830             break;
3831           }
3832         }
3833         if (end == -1)
3834           end = max_value;
3835       }
3836
3837       if (msgStart == end)
3838         continue; // empty
3839
3840       // trim the message
3841       while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
3842         end--;
3843       while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
3844         msgStart++;
3845
3846       // update the end position of the task
3847       this.foundTaskPositions[i][1] = end;
3848
3849       // get the message source
3850       final int messageLength = end - msgStart + 1;
3851       char[] message = new char[messageLength];
3852
3853       System.arraycopy(source, msgStart, message, 0, messageLength);
3854       this.foundTaskMessages[i] = message;
3855     }
3856   }
3857
3858 }