net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/compiler/parser/Scanner.java

   1 /***********************************************************************************************************************************
   2  * Copyright (c) 2000, 2001, 2002 International Business Machines Corp. and others. All rights reserved. This program and the
   3  * accompanying materials are made available under the terms of the Common Public License v0.5 which accompanies this distribution,
   4  * and is available at http://www.eclipse.org/legal/cpl-v05.html
   5  *
   6  * Contributors: IBM Corporation - initial API and implementation
   7  **********************************************************************************************************************************/
   8 package net.sourceforge.phpdt.internal.compiler.parser;
   9
  10 import java.util.ArrayList;
  11 import java.util.Iterator;
  12 import java.util.List;
  13
  14 import net.sourceforge.phpdt.core.compiler.CharOperation;
  15 import net.sourceforge.phpdt.core.compiler.IScanner;
  16 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
  17 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
  18 import net.sourceforge.phpdt.internal.compiler.ast.StringLiteral;
  19 import net.sourceforge.phpdt.internal.compiler.env.ICompilationUnit;
  20
  21 public class Scanner implements IScanner, ITerminalSymbols {
  22         /*
  23          * APIs ares - getNextToken() which return the current type of the token
  24          * (this value is not memorized by the scanner) - getCurrentTokenSource()
  25          * which provides with the token "REAL" source (aka all unicode have been
  26          * transformed into a correct char) - sourceStart gives the position into
  27          * the stream - currentPosition-1 gives the sourceEnd position into the
  28          * stream
  29          */
  30         // 1.4 feature
  31         // private boolean assertMode;
  32         public boolean useAssertAsAnIndentifier = false;
  33
  34         // flag indicating if processed source contains occurrences of keyword
  35         // assert
  36         public boolean containsAssertKeyword = false;
  37
  38         public boolean recordLineSeparator;
  39
  40         public boolean ignorePHPOneLiner = false;
  41
  42         public boolean phpMode = false;
  43
  44         /**
  45          * This token is set to TokenNameecho if a short tag block begins (i.e.
  46          * &gt;?= ... ) Directly after the &quot;=&quot; character the
  47          * getNextToken() method returns TokenNameINLINE_HTML In the next call to
  48          * the getNextToken() method the value of fFillerToken (==TokenNameecho) is
  49          * returned
  50          *
  51          */
  52         int fFillerToken = TokenNameEOF;
  53
  54         public char currentCharacter;
  55
  56         public int startPosition;
  57
  58         public int currentPosition;
  59
  60         public int initialPosition, eofPosition;
  61
  62         // after this position eof are generated instead of real token from the
  63         // source
  64         public boolean tokenizeComments;
  65
  66         public boolean tokenizeWhiteSpace;
  67
  68         public boolean tokenizeStrings;
  69
  70         // source should be viewed as a window (aka a part)
  71         // of a entire very large stream
  72         public char source[];
  73
  74         // unicode support
  75         public char[] withoutUnicodeBuffer;
  76
  77         public int withoutUnicodePtr;
  78
  79         // when == 0 ==> no unicode in the current token
  80         public boolean unicodeAsBackSlash = false;
  81
  82         public boolean scanningFloatLiteral = false;
  83
  84         // support for /** comments
  85         public int[] commentStops = new int[10];
  86
  87         public int[] commentStarts = new int[10];
  88
  89         public int commentPtr = -1; // no comment test with commentPtr value -1
  90
  91         protected int lastCommentLinePosition = -1;
  92
  93         // diet parsing support - jump over some method body when requested
  94         public boolean diet = false;
  95
  96         // support for the poor-line-debuggers ....
  97         // remember the position of the cr/lf
  98         public int[] lineEnds = new int[250];
  99
 100         public int linePtr = -1;
 101
 102         public boolean wasAcr = false;
 103
 104         public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
 105
 106         public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
 107
 108         public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
 109
 110         public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
 111
 112         public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
 113
 114         public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
 115
 116         public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
 117
 118         public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
 119
 120         public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
 121
 122         public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
 123
 124         public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
 125
 126         public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
 127
 128         // ----------------optimized identifier managment------------------
 129         static final char[] charArray_a = new char[] { 'a' },
 130                         charArray_b = new char[] { 'b' }, charArray_c = new char[] { 'c' },
 131                         charArray_d = new char[] { 'd' }, charArray_e = new char[] { 'e' },
 132                         charArray_f = new char[] { 'f' }, charArray_g = new char[] { 'g' },
 133                         charArray_h = new char[] { 'h' }, charArray_i = new char[] { 'i' },
 134                         charArray_j = new char[] { 'j' }, charArray_k = new char[] { 'k' },
 135                         charArray_l = new char[] { 'l' }, charArray_m = new char[] { 'm' },
 136                         charArray_n = new char[] { 'n' }, charArray_o = new char[] { 'o' },
 137                         charArray_p = new char[] { 'p' }, charArray_q = new char[] { 'q' },
 138                         charArray_r = new char[] { 'r' }, charArray_s = new char[] { 's' },
 139                         charArray_t = new char[] { 't' }, charArray_u = new char[] { 'u' },
 140                         charArray_v = new char[] { 'v' }, charArray_w = new char[] { 'w' },
 141                         charArray_x = new char[] { 'x' }, charArray_y = new char[] { 'y' },
 142                         charArray_z = new char[] { 'z' };
 143
 144         static final char[] charArray_va = new char[] { '$', 'a' },
 145                         charArray_vb = new char[] { '$', 'b' }, charArray_vc = new char[] {
 146                                         '$', 'c' }, charArray_vd = new char[] { '$', 'd' },
 147                         charArray_ve = new char[] { '$', 'e' }, charArray_vf = new char[] {
 148                                         '$', 'f' }, charArray_vg = new char[] { '$', 'g' },
 149                         charArray_vh = new char[] { '$', 'h' }, charArray_vi = new char[] {
 150                                         '$', 'i' }, charArray_vj = new char[] { '$', 'j' },
 151                         charArray_vk = new char[] { '$', 'k' }, charArray_vl = new char[] {
 152                                         '$', 'l' }, charArray_vm = new char[] { '$', 'm' },
 153                         charArray_vn = new char[] { '$', 'n' }, charArray_vo = new char[] {
 154                                         '$', 'o' }, charArray_vp = new char[] { '$', 'p' },
 155                         charArray_vq = new char[] { '$', 'q' }, charArray_vr = new char[] {
 156                                         '$', 'r' }, charArray_vs = new char[] { '$', 's' },
 157                         charArray_vt = new char[] { '$', 't' }, charArray_vu = new char[] {
 158                                         '$', 'u' }, charArray_vv = new char[] { '$', 'v' },
 159                         charArray_vw = new char[] { '$', 'w' }, charArray_vx = new char[] {
 160                                         '$', 'x' }, charArray_vy = new char[] { '$', 'y' },
 161                         charArray_vz = new char[] { '$', 'z' };
 162
 163         public final static int MAX_OBVIOUS = 256;
 164
 165         static final int[] ObviousIdentCharNatures = new int[MAX_OBVIOUS];
 166
 167         public final static int C_DOLLAR = 8;
 168
 169         public final static int C_LETTER = 4;
 170
 171         public final static int C_DIGIT = 3;
 172
 173         public final static int C_SEPARATOR = 2;
 174
 175         public final static int C_SPACE = 1;
 176         static {
 177                 for (int i = '0'; i <= '9'; i++)
 178                         ObviousIdentCharNatures[i] = C_DIGIT;
 179
 180                 for (int i = 'a'; i <= 'z'; i++)
 181                         ObviousIdentCharNatures[i] = C_LETTER;
 182                 for (int i = 'A'; i <= 'Z'; i++)
 183                         ObviousIdentCharNatures[i] = C_LETTER;
 184                 ObviousIdentCharNatures['_'] = C_LETTER;
 185                 for (int i = 127; i <= 255; i++)
 186                         ObviousIdentCharNatures[i] = C_LETTER;
 187
 188                 ObviousIdentCharNatures['$'] = C_DOLLAR;
 189
 190                 ObviousIdentCharNatures[10] = C_SPACE; // \ u000a: LINE FEED
 191                 ObviousIdentCharNatures[12] = C_SPACE; // \ u000c: FORM FEED
 192                 ObviousIdentCharNatures[13] = C_SPACE; // \ u000d: CARRIAGE RETURN
 193                 ObviousIdentCharNatures[32] = C_SPACE; // \ u0020: SPACE
 194                 ObviousIdentCharNatures[9] = C_SPACE; // \ u0009: HORIZONTAL
 195                                                                                                 // TABULATION
 196
 197                 ObviousIdentCharNatures['.'] = C_SEPARATOR;
 198                 ObviousIdentCharNatures[':'] = C_SEPARATOR;
 199                 ObviousIdentCharNatures[';'] = C_SEPARATOR;
 200                 ObviousIdentCharNatures[','] = C_SEPARATOR;
 201                 ObviousIdentCharNatures['['] = C_SEPARATOR;
 202                 ObviousIdentCharNatures[']'] = C_SEPARATOR;
 203                 ObviousIdentCharNatures['('] = C_SEPARATOR;
 204                 ObviousIdentCharNatures[')'] = C_SEPARATOR;
 205                 ObviousIdentCharNatures['{'] = C_SEPARATOR;
 206                 ObviousIdentCharNatures['}'] = C_SEPARATOR;
 207                 ObviousIdentCharNatures['+'] = C_SEPARATOR;
 208                 ObviousIdentCharNatures['-'] = C_SEPARATOR;
 209                 ObviousIdentCharNatures['*'] = C_SEPARATOR;
 210                 ObviousIdentCharNatures['/'] = C_SEPARATOR;
 211                 ObviousIdentCharNatures['='] = C_SEPARATOR;
 212                 ObviousIdentCharNatures['&'] = C_SEPARATOR;
 213                 ObviousIdentCharNatures['|'] = C_SEPARATOR;
 214                 ObviousIdentCharNatures['?'] = C_SEPARATOR;
 215                 ObviousIdentCharNatures['<'] = C_SEPARATOR;
 216                 ObviousIdentCharNatures['>'] = C_SEPARATOR;
 217                 ObviousIdentCharNatures['!'] = C_SEPARATOR;
 218                 ObviousIdentCharNatures['%'] = C_SEPARATOR;
 219                 ObviousIdentCharNatures['^'] = C_SEPARATOR;
 220                 ObviousIdentCharNatures['~'] = C_SEPARATOR;
 221                 ObviousIdentCharNatures['"'] = C_SEPARATOR;
 222                 ObviousIdentCharNatures['\''] = C_SEPARATOR;
 223         }
 224
 225         static final char[] initCharArray = new char[] { '\u0000', '\u0000',
 226                         '\u0000', '\u0000', '\u0000', '\u0000' };
 227
 228         static final int TableSize = 30, InternalTableSize = 6;
 229
 230         // 30*6 = 180 entries
 231         public static final int OptimizedLength = 6;
 232
 233         public/* static */
 234         final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][];
 235
 236         // support for detecting non-externalized string literals
 237         int currentLineNr = -1;
 238
 239         int previousLineNr = -1;
 240
 241         NLSLine currentLine = null;
 242
 243         List lines = new ArrayList();
 244
 245         public static final String TAG_PREFIX = "//$NON-NLS-"; //$NON-NLS-1$
 246
 247         public static final int TAG_PREFIX_LENGTH = TAG_PREFIX.length();
 248
 249         public static final String TAG_POSTFIX = "$"; //$NON-NLS-1$
 250
 251         public static final int TAG_POSTFIX_LENGTH = TAG_POSTFIX.length();
 252
 253         public StringLiteral[] nonNLSStrings = null;
 254
 255         public boolean checkNonExternalizedStringLiterals = true;
 256
 257         public boolean wasNonExternalizedStringLiteral = false;
 258
 259         /* static */{
 260                 for (int i = 0; i < 6; i++) {
 261                         for (int j = 0; j < TableSize; j++) {
 262                                 for (int k = 0; k < InternalTableSize; k++) {
 263                                         charArray_length[i][j][k] = initCharArray;
 264                                 }
 265                         }
 266                 }
 267         }
 268
 269         static int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0, newEntry5 = 0,
 270                         newEntry6 = 0;
 271
 272         public static final int RoundBracket = 0;
 273
 274         public static final int SquareBracket = 1;
 275
 276         public static final int CurlyBracket = 2;
 277
 278         public static final int BracketKinds = 3;
 279
 280         // task tag support
 281         public char[][] foundTaskTags = null;
 282
 283         public char[][] foundTaskMessages;
 284
 285         public char[][] foundTaskPriorities = null;
 286
 287         public int[][] foundTaskPositions;
 288
 289         public int foundTaskCount = 0;
 290
 291         public char[][] taskTags = null;
 292
 293         public char[][] taskPriorities = null;
 294
 295         public boolean isTaskCaseSensitive = true;
 296
 297         public static final boolean DEBUG = false;
 298
 299         public static final boolean TRACE = false;
 300
 301         public ICompilationUnit compilationUnit = null;
 302
 303         /**
 304          * Determines if the specified character is permissible as the first
 305          * character in a PHP identifier or variable
 306          *
 307          * The '$' character for PHP variables is regarded as a correct first
 308          * character !
 309          *
 310          */
 311         public static boolean isPHPIdentOrVarStart(char ch) {
 312                 if (ch < MAX_OBVIOUS) {
 313                         return ObviousIdentCharNatures[ch] == C_LETTER
 314                                         || ObviousIdentCharNatures[ch] == C_DOLLAR;
 315                 }
 316                 return false;
 317                 // return Character.isLetter(ch) || (ch == '$') || (ch == '_') || (0x7F
 318                 // <=
 319                 // ch && ch <= 0xFF);
 320         }
 321
 322         /**
 323          * Determines if the specified character is permissible as the first
 324          * character in a PHP identifier.
 325          *
 326          * The '$' character for PHP variables isn't regarded as the first character !
 327          */
 328         public static boolean isPHPIdentifierStart(char ch) {
 329                 if (ch < MAX_OBVIOUS) {
 330                         return ObviousIdentCharNatures[ch] == C_LETTER;
 331                 }
 332                 return false;
 333                 // return Character.isLetter(ch) || (ch == '_') || (0x7F <= ch && ch <=
 334                 // 0xFF);
 335         }
 336
 337         /**
 338          * Determines if the specified character may be part of a PHP identifier as
 339          * other than the first character
 340          */
 341         public static boolean isPHPIdentifierPart(char ch) {
 342                 if (ch < MAX_OBVIOUS) {
 343                         return ObviousIdentCharNatures[ch] == C_LETTER
 344                                         || ObviousIdentCharNatures[ch] == C_DIGIT;
 345                 }
 346                 return false;
 347                 // return Character.isLetterOrDigit(ch) || (ch == '_') || (0x7F <= ch &&
 348                 // ch
 349                 // <= 0xFF);
 350         }
 351
 352         public static boolean isSQLIdentifierPart(char ch) {
 353                 if (ch < MAX_OBVIOUS) {
 354                         return ObviousIdentCharNatures[ch] == C_LETTER
 355                                         || ObviousIdentCharNatures[ch] == C_DIGIT;
 356                 }
 357                 return false;
 358         }
 359
 360         public final boolean atEnd() {
 361                 // This code is not relevant if source is
 362                 // Only a part of the real stream input
 363                 return source.length == currentPosition;
 364         }
 365
 366         public char[] getCurrentIdentifierSource() {
 367                 // return the token REAL source (aka unicodes are precomputed)
 368                 char[] result;
 369                 // if (withoutUnicodePtr != 0)
 370                 // //0 is used as a fast test flag so the real first char is in position
 371                 // 1
 372                 // System.arraycopy(
 373                 // withoutUnicodeBuffer,
 374                 // 1,
 375                 // result = new char[withoutUnicodePtr],
 376                 // 0,
 377                 // withoutUnicodePtr);
 378                 // else {
 379                 int length = currentPosition - startPosition;
 380                 switch (length) { // see OptimizedLength
 381                 case 1:
 382                         return optimizedCurrentTokenSource1();
 383                 case 2:
 384                         return optimizedCurrentTokenSource2();
 385                 case 3:
 386                         return optimizedCurrentTokenSource3();
 387                 case 4:
 388                         return optimizedCurrentTokenSource4();
 389                 case 5:
 390                         return optimizedCurrentTokenSource5();
 391                 case 6:
 392                         return optimizedCurrentTokenSource6();
 393                 }
 394                 // no optimization
 395                 System.arraycopy(source, startPosition, result = new char[length], 0,
 396                                 length);
 397                 // }
 398                 return result;
 399         }
 400
 401         public int getCurrentTokenEndPosition() {
 402                 return this.currentPosition - 1;
 403         }
 404
 405         public final char[] getCurrentTokenSource() {
 406                 // Return the token REAL source (aka unicodes are precomputed)
 407                 char[] result;
 408                 // if (withoutUnicodePtr != 0)
 409                 // // 0 is used as a fast test flag so the real first char is in
 410                 // position 1
 411                 // System.arraycopy(
 412                 // withoutUnicodeBuffer,
 413                 // 1,
 414                 // result = new char[withoutUnicodePtr],
 415                 // 0,
 416                 // withoutUnicodePtr);
 417                 // else {
 418                 int length;
 419                 System.arraycopy(source, startPosition,
 420                                 result = new char[length = currentPosition - startPosition], 0,
 421                                 length);
 422                 // }
 423                 return result;
 424         }
 425
 426         public final char[] getCurrentTokenSource(int startPos) {
 427                 // Return the token REAL source (aka unicodes are precomputed)
 428                 char[] result;
 429                 // if (withoutUnicodePtr != 0)
 430                 // // 0 is used as a fast test flag so the real first char is in
 431                 // position 1
 432                 // System.arraycopy(
 433                 // withoutUnicodeBuffer,
 434                 // 1,
 435                 // result = new char[withoutUnicodePtr],
 436                 // 0,
 437                 // withoutUnicodePtr);
 438                 // else {
 439                 int length;
 440                 System.arraycopy(source, startPos,
 441                                 result = new char[length = currentPosition - startPos], 0,
 442                                 length);
 443                 // }
 444                 return result;
 445         }
 446
 447         public final char[] getCurrentTokenSourceString() {
 448                 // return the token REAL source (aka unicodes are precomputed).
 449                 // REMOVE the two " that are at the beginning and the end.
 450                 char[] result;
 451                 if (withoutUnicodePtr != 0)
 452                         // 0 is used as a fast test flag so the real first char is in
 453                         // position 1
 454                         System.arraycopy(withoutUnicodeBuffer, 2,
 455                                         // 2 is 1 (real start) + 1 (to jump over the ")
 456                                         result = new char[withoutUnicodePtr - 2], 0,
 457                                         withoutUnicodePtr - 2);
 458                 else {
 459                         int length;
 460                         System.arraycopy(source, startPosition + 1,
 461                                         result = new char[length = currentPosition - startPosition
 462                                                         - 2], 0, length);
 463                 }
 464                 return result;
 465         }
 466
 467         public final boolean equalsCurrentTokenSource(char[] word) {
 468                 if (word.length != currentPosition - startPosition) {
 469                         return false;
 470                 }
 471                 for (int i = 0; i < word.length; i++) {
 472                         if (word[i] != source[startPosition + i]) {
 473                                 return false;
 474                         }
 475                 }
 476                 return true;
 477         }
 478
 479         public final char[] getRawTokenSourceEnd() {
 480                 int length = this.eofPosition - this.currentPosition - 1;
 481                 char[] sourceEnd = new char[length];
 482                 System.arraycopy(this.source, this.currentPosition, sourceEnd, 0,
 483                                 length);
 484                 return sourceEnd;
 485         }
 486
 487         public int getCurrentTokenStartPosition() {
 488                 return this.startPosition;
 489         }
 490
 491         public final String getCurrentStringLiteral() {
 492                 char[] result = getCurrentStringLiteralSource();
 493                 return new String(result);
 494         }
 495
 496         public final char[] getCurrentStringLiteralSource() {
 497                 // Return the token REAL source (aka unicodes are precomputed)
 498                 if (startPosition + 1 >= currentPosition) {
 499                         return new char[0];
 500                 }
 501                 char[] result;
 502                 int length;
 503                 System
 504                                 .arraycopy(source, startPosition + 1,
 505                                                 result = new char[length = currentPosition
 506                                                                 - startPosition - 2], 0, length);
 507                 // }
 508                 return result;
 509         }
 510
 511         public final char[] getCurrentStringLiteralSource(int startPos) {
 512                 // Return the token REAL source (aka unicodes are precomputed)
 513                 char[] result;
 514                 int length;
 515                 System.arraycopy(source, startPos + 1,
 516                                 result = new char[length = currentPosition - startPos - 2], 0,
 517                                 length);
 518                 // }
 519                 return result;
 520         }
 521
 522         /*
 523          * Search the source position corresponding to the end of a given line
 524          * number
 525          *
 526          * Line numbers are 1-based, and relative to the scanner initialPosition.
 527          * Character positions are 0-based.
 528          *
 529          * In case the given line number is inconsistent, answers -1.
 530          */
 531         public final int getLineEnd(int lineNumber) {
 532                 if (lineEnds == null)
 533                         return -1;
 534                 if (lineNumber >= lineEnds.length)
 535                         return -1;
 536                 if (lineNumber <= 0)
 537                         return -1;
 538                 if (lineNumber == lineEnds.length - 1)
 539                         return eofPosition;
 540                 return lineEnds[lineNumber - 1];
 541                 // next line start one character behind the lineEnd of the previous line
 542         }
 543
 544         /**
 545          * Search the source position corresponding to the beginning of a given line
 546          * number
 547          *
 548          * Line numbers are 1-based, and relative to the scanner initialPosition.
 549          * Character positions are 0-based.
 550          *
 551          * e.g. getLineStart(1) --> 0 i.e. first line starts at character 0.
 552          *
 553          * In case the given line number is inconsistent, answers -1.
 554          */
 555         public final int getLineStart(int lineNumber) {
 556                 if (lineEnds == null)
 557                         return -1;
 558                 if (lineNumber >= lineEnds.length)
 559                         return -1;
 560                 if (lineNumber <= 0)
 561                         return -1;
 562                 if (lineNumber == 1)
 563                         return initialPosition;
 564                 return lineEnds[lineNumber - 2] + 1;
 565                 // next line start one character behind the lineEnd of the previous line
 566         }
 567
 568         public final boolean getNextChar(char testedChar) {
 569                 // BOOLEAN
 570                 // handle the case of unicode.
 571                 // when a unicode appears then we must use a buffer that holds char
 572                 // internal values
 573                 // At the end of this method currentCharacter holds the new visited char
 574                 // and currentPosition points right next after it
 575                 // Both previous lines are true if the currentCharacter is == to the
 576                 // testedChar
 577                 // On false, no side effect has occured.
 578                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 579                 int temp = currentPosition;
 580                 try {
 581                         currentCharacter = source[currentPosition++];
 582                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 583                         // && (source[currentPosition] == 'u')) {
 584                         // //-------------unicode traitement ------------
 585                         // int c1, c2, c3, c4;
 586                         // int unicodeSize = 6;
 587                         // currentPosition++;
 588                         // while (source[currentPosition] == 'u') {
 589                         // currentPosition++;
 590                         // unicodeSize++;
 591                         // }
 592                         //
 593                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 594                         // > 15
 595                         // || c1 < 0)
 596                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 597                         // 15
 598                         // || c2 < 0)
 599                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 600                         // 15
 601                         // || c3 < 0)
 602                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 603                         // 15
 604                         // || c4 < 0)) {
 605                         // currentPosition = temp;
 606                         // return false;
 607                         // }
 608                         //
 609                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 610                         // if (currentCharacter != testedChar) {
 611                         // currentPosition = temp;
 612                         // return false;
 613                         // }
 614                         // unicodeAsBackSlash = currentCharacter == '\\';
 615                         //
 616                         // //need the unicode buffer
 617                         // if (withoutUnicodePtr == 0) {
 618                         // //buffer all the entries that have been left aside....
 619                         // withoutUnicodePtr = currentPosition - unicodeSize -
 620                         // startPosition;
 621                         // System.arraycopy(
 622                         // source,
 623                         // startPosition,
 624                         // withoutUnicodeBuffer,
 625                         // 1,
 626                         // withoutUnicodePtr);
 627                         // }
 628                         // //fill the buffer with the char
 629                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 630                         // return true;
 631                         //
 632                         // } //-------------end unicode traitement--------------
 633                         // else {
 634                         if (currentCharacter != testedChar) {
 635                                 currentPosition = temp;
 636                                 return false;
 637                         }
 638                         unicodeAsBackSlash = false;
 639                         // if (withoutUnicodePtr != 0)
 640                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 641                         return true;
 642                         // }
 643                 } catch (IndexOutOfBoundsException e) {
 644                         unicodeAsBackSlash = false;
 645                         currentPosition = temp;
 646                         return false;
 647                 }
 648         }
 649
 650         public final int getNextChar(char testedChar1, char testedChar2) {
 651                 // INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
 652                 // test can be done with (x==0) for the first and (x>0) for the second
 653                 // handle the case of unicode.
 654                 // when a unicode appears then we must use a buffer that holds char
 655                 // internal values
 656                 // At the end of this method currentCharacter holds the new visited char
 657                 // and currentPosition points right next after it
 658                 // Both previous lines are true if the currentCharacter is == to the
 659                 // testedChar1/2
 660                 // On false, no side effect has occured.
 661                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 662                 int temp = currentPosition;
 663                 try {
 664                         int result;
 665                         currentCharacter = source[currentPosition++];
 666                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 667                         // && (source[currentPosition] == 'u')) {
 668                         // //-------------unicode traitement ------------
 669                         // int c1, c2, c3, c4;
 670                         // int unicodeSize = 6;
 671                         // currentPosition++;
 672                         // while (source[currentPosition] == 'u') {
 673                         // currentPosition++;
 674                         // unicodeSize++;
 675                         // }
 676                         //
 677                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 678                         // > 15
 679                         // || c1 < 0)
 680                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 681                         // 15
 682                         // || c2 < 0)
 683                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 684                         // 15
 685                         // || c3 < 0)
 686                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 687                         // 15
 688                         // || c4 < 0)) {
 689                         // currentPosition = temp;
 690                         // return 2;
 691                         // }
 692                         //
 693                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 694                         // if (currentCharacter == testedChar1)
 695                         // result = 0;
 696                         // else if (currentCharacter == testedChar2)
 697                         // result = 1;
 698                         // else {
 699                         // currentPosition = temp;
 700                         // return -1;
 701                         // }
 702                         //
 703                         // //need the unicode buffer
 704                         // if (withoutUnicodePtr == 0) {
 705                         // //buffer all the entries that have been left aside....
 706                         // withoutUnicodePtr = currentPosition - unicodeSize -
 707                         // startPosition;
 708                         // System.arraycopy(
 709                         // source,
 710                         // startPosition,
 711                         // withoutUnicodeBuffer,
 712                         // 1,
 713                         // withoutUnicodePtr);
 714                         // }
 715                         // //fill the buffer with the char
 716                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 717                         // return result;
 718                         // } //-------------end unicode traitement--------------
 719                         // else {
 720                         if (currentCharacter == testedChar1)
 721                                 result = 0;
 722                         else if (currentCharacter == testedChar2)
 723                                 result = 1;
 724                         else {
 725                                 currentPosition = temp;
 726                                 return -1;
 727                         }
 728                         // if (withoutUnicodePtr != 0)
 729                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 730                         return result;
 731                         // }
 732                 } catch (IndexOutOfBoundsException e) {
 733                         currentPosition = temp;
 734                         return -1;
 735                 }
 736         }
 737
 738         public final boolean getNextCharAsDigit() {
 739                 // BOOLEAN
 740                 // handle the case of unicode.
 741                 // when a unicode appears then we must use a buffer that holds char
 742                 // internal values
 743                 // At the end of this method currentCharacter holds the new visited char
 744                 // and currentPosition points right next after it
 745                 // Both previous lines are true if the currentCharacter is a digit
 746                 // On false, no side effect has occured.
 747                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 748                 int temp = currentPosition;
 749                 try {
 750                         currentCharacter = source[currentPosition++];
 751                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 752                         // && (source[currentPosition] == 'u')) {
 753                         // //-------------unicode traitement ------------
 754                         // int c1, c2, c3, c4;
 755                         // int unicodeSize = 6;
 756                         // currentPosition++;
 757                         // while (source[currentPosition] == 'u') {
 758                         // currentPosition++;
 759                         // unicodeSize++;
 760                         // }
 761                         //
 762                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 763                         // > 15
 764                         // || c1 < 0)
 765                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 766                         // 15
 767                         // || c2 < 0)
 768                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 769                         // 15
 770                         // || c3 < 0)
 771                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 772                         // 15
 773                         // || c4 < 0)) {
 774                         // currentPosition = temp;
 775                         // return false;
 776                         // }
 777                         //
 778                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 779                         // if (!Character.isDigit(currentCharacter)) {
 780                         // currentPosition = temp;
 781                         // return false;
 782                         // }
 783                         //
 784                         // //need the unicode buffer
 785                         // if (withoutUnicodePtr == 0) {
 786                         // //buffer all the entries that have been left aside....
 787                         // withoutUnicodePtr = currentPosition - unicodeSize -
 788                         // startPosition;
 789                         // System.arraycopy(
 790                         // source,
 791                         // startPosition,
 792                         // withoutUnicodeBuffer,
 793                         // 1,
 794                         // withoutUnicodePtr);
 795                         // }
 796                         // //fill the buffer with the char
 797                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 798                         // return true;
 799                         // } //-------------end unicode traitement--------------
 800                         // else {
 801                         if (!Character.isDigit(currentCharacter)) {
 802                                 currentPosition = temp;
 803                                 return false;
 804                         }
 805                         // if (withoutUnicodePtr != 0)
 806                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 807                         return true;
 808                         // }
 809                 } catch (IndexOutOfBoundsException e) {
 810                         currentPosition = temp;
 811                         return false;
 812                 }
 813         }
 814
 815         public final boolean getNextCharAsDigit(int radix) {
 816                 // BOOLEAN
 817                 // handle the case of unicode.
 818                 // when a unicode appears then we must use a buffer that holds char
 819                 // internal values
 820                 // At the end of this method currentCharacter holds the new visited char
 821                 // and currentPosition points right next after it
 822                 // Both previous lines are true if the currentCharacter is a digit base
 823                 // on
 824                 // radix
 825                 // On false, no side effect has occured.
 826                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 827                 int temp = currentPosition;
 828                 try {
 829                         currentCharacter = source[currentPosition++];
 830                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 831                         // && (source[currentPosition] == 'u')) {
 832                         // //-------------unicode traitement ------------
 833                         // int c1, c2, c3, c4;
 834                         // int unicodeSize = 6;
 835                         // currentPosition++;
 836                         // while (source[currentPosition] == 'u') {
 837                         // currentPosition++;
 838                         // unicodeSize++;
 839                         // }
 840                         //
 841                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 842                         // > 15
 843                         // || c1 < 0)
 844                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 845                         // 15
 846                         // || c2 < 0)
 847                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 848                         // 15
 849                         // || c3 < 0)
 850                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 851                         // 15
 852                         // || c4 < 0)) {
 853                         // currentPosition = temp;
 854                         // return false;
 855                         // }
 856                         //
 857                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 858                         // if (Character.digit(currentCharacter, radix) == -1) {
 859                         // currentPosition = temp;
 860                         // return false;
 861                         // }
 862                         //
 863                         // //need the unicode buffer
 864                         // if (withoutUnicodePtr == 0) {
 865                         // //buffer all the entries that have been left aside....
 866                         // withoutUnicodePtr = currentPosition - unicodeSize -
 867                         // startPosition;
 868                         // System.arraycopy(
 869                         // source,
 870                         // startPosition,
 871                         // withoutUnicodeBuffer,
 872                         // 1,
 873                         // withoutUnicodePtr);
 874                         // }
 875                         // //fill the buffer with the char
 876                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 877                         // return true;
 878                         // } //-------------end unicode traitement--------------
 879                         // else {
 880                         if (Character.digit(currentCharacter, radix) == -1) {
 881                                 currentPosition = temp;
 882                                 return false;
 883                         }
 884                         // if (withoutUnicodePtr != 0)
 885                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 886                         return true;
 887                         // }
 888                 } catch (IndexOutOfBoundsException e) {
 889                         currentPosition = temp;
 890                         return false;
 891                 }
 892         }
 893
 894         public boolean getNextCharAsJavaIdentifierPart() {
 895                 // BOOLEAN
 896                 // handle the case of unicode.
 897                 // when a unicode appears then we must use a buffer that holds char
 898                 // internal values
 899                 // At the end of this method currentCharacter holds the new visited char
 900                 // and currentPosition points right next after it
 901                 // Both previous lines are true if the currentCharacter is a
 902                 // JavaIdentifierPart
 903                 // On false, no side effect has occured.
 904                 // ALL getNextChar.... ARE OPTIMIZED COPIES
 905                 int temp = currentPosition;
 906                 try {
 907                         currentCharacter = source[currentPosition++];
 908                         // if (((currentCharacter = source[currentPosition++]) == '\\')
 909                         // && (source[currentPosition] == 'u')) {
 910                         // //-------------unicode traitement ------------
 911                         // int c1, c2, c3, c4;
 912                         // int unicodeSize = 6;
 913                         // currentPosition++;
 914                         // while (source[currentPosition] == 'u') {
 915                         // currentPosition++;
 916                         // unicodeSize++;
 917                         // }
 918                         //
 919                         // if (((c1 = Character.getNumericValue(source[currentPosition++]))
 920                         // > 15
 921                         // || c1 < 0)
 922                         // || ((c2 = Character.getNumericValue(source[currentPosition++])) >
 923                         // 15
 924                         // || c2 < 0)
 925                         // || ((c3 = Character.getNumericValue(source[currentPosition++])) >
 926                         // 15
 927                         // || c3 < 0)
 928                         // || ((c4 = Character.getNumericValue(source[currentPosition++])) >
 929                         // 15
 930                         // || c4 < 0)) {
 931                         // currentPosition = temp;
 932                         // return false;
 933                         // }
 934                         //
 935                         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
 936                         // if (!isPHPIdentifierPart(currentCharacter)) {
 937                         // currentPosition = temp;
 938                         // return false;
 939                         // }
 940                         //
 941                         // //need the unicode buffer
 942                         // if (withoutUnicodePtr == 0) {
 943                         // //buffer all the entries that have been left aside....
 944                         // withoutUnicodePtr = currentPosition - unicodeSize -
 945                         // startPosition;
 946                         // System.arraycopy(
 947                         // source,
 948                         // startPosition,
 949                         // withoutUnicodeBuffer,
 950                         // 1,
 951                         // withoutUnicodePtr);
 952                         // }
 953                         // //fill the buffer with the char
 954                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 955                         // return true;
 956                         // } //-------------end unicode traitement--------------
 957                         // else {
 958                         if (!isPHPIdentifierPart(currentCharacter)) {
 959                                 currentPosition = temp;
 960                                 return false;
 961                         }
 962                         // if (withoutUnicodePtr != 0)
 963                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
 964                         return true;
 965                         // }
 966                 } catch (IndexOutOfBoundsException e) {
 967                         currentPosition = temp;
 968                         return false;
 969                 }
 970         }
 971
 972         public int getCastOrParen() {
 973                 int tempPosition = currentPosition;
 974                 char tempCharacter = currentCharacter;
 975                 int tempToken = TokenNameLPAREN;
 976                 boolean found = false;
 977                 StringBuffer buf = new StringBuffer();
 978                 try {
 979                         do {
 980                                 currentCharacter = source[currentPosition++];
 981                         } while (currentCharacter == ' ' || currentCharacter == '\t');
 982                         while (ObviousIdentCharNatures[currentCharacter] == C_LETTER) {
 983                                 // while((currentCharacter >= 'a' && currentCharacter <= 'z') ||
 984                                 // (currentCharacter >= 'A' && currentCharacter <= 'Z')) {
 985                                 buf.append(currentCharacter);
 986                                 currentCharacter = source[currentPosition++];
 987                         }
 988                         if (buf.length() >= 3 && buf.length() <= 7) {
 989                                 char[] data = buf.toString().toCharArray();
 990                                 int index = 0;
 991                                 switch (data.length) {
 992                                 case 3:
 993                                         // int
 994                                         if ((data[index] == 'i') && (data[++index] == 'n')
 995                                                         && (data[++index] == 't')) {
 996                                                 found = true;
 997                                                 tempToken = TokenNameintCAST;
 998                                         }
 999                                         break;
1000                                 case 4:
1001                                         // bool real
1002                                         if ((data[index] == 'b') && (data[++index] == 'o')
1003                                                         && (data[++index] == 'o') && (data[++index] == 'l')) {
1004                                                 found = true;
1005                                                 tempToken = TokenNameboolCAST;
1006                                         } else {
1007                                                 index = 0;
1008                                                 if ((data[index] == 'r') && (data[++index] == 'e')
1009                                                                 && (data[++index] == 'a')
1010                                                                 && (data[++index] == 'l')) {
1011                                                         found = true;
1012                                                         tempToken = TokenNamedoubleCAST;
1013                                                 }
1014                                         }
1015                                         break;
1016                                 case 5:
1017                                         // array unset float
1018                                         if ((data[index] == 'a') && (data[++index] == 'r')
1019                                                         && (data[++index] == 'r') && (data[++index] == 'a')
1020                                                         && (data[++index] == 'y')) {
1021                                                 found = true;
1022                                                 tempToken = TokenNamearrayCAST;
1023                                         } else {
1024                                                 index = 0;
1025                                                 if ((data[index] == 'u') && (data[++index] == 'n')
1026                                                                 && (data[++index] == 's')
1027                                                                 && (data[++index] == 'e')
1028                                                                 && (data[++index] == 't')) {
1029                                                         found = true;
1030                                                         tempToken = TokenNameunsetCAST;
1031                                                 } else {
1032                                                         index = 0;
1033                                                         if ((data[index] == 'f') && (data[++index] == 'l')
1034                                                                         && (data[++index] == 'o')
1035                                                                         && (data[++index] == 'a')
1036                                                                         && (data[++index] == 't')) {
1037                                                                 found = true;
1038                                                                 tempToken = TokenNamedoubleCAST;
1039                                                         }
1040                                                 }
1041                                         }
1042                                         break;
1043                                 case 6:
1044                                         // object string double
1045                                         if ((data[index] == 'o') && (data[++index] == 'b')
1046                                                         && (data[++index] == 'j') && (data[++index] == 'e')
1047                                                         && (data[++index] == 'c') && (data[++index] == 't')) {
1048                                                 found = true;
1049                                                 tempToken = TokenNameobjectCAST;
1050                                         } else {
1051                                                 index = 0;
1052                                                 if ((data[index] == 's') && (data[++index] == 't')
1053                                                                 && (data[++index] == 'r')
1054                                                                 && (data[++index] == 'i')
1055                                                                 && (data[++index] == 'n')
1056                                                                 && (data[++index] == 'g')) {
1057                                                         found = true;
1058                                                         tempToken = TokenNamestringCAST;
1059                                                 } else {
1060                                                         index = 0;
1061                                                         if ((data[index] == 'd') && (data[++index] == 'o')
1062                                                                         && (data[++index] == 'u')
1063                                                                         && (data[++index] == 'b')
1064                                                                         && (data[++index] == 'l')
1065                                                                         && (data[++index] == 'e')) {
1066                                                                 found = true;
1067                                                                 tempToken = TokenNamedoubleCAST;
1068                                                         }
1069                                                 }
1070                                         }
1071                                         break;
1072                                 case 7:
1073                                         // boolean integer
1074                                         if ((data[index] == 'b') && (data[++index] == 'o')
1075                                                         && (data[++index] == 'o') && (data[++index] == 'l')
1076                                                         && (data[++index] == 'e') && (data[++index] == 'a')
1077                                                         && (data[++index] == 'n')) {
1078                                                 found = true;
1079                                                 tempToken = TokenNameboolCAST;
1080                                         } else {
1081                                                 index = 0;
1082                                                 if ((data[index] == 'i') && (data[++index] == 'n')
1083                                                                 && (data[++index] == 't')
1084                                                                 && (data[++index] == 'e')
1085                                                                 && (data[++index] == 'g')
1086                                                                 && (data[++index] == 'e')
1087                                                                 && (data[++index] == 'r')) {
1088                                                         found = true;
1089                                                         tempToken = TokenNameintCAST;
1090                                                 }
1091                                         }
1092                                         break;
1093                                 }
1094                                 if (found) {
1095                                         while (currentCharacter == ' ' || currentCharacter == '\t') {
1096                                                 currentCharacter = source[currentPosition++];
1097                                         }
1098                                         if (currentCharacter == ')') {
1099                                                 return tempToken;
1100                                         }
1101                                 }
1102                         }
1103                 } catch (IndexOutOfBoundsException e) {
1104                 }
1105                 currentCharacter = tempCharacter;
1106                 currentPosition = tempPosition;
1107                 return TokenNameLPAREN;
1108         }
1109
1110         public void consumeStringInterpolated() throws InvalidInputException {
1111                 try {
1112                         // consume next character
1113                         unicodeAsBackSlash = false;
1114                         currentCharacter = source[currentPosition++];
1115                         // if (((currentCharacter = source[currentPosition++]) == '\\')
1116                         // && (source[currentPosition] == 'u')) {
1117                         // getNextUnicodeChar();
1118                         // } else {
1119                         // if (withoutUnicodePtr != 0) {
1120                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
1121                         // currentCharacter;
1122                         // }
1123                         // }
1124                         while (currentCharacter != '`') {
1125                                 /** ** in PHP \r and \n are valid in string literals *** */
1126                                 // if ((currentCharacter == '\n')
1127                                 // || (currentCharacter == '\r')) {
1128                                 // // relocate if finding another quote fairly close: thus
1129                                 // unicode
1130                                 // '/u000D' will be fully consumed
1131                                 // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1132                                 // if (currentPosition + lookAhead == source.length)
1133                                 // break;
1134                                 // if (source[currentPosition + lookAhead] == '\n')
1135                                 // break;
1136                                 // if (source[currentPosition + lookAhead] == '\"') {
1137                                 // currentPosition += lookAhead + 1;
1138                                 // break;
1139                                 // }
1140                                 // }
1141                                 // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1142                                 // }
1143                                 if (currentCharacter == '\\') {
1144                                         int escapeSize = currentPosition;
1145                                         boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1146                                         // scanEscapeCharacter make a side effect on this value and
1147                                         // we need
1148                                         // the previous value few lines down this one
1149                                         scanDoubleQuotedEscapeCharacter();
1150                                         escapeSize = currentPosition - escapeSize;
1151                                         if (withoutUnicodePtr == 0) {
1152                                                 // buffer all the entries that have been left aside....
1153                                                 withoutUnicodePtr = currentPosition - escapeSize - 1
1154                                                                 - startPosition;
1155                                                 System.arraycopy(source, startPosition,
1156                                                                 withoutUnicodeBuffer, 1, withoutUnicodePtr);
1157                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1158                                         } else { // overwrite the / in the buffer
1159                                                 withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1160                                                 if (backSlashAsUnicodeInString) { // there are TWO \
1161                                                                                                                         // in the stream
1162                                                         // where only one is correct
1163                                                         withoutUnicodePtr--;
1164                                                 }
1165                                         }
1166                                 } else if ((currentCharacter == '\r')
1167                                                 || (currentCharacter == '\n')) {
1168                                         if (recordLineSeparator) {
1169                                                 pushLineSeparator();
1170                                         }
1171                                 }
1172                                 // consume next character
1173                                 unicodeAsBackSlash = false;
1174                                 currentCharacter = source[currentPosition++];
1175                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
1176                                 // && (source[currentPosition] == 'u')) {
1177                                 // getNextUnicodeChar();
1178                                 // } else {
1179                                 if (withoutUnicodePtr != 0) {
1180                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1181                                 }
1182                                 // }
1183                         }
1184                 } catch (IndexOutOfBoundsException e) {
1185                         // reset end position for error reporting
1186                         currentPosition -= 2;
1187                         throw new InvalidInputException(UNTERMINATED_STRING);
1188                 } catch (InvalidInputException e) {
1189                         if (e.getMessage().equals(INVALID_ESCAPE)) {
1190                                 // relocate if finding another quote fairly close: thus unicode
1191                                 // '/u000D' will be fully consumed
1192                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1193                                         if (currentPosition + lookAhead == source.length)
1194                                                 break;
1195                                         if (source[currentPosition + lookAhead] == '\n')
1196                                                 break;
1197                                         if (source[currentPosition + lookAhead] == '`') {
1198                                                 currentPosition += lookAhead + 1;
1199                                                 break;
1200                                         }
1201                                 }
1202                         }
1203                         throw e; // rethrow
1204                 }
1205                 if (checkNonExternalizedStringLiterals) { // check for presence of NLS
1206                                                                                                         // tags
1207                         // //$NON-NLS-?$ where ? is an
1208                         // int.
1209                         if (currentLine == null) {
1210                                 currentLine = new NLSLine();
1211                                 lines.add(currentLine);
1212                         }
1213                         currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
1214                                         startPosition, currentPosition - 1));
1215                 }
1216         }
1217
1218         public void consumeStringConstant() throws InvalidInputException {
1219                 try {
1220                         // consume next character
1221                         unicodeAsBackSlash = false;
1222                         currentCharacter = source[currentPosition++];
1223                         // if (((currentCharacter = source[currentPosition++]) == '\\')
1224                         // && (source[currentPosition] == 'u')) {
1225                         // getNextUnicodeChar();
1226                         // } else {
1227                         // if (withoutUnicodePtr != 0) {
1228                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
1229                         // currentCharacter;
1230                         // }
1231                         // }
1232                         while (currentCharacter != '\'') {
1233                                 /** ** in PHP \r and \n are valid in string literals *** */
1234                                 // if ((currentCharacter == '\n')
1235                                 // || (currentCharacter == '\r')) {
1236                                 // // relocate if finding another quote fairly close: thus
1237                                 // unicode
1238                                 // '/u000D' will be fully consumed
1239                                 // for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1240                                 // if (currentPosition + lookAhead == source.length)
1241                                 // break;
1242                                 // if (source[currentPosition + lookAhead] == '\n')
1243                                 // break;
1244                                 // if (source[currentPosition + lookAhead] == '\"') {
1245                                 // currentPosition += lookAhead + 1;
1246                                 // break;
1247                                 // }
1248                                 // }
1249                                 // throw new InvalidInputException(INVALID_CHAR_IN_STRING);
1250                                 // }
1251                                 if (currentCharacter == '\\') {
1252                                         int escapeSize = currentPosition;
1253                                         boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1254                                         // scanEscapeCharacter make a side effect on this value and
1255                                         // we need
1256                                         // the previous value few lines down this one
1257                                         scanSingleQuotedEscapeCharacter();
1258                                         escapeSize = currentPosition - escapeSize;
1259                                         if (withoutUnicodePtr == 0) {
1260                                                 // buffer all the entries that have been left aside....
1261                                                 withoutUnicodePtr = currentPosition - escapeSize - 1
1262                                                                 - startPosition;
1263                                                 System.arraycopy(source, startPosition,
1264                                                                 withoutUnicodeBuffer, 1, withoutUnicodePtr);
1265                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1266                                         } else { // overwrite the / in the buffer
1267                                                 withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1268                                                 if (backSlashAsUnicodeInString) { // there are TWO \
1269                                                                                                                         // in the stream
1270                                                         // where only one is correct
1271                                                         withoutUnicodePtr--;
1272                                                 }
1273                                         }
1274                                 } else if ((currentCharacter == '\r')
1275                                                 || (currentCharacter == '\n')) {
1276                                         if (recordLineSeparator) {
1277                                                 pushLineSeparator();
1278                                         }
1279                                 }
1280                                 // consume next character
1281                                 unicodeAsBackSlash = false;
1282                                 currentCharacter = source[currentPosition++];
1283                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
1284                                 // && (source[currentPosition] == 'u')) {
1285                                 // getNextUnicodeChar();
1286                                 // } else {
1287                                 if (withoutUnicodePtr != 0) {
1288                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1289                                 }
1290                                 // }
1291                         }
1292                 } catch (IndexOutOfBoundsException e) {
1293                         // reset end position for error reporting
1294                         currentPosition -= 2;
1295                         throw new InvalidInputException(UNTERMINATED_STRING);
1296                 } catch (InvalidInputException e) {
1297                         if (e.getMessage().equals(INVALID_ESCAPE)) {
1298                                 // relocate if finding another quote fairly close: thus unicode
1299                                 // '/u000D' will be fully consumed
1300                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1301                                         if (currentPosition + lookAhead == source.length)
1302                                                 break;
1303                                         if (source[currentPosition + lookAhead] == '\n')
1304                                                 break;
1305                                         if (source[currentPosition + lookAhead] == '\'') {
1306                                                 currentPosition += lookAhead + 1;
1307                                                 break;
1308                                         }
1309                                 }
1310                         }
1311                         throw e; // rethrow
1312                 }
1313                 if (checkNonExternalizedStringLiterals) { // check for presence of NLS
1314                                                                                                         // tags
1315                         // //$NON-NLS-?$ where ? is an
1316                         // int.
1317                         if (currentLine == null) {
1318                                 currentLine = new NLSLine();
1319                                 lines.add(currentLine);
1320                         }
1321                         currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
1322                                         startPosition, currentPosition - 1));
1323                 }
1324         }
1325
1326         public void consumeStringLiteral() throws InvalidInputException {
1327                 try {
1328                         int openDollarBrace = 0;
1329                         // consume next character
1330                         unicodeAsBackSlash = false;
1331                         currentCharacter = source[currentPosition++];
1332                         while (currentCharacter != '"' || openDollarBrace > 0) {
1333                                 /** ** in PHP \r and \n are valid in string literals *** */
1334                                 if (currentCharacter == '\\') {
1335                                         int escapeSize = currentPosition;
1336                                         boolean backSlashAsUnicodeInString = unicodeAsBackSlash;
1337                                         // scanEscapeCharacter make a side effect on this value and
1338                                         // we need
1339                                         // the previous value few lines down this one
1340                                         scanDoubleQuotedEscapeCharacter();
1341                                         escapeSize = currentPosition - escapeSize;
1342                                         if (withoutUnicodePtr == 0) {
1343                                                 // buffer all the entries that have been left aside....
1344                                                 withoutUnicodePtr = currentPosition - escapeSize - 1
1345                                                                 - startPosition;
1346                                                 System.arraycopy(source, startPosition,
1347                                                                 withoutUnicodeBuffer, 1, withoutUnicodePtr);
1348                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1349                                         } else { // overwrite the / in the buffer
1350                                                 withoutUnicodeBuffer[withoutUnicodePtr] = currentCharacter;
1351                                                 if (backSlashAsUnicodeInString) { // there are TWO \
1352                                                                                                                         // in the stream
1353                                                         // where only one is correct
1354                                                         withoutUnicodePtr--;
1355                                                 }
1356                                         }
1357                                 } else if (currentCharacter == '$'
1358                                                 && source[currentPosition] == '{') {
1359                                         openDollarBrace++;
1360                                 } else if (currentCharacter == '{'
1361                                                 && source[currentPosition] == '$') {
1362                                         openDollarBrace++;
1363                                 } else if (currentCharacter == '}') {
1364                                         openDollarBrace--;
1365                                 } else if ((currentCharacter == '\r')
1366                                                 || (currentCharacter == '\n')) {
1367                                         if (recordLineSeparator) {
1368                                                 pushLineSeparator();
1369                                         }
1370                                 }
1371                                 // consume next character
1372                                 unicodeAsBackSlash = false;
1373                                 currentCharacter = source[currentPosition++];
1374                                 if (withoutUnicodePtr != 0) {
1375                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
1376                                 }
1377                         }
1378                 } catch (IndexOutOfBoundsException e) {
1379                         // reset end position for error reporting
1380                         currentPosition -= 2;
1381                         throw new InvalidInputException(UNTERMINATED_STRING);
1382                 } catch (InvalidInputException e) {
1383                         if (e.getMessage().equals(INVALID_ESCAPE)) {
1384                                 // relocate if finding another quote fairly close: thus unicode
1385                                 // '/u000D' will be fully consumed
1386                                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
1387                                         if (currentPosition + lookAhead == source.length)
1388                                                 break;
1389                                         if (source[currentPosition + lookAhead] == '\n')
1390                                                 break;
1391                                         if (source[currentPosition + lookAhead] == '\"') {
1392                                                 currentPosition += lookAhead + 1;
1393                                                 break;
1394                                         }
1395                                 }
1396                         }
1397                         throw e; // rethrow
1398                 }
1399                 if (checkNonExternalizedStringLiterals) { // check for presence of NLS
1400                                                                                                         // tags
1401                         // //$NON-NLS-?$ where ? is an
1402                         // int.
1403                         if (currentLine == null) {
1404                                 currentLine = new NLSLine();
1405                                 lines.add(currentLine);
1406                         }
1407                         currentLine.add(new StringLiteral(getCurrentTokenSourceString(),
1408                                         startPosition, currentPosition - 1));
1409                 }
1410         }
1411
1412         public int getNextToken() throws InvalidInputException {
1413                 if (!phpMode) {
1414                         return getInlinedHTMLToken(currentPosition);
1415                 } else {
1416                         if (fFillerToken != TokenNameEOF) {
1417                                 int tempToken;
1418                                 startPosition = currentPosition;
1419                                 tempToken = fFillerToken;
1420                                 fFillerToken = TokenNameEOF;
1421                                 return tempToken;
1422                         }
1423                         this.wasAcr = false;
1424                         if (diet) {
1425                                 jumpOverMethodBody();
1426                                 diet = false;
1427                                 return currentPosition > source.length ? TokenNameEOF
1428                                                 : TokenNameRBRACE;
1429                         }
1430                         try {
1431                                 while (true) {
1432                                         withoutUnicodePtr = 0;
1433                                         // ---------Consume white space and handles
1434                                         // startPosition---------
1435                                         int whiteStart = currentPosition;
1436                                         startPosition = currentPosition;
1437                                         currentCharacter = source[currentPosition++];
1438
1439                                         while ((currentCharacter == ' ')
1440                                                         || Character.isWhitespace(currentCharacter)) {
1441                                                 if ((currentCharacter == '\r')
1442                                                                 || (currentCharacter == '\n')) {
1443                                                         checkNonExternalizeString();
1444                                                         if (recordLineSeparator) {
1445                                                                 pushLineSeparator();
1446                                                         } else {
1447                                                                 currentLine = null;
1448                                                         }
1449                                                 }
1450                                                 startPosition = currentPosition;
1451                                                 currentCharacter = source[currentPosition++];
1452                                         }
1453                                         if (tokenizeWhiteSpace
1454                                                         && (whiteStart != currentPosition - 1)) {
1455                                                 // reposition scanner in case we are interested by
1456                                                 // spaces as tokens
1457                                                 currentPosition--;
1458                                                 startPosition = whiteStart;
1459                                                 return TokenNameWHITESPACE;
1460                                         }
1461                                         // little trick to get out in the middle of a source
1462                                         // compuation
1463                                         if (currentPosition > eofPosition)
1464                                                 return TokenNameEOF;
1465                                         // ---------Identify the next token-------------
1466                                         switch (currentCharacter) {
1467                                         case '(':
1468                                                 return getCastOrParen();
1469                                         case ')':
1470                                                 return TokenNameRPAREN;
1471                                         case '{':
1472                                                 return TokenNameLBRACE;
1473                                         case '}':
1474                                                 return TokenNameRBRACE;
1475                                         case '[':
1476                                                 return TokenNameLBRACKET;
1477                                         case ']':
1478                                                 return TokenNameRBRACKET;
1479                                         case ';':
1480                                                 return TokenNameSEMICOLON;
1481                                         case ',':
1482                                                 return TokenNameCOMMA;
1483                                         case '.':
1484                                                 if (getNextChar('='))
1485                                                         return TokenNameDOT_EQUAL;
1486                                                 if (getNextCharAsDigit())
1487                                                         return scanNumber(true);
1488                                                 return TokenNameDOT;
1489                                         case '+': {
1490                                                 int test;
1491                                                 if ((test = getNextChar('+', '=')) == 0)
1492                                                         return TokenNamePLUS_PLUS;
1493                                                 if (test > 0)
1494                                                         return TokenNamePLUS_EQUAL;
1495                                                 return TokenNamePLUS;
1496                                         }
1497                                         case '-': {
1498                                                 int test;
1499                                                 if ((test = getNextChar('-', '=')) == 0)
1500                                                         return TokenNameMINUS_MINUS;
1501                                                 if (test > 0)
1502                                                         return TokenNameMINUS_EQUAL;
1503                                                 if (getNextChar('>'))
1504                                                         return TokenNameMINUS_GREATER;
1505                                                 return TokenNameMINUS;
1506                                         }
1507                                         case '~':
1508                                                 if (getNextChar('='))
1509                                                         return TokenNameTWIDDLE_EQUAL;
1510                                                 return TokenNameTWIDDLE;
1511                                         case '!':
1512                                                 if (getNextChar('=')) {
1513                                                         if (getNextChar('=')) {
1514                                                                 return TokenNameNOT_EQUAL_EQUAL;
1515                                                         }
1516                                                         return TokenNameNOT_EQUAL;
1517                                                 }
1518                                                 return TokenNameNOT;
1519                                         case '*':
1520                                                 if (getNextChar('='))
1521                                                         return TokenNameMULTIPLY_EQUAL;
1522                                                 return TokenNameMULTIPLY;
1523                                         case '%':
1524                                                 if (getNextChar('='))
1525                                                         return TokenNameREMAINDER_EQUAL;
1526                                                 return TokenNameREMAINDER;
1527                                         case '<': {
1528                                                 int oldPosition = currentPosition;
1529                                                 try {
1530                                                         currentCharacter = source[currentPosition++];
1531                                                 } catch (IndexOutOfBoundsException e) {
1532                                                         currentPosition = oldPosition;
1533                                                         return TokenNameLESS;
1534                                                 }
1535                                                 switch (currentCharacter) {
1536                                                 case '=':
1537                                                         return TokenNameLESS_EQUAL;
1538                                                 case '>':
1539                                                         return TokenNameNOT_EQUAL;
1540                                                 case '<':
1541                                                         if (getNextChar('='))
1542                                                                 return TokenNameLEFT_SHIFT_EQUAL;
1543                                                         if (getNextChar('<')) {
1544                                                                 currentCharacter = source[currentPosition++];
1545                                                                 while (Character.isWhitespace(currentCharacter)) {
1546                                                                         currentCharacter = source[currentPosition++];
1547                                                                 }
1548                                                                 int heredocStart = currentPosition - 1;
1549                                                                 int heredocLength = 0;
1550                                                                 if (isPHPIdentifierStart(currentCharacter)) {
1551                                                                         currentCharacter = source[currentPosition++];
1552                                                                 } else {
1553                                                                         return TokenNameERROR;
1554                                                                 }
1555                                                                 while (isPHPIdentifierPart(currentCharacter)) {
1556                                                                         currentCharacter = source[currentPosition++];
1557                                                                 }
1558                                                                 heredocLength = currentPosition - heredocStart
1559                                                                                 - 1;
1560                                                                 // heredoc end-tag determination
1561                                                                 boolean endTag = true;
1562                                                                 char ch;
1563                                                                 do {
1564                                                                         ch = source[currentPosition++];
1565                                                                         if (ch == '\r' || ch == '\n') {
1566                                                                                 if (recordLineSeparator) {
1567                                                                                         pushLineSeparator();
1568                                                                                 } else {
1569                                                                                         currentLine = null;
1570                                                                                 }
1571                                                                                 for (int i = 0; i < heredocLength; i++) {
1572                                                                                         if (source[currentPosition + i] != source[heredocStart
1573                                                                                                         + i]) {
1574                                                                                                 endTag = false;
1575                                                                                                 break;
1576                                                                                         }
1577                                                                                 }
1578                                                                                 if (endTag) {
1579                                                                                         currentPosition += heredocLength - 1;
1580                                                                                         currentCharacter = source[currentPosition++];
1581                                                                                         break; // do...while loop
1582                                                                                 } else {
1583                                                                                         endTag = true;
1584                                                                                 }
1585                                                                         }
1586                                                                 } while (true);
1587                                                                 return TokenNameHEREDOC;
1588                                                         }
1589                                                         return TokenNameLEFT_SHIFT;
1590                                                 }
1591                                                 currentPosition = oldPosition;
1592                                                 return TokenNameLESS;
1593                                         }
1594                                         case '>': {
1595                                                 int test;
1596                                                 if ((test = getNextChar('=', '>')) == 0)
1597                                                         return TokenNameGREATER_EQUAL;
1598                                                 if (test > 0) {
1599                                                         if ((test = getNextChar('=', '>')) == 0)
1600                                                                 return TokenNameRIGHT_SHIFT_EQUAL;
1601                                                         return TokenNameRIGHT_SHIFT;
1602                                                 }
1603                                                 return TokenNameGREATER;
1604                                         }
1605                                         case '=':
1606                                                 if (getNextChar('=')) {
1607                                                         if (getNextChar('=')) {
1608                                                                 return TokenNameEQUAL_EQUAL_EQUAL;
1609                                                         }
1610                                                         return TokenNameEQUAL_EQUAL;
1611                                                 }
1612                                                 if (getNextChar('>'))
1613                                                         return TokenNameEQUAL_GREATER;
1614                                                 return TokenNameEQUAL;
1615                                         case '&': {
1616                                                 int test;
1617                                                 if ((test = getNextChar('&', '=')) == 0)
1618                                                         return TokenNameAND_AND;
1619                                                 if (test > 0)
1620                                                         return TokenNameAND_EQUAL;
1621                                                 return TokenNameAND;
1622                                         }
1623                                         case '|': {
1624                                                 int test;
1625                                                 if ((test = getNextChar('|', '=')) == 0)
1626                                                         return TokenNameOR_OR;
1627                                                 if (test > 0)
1628                                                         return TokenNameOR_EQUAL;
1629                                                 return TokenNameOR;
1630                                         }
1631                                         case '^':
1632                                                 if (getNextChar('='))
1633                                                         return TokenNameXOR_EQUAL;
1634                                                 return TokenNameXOR;
1635                                         case '?':
1636                                                 if (getNextChar('>')) {
1637                                                         phpMode = false;
1638                                                         if (currentPosition == source.length) {
1639                                                                 phpMode = true;
1640                                                                 return TokenNameINLINE_HTML;
1641                                                         }
1642                                                         return getInlinedHTMLToken(currentPosition - 2);
1643                                                 }
1644                                                 return TokenNameQUESTION;
1645                                         case ':':
1646                                                 if (getNextChar(':'))
1647                                                         return TokenNamePAAMAYIM_NEKUDOTAYIM;
1648                                                 return TokenNameCOLON;
1649                                         case '@':
1650                                                 return TokenNameAT;
1651                                         case '\\':
1652                                                 return TokenNameForwardSlash;
1653                                         case '\'':
1654                                                 consumeStringConstant();
1655                                                 return TokenNameStringSingleQuote;
1656                                         case '"':
1657                                                 // if (tokenizeStrings) {
1658                                                 consumeStringLiteral();
1659                                                 return TokenNameStringDoubleQuote;
1660                                                 // }
1661                                                 // return TokenNameEncapsedString2;
1662                                         case '`':
1663                                                 // if (tokenizeStrings) {
1664                                                 consumeStringInterpolated();
1665                                                 return TokenNameStringInterpolated;
1666                                                 // }
1667                                                 // return TokenNameEncapsedString0;
1668                                         case '#':
1669                                         case '/': {
1670                                                 char startChar = currentCharacter;
1671                                                 if (getNextChar('=') && startChar == '/') {
1672                                                         return TokenNameDIVIDE_EQUAL;
1673                                                 }
1674                                                 int test;
1675                                                 if ((startChar == '#')
1676                                                                 || (test = getNextChar('/', '*')) == 0) {
1677                                                         // line comment
1678                                                         this.lastCommentLinePosition = this.currentPosition;
1679                                                         int endPositionForLineComment = 0;
1680                                                         try { // get the next char
1681                                                                 currentCharacter = source[currentPosition++];
1682                                                                 // if (((currentCharacter =
1683                                                                 // source[currentPosition++])
1684                                                                 // == '\\')
1685                                                                 // && (source[currentPosition] == 'u')) {
1686                                                                 // //-------------unicode traitement
1687                                                                 // ------------
1688                                                                 // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1689                                                                 // currentPosition++;
1690                                                                 // while (source[currentPosition] == 'u') {
1691                                                                 // currentPosition++;
1692                                                                 // }
1693                                                                 // if ((c1 =
1694                                                                 // Character.getNumericValue(source[currentPosition++]))
1695                                                                 // > 15
1696                                                                 // || c1 < 0
1697                                                                 // || (c2 =
1698                                                                 // Character.getNumericValue(source[currentPosition++]))
1699                                                                 // > 15
1700                                                                 // || c2 < 0
1701                                                                 // || (c3 =
1702                                                                 // Character.getNumericValue(source[currentPosition++]))
1703                                                                 // > 15
1704                                                                 // || c3 < 0
1705                                                                 // || (c4 =
1706                                                                 // Character.getNumericValue(source[currentPosition++]))
1707                                                                 // > 15
1708                                                                 // || c4 < 0) {
1709                                                                 // throw new
1710                                                                 // InvalidInputException(INVALID_UNICODE_ESCAPE);
1711                                                                 // } else {
1712                                                                 // currentCharacter =
1713                                                                 // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
1714                                                                 // c4);
1715                                                                 // }
1716                                                                 // }
1717                                                                 // handle the \\u case manually into comment
1718                                                                 // if (currentCharacter == '\\') {
1719                                                                 // if (source[currentPosition] == '\\')
1720                                                                 // currentPosition++;
1721                                                                 // } //jump over the \\
1722                                                                 boolean isUnicode = false;
1723                                                                 while (currentCharacter != '\r'
1724                                                                                 && currentCharacter != '\n') {
1725                                                                         this.lastCommentLinePosition = this.currentPosition;
1726                                                                         if (currentCharacter == '?') {
1727                                                                                 if (getNextChar('>')) {
1728                                                                                         // ?> breaks line comments
1729                                                                                         startPosition = currentPosition - 2;
1730                                                                                         phpMode = false;
1731                                                                                         return TokenNameINLINE_HTML;
1732                                                                                 }
1733                                                                         }
1734                                                                         // get the next char
1735                                                                         isUnicode = false;
1736                                                                         currentCharacter = source[currentPosition++];
1737                                                                         // if (((currentCharacter =
1738                                                                         // source[currentPosition++])
1739                                                                         // == '\\')
1740                                                                         // && (source[currentPosition] == 'u')) {
1741                                                                         // isUnicode = true;
1742                                                                         // //-------------unicode traitement
1743                                                                         // ------------
1744                                                                         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
1745                                                                         // currentPosition++;
1746                                                                         // while (source[currentPosition] == 'u') {
1747                                                                         // currentPosition++;
1748                                                                         // }
1749                                                                         // if ((c1 =
1750                                                                         // Character.getNumericValue(source[currentPosition++]))
1751                                                                         // > 15
1752                                                                         // || c1 < 0
1753                                                                         // || (c2 =
1754                                                                         // Character.getNumericValue(
1755                                                                         // source[currentPosition++]))
1756                                                                         // > 15
1757                                                                         // || c2 < 0
1758                                                                         // || (c3 =
1759                                                                         // Character.getNumericValue(
1760                                                                         // source[currentPosition++]))
1761                                                                         // > 15
1762                                                                         // || c3 < 0
1763                                                                         // || (c4 =
1764                                                                         // Character.getNumericValue(
1765                                                                         // source[currentPosition++]))
1766                                                                         // > 15
1767                                                                         // || c4 < 0) {
1768                                                                         // throw new
1769                                                                         // InvalidInputException(INVALID_UNICODE_ESCAPE);
1770                                                                         // } else {
1771                                                                         // currentCharacter =
1772                                                                         // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
1773                                                                         // c4);
1774                                                                         // }
1775                                                                         // }
1776                                                                         // handle the \\u case manually into comment
1777                                                                         // if (currentCharacter == '\\') {
1778                                                                         // if (source[currentPosition] == '\\')
1779                                                                         // currentPosition++;
1780                                                                         // } //jump over the \\
1781                                                                 }
1782                                                                 if (isUnicode) {
1783                                                                         endPositionForLineComment = currentPosition - 6;
1784                                                                 } else {
1785                                                                         endPositionForLineComment = currentPosition - 1;
1786                                                                 }
1787                                                                 // recordComment(false);
1788                                                                 recordComment(TokenNameCOMMENT_LINE);
1789                                                                 if (this.taskTags != null)
1790                                                                         checkTaskTag(this.startPosition,
1791                                                                                         this.currentPosition);
1792                                                                 if ((currentCharacter == '\r')
1793                                                                                 || (currentCharacter == '\n')) {
1794                                                                         checkNonExternalizeString();
1795                                                                         if (recordLineSeparator) {
1796                                                                                 if (isUnicode) {
1797                                                                                         pushUnicodeLineSeparator();
1798                                                                                 } else {
1799                                                                                         pushLineSeparator();
1800                                                                                 }
1801                                                                         } else {
1802                                                                                 currentLine = null;
1803                                                                         }
1804                                                                 }
1805                                                                 if (tokenizeComments) {
1806                                                                         if (!isUnicode) {
1807                                                                                 currentPosition = endPositionForLineComment;
1808                                                                                 // reset one character behind
1809                                                                         }
1810                                                                         return TokenNameCOMMENT_LINE;
1811                                                                 }
1812                                                         } catch (IndexOutOfBoundsException e) { // an eof
1813                                                                                                                                         // will them
1814                                                                 // be generated
1815                                                                 if (tokenizeComments) {
1816                                                                         currentPosition--;
1817                                                                         // reset one character behind
1818                                                                         return TokenNameCOMMENT_LINE;
1819                                                                 }
1820                                                         }
1821                                                         break;
1822                                                 }
1823                                                 if (test > 0) {
1824                                                         // traditional and annotation comment
1825                                                         boolean isJavadoc = false, star = false;
1826                                                         // consume next character
1827                                                         unicodeAsBackSlash = false;
1828                                                         currentCharacter = source[currentPosition++];
1829                                                         // if (((currentCharacter =
1830                                                         // source[currentPosition++]) ==
1831                                                         // '\\')
1832                                                         // && (source[currentPosition] == 'u')) {
1833                                                         // getNextUnicodeChar();
1834                                                         // } else {
1835                                                         // if (withoutUnicodePtr != 0) {
1836                                                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
1837                                                         // currentCharacter;
1838                                                         // }
1839                                                         // }
1840                                                         if (currentCharacter == '*') {
1841                                                                 isJavadoc = true;
1842                                                                 star = true;
1843                                                         }
1844                                                         if ((currentCharacter == '\r')
1845                                                                         || (currentCharacter == '\n')) {
1846                                                                 checkNonExternalizeString();
1847                                                                 if (recordLineSeparator) {
1848                                                                         pushLineSeparator();
1849                                                                 } else {
1850                                                                         currentLine = null;
1851                                                                 }
1852                                                         }
1853                                                         try { // get the next char
1854                                                                 currentCharacter = source[currentPosition++];
1855                                                                 // if (((currentCharacter =
1856                                                                 // source[currentPosition++])
1857                                                                 // == '\\')
1858                                                                 // && (source[currentPosition] == 'u')) {
1859                                                                 // //-------------unicode traitement
1860                                                                 // ------------
1861                                                                 // getNextUnicodeChar();
1862                                                                 // }
1863                                                                 // handle the \\u case manually into comment
1864                                                                 // if (currentCharacter == '\\') {
1865                                                                 // if (source[currentPosition] == '\\')
1866                                                                 // currentPosition++;
1867                                                                 // //jump over the \\
1868                                                                 // }
1869                                                                 // empty comment is not a javadoc /**/
1870                                                                 if (currentCharacter == '/') {
1871                                                                         isJavadoc = false;
1872                                                                 }
1873                                                                 // loop until end of comment */
1874                                                                 while ((currentCharacter != '/') || (!star)) {
1875                                                                         if ((currentCharacter == '\r')
1876                                                                                         || (currentCharacter == '\n')) {
1877                                                                                 checkNonExternalizeString();
1878                                                                                 if (recordLineSeparator) {
1879                                                                                         pushLineSeparator();
1880                                                                                 } else {
1881                                                                                         currentLine = null;
1882                                                                                 }
1883                                                                         }
1884                                                                         star = currentCharacter == '*';
1885                                                                         // get next char
1886                                                                         currentCharacter = source[currentPosition++];
1887                                                                         // if (((currentCharacter =
1888                                                                         // source[currentPosition++])
1889                                                                         // == '\\')
1890                                                                         // && (source[currentPosition] == 'u')) {
1891                                                                         // //-------------unicode traitement
1892                                                                         // ------------
1893                                                                         // getNextUnicodeChar();
1894                                                                         // }
1895                                                                         // handle the \\u case manually into comment
1896                                                                         // if (currentCharacter == '\\') {
1897                                                                         // if (source[currentPosition] == '\\')
1898                                                                         // currentPosition++;
1899                                                                         // } //jump over the \\
1900                                                                 }
1901                                                                 // recordComment(isJavadoc);
1902                                                                 if (isJavadoc) {
1903                                                                         recordComment(TokenNameCOMMENT_PHPDOC);
1904                                                                 } else {
1905                                                                         recordComment(TokenNameCOMMENT_BLOCK);
1906                                                                 }
1907
1908                                                                 if (tokenizeComments) {
1909                                                                         if (isJavadoc)
1910                                                                                 return TokenNameCOMMENT_PHPDOC;
1911                                                                         return TokenNameCOMMENT_BLOCK;
1912                                                                 }
1913
1914                                                                 if (this.taskTags != null) {
1915                                                                         checkTaskTag(this.startPosition,
1916                                                                                         this.currentPosition);
1917                                                                 }
1918                                                         } catch (IndexOutOfBoundsException e) {
1919                                                                 // reset end position for error reporting
1920                                                                 currentPosition -= 2;
1921                                                                 throw new InvalidInputException(
1922                                                                                 UNTERMINATED_COMMENT);
1923                                                         }
1924                                                         break;
1925                                                 }
1926                                                 return TokenNameDIVIDE;
1927                                         }
1928                                         case '\u001a':
1929                                                 if (atEnd())
1930                                                         return TokenNameEOF;
1931                                                 // the atEnd may not be <currentPosition ==
1932                                                 // source.length> if
1933                                                 // source is only some part of a real (external) stream
1934                                                 throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$
1935                                         default:
1936                                                 if (currentCharacter == '$') {
1937                                                         int oldPosition = currentPosition;
1938                                                         try {
1939                                                                 currentCharacter = source[currentPosition++];
1940                                                                 if (isPHPIdentifierStart(currentCharacter)) {
1941                                                                         return scanIdentifierOrKeyword(true);
1942                                                                 } else {
1943                                                                         currentPosition = oldPosition;
1944                                                                         return TokenNameDOLLAR;
1945                                                                 }
1946                                                         } catch (IndexOutOfBoundsException e) {
1947                                                                 currentPosition = oldPosition;
1948                                                                 return TokenNameDOLLAR;
1949                                                         }
1950                                                 }
1951                                                 if (isPHPIdentifierStart(currentCharacter))
1952                                                         return scanIdentifierOrKeyword(false);
1953                                                 if (Character.isDigit(currentCharacter))
1954                                                         return scanNumber(false);
1955                                                 return TokenNameERROR;
1956                                         }
1957                                 }
1958                         } // -----------------end switch while try--------------------
1959                         catch (IndexOutOfBoundsException e) {
1960                         }
1961                 }
1962                 return TokenNameEOF;
1963         }
1964
1965         /**
1966          * @return
1967          * @throws InvalidInputException
1968          */
1969         private int getInlinedHTMLToken(int start) throws InvalidInputException {
1970                 boolean phpShortTag = false; // true, if <?= detected
1971                 if (currentPosition > source.length) {
1972                         currentPosition = source.length;
1973                         return TokenNameEOF;
1974                 }
1975                 startPosition = start;
1976                 try {
1977                         while (!phpMode) {
1978                                 currentCharacter = source[currentPosition++];
1979                                 if (currentCharacter == '<') {
1980                                         if (getNextChar('?')) {
1981                                                 currentCharacter = source[currentPosition++];
1982                                                 if ((currentCharacter != 'P')
1983                                                                 && (currentCharacter != 'p')) {
1984                                                         if (currentCharacter != '=') { // <?=
1985                                                                 currentPosition--;
1986                                                                 phpShortTag = false;
1987                                                         } else {
1988                                                                 phpShortTag = true;
1989                                                         }
1990                                                         // <?
1991                                                         if (ignorePHPOneLiner) { // for CodeFormatter
1992                                                                 if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
1993                                                                         phpMode = true;
1994                                                                         if (phpShortTag) {
1995                                                                                 fFillerToken = TokenNameECHO_INVISIBLE;
1996                                                                         }
1997                                                                         return TokenNameINLINE_HTML;
1998                                                                 }
1999                                                         } else {
2000                                                                 boolean foundXML = false;
2001                                                                 if (getNextChar('X', 'x') >= 0) {
2002                                                                         if (getNextChar('M', 'm') >= 0) {
2003                                                                                 if (getNextChar('L', 'l') >= 0) {
2004                                                                                         foundXML = true;
2005                                                                                 }
2006                                                                         }
2007                                                                 }
2008                                                                 if (!foundXML) {
2009                                                                         phpMode = true;
2010                                                                 }
2011                                                                 if (phpShortTag) {
2012                                                                         fFillerToken = TokenNameECHO_INVISIBLE;
2013                                                                 }
2014                                                                 return TokenNameINLINE_HTML;
2015                                                         }
2016                                                 } else {
2017                                                         if (getNextChar('H', 'h') >= 0) {
2018                                                                 if (getNextChar('P', 'p') >= 0) {
2019                                                                         // <?PHP <?php
2020                                                                         if (ignorePHPOneLiner) {
2021                                                                                 if (lookAheadLinePHPTag() == TokenNameINLINE_HTML) {
2022                                                                                         phpMode = true;
2023                                                                                         return TokenNameINLINE_HTML;
2024                                                                                 }
2025                                                                         } else {
2026                                                                                 phpMode = true;
2027                                                                                 return TokenNameINLINE_HTML;
2028                                                                         }
2029                                                                 }
2030                                                         }
2031                                                         // }
2032                                                 }
2033                                         }
2034                                 }
2035                                 if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
2036                                         if (recordLineSeparator) {
2037                                                 pushLineSeparator();
2038                                         } else {
2039                                                 currentLine = null;
2040                                         }
2041                                 }
2042                         } // -----------------while--------------------
2043                         phpMode = true;
2044                         return TokenNameINLINE_HTML;
2045                 } // -----------------try--------------------
2046                 catch (IndexOutOfBoundsException e) {
2047                         startPosition = start;
2048                         currentPosition--;
2049                 }
2050                 phpMode = true;
2051                 return TokenNameINLINE_HTML;
2052         }
2053
2054         /**
2055          * check if the PHP is only in this line (for CodeFormatter)
2056          *
2057          * @return
2058          */
2059         private int lookAheadLinePHPTag() {
2060                 int currentPositionInLine = currentPosition;
2061                 char previousCharInLine = ' ';
2062                 char currentCharInLine = ' ';
2063                 boolean singleQuotedStringActive = false;
2064                 boolean doubleQuotedStringActive = false;
2065
2066                 try {
2067                         // look ahead in this line
2068                         while (true) {
2069                                 previousCharInLine = currentCharInLine;
2070                                 currentCharInLine = source[currentPositionInLine++];
2071                                 switch (currentCharInLine) {
2072                                 case '>':
2073                                         if (previousCharInLine == '?') {
2074                                                 // update the scanner's current Position in the source
2075                                                 currentPosition = currentPositionInLine;
2076                                                 // use as "dummy" token
2077                                                 return TokenNameEOF;
2078                                         }
2079                                         break;
2080                                 case '\\':
2081                                         if (doubleQuotedStringActive) {
2082                                                 // ignore escaped characters in double quoted strings
2083                                                 previousCharInLine = currentCharInLine;
2084                                                 currentCharInLine = source[currentPositionInLine++];
2085                                         }
2086                                 case '\"':
2087                                         if (doubleQuotedStringActive) {
2088                                                 doubleQuotedStringActive = false;
2089                                         } else {
2090                                                 if (!singleQuotedStringActive) {
2091                                                         doubleQuotedStringActive = true;
2092                                                 }
2093                                         }
2094                                         break;
2095                                 case '\'':
2096                                         if (singleQuotedStringActive) {
2097                                                 if (previousCharInLine != '\\') {
2098                                                         singleQuotedStringActive = false;
2099                                                 }
2100                                         } else {
2101                                                 if (!doubleQuotedStringActive) {
2102                                                         singleQuotedStringActive = true;
2103                                                 }
2104                                         }
2105                                         break;
2106                                 case '\n':
2107                                         phpMode = true;
2108                                         return TokenNameINLINE_HTML;
2109                                 case '#':
2110                                         if (!singleQuotedStringActive && !doubleQuotedStringActive) {
2111                                                 phpMode = true;
2112                                                 return TokenNameINLINE_HTML;
2113                                         }
2114                                         break;
2115                                 case '/':
2116                                         if (previousCharInLine == '/' && !singleQuotedStringActive
2117                                                         && !doubleQuotedStringActive) {
2118                                                 phpMode = true;
2119                                                 return TokenNameINLINE_HTML;
2120                                         }
2121                                         break;
2122                                 case '*':
2123                                         if (previousCharInLine == '/' && !singleQuotedStringActive
2124                                                         && !doubleQuotedStringActive) {
2125                                                 phpMode = true;
2126                                                 return TokenNameINLINE_HTML;
2127                                         }
2128                                         break;
2129                                 }
2130                         }
2131                 } catch (IndexOutOfBoundsException e) {
2132                         phpMode = true;
2133                         currentPosition = currentPositionInLine - 1;
2134                         return TokenNameINLINE_HTML;
2135                 }
2136         }
2137
2138         // public final void getNextUnicodeChar()
2139         // throws IndexOutOfBoundsException, InvalidInputException {
2140         // //VOID
2141         // //handle the case of unicode.
2142         // //when a unicode appears then we must use a buffer that holds char
2143         // internal values
2144         // //At the end of this method currentCharacter holds the new visited char
2145         // //and currentPosition points right next after it
2146         //
2147         // //ALL getNextChar.... ARE OPTIMIZED COPIES
2148         //
2149         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
2150         // currentPosition++;
2151         // while (source[currentPosition] == 'u') {
2152         // currentPosition++;
2153         // unicodeSize++;
2154         // }
2155         //
2156         // if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
2157         // || c1 < 0
2158         // || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
2159         // || c2 < 0
2160         // || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
2161         // || c3 < 0
2162         // || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
2163         // || c4 < 0) {
2164         // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2165         // } else {
2166         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2167         // //need the unicode buffer
2168         // if (withoutUnicodePtr == 0) {
2169         // //buffer all the entries that have been left aside....
2170         // withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
2171         // System.arraycopy(
2172         // source,
2173         // startPosition,
2174         // withoutUnicodeBuffer,
2175         // 1,
2176         // withoutUnicodePtr);
2177         // }
2178         // //fill the buffer with the char
2179         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2180         // }
2181         // unicodeAsBackSlash = currentCharacter == '\\';
2182         // }
2183         /*
2184          * Tokenize a method body, assuming that curly brackets are properly
2185          * balanced.
2186          */
2187         public final void jumpOverMethodBody() {
2188                 this.wasAcr = false;
2189                 int found = 1;
2190                 try {
2191                         while (true) { // loop for jumping over comments
2192                                 // ---------Consume white space and handles
2193                                 // startPosition---------
2194                                 boolean isWhiteSpace;
2195                                 do {
2196                                         startPosition = currentPosition;
2197                                         currentCharacter = source[currentPosition++];
2198                                         // if (((currentCharacter = source[currentPosition++]) ==
2199                                         // '\\')
2200                                         // && (source[currentPosition] == 'u')) {
2201                                         // isWhiteSpace = jumpOverUnicodeWhiteSpace();
2202                                         // } else {
2203                                         if (recordLineSeparator
2204                                                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2205                                                 pushLineSeparator();
2206                                         isWhiteSpace = Character.isWhitespace(currentCharacter);
2207                                         // }
2208                                 } while (isWhiteSpace);
2209                                 // -------consume token until } is found---------
2210                                 switch (currentCharacter) {
2211                                 case '{':
2212                                         found++;
2213                                         break;
2214                                 case '}':
2215                                         found--;
2216                                         if (found == 0)
2217                                                 return;
2218                                         break;
2219                                 case '\'': {
2220                                         boolean test;
2221                                         test = getNextChar('\\');
2222                                         if (test) {
2223                                                 try {
2224                                                         scanDoubleQuotedEscapeCharacter();
2225                                                 } catch (InvalidInputException ex) {
2226                                                 }
2227                                                 ;
2228                                         } else {
2229                                                 // try { // consume next character
2230                                                 unicodeAsBackSlash = false;
2231                                                 currentCharacter = source[currentPosition++];
2232                                                 // if (((currentCharacter = source[currentPosition++])
2233                                                 // == '\\')
2234                                                 // && (source[currentPosition] == 'u')) {
2235                                                 // getNextUnicodeChar();
2236                                                 // } else {
2237                                                 if (withoutUnicodePtr != 0) {
2238                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2239                                                 }
2240                                                 // }
2241                                                 // } catch (InvalidInputException ex) {
2242                                                 // };
2243                                         }
2244                                         getNextChar('\'');
2245                                         break;
2246                                 }
2247                                 case '"':
2248                                         try {
2249                                                 // try { // consume next character
2250                                                 unicodeAsBackSlash = false;
2251                                                 currentCharacter = source[currentPosition++];
2252                                                 // if (((currentCharacter = source[currentPosition++])
2253                                                 // == '\\')
2254                                                 // && (source[currentPosition] == 'u')) {
2255                                                 // getNextUnicodeChar();
2256                                                 // } else {
2257                                                 if (withoutUnicodePtr != 0) {
2258                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2259                                                 }
2260                                                 // }
2261                                                 // } catch (InvalidInputException ex) {
2262                                                 // };
2263                                                 while (currentCharacter != '"') {
2264                                                         if (currentCharacter == '\r') {
2265                                                                 if (source[currentPosition] == '\n')
2266                                                                         currentPosition++;
2267                                                                 break;
2268                                                                 // the string cannot go further that the line
2269                                                         }
2270                                                         if (currentCharacter == '\n') {
2271                                                                 break;
2272                                                                 // the string cannot go further that the line
2273                                                         }
2274                                                         if (currentCharacter == '\\') {
2275                                                                 try {
2276                                                                         scanDoubleQuotedEscapeCharacter();
2277                                                                 } catch (InvalidInputException ex) {
2278                                                                 }
2279                                                                 ;
2280                                                         }
2281                                                         // try { // consume next character
2282                                                         unicodeAsBackSlash = false;
2283                                                         currentCharacter = source[currentPosition++];
2284                                                         // if (((currentCharacter =
2285                                                         // source[currentPosition++]) == '\\')
2286                                                         // && (source[currentPosition] == 'u')) {
2287                                                         // getNextUnicodeChar();
2288                                                         // } else {
2289                                                         if (withoutUnicodePtr != 0) {
2290                                                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2291                                                         }
2292                                                         // }
2293                                                         // } catch (InvalidInputException ex) {
2294                                                         // };
2295                                                 }
2296                                         } catch (IndexOutOfBoundsException e) {
2297                                                 return;
2298                                         }
2299                                         break;
2300                                 case '/': {
2301                                         int test;
2302                                         if ((test = getNextChar('/', '*')) == 0) {
2303                                                 // line comment
2304                                                 try {
2305                                                         // get the next char
2306                                                         currentCharacter = source[currentPosition++];
2307                                                         // if (((currentCharacter =
2308                                                         // source[currentPosition++]) ==
2309                                                         // '\\')
2310                                                         // && (source[currentPosition] == 'u')) {
2311                                                         // //-------------unicode traitement ------------
2312                                                         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2313                                                         // currentPosition++;
2314                                                         // while (source[currentPosition] == 'u') {
2315                                                         // currentPosition++;
2316                                                         // }
2317                                                         // if ((c1 =
2318                                                         // Character.getNumericValue(source[currentPosition++]))
2319                                                         // > 15
2320                                                         // || c1 < 0
2321                                                         // || (c2 =
2322                                                         // Character.getNumericValue(source[currentPosition++]))
2323                                                         // > 15
2324                                                         // || c2 < 0
2325                                                         // || (c3 =
2326                                                         // Character.getNumericValue(source[currentPosition++]))
2327                                                         // > 15
2328                                                         // || c3 < 0
2329                                                         // || (c4 =
2330                                                         // Character.getNumericValue(source[currentPosition++]))
2331                                                         // > 15
2332                                                         // || c4 < 0) {
2333                                                         // //error don't care of the value
2334                                                         // currentCharacter = 'A';
2335                                                         // } //something different from \n and \r
2336                                                         // else {
2337                                                         // currentCharacter =
2338                                                         // (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2339                                                         // }
2340                                                         // }
2341                                                         while (currentCharacter != '\r'
2342                                                                         && currentCharacter != '\n') {
2343                                                                 // get the next char
2344                                                                 currentCharacter = source[currentPosition++];
2345                                                                 // if (((currentCharacter =
2346                                                                 // source[currentPosition++])
2347                                                                 // == '\\')
2348                                                                 // && (source[currentPosition] == 'u')) {
2349                                                                 // //-------------unicode traitement
2350                                                                 // ------------
2351                                                                 // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2352                                                                 // currentPosition++;
2353                                                                 // while (source[currentPosition] == 'u') {
2354                                                                 // currentPosition++;
2355                                                                 // }
2356                                                                 // if ((c1 =
2357                                                                 // Character.getNumericValue(source[currentPosition++]))
2358                                                                 // > 15
2359                                                                 // || c1 < 0
2360                                                                 // || (c2 =
2361                                                                 // Character.getNumericValue(source[currentPosition++]))
2362                                                                 // > 15
2363                                                                 // || c2 < 0
2364                                                                 // || (c3 =
2365                                                                 // Character.getNumericValue(source[currentPosition++]))
2366                                                                 // > 15
2367                                                                 // || c3 < 0
2368                                                                 // || (c4 =
2369                                                                 // Character.getNumericValue(source[currentPosition++]))
2370                                                                 // > 15
2371                                                                 // || c4 < 0) {
2372                                                                 // //error don't care of the value
2373                                                                 // currentCharacter = 'A';
2374                                                                 // } //something different from \n and \r
2375                                                                 // else {
2376                                                                 // currentCharacter =
2377                                                                 // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
2378                                                                 // c4);
2379                                                                 // }
2380                                                                 // }
2381                                                         }
2382                                                         if (recordLineSeparator
2383                                                                         && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2384                                                                 pushLineSeparator();
2385                                                 } catch (IndexOutOfBoundsException e) {
2386                                                 } // an eof will them be generated
2387                                                 break;
2388                                         }
2389                                         if (test > 0) {
2390                                                 // traditional and annotation comment
2391                                                 boolean star = false;
2392                                                 // try { // consume next character
2393                                                 unicodeAsBackSlash = false;
2394                                                 currentCharacter = source[currentPosition++];
2395                                                 // if (((currentCharacter = source[currentPosition++])
2396                                                 // == '\\')
2397                                                 // && (source[currentPosition] == 'u')) {
2398                                                 // getNextUnicodeChar();
2399                                                 // } else {
2400                                                 if (withoutUnicodePtr != 0) {
2401                                                         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2402                                                 }
2403                                                 // };
2404                                                 // } catch (InvalidInputException ex) {
2405                                                 // };
2406                                                 if (currentCharacter == '*') {
2407                                                         star = true;
2408                                                 }
2409                                                 if (recordLineSeparator
2410                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2411                                                         pushLineSeparator();
2412                                                 try { // get the next char
2413                                                         currentCharacter = source[currentPosition++];
2414                                                         // if (((currentCharacter =
2415                                                         // source[currentPosition++]) ==
2416                                                         // '\\')
2417                                                         // && (source[currentPosition] == 'u')) {
2418                                                         // //-------------unicode traitement ------------
2419                                                         // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2420                                                         // currentPosition++;
2421                                                         // while (source[currentPosition] == 'u') {
2422                                                         // currentPosition++;
2423                                                         // }
2424                                                         // if ((c1 =
2425                                                         // Character.getNumericValue(source[currentPosition++]))
2426                                                         // > 15
2427                                                         // || c1 < 0
2428                                                         // || (c2 =
2429                                                         // Character.getNumericValue(source[currentPosition++]))
2430                                                         // > 15
2431                                                         // || c2 < 0
2432                                                         // || (c3 =
2433                                                         // Character.getNumericValue(source[currentPosition++]))
2434                                                         // > 15
2435                                                         // || c3 < 0
2436                                                         // || (c4 =
2437                                                         // Character.getNumericValue(source[currentPosition++]))
2438                                                         // > 15
2439                                                         // || c4 < 0) {
2440                                                         // //error don't care of the value
2441                                                         // currentCharacter = 'A';
2442                                                         // } //something different from * and /
2443                                                         // else {
2444                                                         // currentCharacter =
2445                                                         // (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2446                                                         // }
2447                                                         // }
2448                                                         // loop until end of comment */
2449                                                         while ((currentCharacter != '/') || (!star)) {
2450                                                                 if (recordLineSeparator
2451                                                                                 && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2452                                                                         pushLineSeparator();
2453                                                                 star = currentCharacter == '*';
2454                                                                 // get next char
2455                                                                 currentCharacter = source[currentPosition++];
2456                                                                 // if (((currentCharacter =
2457                                                                 // source[currentPosition++])
2458                                                                 // == '\\')
2459                                                                 // && (source[currentPosition] == 'u')) {
2460                                                                 // //-------------unicode traitement
2461                                                                 // ------------
2462                                                                 // int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
2463                                                                 // currentPosition++;
2464                                                                 // while (source[currentPosition] == 'u') {
2465                                                                 // currentPosition++;
2466                                                                 // }
2467                                                                 // if ((c1 =
2468                                                                 // Character.getNumericValue(source[currentPosition++]))
2469                                                                 // > 15
2470                                                                 // || c1 < 0
2471                                                                 // || (c2 =
2472                                                                 // Character.getNumericValue(source[currentPosition++]))
2473                                                                 // > 15
2474                                                                 // || c2 < 0
2475                                                                 // || (c3 =
2476                                                                 // Character.getNumericValue(source[currentPosition++]))
2477                                                                 // > 15
2478                                                                 // || c3 < 0
2479                                                                 // || (c4 =
2480                                                                 // Character.getNumericValue(source[currentPosition++]))
2481                                                                 // > 15
2482                                                                 // || c4 < 0) {
2483                                                                 // //error don't care of the value
2484                                                                 // currentCharacter = 'A';
2485                                                                 // } //something different from * and /
2486                                                                 // else {
2487                                                                 // currentCharacter =
2488                                                                 // (char) (((c1 * 16 + c2) * 16 + c3) * 16 +
2489                                                                 // c4);
2490                                                                 // }
2491                                                                 // }
2492                                                         }
2493                                                 } catch (IndexOutOfBoundsException e) {
2494                                                         return;
2495                                                 }
2496                                                 break;
2497                                         }
2498                                         break;
2499                                 }
2500                                 default:
2501                                         if (isPHPIdentOrVarStart(currentCharacter)) {
2502                                                 try {
2503                                                         scanIdentifierOrKeyword((currentCharacter == '$'));
2504                                                 } catch (InvalidInputException ex) {
2505                                                 }
2506                                                 ;
2507                                                 break;
2508                                         }
2509                                         if (ObviousIdentCharNatures[currentCharacter] == C_DIGIT) {
2510                                                 // if (Character.isDigit(currentCharacter)) {
2511                                                 try {
2512                                                         scanNumber(false);
2513                                                 } catch (InvalidInputException ex) {
2514                                                 }
2515                                                 ;
2516                                                 break;
2517                                         }
2518                                 }
2519                         }
2520                         // -----------------end switch while try--------------------
2521                 } catch (IndexOutOfBoundsException e) {
2522                 } catch (InvalidInputException e) {
2523                 }
2524                 return;
2525         }
2526
2527         // public final boolean jumpOverUnicodeWhiteSpace()
2528         // throws InvalidInputException {
2529         // //BOOLEAN
2530         // //handle the case of unicode. Jump over the next whiteSpace
2531         // //making startPosition pointing on the next available char
2532         // //On false, the currentCharacter is filled up with a potential
2533         // //correct char
2534         //
2535         // try {
2536         // this.wasAcr = false;
2537         // int c1, c2, c3, c4;
2538         // int unicodeSize = 6;
2539         // currentPosition++;
2540         // while (source[currentPosition] == 'u') {
2541         // currentPosition++;
2542         // unicodeSize++;
2543         // }
2544         //
2545         // if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15
2546         // || c1 < 0)
2547         // || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15
2548         // || c2 < 0)
2549         // || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15
2550         // || c3 < 0)
2551         // || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15
2552         // || c4 < 0)) {
2553         // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2554         // }
2555         //
2556         // currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
2557         // if (recordLineSeparator
2558         // && ((currentCharacter == '\r') || (currentCharacter == '\n')))
2559         // pushLineSeparator();
2560         // if (Character.isWhitespace(currentCharacter))
2561         // return true;
2562         //
2563         // //buffer the new char which is not a white space
2564         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
2565         // //withoutUnicodePtr == 1 is true here
2566         // return false;
2567         // } catch (IndexOutOfBoundsException e) {
2568         // throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
2569         // }
2570         // }
2571         public final int[] getLineEnds() {
2572                 // return a bounded copy of this.lineEnds
2573                 int[] copy;
2574                 System.arraycopy(lineEnds, 0, copy = new int[linePtr + 1], 0,
2575                                 linePtr + 1);
2576                 return copy;
2577         }
2578
2579         public char[] getSource() {
2580                 return this.source;
2581         }
2582
2583         public static boolean isIdentifierOrKeyword(int token) {
2584                 return (token == TokenNameIdentifier) || (token > TokenNameKEYWORD);
2585         }
2586
2587         final char[] optimizedCurrentTokenSource1() {
2588                 // return always the same char[] build only once
2589                 // optimization at no speed cost of 99.5 % of the singleCharIdentifier
2590                 char charOne = source[startPosition];
2591                 switch (charOne) {
2592                 case 'a':
2593                         return charArray_a;
2594                 case 'b':
2595                         return charArray_b;
2596                 case 'c':
2597                         return charArray_c;
2598                 case 'd':
2599                         return charArray_d;
2600                 case 'e':
2601                         return charArray_e;
2602                 case 'f':
2603                         return charArray_f;
2604                 case 'g':
2605                         return charArray_g;
2606                 case 'h':
2607                         return charArray_h;
2608                 case 'i':
2609                         return charArray_i;
2610                 case 'j':
2611                         return charArray_j;
2612                 case 'k':
2613                         return charArray_k;
2614                 case 'l':
2615                         return charArray_l;
2616                 case 'm':
2617                         return charArray_m;
2618                 case 'n':
2619                         return charArray_n;
2620                 case 'o':
2621                         return charArray_o;
2622                 case 'p':
2623                         return charArray_p;
2624                 case 'q':
2625                         return charArray_q;
2626                 case 'r':
2627                         return charArray_r;
2628                 case 's':
2629                         return charArray_s;
2630                 case 't':
2631                         return charArray_t;
2632                 case 'u':
2633                         return charArray_u;
2634                 case 'v':
2635                         return charArray_v;
2636                 case 'w':
2637                         return charArray_w;
2638                 case 'x':
2639                         return charArray_x;
2640                 case 'y':
2641                         return charArray_y;
2642                 case 'z':
2643                         return charArray_z;
2644                 default:
2645                         return new char[] { charOne };
2646                 }
2647         }
2648
2649         final char[] optimizedCurrentTokenSource2() {
2650                 char c0, c1;
2651                 c0 = source[startPosition];
2652                 c1 = source[startPosition + 1];
2653                 if (c0 == '$') {
2654                         // return always the same char[] build only once
2655                         // optimization at no speed cost of 99.5 % of the
2656                         // singleCharIdentifier
2657                         switch (c1) {
2658                         case 'a':
2659                                 return charArray_va;
2660                         case 'b':
2661                                 return charArray_vb;
2662                         case 'c':
2663                                 return charArray_vc;
2664                         case 'd':
2665                                 return charArray_vd;
2666                         case 'e':
2667                                 return charArray_ve;
2668                         case 'f':
2669                                 return charArray_vf;
2670                         case 'g':
2671                                 return charArray_vg;
2672                         case 'h':
2673                                 return charArray_vh;
2674                         case 'i':
2675                                 return charArray_vi;
2676                         case 'j':
2677                                 return charArray_vj;
2678                         case 'k':
2679                                 return charArray_vk;
2680                         case 'l':
2681                                 return charArray_vl;
2682                         case 'm':
2683                                 return charArray_vm;
2684                         case 'n':
2685                                 return charArray_vn;
2686                         case 'o':
2687                                 return charArray_vo;
2688                         case 'p':
2689                                 return charArray_vp;
2690                         case 'q':
2691                                 return charArray_vq;
2692                         case 'r':
2693                                 return charArray_vr;
2694                         case 's':
2695                                 return charArray_vs;
2696                         case 't':
2697                                 return charArray_vt;
2698                         case 'u':
2699                                 return charArray_vu;
2700                         case 'v':
2701                                 return charArray_vv;
2702                         case 'w':
2703                                 return charArray_vw;
2704                         case 'x':
2705                                 return charArray_vx;
2706                         case 'y':
2707                                 return charArray_vy;
2708                         case 'z':
2709                                 return charArray_vz;
2710                         }
2711                 }
2712                 // try to return the same char[] build only once
2713                 int hash = ((c0 << 6) + c1) % TableSize;
2714                 char[][] table = charArray_length[0][hash];
2715                 int i = newEntry2;
2716                 while (++i < InternalTableSize) {
2717                         char[] charArray = table[i];
2718                         if ((c0 == charArray[0]) && (c1 == charArray[1]))
2719                                 return charArray;
2720                 }
2721                 // ---------other side---------
2722                 i = -1;
2723                 int max = newEntry2;
2724                 while (++i <= max) {
2725                         char[] charArray = table[i];
2726                         if ((c0 == charArray[0]) && (c1 == charArray[1]))
2727                                 return charArray;
2728                 }
2729                 // --------add the entry-------
2730                 if (++max >= InternalTableSize)
2731                         max = 0;
2732                 char[] r;
2733                 table[max] = (r = new char[] { c0, c1 });
2734                 newEntry2 = max;
2735                 return r;
2736         }
2737
2738         final char[] optimizedCurrentTokenSource3() {
2739                 // try to return the same char[] build only once
2740                 char c0, c1, c2;
2741                 int hash = (((c0 = source[startPosition]) << 12)
2742                                 + ((c1 = source[startPosition + 1]) << 6) + (c2 = source[startPosition + 2]))
2743                                 % TableSize;
2744                 char[][] table = charArray_length[1][hash];
2745                 int i = newEntry3;
2746                 while (++i < InternalTableSize) {
2747                         char[] charArray = table[i];
2748                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2749                                         && (c2 == charArray[2]))
2750                                 return charArray;
2751                 }
2752                 // ---------other side---------
2753                 i = -1;
2754                 int max = newEntry3;
2755                 while (++i <= max) {
2756                         char[] charArray = table[i];
2757                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2758                                         && (c2 == charArray[2]))
2759                                 return charArray;
2760                 }
2761                 // --------add the entry-------
2762                 if (++max >= InternalTableSize)
2763                         max = 0;
2764                 char[] r;
2765                 table[max] = (r = new char[] { c0, c1, c2 });
2766                 newEntry3 = max;
2767                 return r;
2768         }
2769
2770         final char[] optimizedCurrentTokenSource4() {
2771                 // try to return the same char[] build only once
2772                 char c0, c1, c2, c3;
2773                 long hash = ((((long) (c0 = source[startPosition])) << 18)
2774                                 + ((c1 = source[startPosition + 1]) << 12)
2775                                 + ((c2 = source[startPosition + 2]) << 6) + (c3 = source[startPosition + 3]))
2776                                 % TableSize;
2777                 char[][] table = charArray_length[2][(int) hash];
2778                 int i = newEntry4;
2779                 while (++i < InternalTableSize) {
2780                         char[] charArray = table[i];
2781                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2782                                         && (c2 == charArray[2]) && (c3 == charArray[3]))
2783                                 return charArray;
2784                 }
2785                 // ---------other side---------
2786                 i = -1;
2787                 int max = newEntry4;
2788                 while (++i <= max) {
2789                         char[] charArray = table[i];
2790                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2791                                         && (c2 == charArray[2]) && (c3 == charArray[3]))
2792                                 return charArray;
2793                 }
2794                 // --------add the entry-------
2795                 if (++max >= InternalTableSize)
2796                         max = 0;
2797                 char[] r;
2798                 table[max] = (r = new char[] { c0, c1, c2, c3 });
2799                 newEntry4 = max;
2800                 return r;
2801         }
2802
2803         final char[] optimizedCurrentTokenSource5() {
2804                 // try to return the same char[] build only once
2805                 char c0, c1, c2, c3, c4;
2806                 long hash = ((((long) (c0 = source[startPosition])) << 24)
2807                                 + (((long) (c1 = source[startPosition + 1])) << 18)
2808                                 + ((c2 = source[startPosition + 2]) << 12)
2809                                 + ((c3 = source[startPosition + 3]) << 6) + (c4 = source[startPosition + 4]))
2810                                 % TableSize;
2811                 char[][] table = charArray_length[3][(int) hash];
2812                 int i = newEntry5;
2813                 while (++i < InternalTableSize) {
2814                         char[] charArray = table[i];
2815                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2816                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2817                                         && (c4 == charArray[4]))
2818                                 return charArray;
2819                 }
2820                 // ---------other side---------
2821                 i = -1;
2822                 int max = newEntry5;
2823                 while (++i <= max) {
2824                         char[] charArray = table[i];
2825                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2826                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2827                                         && (c4 == charArray[4]))
2828                                 return charArray;
2829                 }
2830                 // --------add the entry-------
2831                 if (++max >= InternalTableSize)
2832                         max = 0;
2833                 char[] r;
2834                 table[max] = (r = new char[] { c0, c1, c2, c3, c4 });
2835                 newEntry5 = max;
2836                 return r;
2837         }
2838
2839         final char[] optimizedCurrentTokenSource6() {
2840                 // try to return the same char[] build only once
2841                 char c0, c1, c2, c3, c4, c5;
2842                 long hash = ((((long) (c0 = source[startPosition])) << 32)
2843                                 + (((long) (c1 = source[startPosition + 1])) << 24)
2844                                 + (((long) (c2 = source[startPosition + 2])) << 18)
2845                                 + ((c3 = source[startPosition + 3]) << 12)
2846                                 + ((c4 = source[startPosition + 4]) << 6) + (c5 = source[startPosition + 5]))
2847                                 % TableSize;
2848                 char[][] table = charArray_length[4][(int) hash];
2849                 int i = newEntry6;
2850                 while (++i < InternalTableSize) {
2851                         char[] charArray = table[i];
2852                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2853                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2854                                         && (c4 == charArray[4]) && (c5 == charArray[5]))
2855                                 return charArray;
2856                 }
2857                 // ---------other side---------
2858                 i = -1;
2859                 int max = newEntry6;
2860                 while (++i <= max) {
2861                         char[] charArray = table[i];
2862                         if ((c0 == charArray[0]) && (c1 == charArray[1])
2863                                         && (c2 == charArray[2]) && (c3 == charArray[3])
2864                                         && (c4 == charArray[4]) && (c5 == charArray[5]))
2865                                 return charArray;
2866                 }
2867                 // --------add the entry-------
2868                 if (++max >= InternalTableSize)
2869                         max = 0;
2870                 char[] r;
2871                 table[max] = (r = new char[] { c0, c1, c2, c3, c4, c5 });
2872                 newEntry6 = max;
2873                 return r;
2874         }
2875
2876         public final void pushLineSeparator() throws InvalidInputException {
2877                 // see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2878                 final int INCREMENT = 250;
2879                 if (this.checkNonExternalizedStringLiterals) {
2880                         // reinitialize the current line for non externalize strings purpose
2881                         currentLine = null;
2882                 }
2883                 // currentCharacter is at position currentPosition-1
2884                 // cr 000D
2885                 if (currentCharacter == '\r') {
2886                         int separatorPos = currentPosition - 1;
2887                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2888                                 return;
2889                         // System.out.println("CR-" + separatorPos);
2890                         try {
2891                                 lineEnds[++linePtr] = separatorPos;
2892                         } catch (IndexOutOfBoundsException e) {
2893                                 // linePtr value is correct
2894                                 int oldLength = lineEnds.length;
2895                                 int[] old = lineEnds;
2896                                 lineEnds = new int[oldLength + INCREMENT];
2897                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
2898                                 lineEnds[linePtr] = separatorPos;
2899                         }
2900                         // look-ahead for merged cr+lf
2901                         try {
2902                                 if (source[currentPosition] == '\n') {
2903                                         // System.out.println("look-ahead LF-" + currentPosition);
2904                                         lineEnds[linePtr] = currentPosition;
2905                                         currentPosition++;
2906                                         wasAcr = false;
2907                                 } else {
2908                                         wasAcr = true;
2909                                 }
2910                         } catch (IndexOutOfBoundsException e) {
2911                                 wasAcr = true;
2912                         }
2913                 } else {
2914                         // lf 000A
2915                         if (currentCharacter == '\n') {
2916                                 // must merge eventual cr followed by lf
2917                                 if (wasAcr && (lineEnds[linePtr] == (currentPosition - 2))) {
2918                                         // System.out.println("merge LF-" + (currentPosition - 1));
2919                                         lineEnds[linePtr] = currentPosition - 1;
2920                                 } else {
2921                                         int separatorPos = currentPosition - 1;
2922                                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2923                                                 return;
2924                                         // System.out.println("LF-" + separatorPos);
2925                                         try {
2926                                                 lineEnds[++linePtr] = separatorPos;
2927                                         } catch (IndexOutOfBoundsException e) {
2928                                                 // linePtr value is correct
2929                                                 int oldLength = lineEnds.length;
2930                                                 int[] old = lineEnds;
2931                                                 lineEnds = new int[oldLength + INCREMENT];
2932                                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
2933                                                 lineEnds[linePtr] = separatorPos;
2934                                         }
2935                                 }
2936                                 wasAcr = false;
2937                         }
2938                 }
2939         }
2940
2941         public final void pushUnicodeLineSeparator() {
2942                 // isUnicode means that the \r or \n has been read as a unicode
2943                 // character
2944                 // see comment on isLineDelimiter(char) for the use of '\n' and '\r'
2945                 final int INCREMENT = 250;
2946                 // currentCharacter is at position currentPosition-1
2947                 if (this.checkNonExternalizedStringLiterals) {
2948                         // reinitialize the current line for non externalize strings purpose
2949                         currentLine = null;
2950                 }
2951                 // cr 000D
2952                 if (currentCharacter == '\r') {
2953                         int separatorPos = currentPosition - 6;
2954                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2955                                 return;
2956                         // System.out.println("CR-" + separatorPos);
2957                         try {
2958                                 lineEnds[++linePtr] = separatorPos;
2959                         } catch (IndexOutOfBoundsException e) {
2960                                 // linePtr value is correct
2961                                 int oldLength = lineEnds.length;
2962                                 int[] old = lineEnds;
2963                                 lineEnds = new int[oldLength + INCREMENT];
2964                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
2965                                 lineEnds[linePtr] = separatorPos;
2966                         }
2967                         // look-ahead for merged cr+lf
2968                         if (source[currentPosition] == '\n') {
2969                                 // System.out.println("look-ahead LF-" + currentPosition);
2970                                 lineEnds[linePtr] = currentPosition;
2971                                 currentPosition++;
2972                                 wasAcr = false;
2973                         } else {
2974                                 wasAcr = true;
2975                         }
2976                 } else {
2977                         // lf 000A
2978                         if (currentCharacter == '\n') {
2979                                 // must merge eventual cr followed by lf
2980                                 if (wasAcr && (lineEnds[linePtr] == (currentPosition - 7))) {
2981                                         // System.out.println("merge LF-" + (currentPosition - 1));
2982                                         lineEnds[linePtr] = currentPosition - 6;
2983                                 } else {
2984                                         int separatorPos = currentPosition - 6;
2985                                         if ((linePtr > 0) && (lineEnds[linePtr] >= separatorPos))
2986                                                 return;
2987                                         // System.out.println("LF-" + separatorPos);
2988                                         try {
2989                                                 lineEnds[++linePtr] = separatorPos;
2990                                         } catch (IndexOutOfBoundsException e) {
2991                                                 // linePtr value is correct
2992                                                 int oldLength = lineEnds.length;
2993                                                 int[] old = lineEnds;
2994                                                 lineEnds = new int[oldLength + INCREMENT];
2995                                                 System.arraycopy(old, 0, lineEnds, 0, oldLength);
2996                                                 lineEnds[linePtr] = separatorPos;
2997                                         }
2998                                 }
2999                                 wasAcr = false;
3000                         }
3001                 }
3002         }
3003
3004         public void recordComment(int token) {
3005                 // compute position
3006                 int stopPosition = this.currentPosition;
3007                 switch (token) {
3008                 case TokenNameCOMMENT_LINE:
3009                         stopPosition = -this.lastCommentLinePosition;
3010                         break;
3011                 case TokenNameCOMMENT_BLOCK:
3012                         stopPosition = -this.currentPosition;
3013                         break;
3014                 }
3015
3016                 // a new comment is recorded
3017                 int length = this.commentStops.length;
3018                 if (++this.commentPtr >= length) {
3019                         System.arraycopy(this.commentStops, 0,
3020                                         this.commentStops = new int[length + 30], 0, length);
3021                         // grows the positions buffers too
3022                         System.arraycopy(this.commentStarts, 0,
3023                                         this.commentStarts = new int[length + 30], 0, length);
3024                 }
3025                 this.commentStops[this.commentPtr] = stopPosition;
3026                 this.commentStarts[this.commentPtr] = this.startPosition;
3027         }
3028
3029         // public final void recordComment(boolean isJavadoc) {
3030         // // a new annotation comment is recorded
3031         // try {
3032         // commentStops[++commentPtr] = isJavadoc
3033         // ? currentPosition
3034         // : -currentPosition;
3035         // } catch (IndexOutOfBoundsException e) {
3036         // int oldStackLength = commentStops.length;
3037         // int[] oldStack = commentStops;
3038         // commentStops = new int[oldStackLength + 30];
3039         // System.arraycopy(oldStack, 0, commentStops, 0, oldStackLength);
3040         // commentStops[commentPtr] = isJavadoc ? currentPosition :
3041         // -currentPosition;
3042         // //grows the positions buffers too
3043         // int[] old = commentStarts;
3044         // commentStarts = new int[oldStackLength + 30];
3045         // System.arraycopy(old, 0, commentStarts, 0, oldStackLength);
3046         // }
3047         // //the buffer is of a correct size here
3048         // commentStarts[commentPtr] = startPosition;
3049         // }
3050         public void resetTo(int begin, int end) {
3051                 // reset the scanner to a given position where it may rescan again
3052                 diet = false;
3053                 initialPosition = startPosition = currentPosition = begin;
3054                 eofPosition = end < Integer.MAX_VALUE ? end + 1 : end;
3055                 commentPtr = -1; // reset comment stack
3056         }
3057
3058         public final void scanSingleQuotedEscapeCharacter()
3059                         throws InvalidInputException {
3060                 // the string with "\\u" is a legal string of two chars \ and u
3061                 // thus we use a direct access to the source (for regular cases).
3062                 // if (unicodeAsBackSlash) {
3063                 // // consume next character
3064                 // unicodeAsBackSlash = false;
3065                 // if (((currentCharacter = source[currentPosition++]) == '\\')
3066                 // && (source[currentPosition] == 'u')) {
3067                 // getNextUnicodeChar();
3068                 // } else {
3069                 // if (withoutUnicodePtr != 0) {
3070                 // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3071                 // }
3072                 // }
3073                 // } else
3074                 currentCharacter = source[currentPosition++];
3075                 switch (currentCharacter) {
3076                 case '\'':
3077                         currentCharacter = '\'';
3078                         break;
3079                 case '\\':
3080                         currentCharacter = '\\';
3081                         break;
3082                 default:
3083                         currentCharacter = '\\';
3084                         currentPosition--;
3085                 }
3086         }
3087
3088         public final void scanDoubleQuotedEscapeCharacter()
3089                         throws InvalidInputException {
3090                 currentCharacter = source[currentPosition++];
3091                 switch (currentCharacter) {
3092                 // case 'b' :
3093                 // currentCharacter = '\b';
3094                 // break;
3095                 case 't':
3096                         currentCharacter = '\t';
3097                         break;
3098                 case 'n':
3099                         currentCharacter = '\n';
3100                         break;
3101                 // case 'f' :
3102                 // currentCharacter = '\f';
3103                 // break;
3104                 case 'r':
3105                         currentCharacter = '\r';
3106                         break;
3107                 case '\"':
3108                         currentCharacter = '\"';
3109                         break;
3110                 case '\'':
3111                         currentCharacter = '\'';
3112                         break;
3113                 case '\\':
3114                         currentCharacter = '\\';
3115                         break;
3116                 case '$':
3117                         currentCharacter = '$';
3118                         break;
3119                 default:
3120                         // -----------octal escape--------------
3121                         // OctalDigit
3122                         // OctalDigit OctalDigit
3123                         // ZeroToThree OctalDigit OctalDigit
3124                         int number = Character.getNumericValue(currentCharacter);
3125                         if (number >= 0 && number <= 7) {
3126                                 boolean zeroToThreeNot = number > 3;
3127                                 if (Character
3128                                                 .isDigit(currentCharacter = source[currentPosition++])) {
3129                                         int digit = Character.getNumericValue(currentCharacter);
3130                                         if (digit >= 0 && digit <= 7) {
3131                                                 number = (number * 8) + digit;
3132                                                 if (Character
3133                                                                 .isDigit(currentCharacter = source[currentPosition++])) {
3134                                                         if (zeroToThreeNot) { // has read \NotZeroToThree
3135                                                                                                         // OctalDigit
3136                                                                 // Digit --> ignore last character
3137                                                                 currentPosition--;
3138                                                         } else {
3139                                                                 digit = Character
3140                                                                                 .getNumericValue(currentCharacter);
3141                                                                 if (digit >= 0 && digit <= 7) {
3142                                                                         // has read \ZeroToThree OctalDigit
3143                                                                         // OctalDigit
3144                                                                         number = (number * 8) + digit;
3145                                                                 } else { // has read \ZeroToThree OctalDigit
3146                                                                                         // NonOctalDigit
3147                                                                         // --> ignore last character
3148                                                                         currentPosition--;
3149                                                                 }
3150                                                         }
3151                                                 } else { // has read \OctalDigit NonDigit--> ignore
3152                                                                         // last
3153                                                         // character
3154                                                         currentPosition--;
3155                                                 }
3156                                         } else { // has read \OctalDigit NonOctalDigit--> ignore
3157                                                                 // last
3158                                                 // character
3159                                                 currentPosition--;
3160                                         }
3161                                 } else { // has read \OctalDigit --> ignore last character
3162                                         currentPosition--;
3163                                 }
3164                                 if (number > 255)
3165                                         throw new InvalidInputException(INVALID_ESCAPE);
3166                                 currentCharacter = (char) number;
3167                         }
3168                         // else
3169                         // throw new InvalidInputException(INVALID_ESCAPE);
3170                 }
3171         }
3172
3173         // public int scanIdentifierOrKeyword() throws InvalidInputException {
3174         // return scanIdentifierOrKeyword( false );
3175         // }
3176         public int scanIdentifierOrKeyword(boolean isVariable)
3177                         throws InvalidInputException {
3178                 // test keywords
3179                 // first dispatch on the first char.
3180                 // then the length. If there are several
3181                 // keywors with the same length AND the same first char, then do another
3182                 // disptach on the second char :-)...cool....but fast !
3183                 useAssertAsAnIndentifier = false;
3184                 while (getNextCharAsJavaIdentifierPart()) {
3185                 }
3186                 ;
3187                 if (isVariable) {
3188                         // if (new String(getCurrentTokenSource()).equals("$this")) {
3189                         // return TokenNamethis;
3190                         // }
3191                         return TokenNameVariable;
3192                 }
3193                 int index, length;
3194                 char[] data;
3195                 char firstLetter;
3196                 // if (withoutUnicodePtr == 0)
3197                 // quick test on length == 1 but not on length > 12 while most
3198                 // identifier
3199                 // have a length which is <= 12...but there are lots of identifier with
3200                 // only one char....
3201                 // {
3202                 if ((length = currentPosition - startPosition) == 1)
3203                         return TokenNameIdentifier;
3204                 // data = source;
3205                 data = new char[length];
3206                 index = startPosition;
3207                 for (int i = 0; i < length; i++) {
3208                         data[i] = Character.toLowerCase(source[index + i]);
3209                 }
3210                 index = 0;
3211                 // } else {
3212                 // if ((length = withoutUnicodePtr) == 1)
3213                 // return TokenNameIdentifier;
3214                 // // data = withoutUnicodeBuffer;
3215                 // data = new char[withoutUnicodeBuffer.length];
3216                 // for (int i = 0; i < withoutUnicodeBuffer.length; i++) {
3217                 // data[i] = Character.toLowerCase(withoutUnicodeBuffer[i]);
3218                 // }
3219                 // index = 1;
3220                 // }
3221                 firstLetter = data[index];
3222                 switch (firstLetter) {
3223                 case '_':
3224                         switch (length) {
3225                         case 8:
3226                                 // __FILE__
3227                                 if ((data[++index] == '_') && (data[++index] == 'f')
3228                                                 && (data[++index] == 'i') && (data[++index] == 'l')
3229                                                 && (data[++index] == 'e') && (data[++index] == '_')
3230                                                 && (data[++index] == '_'))
3231                                         return TokenNameFILE;
3232                                 index = 0; // __LINE__
3233                                 if ((data[++index] == '_') && (data[++index] == 'l')
3234                                                 && (data[++index] == 'i') && (data[++index] == 'n')
3235                                                 && (data[++index] == 'e') && (data[++index] == '_')
3236                                                 && (data[++index] == '_'))
3237                                         return TokenNameLINE;
3238                                 break;
3239                         case 9:
3240                                 // __CLASS__
3241                                 if ((data[++index] == '_') && (data[++index] == 'c')
3242                                                 && (data[++index] == 'l') && (data[++index] == 'a')
3243                                                 && (data[++index] == 's') && (data[++index] == 's')
3244                                                 && (data[++index] == '_') && (data[++index] == '_'))
3245                                         return TokenNameCLASS_C;
3246                                 break;
3247                         case 11:
3248                                 // __METHOD__
3249                                 if ((data[++index] == '_') && (data[++index] == 'm')
3250                                                 && (data[++index] == 'e') && (data[++index] == 't')
3251                                                 && (data[++index] == 'h') && (data[++index] == 'o')
3252                                                 && (data[++index] == 'd') && (data[++index] == '_')
3253                                                 && (data[++index] == '_'))
3254                                         return TokenNameMETHOD_C;
3255                                 break;
3256                         case 12:
3257                                 // __FUNCTION__
3258                                 if ((data[++index] == '_') && (data[++index] == 'f')
3259                                                 && (data[++index] == 'u') && (data[++index] == 'n')
3260                                                 && (data[++index] == 'c') && (data[++index] == 't')
3261                                                 && (data[++index] == 'i') && (data[++index] == 'o')
3262                                                 && (data[++index] == 'n') && (data[++index] == '_')
3263                                                 && (data[++index] == '_'))
3264                                         return TokenNameFUNC_C;
3265                                 break;
3266                         }
3267                         return TokenNameIdentifier;
3268                 case 'a':
3269                         // as and array abstract
3270                         switch (length) {
3271                         case 2:
3272                                 // as
3273                                 if ((data[++index] == 's')) {
3274                                         return TokenNameas;
3275                                 }
3276                                 return TokenNameIdentifier;
3277                         case 3:
3278                                 // and
3279                                 if ((data[++index] == 'n') && (data[++index] == 'd')) {
3280                                         return TokenNameand;
3281                                 }
3282                                 return TokenNameIdentifier;
3283                         case 5:
3284                                 // array
3285                                 if ((data[++index] == 'r') && (data[++index] == 'r')
3286                                                 && (data[++index] == 'a') && (data[++index] == 'y'))
3287                                         return TokenNamearray;
3288                                 return TokenNameIdentifier;
3289                         case 8:
3290                                 if ((data[++index] == 'b') && (data[++index] == 's')
3291                                                 && (data[++index] == 't') && (data[++index] == 'r')
3292                                                 && (data[++index] == 'a') && (data[++index] == 'c')
3293                                                 && (data[++index] == 't'))
3294                                         return TokenNameabstract;
3295                                 return TokenNameIdentifier;
3296                         }
3297                         return TokenNameIdentifier;
3298                 case 'b':
3299                         // break
3300                         switch (length) {
3301                         case 5:
3302                                 if ((data[++index] == 'r') && (data[++index] == 'e')
3303                                                 && (data[++index] == 'a') && (data[++index] == 'k'))
3304                                         return TokenNamebreak;
3305                                 return TokenNameIdentifier;
3306                         }
3307                         return TokenNameIdentifier;
3308                 case 'c':
3309                         // case catch class clone const continue
3310                         switch (length) {
3311                         case 4:
3312                                 if ((data[++index] == 'a') && (data[++index] == 's')
3313                                                 && (data[++index] == 'e'))
3314                                         return TokenNamecase;
3315                                 return TokenNameIdentifier;
3316                         case 5:
3317                                 if ((data[++index] == 'a') && (data[++index] == 't')
3318                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3319                                         return TokenNamecatch;
3320                                 index = 0;
3321                                 if ((data[++index] == 'l') && (data[++index] == 'a')
3322                                                 && (data[++index] == 's') && (data[++index] == 's'))
3323                                         return TokenNameclass;
3324                                 index = 0;
3325                                 if ((data[++index] == 'l') && (data[++index] == 'o')
3326                                                 && (data[++index] == 'n') && (data[++index] == 'e'))
3327                                         return TokenNameclone;
3328                                 index = 0;
3329                                 if ((data[++index] == 'o') && (data[++index] == 'n')
3330                                                 && (data[++index] == 's') && (data[++index] == 't'))
3331                                         return TokenNameconst;
3332                                 return TokenNameIdentifier;
3333                         case 8:
3334                                 if ((data[++index] == 'o') && (data[++index] == 'n')
3335                                                 && (data[++index] == 't') && (data[++index] == 'i')
3336                                                 && (data[++index] == 'n') && (data[++index] == 'u')
3337                                                 && (data[++index] == 'e'))
3338                                         return TokenNamecontinue;
3339                                 return TokenNameIdentifier;
3340                         }
3341                         return TokenNameIdentifier;
3342                 case 'd':
3343                         // declare default do die
3344                         // TODO delete define ==> no keyword !
3345                         switch (length) {
3346                         case 2:
3347                                 if ((data[++index] == 'o'))
3348                                         return TokenNamedo;
3349                                 return TokenNameIdentifier;
3350                                 // case 6 :
3351                                 // if ((data[++index] == 'e')
3352                                 // && (data[++index] == 'f')
3353                                 // && (data[++index] == 'i')
3354                                 // && (data[++index] == 'n')
3355                                 // && (data[++index] == 'e'))
3356                                 // return TokenNamedefine;
3357                                 // else
3358                                 // return TokenNameIdentifier;
3359                         case 7:
3360                                 if ((data[++index] == 'e') && (data[++index] == 'c')
3361                                                 && (data[++index] == 'l') && (data[++index] == 'a')
3362                                                 && (data[++index] == 'r') && (data[++index] == 'e'))
3363                                         return TokenNamedeclare;
3364                                 index = 0;
3365                                 if ((data[++index] == 'e') && (data[++index] == 'f')
3366                                                 && (data[++index] == 'a') && (data[++index] == 'u')
3367                                                 && (data[++index] == 'l') && (data[++index] == 't'))
3368                                         return TokenNamedefault;
3369                                 return TokenNameIdentifier;
3370                         }
3371                         return TokenNameIdentifier;
3372                 case 'e':
3373                         // echo else exit elseif extends eval
3374                         switch (length) {
3375                         case 4:
3376                                 if ((data[++index] == 'c') && (data[++index] == 'h')
3377                                                 && (data[++index] == 'o'))
3378                                         return TokenNameecho;
3379                                 else if ((data[index] == 'l') && (data[++index] == 's')
3380                                                 && (data[++index] == 'e'))
3381                                         return TokenNameelse;
3382                                 else if ((data[index] == 'x') && (data[++index] == 'i')
3383                                                 && (data[++index] == 't'))
3384                                         return TokenNameexit;
3385                                 else if ((data[index] == 'v') && (data[++index] == 'a')
3386                                                 && (data[++index] == 'l'))
3387                                         return TokenNameeval;
3388                                 return TokenNameIdentifier;
3389                         case 5:
3390                                 // endif empty
3391                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3392                                                 && (data[++index] == 'i') && (data[++index] == 'f'))
3393                                         return TokenNameendif;
3394                                 if ((data[index] == 'm') && (data[++index] == 'p')
3395                                                 && (data[++index] == 't') && (data[++index] == 'y'))
3396                                         return TokenNameempty;
3397                                 return TokenNameIdentifier;
3398                         case 6:
3399                                 // endfor
3400                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3401                                                 && (data[++index] == 'f') && (data[++index] == 'o')
3402                                                 && (data[++index] == 'r'))
3403                                         return TokenNameendfor;
3404                                 else if ((data[index] == 'l') && (data[++index] == 's')
3405                                                 && (data[++index] == 'e') && (data[++index] == 'i')
3406                                                 && (data[++index] == 'f'))
3407                                         return TokenNameelseif;
3408                                 return TokenNameIdentifier;
3409                         case 7:
3410                                 if ((data[++index] == 'x') && (data[++index] == 't')
3411                                                 && (data[++index] == 'e') && (data[++index] == 'n')
3412                                                 && (data[++index] == 'd') && (data[++index] == 's'))
3413                                         return TokenNameextends;
3414                                 return TokenNameIdentifier;
3415                         case 8:
3416                                 // endwhile
3417                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3418                                                 && (data[++index] == 'w') && (data[++index] == 'h')
3419                                                 && (data[++index] == 'i') && (data[++index] == 'l')
3420                                                 && (data[++index] == 'e'))
3421                                         return TokenNameendwhile;
3422                                 return TokenNameIdentifier;
3423                         case 9:
3424                                 // endswitch
3425                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3426                                                 && (data[++index] == 's') && (data[++index] == 'w')
3427                                                 && (data[++index] == 'i') && (data[++index] == 't')
3428                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3429                                         return TokenNameendswitch;
3430                                 return TokenNameIdentifier;
3431                         case 10:
3432                                 // enddeclare
3433                                 if ((data[++index] == 'n') && (data[++index] == 'd')
3434                                                 && (data[++index] == 'd') && (data[++index] == 'e')
3435                                                 && (data[++index] == 'c') && (data[++index] == 'l')
3436                                                 && (data[++index] == 'a') && (data[++index] == 'r')
3437                                                 && (data[++index] == 'e'))
3438                                         return TokenNameenddeclare;
3439                                 index = 0;
3440                                 if ((data[++index] == 'n') // endforeach
3441                                                 && (data[++index] == 'd')
3442                                                 && (data[++index] == 'f')
3443                                                 && (data[++index] == 'o')
3444                                                 && (data[++index] == 'r')
3445                                                 && (data[++index] == 'e')
3446                                                 && (data[++index] == 'a')
3447                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3448                                         return TokenNameendforeach;
3449                                 return TokenNameIdentifier;
3450                         }
3451                         return TokenNameIdentifier;
3452                 case 'f':
3453                         // for false final function
3454                         switch (length) {
3455                         case 3:
3456                                 if ((data[++index] == 'o') && (data[++index] == 'r'))
3457                                         return TokenNamefor;
3458                                 return TokenNameIdentifier;
3459                         case 5:
3460                                 // if ((data[++index] == 'a') && (data[++index] == 'l')
3461                                 // && (data[++index] == 's') && (data[++index] == 'e'))
3462                                 // return TokenNamefalse;
3463                                 if ((data[++index] == 'i') && (data[++index] == 'n')
3464                                                 && (data[++index] == 'a') && (data[++index] == 'l'))
3465                                         return TokenNamefinal;
3466                                 return TokenNameIdentifier;
3467                         case 7:
3468                                 // foreach
3469                                 if ((data[++index] == 'o') && (data[++index] == 'r')
3470                                                 && (data[++index] == 'e') && (data[++index] == 'a')
3471                                                 && (data[++index] == 'c') && (data[++index] == 'h'))
3472                                         return TokenNameforeach;
3473                                 return TokenNameIdentifier;
3474                         case 8:
3475                                 // function
3476                                 if ((data[++index] == 'u') && (data[++index] == 'n')
3477                                                 && (data[++index] == 'c') && (data[++index] == 't')
3478                                                 && (data[++index] == 'i') && (data[++index] == 'o')
3479                                                 && (data[++index] == 'n'))
3480                                         return TokenNamefunction;
3481                                 return TokenNameIdentifier;
3482                         }
3483                         return TokenNameIdentifier;
3484                 case 'g':
3485                         // global
3486                         if (length == 6) {
3487                                 if ((data[++index] == 'l') && (data[++index] == 'o')
3488                                                 && (data[++index] == 'b') && (data[++index] == 'a')
3489                                                 && (data[++index] == 'l')) {
3490                                         return TokenNameglobal;
3491                                 }
3492                         }
3493                         return TokenNameIdentifier;
3494                 case 'i':
3495                         // if int isset include include_once instanceof interface implements
3496                         switch (length) {
3497                         case 2:
3498                                 if (data[++index] == 'f')
3499                                         return TokenNameif;
3500                                 return TokenNameIdentifier;
3501                                 // case 3 :
3502                                 // if ((data[++index] == 'n') && (data[++index] == 't'))
3503                                 // return TokenNameint;
3504                                 // else
3505                                 // return TokenNameIdentifier;
3506                         case 5:
3507                                 if ((data[++index] == 's') && (data[++index] == 's')
3508                                                 && (data[++index] == 'e') && (data[++index] == 't'))
3509                                         return TokenNameisset;
3510                                 return TokenNameIdentifier;
3511                         case 7:
3512                                 if ((data[++index] == 'n') && (data[++index] == 'c')
3513                                                 && (data[++index] == 'l') && (data[++index] == 'u')
3514                                                 && (data[++index] == 'd') && (data[++index] == 'e'))
3515                                         return TokenNameinclude;
3516                                 return TokenNameIdentifier;
3517                         case 9:
3518                                 // interface
3519                                 if ((data[++index] == 'n') && (data[++index] == 't')
3520                                                 && (data[++index] == 'e') && (data[++index] == 'r')
3521                                                 && (data[++index] == 'f') && (data[++index] == 'a')
3522                                                 && (data[++index] == 'c') && (data[++index] == 'e'))
3523                                         return TokenNameinterface;
3524                                 return TokenNameIdentifier;
3525                         case 10:
3526                                 // instanceof
3527                                 if ((data[++index] == 'n') && (data[++index] == 's')
3528                                                 && (data[++index] == 't') && (data[++index] == 'a')
3529                                                 && (data[++index] == 'n') && (data[++index] == 'c')
3530                                                 && (data[++index] == 'e') && (data[++index] == 'o')
3531                                                 && (data[++index] == 'f'))
3532                                         return TokenNameinstanceof;
3533                                 if ((data[index] == 'm') && (data[++index] == 'p')
3534                                                 && (data[++index] == 'l') && (data[++index] == 'e')
3535                                                 && (data[++index] == 'm') && (data[++index] == 'e')
3536                                                 && (data[++index] == 'n') && (data[++index] == 't')
3537                                                 && (data[++index] == 's'))
3538                                         return TokenNameimplements;
3539                                 return TokenNameIdentifier;
3540                         case 12:
3541                                 if ((data[++index] == 'n') && (data[++index] == 'c')
3542                                                 && (data[++index] == 'l') && (data[++index] == 'u')
3543                                                 && (data[++index] == 'd') && (data[++index] == 'e')
3544                                                 && (data[++index] == '_') && (data[++index] == 'o')
3545                                                 && (data[++index] == 'n') && (data[++index] == 'c')
3546                                                 && (data[++index] == 'e'))
3547                                         return TokenNameinclude_once;
3548                                 return TokenNameIdentifier;
3549                         }
3550                         return TokenNameIdentifier;
3551                 case 'l':
3552                         // list
3553                         if (length == 4) {
3554                                 if ((data[++index] == 'i') && (data[++index] == 's')
3555                                                 && (data[++index] == 't')) {
3556                                         return TokenNamelist;
3557                                 }
3558                         }
3559                         return TokenNameIdentifier;
3560                 case 'n':
3561                         // new null
3562                         switch (length) {
3563                         case 3:
3564                                 if ((data[++index] == 'e') && (data[++index] == 'w'))
3565                                         return TokenNamenew;
3566                                 return TokenNameIdentifier;
3567                         case 9:
3568                                 //namespace
3569                                 if ((data[++index] == 'a') && (data[++index] == 'm')
3570                                                 && (data[++index] == 'e') && (data[++index] == 's')
3571                                                 && (data[++index] == 'p') && (data[++index] == 'a')
3572                                                 && (data[++index] == 'c') && (data[++index] == 'e'))
3573                                         return TokenNameNamespace;
3574                                 return TokenNameIdentifier;
3575                                 // case 4 :
3576                                 // if ((data[++index] == 'u') && (data[++index] == 'l')
3577                                 // && (data[++index] == 'l'))
3578                                 // return TokenNamenull;
3579                                 // else
3580                                 // return TokenNameIdentifier;
3581                         }
3582                         return TokenNameIdentifier;
3583                 case 'o':
3584                         // or old_function
3585                         if (length == 2) {
3586                                 if (data[++index] == 'r') {
3587                                         return TokenNameor;
3588                                 }
3589                         }
3590                         // if (length == 12) {
3591                         // if ((data[++index] == 'l')
3592                         // && (data[++index] == 'd')
3593                         // && (data[++index] == '_')
3594                         // && (data[++index] == 'f')
3595                         // && (data[++index] == 'u')
3596                         // && (data[++index] == 'n')
3597                         // && (data[++index] == 'c')
3598                         // && (data[++index] == 't')
3599                         // && (data[++index] == 'i')
3600                         // && (data[++index] == 'o')
3601                         // && (data[++index] == 'n')) {
3602                         // return TokenNameold_function;
3603                         // }
3604                         // }
3605                         return TokenNameIdentifier;
3606                 case 'p':
3607                         // print public private protected
3608                         switch (length) {
3609                         case 5:
3610                                 if ((data[++index] == 'r') && (data[++index] == 'i')
3611                                                 && (data[++index] == 'n') && (data[++index] == 't')) {
3612                                         return TokenNameprint;
3613                                 }
3614                                 return TokenNameIdentifier;
3615                         case 6:
3616                                 if ((data[++index] == 'u') && (data[++index] == 'b')
3617                                                 && (data[++index] == 'l') && (data[++index] == 'i')
3618                                                 && (data[++index] == 'c')) {
3619                                         return TokenNamepublic;
3620                                 }
3621                                 return TokenNameIdentifier;
3622                         case 7:
3623                                 if ((data[++index] == 'r') && (data[++index] == 'i')
3624                                                 && (data[++index] == 'v') && (data[++index] == 'a')
3625                                                 && (data[++index] == 't') && (data[++index] == 'e')) {
3626                                         return TokenNameprivate;
3627                                 }
3628                                 return TokenNameIdentifier;
3629                         case 9:
3630                                 if ((data[++index] == 'r') && (data[++index] == 'o')
3631                                                 && (data[++index] == 't') && (data[++index] == 'e')
3632                                                 && (data[++index] == 'c') && (data[++index] == 't')
3633                                                 && (data[++index] == 'e') && (data[++index] == 'd')) {
3634                                         return TokenNameprotected;
3635                                 }
3636                                 return TokenNameIdentifier;
3637                         }
3638                         return TokenNameIdentifier;
3639                 case 'r':
3640                         // return require require_once
3641                         if (length == 6) {
3642                                 if ((data[++index] == 'e') && (data[++index] == 't')
3643                                                 && (data[++index] == 'u') && (data[++index] == 'r')
3644                                                 && (data[++index] == 'n')) {
3645                                         return TokenNamereturn;
3646                                 }
3647                         } else if (length == 7) {
3648                                 if ((data[++index] == 'e') && (data[++index] == 'q')
3649                                                 && (data[++index] == 'u') && (data[++index] == 'i')
3650                                                 && (data[++index] == 'r') && (data[++index] == 'e')) {
3651                                         return TokenNamerequire;
3652                                 }
3653                         } else if (length == 12) {
3654                                 if ((data[++index] == 'e') && (data[++index] == 'q')
3655                                                 && (data[++index] == 'u') && (data[++index] == 'i')
3656                                                 && (data[++index] == 'r') && (data[++index] == 'e')
3657                                                 && (data[++index] == '_') && (data[++index] == 'o')
3658                                                 && (data[++index] == 'n') && (data[++index] == 'c')
3659                                                 && (data[++index] == 'e')) {
3660                                         return TokenNamerequire_once;
3661                                 }
3662                         }
3663                         return TokenNameIdentifier;
3664                 case 's':
3665                         // self static switch
3666                         switch (length) {
3667                         // case 4:
3668                         // if ((data[++index] == 'e') && (data[++index] == 'l') &&
3669                         // (data[++index]
3670                         // == 'f')) {
3671                         // return TokenNameself;
3672                         // }
3673                         // return TokenNameIdentifier;
3674                         case 6:
3675                                 if (data[++index] == 't')
3676                                         if ((data[++index] == 'a') && (data[++index] == 't')
3677                                                         && (data[++index] == 'i') && (data[++index] == 'c')) {
3678                                                 return TokenNamestatic;
3679                                         } else
3680                                                 return TokenNameIdentifier;
3681                                 else if ((data[index] == 'w') && (data[++index] == 'i')
3682                                                 && (data[++index] == 't') && (data[++index] == 'c')
3683                                                 && (data[++index] == 'h'))
3684                                         return TokenNameswitch;
3685                         }
3686                         return TokenNameIdentifier;
3687                 case 't':
3688                         // try true throw
3689                         switch (length) {
3690                         case 3:
3691                                 if ((data[++index] == 'r') && (data[++index] == 'y'))
3692                                         return TokenNametry;
3693                                 // case 4 :
3694                                 // if ((data[++index] == 'r') && (data[++index] == 'u')
3695                                 // && (data[++index] == 'e'))
3696                                 // return TokenNametrue;
3697                                  else
3698                                          return TokenNameIdentifier;
3699                         case 5:
3700                                 if ((data[++index] == 'h') && (data[++index] == 'r')
3701                                                 && (data[++index] == 'o') && (data[++index] == 'w'))
3702                                         return TokenNamethrow;
3703                         }
3704                         return TokenNameIdentifier;
3705                 case 'u':
3706                         // use unset
3707                         switch (length) {
3708                         case 3:
3709                                 if ((data[++index] == 's') && (data[++index] == 'e'))
3710                                         return TokenNameuse;
3711                                 else
3712                                         return TokenNameIdentifier;
3713                         case 5:
3714                                 if ((data[++index] == 'n') && (data[++index] == 's')
3715                                                 && (data[++index] == 'e') && (data[++index] == 't'))
3716                                         return TokenNameunset;
3717                         }
3718                         return TokenNameIdentifier;
3719                 case 'v':
3720                         // var
3721                         switch (length) {
3722                         case 3:
3723                                 if ((data[++index] == 'a') && (data[++index] == 'r'))
3724                                         return TokenNamevar;
3725                         }
3726                         return TokenNameIdentifier;
3727                 case 'w':
3728                         // while
3729                         switch (length) {
3730                         case 5:
3731                                 if ((data[++index] == 'h') && (data[++index] == 'i')
3732                                                 && (data[++index] == 'l') && (data[++index] == 'e'))
3733                                         return TokenNamewhile;
3734                                 // case 6:if ( (data[++index] =='i') && (data[++index]=='d') &&
3735                                 // (data[++index]=='e') && (data[++index]=='f')&&
3736                                 // (data[++index]=='p'))
3737                                 // return TokenNamewidefp ;
3738                                 // else
3739                                 // return TokenNameIdentifier;
3740                         }
3741                         return TokenNameIdentifier;
3742                 case 'x':
3743                         // xor
3744                         switch (length) {
3745                         case 3:
3746                                 if ((data[++index] == 'o') && (data[++index] == 'r'))
3747                                         return TokenNamexor;
3748                                 else
3749                                         return TokenNameIdentifier;
3750                         }
3751                         return TokenNameIdentifier;
3752                 }
3753                 return TokenNameIdentifier;
3754         }
3755
3756         public int scanNumber(boolean dotPrefix) throws InvalidInputException {
3757                 // when entering this method the currentCharacter is the firt
3758                 // digit of the number , i.e. it may be preceeded by a . when
3759                 // dotPrefix is true
3760                 boolean floating = dotPrefix;
3761                 if ((!dotPrefix) && (currentCharacter == '0')) {
3762                         if (getNextChar('x', 'X') >= 0) { // ----------hexa-----------------
3763                                 // force the first char of the hexa number do exist...
3764                                 // consume next character
3765                                 unicodeAsBackSlash = false;
3766                                 currentCharacter = source[currentPosition++];
3767                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
3768                                 // && (source[currentPosition] == 'u')) {
3769                                 // getNextUnicodeChar();
3770                                 // } else {
3771                                 // if (withoutUnicodePtr != 0) {
3772                                 // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3773                                 // }
3774                                 // }
3775                                 if (Character.digit(currentCharacter, 16) == -1)
3776                                         throw new InvalidInputException(INVALID_HEXA);
3777                                 // ---end forcing--
3778                                 while (getNextCharAsDigit(16)) {
3779                                 }
3780                                 ;
3781                                 // if (getNextChar('l', 'L') >= 0)
3782                                 // return TokenNameLongLiteral;
3783                                 // else
3784                                 return TokenNameIntegerLiteral;
3785                         }
3786                         // there is x or X in the number
3787                         // potential octal ! ... some one may write 000099.0 ! thus 00100 <
3788                         // 00078.0 is true !!!!! crazy language
3789                         if (getNextCharAsDigit()) {
3790                                 // -------------potential octal-----------------
3791                                 while (getNextCharAsDigit()) {
3792                                 }
3793                                 ;
3794                                 // if (getNextChar('l', 'L') >= 0) {
3795                                 // return TokenNameLongLiteral;
3796                                 // }
3797                                 //
3798                                 // if (getNextChar('f', 'F') >= 0) {
3799                                 // return TokenNameFloatingPointLiteral;
3800                                 // }
3801                                 if (getNextChar('d', 'D') >= 0) {
3802                                         return TokenNameDoubleLiteral;
3803                                 } else { // make the distinction between octal and float ....
3804                                         if (getNextChar('.')) { // bingo ! ....
3805                                                 while (getNextCharAsDigit()) {
3806                                                 }
3807                                                 ;
3808                                                 if (getNextChar('e', 'E') >= 0) {
3809                                                         // consume next character
3810                                                         unicodeAsBackSlash = false;
3811                                                         currentCharacter = source[currentPosition++];
3812                                                         // if (((currentCharacter =
3813                                                         // source[currentPosition++]) == '\\')
3814                                                         // && (source[currentPosition] == 'u')) {
3815                                                         // getNextUnicodeChar();
3816                                                         // } else {
3817                                                         // if (withoutUnicodePtr != 0) {
3818                                                         // withoutUnicodeBuffer[++withoutUnicodePtr] =
3819                                                         // currentCharacter;
3820                                                         // }
3821                                                         // }
3822                                                         if ((currentCharacter == '-')
3823                                                                         || (currentCharacter == '+')) {
3824                                                                 // consume next character
3825                                                                 unicodeAsBackSlash = false;
3826                                                                 currentCharacter = source[currentPosition++];
3827                                                                 // if (((currentCharacter =
3828                                                                 // source[currentPosition++]) == '\\')
3829                                                                 // && (source[currentPosition] == 'u')) {
3830                                                                 // getNextUnicodeChar();
3831                                                                 // } else {
3832                                                                 // if (withoutUnicodePtr != 0) {
3833                                                                 // withoutUnicodeBuffer[++withoutUnicodePtr] =
3834                                                                 // currentCharacter;
3835                                                                 // }
3836                                                                 // }
3837                                                         }
3838                                                         if (!Character.isDigit(currentCharacter))
3839                                                                 throw new InvalidInputException(INVALID_FLOAT);
3840                                                         while (getNextCharAsDigit()) {
3841                                                         }
3842                                                         ;
3843                                                 }
3844                                                 // if (getNextChar('f', 'F') >= 0)
3845                                                 // return TokenNameFloatingPointLiteral;
3846                                                 getNextChar('d', 'D'); // jump over potential d or D
3847                                                 return TokenNameDoubleLiteral;
3848                                         } else {
3849                                                 return TokenNameIntegerLiteral;
3850                                         }
3851                                 }
3852                         } else {
3853                                 /* carry on */
3854                         }
3855                 }
3856                 while (getNextCharAsDigit()) {
3857                 }
3858                 ;
3859                 // if ((!dotPrefix) && (getNextChar('l', 'L') >= 0))
3860                 // return TokenNameLongLiteral;
3861                 if ((!dotPrefix) && (getNextChar('.'))) { // decimal part that can be
3862                                                                                                         // empty
3863                         while (getNextCharAsDigit()) {
3864                         }
3865                         ;
3866                         floating = true;
3867                 }
3868                 // if floating is true both exponant and suffix may be optional
3869                 if (getNextChar('e', 'E') >= 0) {
3870                         floating = true;
3871                         // consume next character
3872                         unicodeAsBackSlash = false;
3873                         currentCharacter = source[currentPosition++];
3874                         // if (((currentCharacter = source[currentPosition++]) == '\\')
3875                         // && (source[currentPosition] == 'u')) {
3876                         // getNextUnicodeChar();
3877                         // } else {
3878                         // if (withoutUnicodePtr != 0) {
3879                         // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3880                         // }
3881                         // }
3882                         if ((currentCharacter == '-') || (currentCharacter == '+')) { // consume
3883                                 // next
3884                                 // character
3885                                 unicodeAsBackSlash = false;
3886                                 currentCharacter = source[currentPosition++];
3887                                 // if (((currentCharacter = source[currentPosition++]) == '\\')
3888                                 // && (source[currentPosition] == 'u')) {
3889                                 // getNextUnicodeChar();
3890                                 // } else {
3891                                 // if (withoutUnicodePtr != 0) {
3892                                 // withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
3893                                 // }
3894                                 // }
3895                         }
3896                         if (!Character.isDigit(currentCharacter))
3897                                 throw new InvalidInputException(INVALID_FLOAT);
3898                         while (getNextCharAsDigit()) {
3899                         }
3900                         ;
3901                 }
3902                 if (getNextChar('d', 'D') >= 0)
3903                         return TokenNameDoubleLiteral;
3904                 // if (getNextChar('f', 'F') >= 0)
3905                 // return TokenNameFloatingPointLiteral;
3906                 // the long flag has been tested before
3907                 return floating ? TokenNameDoubleLiteral : TokenNameIntegerLiteral;
3908         }
3909
3910         /**
3911          * Search the line number corresponding to a specific position
3912          *
3913          */
3914         public final int getLineNumber(int position) {
3915                 if (lineEnds == null)
3916                         return 1;
3917                 int length = linePtr + 1;
3918                 if (length == 0)
3919                         return 1;
3920                 int g = 0, d = length - 1;
3921                 int m = 0;
3922                 while (g <= d) {
3923                         m = (g + d) / 2;
3924                         if (position < lineEnds[m]) {
3925                                 d = m - 1;
3926                         } else if (position > lineEnds[m]) {
3927                                 g = m + 1;
3928                         } else {
3929                                 return m + 1;
3930                         }
3931                 }
3932                 if (position < lineEnds[m]) {
3933                         return m + 1;
3934                 }
3935                 return m + 2;
3936         }
3937
3938         public void setPHPMode(boolean mode) {
3939                 phpMode = mode;
3940         }
3941
3942         public final void setSource(char[] source) {
3943                 setSource(null, source);
3944         }
3945
3946         public final void setSource(ICompilationUnit compilationUnit, char[] source) {
3947                 // the source-buffer is set to sourceString
3948                 this.compilationUnit = compilationUnit;
3949                 if (source == null) {
3950                         this.source = new char[0];
3951                 } else {
3952                         this.source = source;
3953                 }
3954                 startPosition = -1;
3955                 initialPosition = currentPosition = 0;
3956                 containsAssertKeyword = false;
3957                 withoutUnicodeBuffer = new char[this.source.length];
3958                 fFillerToken = TokenNameEOF;
3959                 // encapsedStringStack = new Stack();
3960         }
3961
3962         public String toString() {
3963                 if (startPosition == source.length)
3964                         return "EOF\n\n" + new String(source); //$NON-NLS-1$
3965                 if (currentPosition > source.length)
3966                         return "behind the EOF :-( ....\n\n" + new String(source); //$NON-NLS-1$
3967                 char front[] = new char[startPosition];
3968                 System.arraycopy(source, 0, front, 0, startPosition);
3969                 int middleLength = (currentPosition - 1) - startPosition + 1;
3970                 char middle[];
3971                 if (middleLength > -1) {
3972                         middle = new char[middleLength];
3973                         System.arraycopy(source, startPosition, middle, 0, middleLength);
3974                 } else {
3975                         middle = new char[0];
3976                 }
3977                 char end[] = new char[source.length - (currentPosition - 1)];
3978                 System.arraycopy(source, (currentPosition - 1) + 1, end, 0,
3979                                 source.length - (currentPosition - 1) - 1);
3980                 return new String(front)
3981                                 + "\n===============================\nStarts here -->" //$NON-NLS-1$
3982                                 + new String(middle)
3983                                 + "<-- Ends here\n===============================\n" //$NON-NLS-1$
3984                                 + new String(end);
3985         }
3986
3987         public final String toStringAction(int act) {
3988                 switch (act) {
3989                 case TokenNameERROR:
3990                         return "ScannerError"; // + new String(getCurrentTokenSource()) +
3991                                                                         // ")";
3992                         // //$NON-NLS-1$
3993                 case TokenNameINLINE_HTML:
3994                         return "Inline-HTML(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
3995                 case TokenNameECHO_INVISIBLE:
3996                         // 0-length token
3997                         return "";
3998                 case TokenNameIdentifier:
3999                         return "Identifier(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4000                 case TokenNameVariable:
4001                         return "Variable(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4002                 case TokenNameabstract:
4003                         return "abstract"; //$NON-NLS-1$
4004                 case TokenNameand:
4005                         return "AND"; //$NON-NLS-1$
4006                 case TokenNamearray:
4007                         return "array"; //$NON-NLS-1$
4008                 case TokenNameas:
4009                         return "as"; //$NON-NLS-1$
4010                 case TokenNamebreak:
4011                         return "break"; //$NON-NLS-1$
4012                 case TokenNamecase:
4013                         return "case"; //$NON-NLS-1$
4014                 case TokenNameclass:
4015                         return "class"; //$NON-NLS-1$
4016                 case TokenNamecatch:
4017                         return "catch"; //$NON-NLS-1$
4018                 case TokenNameclone:
4019                         //$NON-NLS-1$
4020                         return "clone";
4021                 case TokenNameconst:
4022                         //$NON-NLS-1$
4023                         return "const";
4024                 case TokenNamecontinue:
4025                         return "continue"; //$NON-NLS-1$
4026                 case TokenNamedefault:
4027                         return "default"; //$NON-NLS-1$
4028                         // case TokenNamedefine :
4029                         // return "define"; //$NON-NLS-1$
4030                 case TokenNamedo:
4031                         return "do"; //$NON-NLS-1$
4032                 case TokenNameecho:
4033                         return "echo"; //$NON-NLS-1$
4034                 case TokenNameelse:
4035                         return "else"; //$NON-NLS-1$
4036                 case TokenNameelseif:
4037                         return "elseif"; //$NON-NLS-1$
4038                 case TokenNameendfor:
4039                         return "endfor"; //$NON-NLS-1$
4040                 case TokenNameendforeach:
4041                         return "endforeach"; //$NON-NLS-1$
4042                 case TokenNameendif:
4043                         return "endif"; //$NON-NLS-1$
4044                 case TokenNameendswitch:
4045                         return "endswitch"; //$NON-NLS-1$
4046                 case TokenNameendwhile:
4047                         return "endwhile"; //$NON-NLS-1$
4048                 case TokenNameexit:
4049                         return "exit";
4050                 case TokenNameextends:
4051                         return "extends"; //$NON-NLS-1$
4052                         // case TokenNamefalse :
4053                         // return "false"; //$NON-NLS-1$
4054                 case TokenNamefinal:
4055                         return "final"; //$NON-NLS-1$
4056                 case TokenNamefor:
4057                         return "for"; //$NON-NLS-1$
4058                 case TokenNameforeach:
4059                         return "foreach"; //$NON-NLS-1$
4060                 case TokenNamefunction:
4061                         return "function"; //$NON-NLS-1$
4062                 case TokenNameglobal:
4063                         return "global"; //$NON-NLS-1$
4064                 case TokenNameif:
4065                         return "if"; //$NON-NLS-1$
4066                 case TokenNameimplements:
4067                         return "implements"; //$NON-NLS-1$
4068                 case TokenNameinclude:
4069                         return "include"; //$NON-NLS-1$
4070                 case TokenNameinclude_once:
4071                         return "include_once"; //$NON-NLS-1$
4072                 case TokenNameinstanceof:
4073                         return "instanceof"; //$NON-NLS-1$
4074                 case TokenNameinterface:
4075                         return "interface"; //$NON-NLS-1$
4076                 case TokenNameisset:
4077                         return "isset"; //$NON-NLS-1$
4078                 case TokenNamelist:
4079                         return "list"; //$NON-NLS-1$
4080                 case TokenNamenew:
4081                         return "new"; //$NON-NLS-1$
4082                         // case TokenNamenull :
4083                         // return "null"; //$NON-NLS-1$
4084                 case TokenNameor:
4085                         return "OR"; //$NON-NLS-1$
4086                 case TokenNameprint:
4087                         return "print"; //$NON-NLS-1$
4088                 case TokenNameprivate:
4089                         return "private"; //$NON-NLS-1$
4090                 case TokenNameprotected:
4091                         return "protected"; //$NON-NLS-1$
4092                 case TokenNamepublic:
4093                         return "public"; //$NON-NLS-1$
4094                 case TokenNamerequire:
4095                         return "require"; //$NON-NLS-1$
4096                 case TokenNamerequire_once:
4097                         return "require_once"; //$NON-NLS-1$
4098                 case TokenNameNamespace:
4099                         return "namespace"; //$NON-NLS-1$
4100                 case TokenNamereturn:
4101                         return "return"; //$NON-NLS-1$
4102                         // case TokenNameself:
4103                         // return "self"; //$NON-NLS-1$
4104                 case TokenNamestatic:
4105                         return "static"; //$NON-NLS-1$
4106                 case TokenNameswitch:
4107                         return "switch"; //$NON-NLS-1$
4108                         // case TokenNametrue :
4109                         // return "true"; //$NON-NLS-1$
4110                 case TokenNameunset:
4111                         return "unset"; //$NON-NLS-1$
4112                 case TokenNamevar:
4113                         return "var"; //$NON-NLS-1$
4114                 case TokenNamewhile:
4115                         return "while"; //$NON-NLS-1$
4116                 case TokenNamexor:
4117                         return "XOR"; //$NON-NLS-1$
4118                         // case TokenNamethis :
4119                         // return "$this"; //$NON-NLS-1$
4120                 case TokenNameIntegerLiteral:
4121                         return "Integer(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4122                 case TokenNameDoubleLiteral:
4123                         return "Double(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4124                 case TokenNameStringDoubleQuote:
4125                         return "StringLiteral(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4126                 case TokenNameStringSingleQuote:
4127                         return "StringConstant(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4128                 case TokenNameStringInterpolated:
4129                         return "StringInterpolated(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4130                 case TokenNameEncapsedString0:
4131                         return "`"; //$NON-NLS-1$
4132                         // case TokenNameEncapsedString1:
4133                         // return "\'"; //$NON-NLS-1$
4134                         // case TokenNameEncapsedString2:
4135                         // return "\""; //$NON-NLS-1$
4136                 case TokenNameSTRING:
4137                         return "STRING_DQ(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$ //$NON-NLS-2$
4138                 case TokenNameHEREDOC:
4139                         return "HEREDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4140                 case TokenNamePLUS_PLUS:
4141                         return "++"; //$NON-NLS-1$
4142                 case TokenNameMINUS_MINUS:
4143                         return "--"; //$NON-NLS-1$
4144                 case TokenNameEQUAL_EQUAL:
4145                         return "=="; //$NON-NLS-1$
4146                 case TokenNameEQUAL_EQUAL_EQUAL:
4147                         return "==="; //$NON-NLS-1$
4148                 case TokenNameEQUAL_GREATER:
4149                         return "=>"; //$NON-NLS-1$
4150                 case TokenNameLESS_EQUAL:
4151                         return "<="; //$NON-NLS-1$
4152                 case TokenNameGREATER_EQUAL:
4153                         return ">="; //$NON-NLS-1$
4154                 case TokenNameNOT_EQUAL:
4155                         return "!="; //$NON-NLS-1$
4156                 case TokenNameNOT_EQUAL_EQUAL:
4157                         return "!=="; //$NON-NLS-1$
4158                 case TokenNameLEFT_SHIFT:
4159                         return "<<"; //$NON-NLS-1$
4160                 case TokenNameRIGHT_SHIFT:
4161                         return ">>"; //$NON-NLS-1$
4162                 case TokenNamePLUS_EQUAL:
4163                         return "+="; //$NON-NLS-1$
4164                 case TokenNameMINUS_EQUAL:
4165                         return "-="; //$NON-NLS-1$
4166                 case TokenNameMULTIPLY_EQUAL:
4167                         return "*="; //$NON-NLS-1$
4168                 case TokenNameDIVIDE_EQUAL:
4169                         return "/="; //$NON-NLS-1$
4170                 case TokenNameAND_EQUAL:
4171                         return "&="; //$NON-NLS-1$
4172                 case TokenNameOR_EQUAL:
4173                         return "|="; //$NON-NLS-1$
4174                 case TokenNameXOR_EQUAL:
4175                         return "^="; //$NON-NLS-1$
4176                 case TokenNameREMAINDER_EQUAL:
4177                         return "%="; //$NON-NLS-1$
4178                 case TokenNameDOT_EQUAL:
4179                         return ".="; //$NON-NLS-1$
4180                 case TokenNameLEFT_SHIFT_EQUAL:
4181                         return "<<="; //$NON-NLS-1$
4182                 case TokenNameRIGHT_SHIFT_EQUAL:
4183                         return ">>="; //$NON-NLS-1$
4184                 case TokenNameOR_OR:
4185                         return "||"; //$NON-NLS-1$
4186                 case TokenNameAND_AND:
4187                         return "&&"; //$NON-NLS-1$
4188                 case TokenNamePLUS:
4189                         return "+"; //$NON-NLS-1$
4190                 case TokenNameMINUS:
4191                         return "-"; //$NON-NLS-1$
4192                 case TokenNameMINUS_GREATER:
4193                         return "->";
4194                 case TokenNameNOT:
4195                         return "!"; //$NON-NLS-1$
4196                 case TokenNameREMAINDER:
4197                         return "%"; //$NON-NLS-1$
4198                 case TokenNameXOR:
4199                         return "^"; //$NON-NLS-1$
4200                 case TokenNameAND:
4201                         return "&"; //$NON-NLS-1$
4202                 case TokenNameMULTIPLY:
4203                         return "*"; //$NON-NLS-1$
4204                 case TokenNameOR:
4205                         return "|"; //$NON-NLS-1$
4206                 case TokenNameTWIDDLE:
4207                         return "~"; //$NON-NLS-1$
4208                 case TokenNameTWIDDLE_EQUAL:
4209                         return "~="; //$NON-NLS-1$
4210                 case TokenNameDIVIDE:
4211                         return "/"; //$NON-NLS-1$
4212                 case TokenNameGREATER:
4213                         return ">"; //$NON-NLS-1$
4214                 case TokenNameLESS:
4215                         return "<"; //$NON-NLS-1$
4216                 case TokenNameLPAREN:
4217                         return "("; //$NON-NLS-1$
4218                 case TokenNameRPAREN:
4219                         return ")"; //$NON-NLS-1$
4220                 case TokenNameLBRACE:
4221                         return "{"; //$NON-NLS-1$
4222                 case TokenNameRBRACE:
4223                         return "}"; //$NON-NLS-1$
4224                 case TokenNameLBRACKET:
4225                         return "["; //$NON-NLS-1$
4226                 case TokenNameRBRACKET:
4227                         return "]"; //$NON-NLS-1$
4228                 case TokenNameSEMICOLON:
4229                         return ";"; //$NON-NLS-1$
4230                 case TokenNameQUESTION:
4231                         return "?"; //$NON-NLS-1$
4232                 case TokenNameCOLON:
4233                         return ":"; //$NON-NLS-1$
4234                 case TokenNameCOMMA:
4235                         return ","; //$NON-NLS-1$
4236                 case TokenNameDOT:
4237                         return "."; //$NON-NLS-1$
4238                 case TokenNameEQUAL:
4239                         return "="; //$NON-NLS-1$
4240                 case TokenNameAT:
4241                         return "@";
4242                 case TokenNameDOLLAR:
4243                         return "$";
4244                 case TokenNameDOLLAR_LBRACE:
4245                         return "${";
4246                 case TokenNameLBRACE_DOLLAR:
4247                         return "{$";
4248                 case TokenNameEOF:
4249                         return "EOF"; //$NON-NLS-1$
4250                 case TokenNameWHITESPACE:
4251                         return "WHITESPACE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4252                 case TokenNameCOMMENT_LINE:
4253                         return "COMMENT_LINE(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4254                 case TokenNameCOMMENT_BLOCK:
4255                         return "COMMENT_BLOCK(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4256                 case TokenNameCOMMENT_PHPDOC:
4257                         return "COMMENT_PHPDOC(" + new String(getCurrentTokenSource()) + ")"; //$NON-NLS-1$
4258                         // case TokenNameHTML :
4259                         // return "HTML(" + new String(getCurrentTokenSource()) + ")";
4260                         // //$NON-NLS-1$
4261                 case TokenNameFILE:
4262                         return "__FILE__"; //$NON-NLS-1$
4263                 case TokenNameLINE:
4264                         return "__LINE__"; //$NON-NLS-1$
4265                 case TokenNameCLASS_C:
4266                         return "__CLASS__"; //$NON-NLS-1$
4267                 case TokenNameMETHOD_C:
4268                         return "__METHOD__"; //$NON-NLS-1$
4269                 case TokenNameFUNC_C:
4270                         return "__FUNCTION__"; //$NON-NLS-1
4271                 case TokenNameboolCAST:
4272                         return "( bool )"; //$NON-NLS-1$
4273                 case TokenNameintCAST:
4274                         return "( int )"; //$NON-NLS-1$
4275                 case TokenNamedoubleCAST:
4276                         return "( double )"; //$NON-NLS-1$
4277                 case TokenNameobjectCAST:
4278                         return "( object )"; //$NON-NLS-1$
4279                 case TokenNamestringCAST:
4280                         return "( string )"; //$NON-NLS-1$
4281                 default:
4282                         return "not-a-token(" + (new Integer(act)) + ") " + new String(getCurrentTokenSource()); //$NON-NLS-1$
4283                 }
4284         }
4285
4286         public Scanner() {
4287                 this(false, false);
4288         }
4289
4290         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
4291                 this(tokenizeComments, tokenizeWhiteSpace, false);
4292         }
4293
4294         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
4295                         boolean checkNonExternalizedStringLiterals) {
4296                 this(tokenizeComments, tokenizeWhiteSpace,
4297                                 checkNonExternalizedStringLiterals, false);
4298         }
4299
4300         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
4301                         boolean checkNonExternalizedStringLiterals, boolean assertMode) {
4302                 this(tokenizeComments, tokenizeWhiteSpace,
4303                                 checkNonExternalizedStringLiterals, assertMode, false, null,
4304                                 null, true);
4305         }
4306
4307         public Scanner(boolean tokenizeComments, boolean tokenizeWhiteSpace,
4308                         boolean checkNonExternalizedStringLiterals, boolean assertMode,
4309                         boolean tokenizeStrings, char[][] taskTags,
4310                         char[][] taskPriorities, boolean isTaskCaseSensitive) {
4311                 this.eofPosition = Integer.MAX_VALUE;
4312                 this.tokenizeComments = tokenizeComments;
4313                 this.tokenizeWhiteSpace = tokenizeWhiteSpace;
4314                 this.tokenizeStrings = tokenizeStrings;
4315                 this.checkNonExternalizedStringLiterals = checkNonExternalizedStringLiterals;
4316                 // this.assertMode = assertMode;
4317                 // this.encapsedStringStack = null;
4318                 this.taskTags = taskTags;
4319                 this.taskPriorities = taskPriorities;
4320         }
4321
4322         private void checkNonExternalizeString() throws InvalidInputException {
4323                 if (currentLine == null)
4324                         return;
4325                 parseTags(currentLine);
4326         }
4327
4328         private void parseTags(NLSLine line) throws InvalidInputException {
4329                 String s = new String(getCurrentTokenSource());
4330                 int pos = s.indexOf(TAG_PREFIX);
4331                 int lineLength = line.size();
4332                 while (pos != -1) {
4333                         int start = pos + TAG_PREFIX_LENGTH;
4334                         int end = s.indexOf(TAG_POSTFIX, start);
4335                         String index = s.substring(start, end);
4336                         int i = 0;
4337                         try {
4338                                 i = Integer.parseInt(index) - 1;
4339                                 // Tags are one based not zero based.
4340                         } catch (NumberFormatException e) {
4341                                 i = -1; // we don't want to consider this as a valid NLS tag
4342                         }
4343                         if (line.exists(i)) {
4344                                 line.set(i, null);
4345                         }
4346                         pos = s.indexOf(TAG_PREFIX, start);
4347                 }
4348                 this.nonNLSStrings = new StringLiteral[lineLength];
4349                 int nonNLSCounter = 0;
4350                 for (Iterator iterator = line.iterator(); iterator.hasNext();) {
4351                         StringLiteral literal = (StringLiteral) iterator.next();
4352                         if (literal != null) {
4353                                 this.nonNLSStrings[nonNLSCounter++] = literal;
4354                         }
4355                 }
4356                 if (nonNLSCounter == 0) {
4357                         this.nonNLSStrings = null;
4358                         currentLine = null;
4359                         return;
4360                 }
4361                 this.wasNonExternalizedStringLiteral = true;
4362                 if (nonNLSCounter != lineLength) {
4363                         System.arraycopy(this.nonNLSStrings, 0,
4364                                         (this.nonNLSStrings = new StringLiteral[nonNLSCounter]), 0,
4365                                         nonNLSCounter);
4366                 }
4367                 currentLine = null;
4368         }
4369
4370         public final void scanEscapeCharacter() throws InvalidInputException {
4371                 // the string with "\\u" is a legal string of two chars \ and u
4372                 // thus we use a direct access to the source (for regular cases).
4373                 if (unicodeAsBackSlash) {
4374                         // consume next character
4375                         unicodeAsBackSlash = false;
4376                         // if (((currentCharacter = source[currentPosition++]) == '\\') &&
4377                         // (source[currentPosition] == 'u')) {
4378                         // getNextUnicodeChar();
4379                         // } else {
4380                         if (withoutUnicodePtr != 0) {
4381                                 withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
4382                                 // }
4383                         }
4384                 } else
4385                         currentCharacter = source[currentPosition++];
4386                 switch (currentCharacter) {
4387                 case 'b':
4388                         currentCharacter = '\b';
4389                         break;
4390                 case 't':
4391                         currentCharacter = '\t';
4392                         break;
4393                 case 'n':
4394                         currentCharacter = '\n';
4395                         break;
4396                 case 'f':
4397                         currentCharacter = '\f';
4398                         break;
4399                 case 'r':
4400                         currentCharacter = '\r';
4401                         break;
4402                 case '\"':
4403                         currentCharacter = '\"';
4404                         break;
4405                 case '\'':
4406                         currentCharacter = '\'';
4407                         break;
4408                 case '\\':
4409                         currentCharacter = '\\';
4410                         break;
4411                 default:
4412                         // -----------octal escape--------------
4413                         // OctalDigit
4414                         // OctalDigit OctalDigit
4415                         // ZeroToThree OctalDigit OctalDigit
4416                         int number = Character.getNumericValue(currentCharacter);
4417                         if (number >= 0 && number <= 7) {
4418                                 boolean zeroToThreeNot = number > 3;
4419                                 if (Character
4420                                                 .isDigit(currentCharacter = source[currentPosition++])) {
4421                                         int digit = Character.getNumericValue(currentCharacter);
4422                                         if (digit >= 0 && digit <= 7) {
4423                                                 number = (number * 8) + digit;
4424                                                 if (Character
4425                                                                 .isDigit(currentCharacter = source[currentPosition++])) {
4426                                                         if (zeroToThreeNot) { // has read \NotZeroToThree
4427                                                                                                         // OctalDigit
4428                                                                 // Digit --> ignore last character
4429                                                                 currentPosition--;
4430                                                         } else {
4431                                                                 digit = Character
4432                                                                                 .getNumericValue(currentCharacter);
4433                                                                 if (digit >= 0 && digit <= 7) { // has read
4434                                                                                                                                 // \ZeroToThree
4435                                                                         // OctalDigit OctalDigit
4436                                                                         number = (number * 8) + digit;
4437                                                                 } else { // has read \ZeroToThree OctalDigit
4438                                                                                         // NonOctalDigit
4439                                                                         // --> ignore last character
4440                                                                         currentPosition--;
4441                                                                 }
4442                                                         }
4443                                                 } else { // has read \OctalDigit NonDigit--> ignore
4444                                                                         // last
4445                                                         // character
4446                                                         currentPosition--;
4447                                                 }
4448                                         } else { // has read \OctalDigit NonOctalDigit--> ignore
4449                                                                 // last
4450                                                 // character
4451                                                 currentPosition--;
4452                                         }
4453                                 } else { // has read \OctalDigit --> ignore last character
4454                                         currentPosition--;
4455                                 }
4456                                 if (number > 255)
4457                                         throw new InvalidInputException(INVALID_ESCAPE);
4458                                 currentCharacter = (char) number;
4459                         } else
4460                                 throw new InvalidInputException(INVALID_ESCAPE);
4461                 }
4462         }
4463
4464         // chech presence of task: tags
4465         // TODO (frederic) see if we need to take unicode characters into account...
4466         public void checkTaskTag(int commentStart, int commentEnd) {
4467                 char[] src = this.source;
4468
4469                 // only look for newer task: tags
4470                 if (this.foundTaskCount > 0
4471                                 && this.foundTaskPositions[this.foundTaskCount - 1][0] >= commentStart) {
4472                         return;
4473                 }
4474                 int foundTaskIndex = this.foundTaskCount;
4475                 char previous = src[commentStart + 1]; // should be '*' or '/'
4476                 nextChar: for (int i = commentStart + 2; i < commentEnd
4477                                 && i < this.eofPosition; i++) {
4478                         char[] tag = null;
4479                         char[] priority = null;
4480                         // check for tag occurrence only if not ambiguous with javadoc tag
4481                         if (previous != '@') {
4482                                 nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
4483                                         tag = this.taskTags[itag];
4484                                         int tagLength = tag.length;
4485                                         if (tagLength == 0)
4486                                                 continue nextTag;
4487
4488                                         // ensure tag is not leaded with letter if tag starts with a
4489                                         // letter
4490                                         if (Scanner.isPHPIdentifierStart(tag[0])) {
4491                                                 if (Scanner.isPHPIdentifierPart(previous)) {
4492                                                         continue nextTag;
4493                                                 }
4494                                         }
4495
4496                                         for (int t = 0; t < tagLength; t++) {
4497                                                 char sc, tc;
4498                                                 int x = i + t;
4499                                                 if (x >= this.eofPosition || x >= commentEnd)
4500                                                         continue nextTag;
4501                                                 if ((sc = src[i + t]) != (tc = tag[t])) { // case
4502                                                                                                                                         // sensitive
4503                                                                                                                                         // check
4504                                                         if (this.isTaskCaseSensitive
4505                                                                         || (Character.toLowerCase(sc) != Character
4506                                                                                         .toLowerCase(tc))) { // case
4507                                                                 // insensitive
4508                                                                 // check
4509                                                                 continue nextTag;
4510                                                         }
4511                                                 }
4512                                         }
4513                                         // ensure tag is not followed with letter if tag finishes
4514                                         // with a
4515                                         // letter
4516                                         if (i + tagLength < commentEnd
4517                                                         && Scanner.isPHPIdentifierPart(src[i + tagLength
4518                                                                         - 1])) {
4519                                                 if (Scanner.isPHPIdentifierPart(src[i + tagLength]))
4520                                                         continue nextTag;
4521                                         }
4522                                         if (this.foundTaskTags == null) {
4523                                                 this.foundTaskTags = new char[5][];
4524                                                 this.foundTaskMessages = new char[5][];
4525                                                 this.foundTaskPriorities = new char[5][];
4526                                                 this.foundTaskPositions = new int[5][];
4527                                         } else if (this.foundTaskCount == this.foundTaskTags.length) {
4528                                                 System
4529                                                                 .arraycopy(
4530                                                                                 this.foundTaskTags,
4531                                                                                 0,
4532                                                                                 this.foundTaskTags = new char[this.foundTaskCount * 2][],
4533                                                                                 0, this.foundTaskCount);
4534                                                 System
4535                                                                 .arraycopy(
4536                                                                                 this.foundTaskMessages,
4537                                                                                 0,
4538                                                                                 this.foundTaskMessages = new char[this.foundTaskCount * 2][],
4539                                                                                 0, this.foundTaskCount);
4540                                                 System
4541                                                                 .arraycopy(
4542                                                                                 this.foundTaskPriorities,
4543                                                                                 0,
4544                                                                                 this.foundTaskPriorities = new char[this.foundTaskCount * 2][],
4545                                                                                 0, this.foundTaskCount);
4546                                                 System
4547                                                                 .arraycopy(
4548                                                                                 this.foundTaskPositions,
4549                                                                                 0,
4550                                                                                 this.foundTaskPositions = new int[this.foundTaskCount * 2][],
4551                                                                                 0, this.foundTaskCount);
4552                                         }
4553
4554                                         priority = this.taskPriorities != null
4555                                                         && itag < this.taskPriorities.length ? this.taskPriorities[itag]
4556                                                         : null;
4557
4558                                         this.foundTaskTags[this.foundTaskCount] = tag;
4559                                         this.foundTaskPriorities[this.foundTaskCount] = priority;
4560                                         this.foundTaskPositions[this.foundTaskCount] = new int[] {
4561                                                         i, i + tagLength - 1 };
4562                                         this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
4563                                         this.foundTaskCount++;
4564                                         i += tagLength - 1; // will be incremented when looping
4565                                         break nextTag;
4566                                 }
4567                         }
4568                         previous = src[i];
4569                 }
4570                 for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
4571                         // retrieve message start and end positions
4572                         int msgStart = this.foundTaskPositions[i][0]
4573                                         + this.foundTaskTags[i].length;
4574                         int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i + 1][0] - 1
4575                                         : commentEnd - 1;
4576                         // at most beginning of next task
4577                         if (max_value < msgStart) {
4578                                 max_value = msgStart; // would only occur if tag is before
4579                                                                                 // EOF.
4580                         }
4581                         int end = -1;
4582                         char c;
4583                         for (int j = msgStart; j < max_value; j++) {
4584                                 if ((c = src[j]) == '\n' || c == '\r') {
4585                                         end = j - 1;
4586                                         break;
4587                                 }
4588                         }
4589                         if (end == -1) {
4590                                 for (int j = max_value; j > msgStart; j--) {
4591                                         if ((c = src[j]) == '*') {
4592                                                 end = j - 1;
4593                                                 break;
4594                                         }
4595                                 }
4596                                 if (end == -1)
4597                                         end = max_value;
4598                         }
4599                         if (msgStart == end)
4600                                 continue; // empty
4601                         // trim the message
4602                         while (CharOperation.isWhitespace(src[end]) && msgStart <= end)
4603                                 end--;
4604                         while (CharOperation.isWhitespace(src[msgStart]) && msgStart <= end)
4605                                 msgStart++;
4606                         // update the end position of the task
4607                         this.foundTaskPositions[i][1] = end;
4608                         // get the message source
4609                         final int messageLength = end - msgStart + 1;
4610                         char[] message = new char[messageLength];
4611                         System.arraycopy(src, msgStart, message, 0, messageLength);
4612                         this.foundTaskMessages[i] = message;
4613                 }
4614         }
4615
4616         // chech presence of task: tags
4617         // public void checkTaskTag(int commentStart, int commentEnd) {
4618         // // only look for newer task: tags
4619         // if (this.foundTaskCount > 0 &&
4620         // this.foundTaskPositions[this.foundTaskCount
4621         // - 1][0] >= commentStart) {
4622         // return;
4623         // }
4624         // int foundTaskIndex = this.foundTaskCount;
4625         // nextChar: for (int i = commentStart; i < commentEnd && i <
4626         // this.eofPosition; i++) {
4627         // char[] tag = null;
4628         // char[] priority = null;
4629         // // check for tag occurrence
4630         // nextTag: for (int itag = 0; itag < this.taskTags.length; itag++) {
4631         // tag = this.taskTags[itag];
4632         // priority = this.taskPriorities != null && itag <
4633         // this.taskPriorities.length
4634         // ? this.taskPriorities[itag] : null;
4635         // int tagLength = tag.length;
4636         // for (int t = 0; t < tagLength; t++) {
4637         // if (this.source[i + t] != tag[t])
4638         // continue nextTag;
4639         // }
4640         // if (this.foundTaskTags == null) {
4641         // this.foundTaskTags = new char[5][];
4642         // this.foundTaskMessages = new char[5][];
4643         // this.foundTaskPriorities = new char[5][];
4644         // this.foundTaskPositions = new int[5][];
4645         // } else if (this.foundTaskCount == this.foundTaskTags.length) {
4646         // System.arraycopy(this.foundTaskTags, 0, this.foundTaskTags = new
4647         // char[this.foundTaskCount * 2][], 0, this.foundTaskCount);
4648         // System.arraycopy(this.foundTaskMessages, 0, this.foundTaskMessages = new
4649         // char[this.foundTaskCount * 2][], 0,
4650         // this.foundTaskCount);
4651         // System.arraycopy(this.foundTaskPriorities, 0, this.foundTaskPriorities =
4652         // new char[this.foundTaskCount * 2][], 0,
4653         // this.foundTaskCount);
4654         // System.arraycopy(this.foundTaskPositions, 0, this.foundTaskPositions =
4655         // new
4656         // int[this.foundTaskCount * 2][], 0,
4657         // this.foundTaskCount);
4658         // }
4659         // this.foundTaskTags[this.foundTaskCount] = tag;
4660         // this.foundTaskPriorities[this.foundTaskCount] = priority;
4661         // this.foundTaskPositions[this.foundTaskCount] = new int[] { i, i +
4662         // tagLength
4663         // - 1 };
4664         // this.foundTaskMessages[this.foundTaskCount] = CharOperation.NO_CHAR;
4665         // this.foundTaskCount++;
4666         // i += tagLength - 1; // will be incremented when looping
4667         // }
4668         // }
4669         // for (int i = foundTaskIndex; i < this.foundTaskCount; i++) {
4670         // // retrieve message start and end positions
4671         // int msgStart = this.foundTaskPositions[i][0] +
4672         // this.foundTaskTags[i].length;
4673         // int max_value = i + 1 < this.foundTaskCount ? this.foundTaskPositions[i +
4674         // 1][0] - 1 : commentEnd - 1;
4675         // // at most beginning of next task
4676         // if (max_value < msgStart)
4677         // max_value = msgStart; // would only occur if tag is before EOF.
4678         // int end = -1;
4679         // char c;
4680         // for (int j = msgStart; j < max_value; j++) {
4681         // if ((c = this.source[j]) == '\n' || c == '\r') {
4682         // end = j - 1;
4683         // break;
4684         // }
4685         // }
4686         // if (end == -1) {
4687         // for (int j = max_value; j > msgStart; j--) {
4688         // if ((c = this.source[j]) == '*') {
4689         // end = j - 1;
4690         // break;
4691         // }
4692         // }
4693         // if (end == -1)
4694         // end = max_value;
4695         // }
4696         // if (msgStart == end)
4697         // continue; // empty
4698         // // trim the message
4699         // while (CharOperation.isWhitespace(source[end]) && msgStart <= end)
4700         // end--;
4701         // while (CharOperation.isWhitespace(source[msgStart]) && msgStart <= end)
4702         // msgStart++;
4703         // // update the end position of the task
4704         // this.foundTaskPositions[i][1] = end;
4705         // // get the message source
4706         // final int messageLength = end - msgStart + 1;
4707         // char[] message = new char[messageLength];
4708         // System.arraycopy(source, msgStart, message, 0, messageLength);
4709         // this.foundTaskMessages[i] = message;
4710         // }
4711         // }
4712 }