1) Fixed html syntax highlighting.
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / compiler / parser / AbstractCommentParser.java
1 /*******************************************************************************
2  * Copyright (c) 2000, 2004 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials 
4  * are made available under the terms of the Common Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v10.html
7  * 
8  * Contributors:
9  *     IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package net.sourceforge.phpdt.internal.compiler.parser;
12
13 import java.util.ArrayList;
14 import java.util.List;
15
16 import net.sourceforge.phpdt.core.compiler.CharOperation;
17 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols;
18 import net.sourceforge.phpdt.core.compiler.ITerminalSymbols.TokenName;
19 import net.sourceforge.phpdt.core.compiler.InvalidInputException;
20
21 /**
22  * Parser specialized for decoding javadoc comments
23  */
24 public abstract class AbstractCommentParser {
25
26         // recognized tags
27         public static final char[] TAG_DEPRECATED = "deprecated".toCharArray(); //$NON-NLS-1$
28
29         public static final char[] TAG_PARAM = "param".toCharArray(); //$NON-NLS-1$
30
31         public static final char[] TAG_RETURN = "return".toCharArray(); //$NON-NLS-1$
32
33         public static final char[] TAG_THROWS = "throws".toCharArray(); //$NON-NLS-1$
34
35         public static final char[] TAG_EXCEPTION = "exception".toCharArray(); //$NON-NLS-1$
36
37         public static final char[] TAG_SEE = "see".toCharArray(); //$NON-NLS-1$
38
39         public static final char[] TAG_LINK = "link".toCharArray(); //$NON-NLS-1$
40
41         public static final char[] TAG_LINKPLAIN = "linkplain".toCharArray(); //$NON-NLS-1$
42
43         public static final char[] TAG_INHERITDOC = "inheritDoc".toCharArray(); //$NON-NLS-1$
44
45         // tags expected positions
46         public final static int ORDERED_TAGS_NUMBER = 3;
47
48         public final static int PARAM_TAG_EXPECTED_ORDER = 0;
49
50         public final static int THROWS_TAG_EXPECTED_ORDER = 1;
51
52         public final static int SEE_TAG_EXPECTED_ORDER = 2;
53
54         // Kind of comment parser
55         public final static int COMPIL_PARSER = 0x00000001;
56
57         public final static int DOM_PARSER = 0x00000002;
58
59         // Public fields
60         public Scanner scanner;
61
62         public boolean checkDocComment = false;
63
64         // Protected fields
65         protected boolean inherited, deprecated;
66
67         protected char[] source;
68
69         protected int index, endComment, lineEnd;
70
71         protected int tokenPreviousPosition, lastIdentifierEndPosition,
72                         starPosition;
73
74         protected int textStart, memberStart;
75
76         protected int tagSourceStart, tagSourceEnd;
77
78         protected int inlineTagStart;
79
80         protected Parser sourceParser;
81
82         protected Object returnStatement;
83
84         protected boolean lineStarted = false, inlineTagStarted = false;
85
86         protected int kind;
87
88         protected int[] lineEnds;
89
90         // Private fields
91         private TokenName currentTokenType = ITerminalSymbols.TokenName.NONE;
92
93         // Line pointers
94         private int linePtr, lastLinePtr;
95
96         // Identifier stack
97         protected int identifierPtr;
98
99         protected char[][] identifierStack;
100
101         protected int identifierLengthPtr;
102
103         protected int[] identifierLengthStack;
104
105         protected long[] identifierPositionStack;
106
107         // Ast stack
108         protected static int AstStackIncrement = 10;
109
110         protected int astPtr;
111
112         protected Object[] astStack;
113
114         protected int astLengthPtr;
115
116         protected int[] astLengthStack;
117
118         protected AbstractCommentParser(Parser sourceParser) {
119                 this.sourceParser = sourceParser;
120                 this.scanner = new Scanner(false, false, false, false, false, null,
121                                 null, false);
122
123                 this.identifierStack = new char[20][];
124                 this.identifierPositionStack = new long[20];
125                 this.identifierLengthStack = new int[10];
126                 this.astStack = new Object[30];
127                 this.astLengthStack = new int[20];
128         }
129
130         /*
131          * (non-Javadoc) Returns true if tag
132          * 
133          * @deprecated is present in javadoc comment.
134          * 
135          * If javadoc checking is enabled, will also construct an Javadoc node,
136          * which will be stored into Parser.javadoc slot for being consumed later
137          * on.
138          */
139         protected boolean parseComment(int javadocStart, int javadocEnd) {
140
141                 boolean validComment = true;
142                 try {
143                         // Init scanner position
144                         this.scanner.resetTo(javadocStart, javadocEnd);
145                         this.endComment = javadocEnd;
146                         this.index = javadocStart;
147                         readChar(); // starting '/'
148                         int previousPosition = this.index;
149                         readChar(); // first '*'
150                         char nextCharacter = readChar(); // second '*'
151
152                         // Init local variables
153                         this.astLengthPtr = -1;
154                         this.astPtr = -1;
155                         this.currentTokenType = ITerminalSymbols.TokenName.NONE;
156                         this.inlineTagStarted = false;
157                         this.inlineTagStart = -1;
158                         this.lineStarted = false;
159                         this.returnStatement = null;
160                         this.inherited = false;
161                         this.deprecated = false;
162                         this.linePtr = getLineNumber(javadocStart);
163                         this.lastLinePtr = getLineNumber(javadocEnd);
164                         this.lineEnd = (this.linePtr == this.lastLinePtr) ? this.endComment
165                                         : this.scanner.getLineEnd(this.linePtr);
166                         this.textStart = -1;
167                         char previousChar = 0;
168                         int invalidTagLineEnd = -1;
169                         int invalidInlineTagLineEnd = -1;
170
171                         // Loop on each comment character
172                         while (this.index < this.endComment) {
173                                 previousPosition = this.index;
174                                 previousChar = nextCharacter;
175
176                                 // Calculate line end (cannot use this.scanner.linePtr as
177                                 // scanner does not parse line ends again)
178                                 if (this.index > (this.lineEnd + 1)) {
179                                         updateLineEnd();
180                                 }
181
182                                 // Read next char only if token was consumed
183                                 if (this.currentTokenType.compareTo (ITerminalSymbols.TokenName.NONE) <= 0) {
184                                         nextCharacter = readChar(); // consider unicodes
185                                 } else {
186                                         previousPosition = this.scanner
187                                                         .getCurrentTokenStartPosition();
188                                         switch (this.currentTokenType) {
189                                         case RBRACE:
190                                                 nextCharacter = '}';
191                                                 break;
192                                         case MULTIPLY:
193                                                 nextCharacter = '*';
194                                                 break;
195                                         default:
196                                                 nextCharacter = this.scanner.currentCharacter;
197                                         }
198                                         consumeToken();
199                                 }
200
201                                 if (this.index >= this.endComment) {
202                                         break;
203                                 }
204
205                                 switch (nextCharacter) {
206                                 case '@':
207                                         boolean valid = false;
208                                         // Start tag parsing only if we have a java identifier start
209                                         // character and if we are on line beginning or at inline
210                                         // tag beginning
211                                         if ((!this.lineStarted || previousChar == '{')) {
212                                                 this.lineStarted = true;
213                                                 if (this.inlineTagStarted) {
214                                                         this.inlineTagStarted = false;
215                                                         // bug
216                                                         // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
217                                                         // Cannot have @ inside inline comment
218                                                         if (this.sourceParser != null) {
219                                                                 int end = previousPosition < invalidInlineTagLineEnd ? previousPosition
220                                                                                 : invalidInlineTagLineEnd;
221                                                                 this.sourceParser.problemReporter()
222                                                                                 .javadocUnterminatedInlineTag(
223                                                                                                 this.inlineTagStart, end);
224                                                         }
225                                                         validComment = false;
226                                                         if (this.lineStarted && this.textStart != -1
227                                                                         && this.textStart < previousPosition) {
228                                                                 pushText(this.textStart, previousPosition);
229                                                         }
230                                                         if (this.kind == DOM_PARSER)
231                                                                 refreshInlineTagPosition(previousPosition);
232                                                 }
233                                                 if (previousChar == '{') {
234                                                         if (this.textStart != -1
235                                                                         && this.textStart < this.inlineTagStart) {
236                                                                 pushText(this.textStart, this.inlineTagStart);
237                                                         }
238                                                         this.inlineTagStarted = true;
239                                                         invalidInlineTagLineEnd = this.lineEnd;
240                                                 } else if (this.textStart != -1
241                                                                 && this.textStart < invalidTagLineEnd) {
242                                                         pushText(this.textStart, invalidTagLineEnd);
243                                                 }
244                                                 this.scanner.resetTo(this.index, this.endComment);
245                                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE; // flush token cache at line
246                                                                                                         // begin
247                                                 try {
248                                                         TokenName token = readTokenAndConsume();
249                                                         this.tagSourceStart = this.scanner
250                                                                         .getCurrentTokenStartPosition();
251                                                         this.tagSourceEnd = this.scanner
252                                                                         .getCurrentTokenEndPosition();
253                                                         char[] tag = this.scanner
254                                                                         .getCurrentIdentifierSource(); // first
255                                                                                                                                         // token is
256                                                                                                                                         // either an
257                                                                                                                                         // identifier
258                                                                                                                                         // or a
259                                                                                                                                         // keyword
260                                                         if (this.kind == DOM_PARSER) {
261                                                                 // For DOM parser, try to get tag name other
262                                                                 // than java identifier
263                                                                 // (see bug
264                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51660)
265                                                                 TokenName tk = token;
266                                                                 int le = this.lineEnd;
267                                                                 char pc = peekChar();
268                                                                 tagNameToken: while (tk != ITerminalSymbols.TokenName.EOF) {
269                                                                         this.tagSourceEnd = this.scanner
270                                                                                         .getCurrentTokenEndPosition();
271                                                                         token = tk;
272                                                                         // !, ", #, %, &, ', -, :, <, >, * chars and
273                                                                         // spaces are not allowed in tag names
274                                                                         switch (pc) {
275                                                                         case '}':
276                                                                         case '!':
277                                                                         case '#':
278                                                                         case '%':
279                                                                         case '&':
280                                                                         case '\'':
281                                                                         case ':':
282                                                                                 // case '-': allowed in tag names as
283                                                                                 // this character is often used in
284                                                                                 // doclets (bug 68087)
285                                                                         case '<':
286                                                                         case '>':
287                                                                         case '*': // break for '*' as this is
288                                                                                                 // perhaps the end of comment
289                                                                                                 // (bug 65288)
290                                                                                 break tagNameToken;
291                                                                         default:
292                                                                                 if (pc == ' '
293                                                                                                 || Character.isWhitespace(pc))
294                                                                                         break tagNameToken;
295                                                                         }
296                                                                         tk = readTokenAndConsume();
297                                                                         pc = peekChar();
298                                                                 }
299                                                                 int length = this.tagSourceEnd
300                                                                                 - this.tagSourceStart + 1;
301                                                                 tag = new char[length];
302                                                                 System.arraycopy(this.source,
303                                                                                 this.tagSourceStart, tag, 0, length);
304                                                                 this.index = this.tagSourceEnd + 1;
305                                                                 this.scanner.currentPosition = this.tagSourceEnd + 1;
306                                                                 this.tagSourceStart = previousPosition;
307                                                                 this.lineEnd = le;
308                                                         }
309                                                         switch (token) {
310                                                         case IDENTIFIER:
311                                                                 if (CharOperation.equals(tag, TAG_DEPRECATED)) {
312                                                                         this.deprecated = true;
313                                                                         if (this.kind == DOM_PARSER) {
314                                                                                 valid = parseTag();
315                                                                         } else {
316                                                                                 valid = true;
317                                                                         }
318                                                                 } else if (CharOperation.equals(tag,
319                                                                                 TAG_INHERITDOC)) {
320                                                                         // inhibits inherited flag when tags have
321                                                                         // been already stored
322                                                                         // see bug
323                                                                         // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51606
324                                                                         // Note that for DOM_PARSER, nodes stack may
325                                                                         // be not empty even no '@' tag
326                                                                         // was encountered in comment. But it cannot
327                                                                         // be the case for COMPILER_PARSER
328                                                                         // and so is enough as it is only this
329                                                                         // parser which signals the missing tag
330                                                                         // warnings...
331                                                                         this.inherited = this.astPtr == -1;
332                                                                         if (this.kind == DOM_PARSER) {
333                                                                                 valid = parseTag();
334                                                                         } else {
335                                                                                 valid = true;
336                                                                         }
337                                                                 } else if (CharOperation.equals(tag, TAG_PARAM)) {
338                                                                         valid = parseParam();
339                                                                 } else if (CharOperation.equals(tag,
340                                                                                 TAG_EXCEPTION)) {
341                                                                         valid = parseThrows(false);
342                                                                 } else if (CharOperation.equals(tag, TAG_SEE)) {
343                                                                         if (this.inlineTagStarted) {
344                                                                                 // bug
345                                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
346                                                                                 // Cannot have @see inside inline
347                                                                                 // comment
348                                                                                 valid = false;
349                                                                                 if (this.sourceParser != null)
350                                                                                         this.sourceParser
351                                                                                                         .problemReporter()
352                                                                                                         .javadocUnexpectedTag(
353                                                                                                                         this.tagSourceStart,
354                                                                                                                         this.tagSourceEnd);
355                                                                         } else {
356                                                                                 valid = parseSee(false);
357                                                                         }
358                                                                 } else if (CharOperation.equals(tag, TAG_LINK)) {
359                                                                         if (this.inlineTagStarted) {
360                                                                                 valid = parseSee(false);
361                                                                         } else {
362                                                                                 // bug
363                                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53290
364                                                                                 // Cannot have @link outside inline
365                                                                                 // comment
366                                                                                 valid = false;
367                                                                                 if (this.sourceParser != null)
368                                                                                         this.sourceParser
369                                                                                                         .problemReporter()
370                                                                                                         .javadocUnexpectedTag(
371                                                                                                                         this.tagSourceStart,
372                                                                                                                         this.tagSourceEnd);
373                                                                         }
374                                                                 } else if (CharOperation.equals(tag,
375                                                                                 TAG_LINKPLAIN)) {
376                                                                         if (this.inlineTagStarted) {
377                                                                                 valid = parseSee(true);
378                                                                         } else {
379                                                                                 valid = parseTag();
380                                                                         }
381                                                                 } else {
382                                                                         valid = parseTag();
383                                                                 }
384                                                                 break;
385                                                         case RETURN:
386                                                                 valid = parseReturn();
387                                                                 // verify characters after return tag (we're
388                                                                 // expecting text description)
389                                                                 if (!verifyCharsAfterReturnTag(this.index)) {
390                                                                         if (this.sourceParser != null) {
391                                                                                 int end = this.starPosition == -1
392                                                                                                 || this.lineEnd < this.starPosition ? this.lineEnd
393                                                                                                 : this.starPosition;
394                                                                                 this.sourceParser.problemReporter()
395                                                                                                 .javadocInvalidTag(
396                                                                                                                 this.tagSourceStart,
397                                                                                                                 end);
398                                                                         }
399                                                                 }
400                                                                 break;
401                                                         // case ITerminalSymbols.TokenName.throws :
402                                                         // valid = parseThrows(true);
403                                                         // break;
404                                                         default:
405                                                                 if (this.kind == DOM_PARSER) {
406                                                                         switch (token) {
407                                                                         case ABSTRACT:
408                                                                                 // case
409                                                                                 // ITerminalSymbols.TokenName.assert:
410                                                                                 // case
411                                                                                 // ITerminalSymbols.TokenName.boolean:
412                                                                         case BREAK:
413                                                                                 // case byte:
414                                                                         case CASE:
415                                                                         case CATCH:
416                                                                                 // case char:
417                                                                         case CLASS:
418                                                                         case CONTINUE:
419                                                                         case DEFAULT:
420                                                                         case DO:
421                                                                                 // case
422                                                                                 // double:
423                                                                         case ELSE:
424                                                                         case EXTENDS:
425                                                                                 // case false:
426                                                                         case FINAL:
427                                                                         case FINALLY:
428                                                                                 // case float:
429                                                                         case FOR:
430                                                                         case IF:
431                                                                         case IMPLEMENTS:
432                                                                                 // case
433                                                                                 // import:
434                                                                         case INSTANCEOF:
435                                                                                 // case int:
436                                                                         case INTERFACE:
437                                                                                 // case long:
438                                                                                 // case
439                                                                                 // native:
440                                                                         case NEW:
441                                                                                 // case null:
442                                                                                 // case
443                                                                                 // package:
444                                                                         case PRIVATE:
445                                                                         case PROTECTED:
446                                                                         case PUBLIC:
447                                                                                 // case short:
448                                                                         case STATIC:
449                                                                                 // case
450                                                                                 // strictfp:
451                                                                         case SUPER:
452                                                                         case SWITCH:
453                                                                                 // case
454                                                                                 // synchronized:
455                                                                                 // case this:
456                                                                         case THROW:
457                                                                                 // case
458                                                                                 // transient:
459                                                                                 // case true:
460                                                                         case TRY:
461                                                                                 // case void:
462                                                                                 // case
463                                                                                 // volatile:
464                                                                         case WHILE:
465                                                                                 valid = parseTag();
466                                                                                 break;
467                                                                         }
468                                                                 }
469                                                         }
470                                                         this.textStart = this.index;
471                                                         if (!valid) {
472                                                                 // bug
473                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
474                                                                 // do not stop the inline tag when error is
475                                                                 // encountered to get text after
476                                                                 validComment = false;
477                                                                 // bug
478                                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=51600
479                                                                 // for DOM AST node, store tag as text in case
480                                                                 // of invalid syntax
481                                                                 if (this.kind == DOM_PARSER) {
482                                                                         parseTag();
483                                                                         this.textStart = this.tagSourceEnd + 1;
484                                                                         invalidTagLineEnd = this.lineEnd;
485                                                                 }
486                                                         }
487                                                 } catch (InvalidInputException e) {
488                                                         consumeToken();
489                                                 }
490                                         }
491                                         break;
492                                 case '\r':
493                                 case '\n':
494                                         if (this.lineStarted && this.textStart < previousPosition) {
495                                                 pushText(this.textStart, previousPosition);
496                                         }
497                                         this.lineStarted = false;
498                                         // Fix bug 51650
499                                         this.textStart = -1;
500                                         break;
501                                 case '}':
502                                         if (this.inlineTagStarted) {
503                                                 if (this.lineStarted && this.textStart != -1
504                                                                 && this.textStart < previousPosition) {
505                                                         pushText(this.textStart, previousPosition);
506                                                 }
507                                                 if (this.kind == DOM_PARSER)
508                                                         refreshInlineTagPosition(previousPosition);
509                                                 this.textStart = this.index;
510                                                 this.inlineTagStarted = false;
511                                         } else {
512                                                 if (!this.lineStarted) {
513                                                         this.textStart = previousPosition;
514                                                 }
515                                         }
516                                         this.lineStarted = true;
517                                         break;
518                                 case '{':
519                                         if (this.inlineTagStarted) {
520                                                 this.inlineTagStarted = false;
521                                                 // bug
522                                                 // https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
523                                                 // Cannot have opening brace in inline comment
524                                                 if (this.sourceParser != null) {
525                                                         int end = previousPosition < invalidInlineTagLineEnd ? previousPosition
526                                                                         : invalidInlineTagLineEnd;
527                                                         this.sourceParser.problemReporter()
528                                                                         .javadocUnterminatedInlineTag(
529                                                                                         this.inlineTagStart, end);
530                                                 }
531                                                 if (this.lineStarted && this.textStart != -1
532                                                                 && this.textStart < previousPosition) {
533                                                         pushText(this.textStart, previousPosition);
534                                                 }
535                                                 if (this.kind == DOM_PARSER)
536                                                         refreshInlineTagPosition(previousPosition);
537                                         }
538                                         if (!this.lineStarted) {
539                                                 this.textStart = previousPosition;
540                                         }
541                                         this.lineStarted = true;
542                                         this.inlineTagStart = previousPosition;
543                                         break;
544                                 case '*':
545                                 case '\u000c': /* FORM FEED */
546                                 case ' ': /* SPACE */
547                                 case '\t': /* HORIZONTAL TABULATION */
548                                         // do nothing for space or '*' characters
549                                         break;
550                                 default:
551                                         if (!this.lineStarted) {
552                                                 this.textStart = previousPosition;
553                                         }
554                                         this.lineStarted = true;
555                                         break;
556                                 }
557                         }
558                         // bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=53279
559                         // Cannot leave comment inside inline comment
560                         if (this.inlineTagStarted) {
561                                 this.inlineTagStarted = false;
562                                 if (this.sourceParser != null) {
563                                         int end = previousPosition < invalidInlineTagLineEnd ? previousPosition
564                                                         : invalidInlineTagLineEnd;
565                                         if (this.index >= this.endComment)
566                                                 end = invalidInlineTagLineEnd;
567                                         this.sourceParser.problemReporter()
568                                                         .javadocUnterminatedInlineTag(this.inlineTagStart,
569                                                                         end);
570                                 }
571                                 if (this.lineStarted && this.textStart != -1
572                                                 && this.textStart < previousPosition) {
573                                         pushText(this.textStart, previousPosition);
574                                 }
575                                 if (this.kind == DOM_PARSER) {
576                                         refreshInlineTagPosition(previousPosition);
577                                 }
578                         } else if (this.lineStarted && this.textStart < previousPosition) {
579                                 pushText(this.textStart, previousPosition);
580                         }
581                         updateDocComment();
582                 } catch (Exception ex) {
583                         validComment = false;
584                 }
585                 return validComment;
586         }
587
588         private void consumeToken() {
589                 this.currentTokenType = ITerminalSymbols.TokenName.NONE; // flush token cache
590                 updateLineEnd();
591         }
592
593         protected abstract Object createArgumentReference(char[] name, int dim,
594                         Object typeRef, long[] dimPos, long argNamePos)
595                         throws InvalidInputException;
596
597         protected abstract Object createFieldReference(Object receiver)
598                         throws InvalidInputException;
599
600         protected abstract Object createMethodReference(Object receiver,
601                         List arguments) throws InvalidInputException;
602
603         protected Object createReturnStatement() {
604                 return null;
605         }
606
607         protected abstract Object createTypeReference(int primitiveToken);
608
609         private int getEndPosition() {
610                 if (this.scanner.getCurrentTokenEndPosition() > this.lineEnd) {
611                         return this.lineEnd;
612                 } else {
613                         return this.scanner.getCurrentTokenEndPosition();
614                 }
615         }
616
617         /*
618          * Search the source position corresponding to the end of a given line
619          * number. Warning: returned position is 1-based index!
620          * 
621          * @see Scanner#getLineEnd(int) We cannot directly use this method when
622          *      linePtr field is not initialized.
623          * 
624          * private int getLineEnd(int lineNumber) {
625          * 
626          * if (this.scanner.linePtr != -1) { return
627          * this.scanner.getLineEnd(lineNumber); } if (this.lineEnds == null) return
628          * -1; if (lineNumber > this.lineEnds.length+1) return -1; if (lineNumber <=
629          * 0) return -1; if (lineNumber == this.lineEnds.length + 1) return
630          * this.scanner.eofPosition; return this.lineEnds[lineNumber-1]; // next
631          * line start one character behind the lineEnd of the previous line }
632          */
633
634         /**
635          * Search the line number corresponding to a specific position. Warning:
636          * returned position is 1-based index!
637          * 
638          * @see Scanner#getLineNumber(int) We cannot directly use this method when
639          *      linePtr field is not initialized.
640          */
641         private int getLineNumber(int position) {
642
643                 if (this.scanner.linePtr != -1) {
644                         return this.scanner.getLineNumber(position);
645                 }
646                 if (this.lineEnds == null)
647                         return 1;
648                 int length = this.lineEnds.length;
649                 if (length == 0)
650                         return 1;
651                 int g = 0, d = length - 1;
652                 int m = 0;
653                 while (g <= d) {
654                         m = (g + d) / 2;
655                         if (position < this.lineEnds[m]) {
656                                 d = m - 1;
657                         } else if (position > this.lineEnds[m]) {
658                                 g = m + 1;
659                         } else {
660                                 return m + 1;
661                         }
662                 }
663                 if (position < this.lineEnds[m]) {
664                         return m + 1;
665                 }
666                 return m + 2;
667         }
668
669         /*
670          * Parse argument in
671          * 
672          * @see tag method reference
673          */
674         private Object parseArguments(Object receiver) throws InvalidInputException {
675
676                 // Init
677                 int modulo = 0; // should be 2 for (Type,Type,...) or 3 for (Type
678                                                 // arg,Type arg,...)
679                 int iToken = 0;
680                 char[] argName = null;
681                 List arguments = new ArrayList(10);
682                 int start = this.scanner.getCurrentTokenStartPosition();
683
684                 // Parse arguments declaration if method reference
685                 nextArg: while (this.index < this.scanner.eofPosition) {
686
687                         // Read argument type reference
688                         Object typeRef;
689                         try {
690                                 typeRef = parseQualifiedName(false);
691                         } catch (InvalidInputException e) {
692                                 break nextArg;
693                         }
694                         boolean firstArg = modulo == 0;
695                         if (firstArg) { // verify position
696                                 if (iToken != 0)
697                                         break nextArg;
698                         } else if ((iToken % modulo) != 0) {
699                                 break nextArg;
700                         }
701                         if (typeRef == null) {
702                                 if (firstArg
703                                                 && this.currentTokenType == ITerminalSymbols.TokenName.RPAREN) {
704                                         // verify characters after arguments declaration (expecting
705                                         // white space or end comment)
706                                         if (!verifySpaceOrEndComment()) {
707                                                 int end = this.starPosition == -1 ? this.lineEnd
708                                                                 : this.starPosition;
709                                                 if (this.source[end] == '\n')
710                                                         end--;
711                                                 if (this.sourceParser != null)
712                                                         this.sourceParser.problemReporter()
713                                                                         .javadocMalformedSeeReference(start, end);
714                                                 return null;
715                                         }
716                                         this.lineStarted = true;
717                                         return createMethodReference(receiver, null);
718                                 }
719                                 break nextArg;
720                         }
721                         iToken++;
722
723                         // Read possible array declaration
724                         int dim = 0;
725                         long[] dimPositions = new long[20]; // assume that there won't be
726                                                                                                 // more than 20 dimensions...
727                         if (readToken() == ITerminalSymbols.TokenName.LBRACKET) {
728                                 int dimStart = this.scanner.getCurrentTokenStartPosition();
729                                 while (readToken() == ITerminalSymbols.TokenName.LBRACKET) {
730                                         consumeToken();
731                                         if (readToken() != ITerminalSymbols.TokenName.RBRACKET) {
732                                                 break nextArg;
733                                         }
734                                         consumeToken();
735                                         dimPositions[dim++] = (((long) dimStart) << 32)
736                                                         + this.scanner.getCurrentTokenEndPosition();
737                                 }
738                         }
739
740                         // Read argument name
741                         long argNamePos = -1;
742                         if (readToken() == ITerminalSymbols.TokenName.IDENTIFIER) {
743                                 consumeToken();
744                                 if (firstArg) { // verify position
745                                         if (iToken != 1)
746                                                 break nextArg;
747                                 } else if ((iToken % modulo) != 1) {
748                                         break nextArg;
749                                 }
750                                 if (argName == null) { // verify that all arguments name are
751                                                                                 // declared
752                                         if (!firstArg) {
753                                                 break nextArg;
754                                         }
755                                 }
756                                 argName = this.scanner.getCurrentIdentifierSource();
757                                 argNamePos = (((long) this.scanner
758                                                 .getCurrentTokenStartPosition()) << 32)
759                                                 + this.scanner.getCurrentTokenEndPosition();
760                                 iToken++;
761                         } else if (argName != null) { // verify that no argument name is
762                                                                                         // declared
763                                 break nextArg;
764                         }
765
766                         // Verify token position
767                         if (firstArg) {
768                                 modulo = iToken + 1;
769                         } else {
770                                 if ((iToken % modulo) != (modulo - 1)) {
771                                         break nextArg;
772                                 }
773                         }
774
775                         // Read separator or end arguments declaration
776                         TokenName token = readToken();
777                         char[] name = argName == null ? new char[0] : argName;
778                         if (token == ITerminalSymbols.TokenName.COMMA) {
779                                 // Create new argument
780                                 Object argument = createArgumentReference(name, dim, typeRef,
781                                                 dimPositions, argNamePos);
782                                 arguments.add(argument);
783                                 consumeToken();
784                                 iToken++;
785                         } else if (token == ITerminalSymbols.TokenName.RPAREN) {
786                                 // verify characters after arguments declaration (expecting
787                                 // white space or end comment)
788                                 if (!verifySpaceOrEndComment()) {
789                                         int end = this.starPosition == -1 ? this.lineEnd
790                                                         : this.starPosition;
791                                         if (this.source[end] == '\n')
792                                                 end--;
793                                         if (this.sourceParser != null)
794                                                 this.sourceParser.problemReporter()
795                                                                 .javadocMalformedSeeReference(start, end);
796                                         return null;
797                                 }
798                                 // Create new argument
799                                 Object argument = createArgumentReference(name, dim, typeRef,
800                                                 dimPositions, argNamePos);
801                                 arguments.add(argument);
802                                 consumeToken();
803                                 return createMethodReference(receiver, arguments);
804                         } else {
805                                 break nextArg;
806                         }
807                 }
808
809                 // Something wrong happened => Invalid input
810                 throw new InvalidInputException();
811         }
812
813         /*
814          * Parse an URL link reference in
815          * 
816          * @see tag
817          */
818         private boolean parseHref() throws InvalidInputException {
819                 int start = this.scanner.getCurrentTokenStartPosition();
820                 if (Character.toLowerCase(readChar()) == 'a') {
821                         this.scanner.currentPosition = this.index;
822                         if (readToken() == ITerminalSymbols.TokenName.IDENTIFIER) {
823                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE; // do not update line end
824                                 try {
825                                         if (CharOperation.equals(this.scanner
826                                                         .getCurrentIdentifierSource(), new char[] { 'h',
827                                                         'r', 'e', 'f' }, false)
828                                                         && readToken() == ITerminalSymbols.TokenName.EQUAL) {
829                                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE; // do not update line end
830                                                 if (readToken() == ITerminalSymbols.TokenName.STRINGDOUBLEQUOTE
831                                                                 || readToken() == ITerminalSymbols.TokenName.STRINGSINGLEQUOTE) {
832                                                         this.currentTokenType = ITerminalSymbols.TokenName.NONE; // do not update line
833                                                                                                                 // end
834                                                         // Skip all characters after string literal until
835                                                         // closing '>' (see bug 68726)
836                                                         while (this.index <= this.lineEnd
837                                                                         && readToken() != ITerminalSymbols.TokenName.GREATER) {
838                                                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE; // do not update
839                                                                                                                         // line end
840                                                         }
841                                                         if (this.currentTokenType == ITerminalSymbols.TokenName.GREATER) {
842                                                                 consumeToken(); // update line end as new lines
843                                                                                                 // are allowed in URL
844                                                                                                 // description
845                                                                 while (readToken() != ITerminalSymbols.TokenName.LESS) {
846                                                                         if (this.scanner.currentPosition >= this.scanner.eofPosition
847                                                                                         || this.scanner.currentCharacter == '@') {
848                                                                                 // Reset position: we want to rescan
849                                                                                 // last token
850                                                                                 this.index = this.tokenPreviousPosition;
851                                                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
852                                                                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
853                                                                                 // Signal syntax error
854                                                                                 if (this.sourceParser != null)
855                                                                                         this.sourceParser
856                                                                                                         .problemReporter()
857                                                                                                         .javadocInvalidSeeUrlReference(
858                                                                                                                         start, this.lineEnd);
859                                                                                 return false;
860                                                                         }
861                                                                         consumeToken();
862                                                                 }
863                                                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE; // do not update
864                                                                                                                         // line end
865                                                                 if (readChar() == '/') {
866                                                                         if (Character.toLowerCase(readChar()) == 'a') {
867                                                                                 if (readChar() == '>') {
868                                                                                         // Valid href
869                                                                                         return true;
870                                                                                 }
871                                                                         }
872                                                                 }
873                                                         }
874                                                 }
875                                         }
876                                 } catch (InvalidInputException ex) {
877                                         // Do nothing as we want to keep positions for error message
878                                 }
879                         }
880                 }
881                 // Reset position: we want to rescan last token
882                 this.index = this.tokenPreviousPosition;
883                 this.scanner.currentPosition = this.tokenPreviousPosition;
884                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
885                 // Signal syntax error
886                 if (this.sourceParser != null)
887                         this.sourceParser.problemReporter().javadocInvalidSeeUrlReference(
888                                         start, this.lineEnd);
889                 return false;
890         }
891
892         /*
893          * Parse a method reference in
894          * 
895          * @see tag
896          */
897         private Object parseMember(Object receiver) throws InvalidInputException {
898                 // Init
899                 this.identifierPtr = -1;
900                 this.identifierLengthPtr = -1;
901                 int start = this.scanner.getCurrentTokenStartPosition();
902                 this.memberStart = start;
903
904                 // Get member identifier
905                 if (readToken() == ITerminalSymbols.TokenName.IDENTIFIER) {
906                         consumeToken();
907                         pushIdentifier(true);
908                         // Look for next token to know whether it's a field or method
909                         // reference
910                         int previousPosition = this.index;
911                         if (readToken() == ITerminalSymbols.TokenName.LPAREN) {
912                                 consumeToken();
913                                 start = this.scanner.getCurrentTokenStartPosition();
914                                 try {
915                                         return parseArguments(receiver);
916                                 } catch (InvalidInputException e) {
917                                         int end = this.scanner.getCurrentTokenEndPosition() < this.lineEnd ? this.scanner
918                                                         .getCurrentTokenEndPosition()
919                                                         : this.scanner.getCurrentTokenStartPosition();
920                                         end = end < this.lineEnd ? end : this.lineEnd;
921                                         if (this.sourceParser != null)
922                                                 this.sourceParser.problemReporter()
923                                                                 .javadocInvalidSeeReferenceArgs(start, end);
924                                 }
925                                 return null;
926                         }
927
928                         // Reset position: we want to rescan last token
929                         this.index = previousPosition;
930                         this.scanner.currentPosition = previousPosition;
931                         this.currentTokenType = ITerminalSymbols.TokenName.NONE;
932
933                         // Verify character(s) after identifier (expecting space or end
934                         // comment)
935                         if (!verifySpaceOrEndComment()) {
936                                 int end = this.starPosition == -1 ? this.lineEnd
937                                                 : this.starPosition;
938                                 if (this.source[end] == '\n')
939                                         end--;
940                                 if (this.sourceParser != null)
941                                         this.sourceParser.problemReporter()
942                                                         .javadocMalformedSeeReference(start, end);
943                                 return null;
944                         }
945                         return createFieldReference(receiver);
946                 }
947                 int end = getEndPosition() - 1;
948                 end = start > end ? start : end;
949                 if (this.sourceParser != null)
950                         this.sourceParser.problemReporter().javadocInvalidSeeReference(
951                                         start, end);
952                 // Reset position: we want to rescan last token
953                 this.index = this.tokenPreviousPosition;
954                 this.scanner.currentPosition = this.tokenPreviousPosition;
955                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
956                 return null;
957         }
958
959         /*
960          * Parse @param tag declaration
961          */
962         protected boolean parseParam() {
963
964                 // Store current token state
965                 int start = this.tagSourceStart;
966                 int end = this.tagSourceEnd;
967
968                 try {
969                         // Push identifier next
970                         TokenName token = readToken();
971                         switch (token) {
972                         case IDENTIFIER:
973                                 consumeToken();
974                                 return pushParamName();
975                         case EOF:
976                                 break;
977                         default:
978                                 start = this.scanner.getCurrentTokenStartPosition();
979                                 end = getEndPosition();
980                                 if (end < start)
981                                         start = this.tagSourceStart;
982                                 break;
983                         }
984                 } catch (InvalidInputException e) {
985                         end = getEndPosition();
986                 }
987
988                 // Reset position to avoid missing tokens when new line was encountered
989                 this.index = this.tokenPreviousPosition;
990                 this.scanner.currentPosition = this.tokenPreviousPosition;
991                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
992
993                 // Report problem
994                 if (this.sourceParser != null)
995                         this.sourceParser.problemReporter().javadocMissingParamName(start,
996                                         end);
997                 return false;
998         }
999
1000         /*
1001          * Parse a qualified name and built a type reference if the syntax is valid.
1002          */
1003         protected Object parseQualifiedName(boolean reset)
1004                         throws InvalidInputException {
1005
1006                 // Reset identifier stack if requested
1007                 if (reset) {
1008                         this.identifierPtr = -1;
1009                         this.identifierLengthPtr = -1;
1010                 }
1011
1012                 // Scan tokens
1013                 int primitiveToken = -1;
1014                 nextToken: for (int iToken = 0;; iToken++) {
1015                         TokenName token = readToken();
1016                         switch (token) {
1017                         case IDENTIFIER:
1018                                 if (((iToken % 2) > 0)) { // identifiers must be odd tokens
1019                                         break nextToken;
1020                                 }
1021                                 pushIdentifier(iToken == 0);
1022                                 consumeToken();
1023                                 break;
1024
1025                         case DOT:
1026                                 if ((iToken % 2) == 0) { // dots must be even tokens
1027                                         throw new InvalidInputException();
1028                                 }
1029                                 consumeToken();
1030                                 break;
1031
1032                         // case ITerminalSymbols.TokenName.void :
1033                         // case ITerminalSymbols.TokenName.boolean :
1034                         // case ITerminalSymbols.TokenName.byte :
1035                         // case ITerminalSymbols.TokenName.char :
1036                         // case ITerminalSymbols.TokenName.double :
1037                         // case ITerminalSymbols.TokenName.float :
1038                         // case ITerminalSymbols.TokenName.int :
1039                         // case ITerminalSymbols.TokenName.long :
1040                         // case ITerminalSymbols.TokenName.short :
1041                         // if (iToken > 0) {
1042                         // throw new InvalidInputException();
1043                         // }
1044                         // pushIdentifier(true);
1045                         // primitiveToken = token;
1046                         // consumeToken();
1047                         // break nextToken;
1048
1049                         default:
1050                                 if (iToken == 0) {
1051                                         return null;
1052                                 }
1053                                 if ((iToken % 2) == 0) { // cannot leave on a dot
1054                                         // Reset position: we want to rescan last token
1055                                         if (this.kind == DOM_PARSER && this.currentTokenType.compareTo (ITerminalSymbols.TokenName.NONE) > 0) {
1056                                                 this.index = this.tokenPreviousPosition;
1057                                                 this.scanner.currentPosition = this.tokenPreviousPosition;
1058                                                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
1059                                         }
1060                                         throw new InvalidInputException();
1061                                 }
1062                                 break nextToken;
1063                         }
1064                 }
1065                 // Reset position: we want to rescan last token
1066                 if (this.currentTokenType.compareTo (ITerminalSymbols.TokenName.NONE) > 0) {
1067                         this.index = this.tokenPreviousPosition;
1068                         this.scanner.currentPosition = this.tokenPreviousPosition;
1069                         this.currentTokenType = ITerminalSymbols.TokenName.NONE;
1070                 }
1071                 this.lastIdentifierEndPosition = (int) this.identifierPositionStack[this.identifierPtr];
1072                 return createTypeReference(primitiveToken);
1073         }
1074
1075         /*
1076          * Parse a reference in
1077          * 
1078          * @see tag
1079          */
1080         protected boolean parseReference(boolean plain)
1081                         throws InvalidInputException {
1082                 Object typeRef = null;
1083                 Object reference = null;
1084                 int previousPosition = -1;
1085                 int typeRefStartPosition = -1;
1086                 nextToken: while (this.index < this.scanner.eofPosition) {
1087                         previousPosition = this.index;
1088                         TokenName token = readToken();
1089                         switch (token) {
1090                         case STRINGDOUBLEQUOTE: // @see "string"
1091                         case STRINGSINGLEQUOTE:
1092                                 int start = this.scanner.getCurrentTokenStartPosition();
1093                                 consumeToken();
1094                                 // If typeRef != null we may raise a warning here to let user
1095                                 // know there's an unused reference...
1096                                 // Currently as javadoc 1.4.2 ignore it, we do the same (see bug
1097                                 // 69302)
1098                                 if (typeRef != null) {
1099                                         start = this.tagSourceEnd + 1;
1100                                         previousPosition = start;
1101                                         typeRef = null;
1102                                 }
1103                                 // verify end line (expecting empty or end comment)
1104                                 if (verifyEndLine(previousPosition)) {
1105                                         return true;
1106                                 }
1107                                 if (this.sourceParser != null)
1108                                         this.sourceParser.problemReporter()
1109                                                         .javadocInvalidSeeReference(start, this.lineEnd);
1110                                 return false;
1111                         case LESS: // @see "<a href="URL#Value">label</a>
1112                                 consumeToken();
1113                                 start = this.scanner.getCurrentTokenStartPosition();
1114                                 if (parseHref()) {
1115                                         consumeToken();
1116                                         // If typeRef != null we may raise a warning here to let
1117                                         // user know there's an unused reference...
1118                                         // Currently as javadoc 1.4.2 ignore it, we do the same (see
1119                                         // bug 69302)
1120                                         if (typeRef != null) {
1121                                                 start = this.tagSourceEnd + 1;
1122                                                 previousPosition = start;
1123                                                 typeRef = null;
1124                                         }
1125                                         // verify end line (expecting empty or end comment)
1126                                         if (verifyEndLine(previousPosition)) {
1127                                                 return true;
1128                                         }
1129                                         if (this.sourceParser != null)
1130                                                 this.sourceParser
1131                                                                 .problemReporter()
1132                                                                 .javadocInvalidSeeReference(start, this.lineEnd);
1133                                 }
1134                                 return false;
1135                         case ERROR:
1136                                 if (this.scanner.currentCharacter == '#') { // @see ...#member
1137                                         consumeToken();
1138                                         reference = parseMember(typeRef);
1139                                         if (reference != null) {
1140                                                 return pushSeeRef(reference, plain);
1141                                         }
1142                                         return false;
1143                                 }
1144                                 break nextToken;
1145                         case IDENTIFIER:
1146                                 if (typeRef == null) {
1147                                         typeRefStartPosition = this.scanner
1148                                                         .getCurrentTokenStartPosition();
1149                                         typeRef = parseQualifiedName(true);
1150                                         break;
1151                                 }
1152                                 break nextToken;
1153                         default:
1154                                 break nextToken;
1155                         }
1156                 }
1157
1158                 // Verify that we got a reference
1159                 if (reference == null)
1160                         reference = typeRef;
1161                 if (reference == null) {
1162                         this.index = this.tokenPreviousPosition;
1163                         this.scanner.currentPosition = this.tokenPreviousPosition;
1164                         this.currentTokenType = ITerminalSymbols.TokenName.NONE;
1165                         if (this.sourceParser != null)
1166                                 this.sourceParser.problemReporter().javadocMissingSeeReference(
1167                                                 this.tagSourceStart, this.tagSourceEnd);
1168                         return false;
1169                 }
1170
1171                 // Reset position at the end of type reference
1172                 this.index = this.lastIdentifierEndPosition + 1;
1173                 this.scanner.currentPosition = this.index;
1174                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
1175
1176                 // Verify that line end does not start with an open parenthese (which
1177                 // could be a constructor reference wrongly written...)
1178                 // See bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=47215
1179                 char ch = peekChar();
1180                 if (ch == '(') {
1181                         if (this.sourceParser != null)
1182                                 this.sourceParser.problemReporter().javadocInvalidSeeReference(
1183                                                 typeRefStartPosition, this.lineEnd);
1184                         return false;
1185                 }
1186
1187                 // Verify that we get white space after reference
1188                 if (!verifySpaceOrEndComment()) {
1189                         this.index = this.tokenPreviousPosition;
1190                         this.scanner.currentPosition = this.tokenPreviousPosition;
1191                         this.currentTokenType = ITerminalSymbols.TokenName.NONE;
1192                         int end = this.starPosition == -1 ? this.lineEnd
1193                                         : this.starPosition;
1194                         if (this.source[end] == '\n')
1195                                 end--;
1196                         if (this.sourceParser != null)
1197                                 this.sourceParser
1198                                                 .problemReporter()
1199                                                 .javadocMalformedSeeReference(typeRefStartPosition, end);
1200                         return false;
1201                 }
1202
1203                 // Everything is OK, store reference
1204                 return pushSeeRef(reference, plain);
1205         }
1206
1207         /*
1208          * Parse @return tag declaration
1209          */
1210         protected abstract boolean parseReturn();
1211
1212         /*
1213          * Parse
1214          * 
1215          * @see tag declaration
1216          */
1217         protected boolean parseSee(boolean plain) {
1218                 int start = this.scanner.currentPosition;
1219                 try {
1220                         return parseReference(plain);
1221                 } catch (InvalidInputException ex) {
1222                         if (this.sourceParser != null)
1223                                 this.sourceParser.problemReporter().javadocInvalidSeeReference(
1224                                                 start, getEndPosition());
1225                 }
1226                 // Reset position to avoid missing tokens when new line was encountered
1227                 this.index = this.tokenPreviousPosition;
1228                 this.scanner.currentPosition = this.tokenPreviousPosition;
1229                 this.currentTokenType = ITerminalSymbols.TokenName.NONE;
1230                 return false;
1231         }
1232
1233         /*
1234          * Parse @return tag declaration
1235          */
1236         protected abstract boolean parseTag();
1237
1238         /*
1239          * Parse @throws tag declaration
1240          */
1241         protected boolean parseThrows(boolean real) {
1242                 int start = this.scanner.currentPosition;
1243                 try {
1244                         Object typeRef = parseQualifiedName(true);
1245                         if (typeRef == null) {
1246                                 if (this.sourceParser != null)
1247                                         this.sourceParser.problemReporter()
1248                                                         .javadocMissingThrowsClassName(this.tagSourceStart,
1249                                                                         this.tagSourceEnd);
1250                         } else {
1251                                 return pushThrowName(typeRef, real);
1252                         }
1253                 } catch (InvalidInputException ex) {
1254                         if (this.sourceParser != null)
1255                                 this.sourceParser.problemReporter().javadocInvalidThrowsClass(
1256                                                 start, getEndPosition());
1257                 }
1258                 return false;
1259         }
1260
1261         /*
1262          * Return current character without move index position.
1263          */
1264         private char peekChar() {
1265                 int idx = this.index;
1266                 char c = this.source[idx++];
1267                 if (c == '\\' && this.source[idx] == 'u') {
1268                         int c1, c2, c3, c4;
1269                         idx++;
1270                         while (this.source[idx] == 'u')
1271                                 idx++;
1272                         if (!(((c1 = Character.getNumericValue(this.source[idx++])) > 15 || c1 < 0)
1273                                         || ((c2 = Character.getNumericValue(this.source[idx++])) > 15 || c2 < 0)
1274                                         || ((c3 = Character.getNumericValue(this.source[idx++])) > 15 || c3 < 0) || ((c4 = Character
1275                                         .getNumericValue(this.source[idx++])) > 15 || c4 < 0))) {
1276                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1277                         }
1278                 }
1279                 return c;
1280         }
1281
1282         /*
1283          * push the consumeToken on the identifier stack. Increase the total number
1284          * of identifier in the stack.
1285          */
1286         protected void pushIdentifier(boolean newLength) {
1287
1288                 int stackLength = this.identifierStack.length;
1289                 if (++this.identifierPtr >= stackLength) {
1290                         System.arraycopy(this.identifierStack, 0,
1291                                         this.identifierStack = new char[stackLength + 10][], 0,
1292                                         stackLength);
1293                         System.arraycopy(this.identifierPositionStack, 0,
1294                                         this.identifierPositionStack = new long[stackLength + 10],
1295                                         0, stackLength);
1296                 }
1297                 this.identifierStack[this.identifierPtr] = this.scanner
1298                                 .getCurrentIdentifierSource();
1299                 this.identifierPositionStack[this.identifierPtr] = (((long) this.scanner.startPosition) << 32)
1300                                 + (this.scanner.currentPosition - 1);
1301
1302                 if (newLength) {
1303                         stackLength = this.identifierLengthStack.length;
1304                         if (++this.identifierLengthPtr >= stackLength) {
1305                                 System.arraycopy(this.identifierLengthStack, 0,
1306                                                 this.identifierLengthStack = new int[stackLength + 10],
1307                                                 0, stackLength);
1308                         }
1309                         this.identifierLengthStack[this.identifierLengthPtr] = 1;
1310                 } else {
1311                         this.identifierLengthStack[this.identifierLengthPtr]++;
1312                 }
1313         }
1314
1315         /*
1316          * Add a new obj on top of the ast stack. If new length is required, then
1317          * add also a new length in length stack.
1318          */
1319         protected void pushOnAstStack(Object node, boolean newLength) {
1320
1321                 if (node == null) {
1322                         this.astLengthStack[++this.astLengthPtr] = 0;
1323                         return;
1324                 }
1325
1326                 int stackLength = this.astStack.length;
1327                 if (++this.astPtr >= stackLength) {
1328                         System
1329                                         .arraycopy(this.astStack, 0,
1330                                                         this.astStack = new Object[stackLength
1331                                                                         + AstStackIncrement], 0, stackLength);
1332                         this.astPtr = stackLength;
1333                 }
1334                 this.astStack[this.astPtr] = node;
1335
1336                 if (newLength) {
1337                         stackLength = this.astLengthStack.length;
1338                         if (++this.astLengthPtr >= stackLength) {
1339                                 System.arraycopy(this.astLengthStack, 0,
1340                                                 this.astLengthStack = new int[stackLength
1341                                                                 + AstStackIncrement], 0, stackLength);
1342                         }
1343                         this.astLengthStack[this.astLengthPtr] = 1;
1344                 } else {
1345                         this.astLengthStack[this.astLengthPtr]++;
1346                 }
1347         }
1348
1349         /*
1350          * Push a param name in ast node stack.
1351          */
1352         protected abstract boolean pushParamName();
1353
1354         /*
1355          * Push a reference statement in ast node stack.
1356          */
1357         protected abstract boolean pushSeeRef(Object statement, boolean plain);
1358
1359         /*
1360          * Push a text element in ast node stack
1361          */
1362         protected abstract void pushText(int start, int end);
1363
1364         /*
1365          * Push a throws type ref in ast node stack.
1366          */
1367         protected abstract boolean pushThrowName(Object typeRef, boolean real);
1368
1369         /*
1370          * Read current character and move index position. Warning: scanner position
1371          * is unchanged using this method!
1372          */
1373         protected char readChar() {
1374
1375                 char c = this.source[this.index++];
1376                 if (c == '\\' && this.source[this.index] == 'u') {
1377                         int c1, c2, c3, c4;
1378                         int pos = this.index;
1379                         this.index++;
1380                         while (this.source[this.index] == 'u')
1381                                 this.index++;
1382                         if (!(((c1 = Character.getNumericValue(this.source[this.index++])) > 15 || c1 < 0)
1383                                         || ((c2 = Character
1384                                                         .getNumericValue(this.source[this.index++])) > 15 || c2 < 0)
1385                                         || ((c3 = Character
1386                                                         .getNumericValue(this.source[this.index++])) > 15 || c3 < 0) || ((c4 = Character
1387                                         .getNumericValue(this.source[this.index++])) > 15 || c4 < 0))) {
1388                                 c = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
1389                         } else {
1390                                 // TODO (frederic) currently reset to previous position, perhaps
1391                                 // signal a syntax error would be more appropriate
1392                                 this.index = pos;
1393                         }
1394                 }
1395                 return c;
1396         }
1397
1398         /*
1399          * Read token only if previous was consumed
1400          */
1401         private TokenName readToken() throws InvalidInputException {
1402                 if (this.currentTokenType.compareTo (ITerminalSymbols.TokenName.NONE) <= 0) {
1403                         this.tokenPreviousPosition = this.scanner.currentPosition;
1404                         this.currentTokenType = this.scanner.getNextToken();
1405                         if (this.scanner.currentPosition > (this.lineEnd + 1)) { // be
1406                                                                                                                                                 // sure
1407                                                                                                                                                 // to be
1408                                                                                                                                                 // on
1409                                                                                                                                                 // next
1410                                                                                                                                                 // line
1411                                                                                                                                                 // (lineEnd
1412                                                                                                                                                 // is
1413                                                                                                                                                 // still
1414                                                                                                                                                 // on
1415                                                                                                                                                 // the
1416                                                                                                                                                 // same
1417                                                                                                                                                 // line)
1418                                 this.lineStarted = false;
1419                                 while (this.currentTokenType == ITerminalSymbols.TokenName.MULTIPLY) {
1420                                         this.currentTokenType = this.scanner.getNextToken();
1421                                 }
1422                         }
1423                         this.index = this.scanner.currentPosition;
1424                         this.lineStarted = true; // after having read a token, line is
1425                                                                                 // obviously started...
1426                 }
1427                 return this.currentTokenType;
1428         }
1429
1430         private TokenName readTokenAndConsume() throws InvalidInputException {
1431                 TokenName token = readToken();
1432                 consumeToken();
1433                 return token;
1434         }
1435
1436         /*
1437          * Refresh start position and length of an inline tag.
1438          */
1439         protected void refreshInlineTagPosition(int previousPosition) {
1440                 // do nothing by default
1441         }
1442
1443         public String toString() {
1444                 StringBuffer buffer = new StringBuffer();
1445                 int startPos = this.scanner.currentPosition < this.index ? this.scanner.currentPosition
1446                                 : this.index;
1447                 int endPos = this.scanner.currentPosition < this.index ? this.index
1448                                 : this.scanner.currentPosition;
1449                 if (startPos == this.source.length)
1450                         return "EOF\n\n" + new String(this.source); //$NON-NLS-1$
1451                 if (endPos > this.source.length)
1452                         return "behind the EOF\n\n" + new String(this.source); //$NON-NLS-1$
1453
1454                 char front[] = new char[startPos];
1455                 System.arraycopy(this.source, 0, front, 0, startPos);
1456
1457                 int middleLength = (endPos - 1) - startPos + 1;
1458                 char middle[];
1459                 if (middleLength > -1) {
1460                         middle = new char[middleLength];
1461                         System.arraycopy(this.source, startPos, middle, 0, middleLength);
1462                 } else {
1463                         middle = CharOperation.NO_CHAR;
1464                 }
1465
1466                 char end[] = new char[this.source.length - (endPos - 1)];
1467                 System.arraycopy(this.source, (endPos - 1) + 1, end, 0,
1468                                 this.source.length - (endPos - 1) - 1);
1469
1470                 buffer.append(front);
1471                 if (this.scanner.currentPosition < this.index) {
1472                         buffer
1473                                         .append("\n===============================\nScanner current position here -->"); //$NON-NLS-1$
1474                 } else {
1475                         buffer
1476                                         .append("\n===============================\nParser index here -->"); //$NON-NLS-1$
1477                 }
1478                 buffer.append(middle);
1479                 if (this.scanner.currentPosition < this.index) {
1480                         buffer
1481                                         .append("<-- Parser index here\n===============================\n"); //$NON-NLS-1$
1482                 } else {
1483                         buffer
1484                                         .append("<-- Scanner current position here\n===============================\n"); //$NON-NLS-1$
1485                 }
1486                 buffer.append(end);
1487
1488                 return buffer.toString();
1489         }
1490
1491         /*
1492          * Update
1493          */
1494         protected abstract void updateDocComment();
1495
1496         /*
1497          * Update line end
1498          */
1499         protected void updateLineEnd() {
1500                 while (this.index > (this.lineEnd + 1)) { // be sure to be on next
1501                                                                                                         // line (lineEnd is still on
1502                                                                                                         // the same line)
1503                         if (this.linePtr < this.lastLinePtr) {
1504                                 this.lineEnd = this.scanner.getLineEnd(++this.linePtr) - 1;
1505                         } else {
1506                                 this.lineEnd = this.endComment;
1507                                 return;
1508                         }
1509                 }
1510         }
1511
1512         /*
1513          * Verify that end of the line only contains space characters or end of
1514          * comment. Note that end of comment may be preceeding by several contiguous
1515          * '*' chars.
1516          */
1517         private boolean verifyEndLine(int textPosition) {
1518                 int startPosition = this.index;
1519                 int previousPosition = this.index;
1520                 this.starPosition = -1;
1521                 char ch = readChar();
1522                 nextChar: while (true) {
1523                         switch (ch) {
1524                         case '\r':
1525                         case '\n':
1526                                 if (this.kind == DOM_PARSER) {
1527                                         parseTag();
1528                                         pushText(textPosition, previousPosition);
1529                                 }
1530                                 this.index = previousPosition;
1531                                 return true;
1532                         case '\u000c': /* FORM FEED */
1533                         case ' ': /* SPACE */
1534                         case '\t': /* HORIZONTAL TABULATION */
1535                                 if (this.starPosition >= 0)
1536                                         break nextChar;
1537                                 break;
1538                         case '*':
1539                                 this.starPosition = previousPosition;
1540                                 break;
1541                         case '/':
1542                                 if (this.starPosition >= textPosition) {
1543                                         if (this.kind == DOM_PARSER) {
1544                                                 parseTag();
1545                                                 pushText(textPosition, this.starPosition);
1546                                         }
1547                                         return true;
1548                                 }
1549                         default:
1550                                 // leave loop
1551                                 break nextChar;
1552
1553                         }
1554                         previousPosition = this.index;
1555                         ch = readChar();
1556                 }
1557                 this.index = startPosition;
1558                 return false;
1559         }
1560
1561         /*
1562          * Verify that some text exists after a @return tag. Text must be different
1563          * than end of comment which may be preceeding by several '*' chars.
1564          */
1565         private boolean verifyCharsAfterReturnTag(int startPosition) {
1566                 // Whitespace or inline tag closing brace
1567                 int previousPosition = this.index;
1568                 char ch = readChar();
1569                 boolean malformed = true;
1570                 while (Character.isWhitespace(ch)) {
1571                         malformed = false;
1572                         previousPosition = this.index;
1573                         ch = readChar();
1574                 }
1575                 // End of comment
1576                 this.starPosition = -1;
1577                 nextChar: while (this.index < this.source.length) {
1578                         switch (ch) {
1579                         case '*':
1580                                 // valid whatever the number of star before last '/'
1581                                 this.starPosition = previousPosition;
1582                                 break;
1583                         case '/':
1584                                 if (this.starPosition >= startPosition) { // valid only if a
1585                                                                                                                         // star was previous
1586                                                                                                                         // character
1587                                         return false;
1588                                 }
1589                         default:
1590                                 // valid if any other character is encountered, even white
1591                                 // spaces
1592                                 this.index = startPosition;
1593                                 return !malformed;
1594
1595                         }
1596                         previousPosition = this.index;
1597                         ch = readChar();
1598                 }
1599                 this.index = startPosition;
1600                 return false;
1601         }
1602
1603         /*
1604          * Verify characters after a name matches one of following conditions: 1-
1605          * first character is a white space 2- first character is a closing brace
1606          * *and* we're currently parsing an inline tag 3- are the end of comment
1607          * (several contiguous star ('*') characters may be found before the last
1608          * slash ('/') character).
1609          */
1610         private boolean verifySpaceOrEndComment() {
1611                 int startPosition = this.index;
1612                 // Whitespace or inline tag closing brace
1613                 char ch = peekChar();
1614                 switch (ch) {
1615                 case '}':
1616                         return this.inlineTagStarted;
1617                 default:
1618                         if (Character.isWhitespace(ch)) {
1619                                 return true;
1620                         }
1621                 }
1622                 // End of comment
1623                 int previousPosition = this.index;
1624                 this.starPosition = -1;
1625                 ch = readChar();
1626                 nextChar: while (this.index < this.source.length) {
1627                         switch (ch) {
1628                         case '*':
1629                                 // valid whatever the number of star before last '/'
1630                                 this.starPosition = previousPosition;
1631                                 break;
1632                         case '/':
1633                                 if (this.starPosition >= startPosition) { // valid only if a
1634                                                                                                                         // star was previous
1635                                                                                                                         // character
1636                                         return true;
1637                                 }
1638                         default:
1639                                 // invalid whatever other character, even white spaces
1640                                 this.index = startPosition;
1641                                 return false;
1642
1643                         }
1644                         previousPosition = this.index;
1645                         ch = readChar();
1646                 }
1647                 this.index = startPosition;
1648                 return false;
1649         }
1650 }