net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/ui/text/FastJavaPartitionScanner.java

   1 package net.sourceforge.phpdt.internal.ui.text;
   2
   3 /*
   4  * (c) Copyright IBM Corp. 2000, 2001.
   5  * All Rights Reserved.
   6  */
   7
   8 import org.eclipse.jface.text.IDocument;
   9 import org.eclipse.jface.text.rules.ICharacterScanner;
  10 import org.eclipse.jface.text.rules.IPartitionTokenScanner;
  11 import org.eclipse.jface.text.rules.IToken;
  12 import org.eclipse.jface.text.rules.Token;
  13
  14
  15 /**
  16  * This scanner recognizes the JavaDoc comments, Java multi line comments, Java single line comments,
  17  * Java strings and Java characters.
  18  */
  19 public class FastJavaPartitionScanner implements IPartitionTokenScanner {
  20
  21         private final static String SKIP= "__skip"; //$NON-NLS-1$
  22         public final static String JAVA_STRING= "__php_string"; //$NON-NLS-1$
  23         public final static String JAVA_SINGLE_LINE_COMMENT= "__php_singleline_comment"; //$NON-NLS-1$
  24         public final static String JAVA_MULTI_LINE_COMMENT= "__php_multiline_comment"; //$NON-NLS-1$
  25         public final static String JAVA_DOC= "__php_phpdoc"; //$NON-NLS-1$
  26
  27         // states
  28         private static final int JAVA= 0;
  29         private static final int SINGLE_LINE_COMMENT= 1;
  30         private static final int MULTI_LINE_COMMENT= 2;
  31         private static final int JAVADOC= 3;
  32         private static final int CHARACTER= 4;
  33         private static final int STRING= 5;
  34
  35         // beginning of prefixes and postfixes
  36         private static final int NONE= 0;
  37         private static final int BACKSLASH= 1; // postfix for STRING and CHARACTER
  38         private static final int SLASH= 2; // prefix for SINGLE_LINE or MULTI_LINE or JAVADOC
  39         private static final int SLASH_STAR= 3; // prefix for MULTI_LINE_COMMENT or JAVADOC
  40         private static final int SLASH_STAR_STAR= 4; // prefix for MULTI_LINE_COMMENT or JAVADOC
  41         private static final int STAR= 5; // postfix for MULTI_LINE_COMMENT or JAVADOC
  42         private static final int CARRIAGE_RETURN=6; // postfix for STRING, CHARACTER and SINGLE_LINE_COMMENT
  43
  44         /** The scanner. */
  45 //      private final BufferedRuleBasedScanner fScanner= new BufferedRuleBasedScanner(1000);
  46         private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);      // faster implementation
  47
  48         /** The offset of the last returned token. */
  49         private int fTokenOffset;
  50         /** The length of the last returned token. */
  51         private int fTokenLength;
  52
  53         /** The state of the scanner. */
  54         private int fState;
  55         /** The last significant characters read. */
  56         private int fLast;
  57         /** The amount of characters already read on first call to nextToken(). */
  58         private int fPrefixLength;
  59
  60         // emulate JavaPartitionScanner
  61         private static final boolean fgEmulate= false;
  62         private int fJavaOffset;
  63         private int fJavaLength;
  64
  65         private final IToken[] fTokens= new IToken[] {
  66                 new Token(null),
  67                 new Token(JAVA_SINGLE_LINE_COMMENT),
  68                 new Token(JAVA_MULTI_LINE_COMMENT),
  69                 new Token(JAVA_DOC),
  70                 new Token(SKIP),
  71                 new Token(JAVA_STRING)
  72         };
  73
  74         /*
  75          * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
  76          */
  77         public IToken nextToken() {
  78
  79                 // emulate JavaPartitionScanner
  80                 if (fgEmulate) {
  81                         if (fJavaOffset != -1 && fTokenOffset + fTokenLength != fJavaOffset + fJavaLength) {
  82                                 fTokenOffset += fTokenLength;
  83                                 return fTokens[JAVA];
  84                         } else {
  85                                 fJavaOffset= -1;
  86                                 fJavaLength= 0;
  87                         }
  88                 }
  89
  90                 fTokenOffset += fTokenLength;
  91                 fTokenLength= fPrefixLength;
  92
  93                 while (true) {
  94                         final int ch= fScanner.read();
  95
  96                         // characters
  97                         switch (ch) {
  98                         case ICharacterScanner.EOF:
  99                                 if (fTokenLength > 0) {
 100                                         fLast= NONE; // ignore last
 101                                         return preFix(fState, JAVA, NONE, 0);
 102
 103                                 } else {
 104                                         fLast= NONE;
 105                                         fPrefixLength= 0;
 106                                         return Token.EOF;
 107                                 }
 108
 109                         case '\r':
 110                                 // emulate JavaPartitionScanner
 111                                 if (!fgEmulate && fLast != CARRIAGE_RETURN) {
 112                                                 fLast= CARRIAGE_RETURN;
 113                                                 fTokenLength++;
 114                                                 continue;
 115
 116                                 } else {
 117
 118                                         switch (fState) {
 119                                         case SINGLE_LINE_COMMENT:
 120                                         case CHARACTER:
 121                                         case STRING:
 122                                                 if (fTokenLength > 0) {
 123                                                         IToken token= fTokens[fState];
 124
 125                                                         // emulate JavaPartitionScanner
 126                                                         if (fgEmulate) {
 127                                                                 fTokenLength++;
 128                                                                 fLast= NONE;
 129                                                                 fPrefixLength= 0;
 130                                                         } else {
 131                                                                 fLast= CARRIAGE_RETURN;
 132                                                                 fPrefixLength= 1;
 133                                                         }
 134
 135                                                         fState= JAVA;
 136                                                         return token;
 137
 138                                                 } else {
 139                                                         consume();
 140                                                         continue;
 141                                                 }
 142
 143                                         default:
 144                                                 consume();
 145                                                 continue;
 146                                         }
 147                                 }
 148
 149                         case '\n':
 150                                 switch (fState) {
 151                                 case SINGLE_LINE_COMMENT:
 152                                 case CHARACTER:
 153                                 case STRING:
 154                                         // assert(fTokenLength > 0);
 155                                         return postFix(fState);
 156
 157                                 default:
 158                                         consume();
 159                                         continue;
 160                                 }
 161
 162                         default:
 163                                 if (!fgEmulate && fLast == CARRIAGE_RETURN) {
 164                                         switch (fState) {
 165                                         case SINGLE_LINE_COMMENT:
 166                                         case CHARACTER:
 167                                         case STRING:
 168
 169                                                 int last;
 170                                                 int newState;
 171                                                 switch (ch) {
 172                                                 case '/':
 173                                                         last= SLASH;
 174                                                         newState= JAVA;
 175                                                         break;
 176
 177                                                 case '*':
 178                                                         last= STAR;
 179                                                         newState= JAVA;
 180                                                         break;
 181
 182                                                 case '\'':
 183                                                         last= NONE;
 184                                                         newState= CHARACTER;
 185                                                         break;
 186
 187                                                 case '"':
 188                                                         last= NONE;
 189                                                         newState= STRING;
 190                                                         break;
 191
 192                                                 case '\r':
 193                                                         last= CARRIAGE_RETURN;
 194                                                         newState= JAVA;
 195                                                         break;
 196
 197                                                 case '\\':
 198                                                         last= BACKSLASH;
 199                                                         newState= JAVA;
 200                                                         break;
 201
 202                                                 default:
 203                                                         last= NONE;
 204                                                         newState= JAVA;
 205                                                         break;
 206                                                 }
 207
 208                                                 fLast= NONE; // ignore fLast
 209                                                 return preFix(fState, newState, last, 1);
 210
 211                                         default:
 212                                                 break;
 213                                         }
 214                                 }
 215                         }
 216
 217                         // states
 218                         switch (fState) {
 219                         case JAVA:
 220                                 switch (ch) {
 221                                 case '/':
 222                                         if (fLast == SLASH) {
 223                                                 if (fTokenLength - getLastLength(fLast) > 0) {
 224                                                         return preFix(JAVA, SINGLE_LINE_COMMENT, NONE, 2);
 225                                                 } else {
 226                                                         preFix(JAVA, SINGLE_LINE_COMMENT, NONE, 2);
 227                                                         fTokenOffset += fTokenLength;
 228                                                         fTokenLength= fPrefixLength;
 229                                                         break;
 230                                                 }
 231
 232                                         } else {
 233                                                 fTokenLength++;
 234                                                 fLast= SLASH;
 235                                                 break;
 236                                         }
 237
 238                                 case '*':
 239                                         if (fLast == SLASH) {
 240                                                 if (fTokenLength - getLastLength(fLast) > 0)
 241                                                         return preFix(JAVA, MULTI_LINE_COMMENT, SLASH_STAR, 2);
 242                                                 else {
 243                                                         preFix(JAVA, MULTI_LINE_COMMENT, SLASH_STAR, 2);
 244                                                         fTokenOffset += fTokenLength;
 245                                                         fTokenLength= fPrefixLength;
 246                                                         break;
 247                                                 }
 248
 249                                         } else {
 250                                                 consume();
 251                                                 break;
 252                                         }
 253
 254                                 case '\'':
 255                                         fLast= NONE; // ignore fLast
 256                                         if (fTokenLength > 0)
 257                                                 return preFix(JAVA, CHARACTER, NONE, 1);
 258                                         else {
 259                                                 preFix(JAVA, CHARACTER, NONE, 1);
 260                                                 fTokenOffset += fTokenLength;
 261                                                 fTokenLength= fPrefixLength;
 262                                                 break;
 263                                         }
 264
 265                                 case '"':
 266                                         fLast= NONE; // ignore fLast
 267                                         if (fTokenLength > 0)
 268                                                 return preFix(JAVA, STRING, NONE, 1);
 269                                         else {
 270                                                 preFix(JAVA, STRING, NONE, 1);
 271                                                 fTokenOffset += fTokenLength;
 272                                                 fTokenLength= fPrefixLength;
 273                                                 break;
 274                                         }
 275
 276                                 default:
 277                                         consume();
 278                                         break;
 279                                 }
 280                                 break;
 281
 282                         case SINGLE_LINE_COMMENT:
 283                                 consume();
 284                                 break;
 285
 286                         case JAVADOC:
 287                                 switch (ch) {
 288                                 case '/':
 289                                         switch (fLast) {
 290                                         case SLASH_STAR_STAR:
 291                                                 return postFix(MULTI_LINE_COMMENT);
 292
 293                                         case STAR:
 294                                                 return postFix(JAVADOC);
 295
 296                                         default:
 297                                                 consume();
 298                                                 break;
 299                                         }
 300                                         break;
 301
 302                                 case '*':
 303                                         fTokenLength++;
 304                                         fLast= STAR;
 305                                         break;
 306
 307                                 default:
 308                                         consume();
 309                                         break;
 310                                 }
 311                                 break;
 312
 313                         case MULTI_LINE_COMMENT:
 314                                 switch (ch) {
 315                                 case '*':
 316                                         if (fLast == SLASH_STAR) {
 317                                                 fLast= SLASH_STAR_STAR;
 318                                                 fTokenLength++;
 319                                                 fState= JAVADOC;
 320                                         } else {
 321                                                 fTokenLength++;
 322                                                 fLast= STAR;
 323                                         }
 324                                         break;
 325
 326                                 case '/':
 327                                         if (fLast == STAR) {
 328                                                 return postFix(MULTI_LINE_COMMENT);
 329                                         } else {
 330                                                 consume();
 331                                                 break;
 332                                         }
 333
 334                                 default:
 335                                         consume();
 336                                         break;
 337                                 }
 338                                 break;
 339
 340                         case STRING:
 341                                 switch (ch) {
 342                                 case '\\':
 343                                         fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
 344                                         fTokenLength++;
 345                                         break;
 346
 347                                 case '\"':
 348                                         if (fLast != BACKSLASH) {
 349                                                 return postFix(STRING);
 350
 351                                         } else {
 352                                                 consume();
 353                                                 break;
 354                                         }
 355
 356                                 default:
 357                                         consume();
 358                                         break;
 359                                 }
 360                                 break;
 361
 362                         case CHARACTER:
 363                                 switch (ch) {
 364                                 case '\\':
 365                                         fLast= (fLast == BACKSLASH) ? NONE : BACKSLASH;
 366                                         fTokenLength++;
 367                                         break;
 368
 369                                 case '\'':
 370                                         if (fLast != BACKSLASH) {
 371                                                 return postFix(CHARACTER);
 372
 373                                         } else {
 374                                                 consume();
 375                                                 break;
 376                                         }
 377
 378                                 default:
 379                                         consume();
 380                                         break;
 381                                 }
 382                                 break;
 383                         }
 384                 }
 385         }
 386
 387         private static final int getLastLength(int last) {
 388                 switch (last) {
 389                 default:
 390                         return -1;
 391
 392                 case NONE:
 393                         return 0;
 394
 395                 case CARRIAGE_RETURN:
 396                 case BACKSLASH:
 397                 case SLASH:
 398                 case STAR:
 399                         return 1;
 400
 401                 case SLASH_STAR:
 402                         return 2;
 403
 404                 case SLASH_STAR_STAR:
 405                         return 3;
 406                 }
 407         }
 408
 409         private final void consume() {
 410                 fTokenLength++;
 411                 fLast= NONE;
 412         }
 413
 414         private final IToken postFix(int state) {
 415                 fTokenLength++;
 416                 fLast= NONE;
 417                 fState= JAVA;
 418                 fPrefixLength= 0;
 419                 return fTokens[state];
 420         }
 421
 422         private final IToken preFix(int state, int newState, int last, int prefixLength) {
 423                 // emulate JavaPartitionScanner
 424                 if (fgEmulate && state == JAVA && (fTokenLength - getLastLength(fLast) > 0)) {
 425                         fTokenLength -= getLastLength(fLast);
 426                         fJavaOffset= fTokenOffset;
 427                         fJavaLength= fTokenLength;
 428                         fTokenLength= 1;
 429                         fState= newState;
 430                         fPrefixLength= prefixLength;
 431                         fLast= last;
 432                         return fTokens[state];
 433
 434                 } else {
 435                         fTokenLength -= getLastLength(fLast);
 436                         fLast= last;
 437                         fPrefixLength= prefixLength;
 438                         IToken token= fTokens[state];
 439                         fState= newState;
 440                         return token;
 441                 }
 442         }
 443
 444         private static int getState(String contentType) {
 445
 446                 if (contentType == null)
 447                         return JAVA;
 448
 449                 else if (contentType.equals(JAVA_SINGLE_LINE_COMMENT))
 450                         return SINGLE_LINE_COMMENT;
 451
 452                 else if (contentType.equals(JAVA_MULTI_LINE_COMMENT))
 453                         return MULTI_LINE_COMMENT;
 454
 455                 else if (contentType.equals(JAVA_DOC))
 456                         return JAVADOC;
 457
 458                 else if (contentType.equals(JAVA_STRING))
 459                         return STRING;
 460
 461                 else if (contentType.equals(SKIP))
 462                         return CHARACTER;
 463
 464                 else
 465                         return JAVA;
 466         }
 467
 468         /*
 469          * @see IPartitionTokenScanner#setPartialRange(IDocument, int, int, String, int)
 470          */
 471         public void setPartialRange(IDocument document, int offset, int length, String contentType, int partitionOffset) {
 472
 473                 fScanner.setRange(document, offset, length);
 474                 fTokenOffset= partitionOffset;
 475                 fTokenLength= 0;
 476                 fPrefixLength= offset - partitionOffset;
 477                 fLast= NONE;
 478
 479                 if (offset == partitionOffset) {
 480                         // restart at beginning of partition
 481                         fState= JAVA;
 482                 } else {
 483                         fState= getState(contentType);
 484                 }
 485
 486                 // emulate JavaPartitionScanner
 487                 if (fgEmulate) {
 488                         fJavaOffset= -1;
 489                         fJavaLength= 0;
 490                 }
 491         }
 492
 493         /*
 494          * @see ITokenScanner#setRange(IDocument, int, int)
 495          */
 496         public void setRange(IDocument document, int offset, int length) {
 497
 498                 fScanner.setRange(document, offset, length);
 499                 fTokenOffset= offset;
 500                 fTokenLength= 0;
 501                 fPrefixLength= 0;
 502                 fLast= NONE;
 503                 fState= JAVA;
 504
 505                 // emulate JavaPartitionScanner
 506                 if (fgEmulate) {
 507                         fJavaOffset= -1;
 508                         fJavaLength= 0;
 509                 }
 510         }
 511
 512         /*
 513          * @see ITokenScanner#getTokenLength()
 514          */
 515         public int getTokenLength() {
 516                 return fTokenLength;
 517         }
 518
 519         /*
 520          * @see ITokenScanner#getTokenOffset()
 521          */
 522         public int getTokenOffset() {
 523                 return fTokenOffset;
 524         }
 525
 526 }