net.sourceforge.phpeclipse/src/net/sourceforge/phpdt/internal/ui/text/FastJavaPartitionScanner.java

   1 /*******************************************************************************
   2  * Copyright (c) 2000, 2004 IBM Corporation and others.
   3  * All rights reserved. This program and the accompanying materials
   4  * are made available under the terms of the Common Public License v1.0
   5  * which accompanies this distribution, and is available at
   6  * http://www.eclipse.org/legal/cpl-v10.html
   7  *
   8  * Contributors:
   9  *     IBM Corporation - initial API and implementation
  10  *******************************************************************************/
  11 package net.sourceforge.phpdt.internal.ui.text;
  12
  13 import net.sourceforge.phpeclipse.ui.text.rules.AbstractPartitioner;
  14
  15 //incastrix
  16 //import org.eclipse.jface.text.Assert;
  17 import org.eclipse.core.runtime.Assert;
  18 import org.eclipse.jface.text.IDocument;
  19 import org.eclipse.jface.text.rules.ICharacterScanner;
  20 import org.eclipse.jface.text.rules.IPartitionTokenScanner;
  21 import org.eclipse.jface.text.rules.IToken;
  22 import org.eclipse.jface.text.rules.Token;
  23
  24 /**
  25  * This scanner recognizes the JavaDoc comments, Java multi line comments, Java
  26  * single line comments, Java strings.
  27  */
  28 public class FastJavaPartitionScanner implements IPartitionTokenScanner,
  29                 IPHPPartitions {
  30
  31         // states
  32
  33         private static enum PartState {
  34                 PHP,
  35                 SINGLE_LINE_COMMENT,
  36                 MULTI_LINE_COMMENT,
  37                 PHPDOC,
  38                 STRING_DQ,
  39                 STRING_SQ,
  40                 STRING_HEREDOC,
  41         };
  42
  43         private static enum ScanState {
  44                 NONE,
  45                 BACKSLASH,                                                                      // postfix for STRING_DQ and CHARACTER
  46                 SLASH,                                                                          // prefix for SINGLE_LINE or MULTI_LINE or JAVADOC
  47                 SLASH_STAR,                                                                     // prefix for MULTI_LINE_COMMENT or JAVADOC
  48                 SLASH_STAR_STAR,                                                        // prefix for MULTI_LINE_COMMENT or JAVADOC
  49                 STAR,                                                                           // postfix for MULTI_LINE_COMMENT or JAVADOC
  50                 CARRIAGE_RETURN,                                                        // postfix for STRING_DQ, CHARACTER and SINGLE_LINE_COMMENT
  51                 LESS,                                                                           // Found a '<'
  52                 LESS_LESS,                                                                      // Found a '<<'
  53                 LESS_LESS_LESS,                                                         // Found a '<<<'
  54                 HEREDOC_ID,                                                                     // Found a '<<<' and scanning ID (till and of id, which is a ' ')
  55                 HEREDOC,                                                                        // Found a '<<<' and ID
  56                 HEREDOC_ID_END,                                                         // Searching the heredoc end ID
  57         };
  58
  59         /** The heredoc ID string */
  60         private String fHeredocId;
  61
  62         /** The possible heredoc ID string which is read right after a new line. Ends with a ';' and should
  63          * match the heredoc ID string fHeredocId
  64          */
  65         private String fHeredocIdEnd;
  66
  67         /** The scanner. */
  68         private final BufferedDocumentScanner fScanner = new BufferedDocumentScanner (1000); // faster implementation
  69
  70         /** The offset of the last returned token. */
  71         private int fTokenOffset;
  72
  73         /** The length of the last returned token. */
  74         private int fTokenLength;
  75
  76         /** The state of the scanner. */
  77         private PartState fState;
  78
  79         /** The last significant characters read. */
  80         private ScanState fLast;
  81
  82         /** The amount of characters already read on first call to nextToken(). */
  83         private int fPrefixLength;
  84
  85         // emulate JavaPartitionScanner
  86         private boolean fEmulate = false;
  87
  88         private int fJavaOffset;
  89
  90         private int fJavaLength;
  91
  92         private final IToken[] fTokens = new IToken[] {
  93                         new Token (null),
  94                         new Token (PHP_SINGLELINE_COMMENT),
  95                         new Token (PHP_MULTILINE_COMMENT),
  96                         new Token (PHP_PHPDOC_COMMENT),
  97                         new Token (PHP_STRING_DQ),
  98                         new Token (PHP_STRING_SQ),
  99                         new Token (PHP_STRING_HEREDOC)};
 100
 101         public FastJavaPartitionScanner(boolean emulate) {
 102                 fEmulate = emulate;
 103         }
 104
 105         public FastJavaPartitionScanner() {
 106                 this(false);
 107         }
 108
 109         /**
 110          * Emulate JavaPartitionScanner
 111          *
 112          * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
 113          */
 114         public IToken nextToken() {
 115                 if (fEmulate) {
 116                         if ((fJavaOffset != -1) && (fTokenOffset + fTokenLength != fJavaOffset + fJavaLength)) {
 117                                 fTokenOffset += fTokenLength;
 118
 119                                 return fTokens[PartState.PHP.ordinal()];
 120                         }
 121                         else {
 122                                 fJavaOffset = -1;
 123                                 fJavaLength = 0;
 124                         }
 125                 }
 126
 127                 fTokenOffset += fTokenLength;                           // The new token offset is the offset of the previous partition + length of previous partition
 128                 fTokenLength  = fPrefixLength;                          // The new partition is at least the length of the start of the new partition
 129
 130                 while (true) {
 131                         final int ch = fScanner.read();
 132
 133                         switch (ch) {
 134                                 case ICharacterScanner.EOF:
 135                                         if (fTokenLength > 0) {
 136                                                 fLast = ScanState.NONE; // ignore last
 137                                                 return preFix (fState, PartState.PHP, ScanState.NONE, 0);
 138                                         }
 139                                         else {
 140                                                 fLast = ScanState.NONE;
 141                                                 fPrefixLength = 0;
 142                                                 return Token.EOF;
 143                                         }
 144
 145                                 case '\r':                                                                              // Found a carriage return
 146                                         // emulate JavaPartitionScanner
 147                                         if (!fEmulate && (fLast != ScanState.CARRIAGE_RETURN)) {
 148                                                 fLast = ScanState.CARRIAGE_RETURN;              // Set to what we currently found
 149                                                 fTokenLength++;                                                 // and count the partition length
 150
 151                                                 continue;                                                               // Go for the next character to read
 152                                         }
 153                                         else {
 154                                                 switch (fState) {
 155                                                         case SINGLE_LINE_COMMENT:
 156                                                                 if (fTokenLength > 0) {
 157                                                                         IToken token = fTokens[fState.ordinal()];
 158
 159                                                                         // emulate JavaPartitionScanner
 160                                                                         if (fEmulate) {
 161                                                                                 fTokenLength++;
 162                                                                                 fLast = ScanState.NONE;
 163                                                                                 fPrefixLength = 0;
 164                                                                         }
 165                                                                         else {
 166                                                                                 fLast = ScanState.CARRIAGE_RETURN;
 167                                                                                 fPrefixLength = 1;
 168                                                                         }
 169
 170                                                                         fState = PartState.PHP;
 171                                                                         return token;
 172                                                                 }
 173                                                                 else {
 174                                                                         consume();
 175                                                                         continue;
 176                                                                 }
 177
 178                                                         default:
 179                                                                 consume();
 180                                                                 continue;
 181                                                 }
 182                                         }
 183
 184                                 case '\n':                                                                                      // Found a line feed
 185                                         switch (fState) {
 186                                                 case SINGLE_LINE_COMMENT:                                       // If we running within a single line comment,
 187                                                         return postFix (fState);                                // this is the end my friend
 188
 189                                                 case STRING_HEREDOC:                                            // If we running within a heredoc string
 190                                                         fTokenLength++;                                                 // Count the character
 191                                                         fLast = ScanState.CARRIAGE_RETURN;              // and state is still new line
 192                                                         continue;
 193
 194                                                 default:                                                                        // If running anywhere else than on a single line comment
 195                                                         consume();                                                              // count the length of the current partition
 196                                                         continue;
 197                                         }
 198
 199                                 case '?':
 200                                         if (fState == PartState.SINGLE_LINE_COMMENT) {
 201                                                 int nextch = fScanner.read();
 202
 203                                                 if (nextch == '>') {
 204                                                         // <h1>This is an <?php # echo 'simple' ?> example.</h1>
 205                                                         fTokenLength--;
 206                                                         fScanner.unread();
 207                                                         fScanner.unread();
 208
 209                                                         return postFix (fState);
 210                                                 }
 211                                                 else {
 212                                                         // bug #1404228: Crash on <?php // comment ?>
 213                                                         if (nextch != ICharacterScanner.EOF) {
 214                                                                 fScanner.unread();
 215                                                         }
 216                                                 }
 217                                         }
 218
 219                                 default:
 220                                         if (!fEmulate && (fLast == ScanState.CARRIAGE_RETURN)) {
 221                                                 switch (fState) {
 222                                                         case SINGLE_LINE_COMMENT:
 223                                                                 // case CHARACTER:
 224                                                                 // case STRING_DQ:
 225                                                                 // case STRING_SQ:
 226                                                                 ScanState last;
 227                                                                 PartState newState;
 228
 229                                                                 switch (ch) {
 230                                                                         case '/':
 231                                                                                 last = ScanState.SLASH;
 232                                                                                 newState = PartState.PHP;
 233                                                                                 break;
 234
 235                                                                         case '*':
 236                                                                                 last = ScanState.STAR;
 237                                                                                 newState = PartState.PHP;
 238                                                                                 break;
 239
 240                                                                         case '\'':
 241                                                                                 last = ScanState.NONE;
 242                                                                                 newState = PartState.STRING_SQ;
 243                                                                                 break;
 244
 245                                                                         case '"':
 246                                                                                 last = ScanState.NONE;
 247                                                                                 newState = PartState.STRING_DQ;
 248                                                                                 break;
 249
 250                                                                         case '\r':
 251                                                                                 last = ScanState.CARRIAGE_RETURN;
 252                                                                                 newState = PartState.PHP;
 253                                                                                 break;
 254
 255                                                                         case '\\':
 256                                                                                 last = ScanState.BACKSLASH;
 257                                                                                 newState = PartState.PHP;
 258                                                                                 break;
 259
 260                                                                         default:
 261                                                                                 last = ScanState.NONE;
 262                                                                                 newState = PartState.PHP;
 263                                                                                 break;
 264                                                                 }
 265
 266                                                                 fLast = ScanState.NONE; // ignore fLast
 267                                                                 return preFix (fState, newState, last, 1);
 268
 269                                                         default:
 270                                                                 break;
 271                                                 }
 272                                         }
 273                         }
 274
 275                         // states
 276                         switch (fState) {
 277                                 case PHP:
 278                                         switch (ch) {
 279                                                 case '#':                                                       // Start of a single line comment
 280                                                         if (fTokenLength > 0) {
 281                                                                 return preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 1);
 282                                                         }
 283                                                         else {
 284                                                                 preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 1);
 285                                                                 fTokenOffset += fTokenLength;
 286                                                                 fTokenLength = fPrefixLength;
 287                                                         }
 288                                                         break;
 289
 290                                                 case '<':
 291                                                         if (fLast == ScanState.LESS) {
 292                                                                 fTokenLength++;
 293                                                                 fLast = ScanState.LESS_LESS;
 294                                                         }
 295                                                         else if (fLast == ScanState.LESS_LESS) {
 296                                                                 if (fTokenLength - getLastLength(fLast) > 0) {  // this is the start of a single line comment
 297                                                                         return preFix (PartState.PHP, PartState.STRING_HEREDOC, ScanState.LESS_LESS_LESS, 3);
 298                                                                 }
 299                                                                 else {
 300                                                                         preFix (PartState.PHP, PartState.STRING_HEREDOC, ScanState.LESS_LESS_LESS, 3);
 301                                                                         fTokenOffset += fTokenLength;
 302                                                                         fTokenLength = fPrefixLength;
 303                                                                 }
 304                                                         }
 305                                                         else {
 306                                                                 fTokenLength++;
 307                                                                 fLast = ScanState.LESS;
 308                                                         }
 309                                                         break;
 310
 311                                                 case '/':                                                                                               // Start of single line comment?
 312                                                         if (fLast == ScanState.SLASH) {                                         // If previous character was already a slash,
 313                                                                 if (fTokenLength - getLastLength(fLast) > 0) {  // this is the start of a single line comment
 314                                                                         return preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 2);
 315                                                                 }
 316                                                                 else {
 317                                                                         preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 2);
 318                                                                         fTokenOffset += fTokenLength;
 319                                                                         fTokenLength = fPrefixLength;
 320                                                                 }
 321                                                         }
 322                                                         else {
 323                                                                 fTokenLength++;
 324                                                                 fLast = ScanState.SLASH;                                                // We currently found a slash
 325                                                         }
 326                                                         break;
 327
 328                                                 case '*':
 329                                                         if (fLast == ScanState.SLASH) {                                         // If previous character was a slash
 330                                                                 if (fTokenLength - getLastLength (fLast) > 0) { // this is the start of a comment /*
 331                                                                         return preFix (PartState.PHP, PartState.MULTI_LINE_COMMENT, ScanState.SLASH_STAR, 2);
 332                                                                 }
 333                                                                 else {
 334                                                                         preFix (PartState.PHP, PartState.MULTI_LINE_COMMENT, ScanState.SLASH_STAR, 2);
 335                                                                         fTokenOffset += fTokenLength;
 336                                                                         fTokenLength = fPrefixLength;
 337                                                                 }
 338                                                         }
 339                                                         else {                                                                                          // No slash before the '*', so it's a normal character
 340                                                                 consume ();
 341                                                         }
 342                                                         break;
 343
 344                                                 case '\'':                                                                                              // The start of a single quoted string
 345                                                         fLast = ScanState.NONE; // ignore fLast
 346
 347                                                         if (fTokenLength > 0) {
 348                                                                 return preFix (PartState.PHP, PartState.STRING_SQ, ScanState.NONE, 1);
 349                                                         }
 350                                                         else {
 351                                                                 preFix (PartState.PHP, PartState.STRING_SQ, ScanState.NONE, 1);
 352                                                                 fTokenOffset += fTokenLength;
 353                                                                 fTokenLength = fPrefixLength;
 354                                                         }
 355                                                         break;
 356
 357                                                 case '"':                                                                                               // The start of a double quoted string
 358                                                         fLast = ScanState.NONE; // ignore fLast
 359
 360                                                         if (fTokenLength > 0) {
 361                                                                 return preFix (PartState.PHP, PartState.STRING_DQ, ScanState.NONE, 1);
 362                                                         }
 363                                                         else {
 364                                                                 preFix (PartState.PHP, PartState.STRING_DQ, ScanState.NONE, 1);
 365                                                                 fTokenOffset += fTokenLength;
 366                                                                 fTokenLength = fPrefixLength;
 367                                                         }
 368                                                         break;
 369
 370                                                 default:                                                                                                // Just a normal character with no special meaning
 371                                                         consume ();
 372                                                         break;
 373                                         }
 374                                         break;
 375
 376                                 case SINGLE_LINE_COMMENT:                                                                               // We are just running within a single line comment (started with // or #)
 377                                         consume();
 378                                         break;
 379
 380                                 case PHPDOC:                                                                                                    // We are just running within a php doc comment
 381                                         switch (ch) {
 382                                                 case '/':
 383                                                         switch (fLast) {
 384                                                                 case SLASH_STAR_STAR:
 385                                                                         return postFix (PartState.MULTI_LINE_COMMENT);
 386
 387                                                                 case STAR:
 388                                                                         return postFix (PartState.PHPDOC);                      // Found the end of the php doc (multi line) comment
 389
 390                                                                 default:
 391                                                                         consume();
 392                                                                         break;
 393                                                         }
 394                                                         break;
 395
 396                                                 case '*':                                                                                               // Found a '*'
 397                                                         fTokenLength++;
 398                                                         fLast = ScanState.STAR;                                                         // Remember that we found a '*'
 399                                                         break;
 400
 401                                                 default:
 402                                                         consume();
 403                                                         break;
 404                                         }
 405                                         break;
 406
 407                                 case MULTI_LINE_COMMENT:                                                                        // We are currently running through a (possible) multi line comment
 408                                         switch (ch) {
 409                                                 case '*':                                                                                       // and we found a '*'
 410                                                         if (fLast == ScanState.SLASH_STAR) {                    // If the previous characters have been a /*
 411                                                                 fLast = ScanState.SLASH_STAR_STAR;
 412                                                                 fTokenLength++;
 413                                                                 fState = PartState.PHPDOC;
 414                                                         }
 415                                                         else {
 416                                                                 fTokenLength++;
 417                                                                 fLast = ScanState.STAR;
 418                                                         }
 419                                                         break;
 420
 421                                                 case '/':
 422                                                         if (fLast == ScanState.STAR) {
 423                                                                 return postFix (PartState.MULTI_LINE_COMMENT);
 424                                                         }
 425                                                         else {
 426                                                                 consume();
 427                                                                 break;
 428                                                         }
 429
 430                                                 default:
 431                                                         consume();
 432                                                         break;
 433                                         }
 434                                         break;
 435
 436                                 case STRING_DQ:
 437                                         switch (ch) {
 438                                                 case '\\':
 439                                                         fLast = (fLast == ScanState.BACKSLASH) ? ScanState.NONE : ScanState.BACKSLASH;
 440                                                         fTokenLength++;
 441                                                         break;
 442
 443                                                 case '\"':
 444                                                         if (fLast != ScanState.BACKSLASH) {
 445                                                                 return postFix (PartState.STRING_DQ);
 446                                                         }
 447                                                         else {
 448                                                                 consume();
 449                                                         }
 450                                                         break;
 451
 452                                                 default:
 453                                                         consume();
 454                                                         break;
 455                                         }
 456                                         break;
 457
 458                                 case STRING_SQ:
 459                                         switch (ch) {
 460                                                 case '\\':
 461                                                         fLast = (fLast == ScanState.BACKSLASH) ? ScanState.NONE : ScanState.BACKSLASH;
 462                                                         fTokenLength++;
 463                                                         break;
 464
 465                                                 case '\'':
 466                                                         if (fLast != ScanState.BACKSLASH) {
 467                                                                 return postFix (PartState.STRING_SQ);
 468                                                         }
 469                                                         else {
 470                                                                 consume();
 471                                                         }
 472                                                         break;
 473
 474                                                 default:
 475                                                         consume();
 476                                                         break;
 477                                         }
 478                                         break;
 479
 480                                 case STRING_HEREDOC:                                                                                                    // We are just running within a heredoc string
 481                                         switch (fLast) {
 482                                                 case LESS_LESS_LESS:                                                                                    // The first time after we recognized the '<<<'
 483                                                         fLast = ScanState.HEREDOC_ID;                                                           // We do a scan of the heredoc id string
 484                                                         fHeredocId  = "";
 485                                                         fHeredocId += (char) ch;
 486                                                         fTokenLength++;
 487                                                         break;
 488
 489                                                 case HEREDOC_ID:                                                                                                // Scan the starting heredoc ID
 490                                                         if (ch == ' ') {
 491                                                                 fLast = ScanState.HEREDOC;
 492                                                                 fTokenLength++;
 493                                                         }
 494                                                         else {
 495                                                                 fHeredocId += (char) ch;
 496                                                                 fTokenLength++;
 497                                                         }
 498                                                         break;
 499
 500                                                 case CARRIAGE_RETURN:                                                                                   // We previously found a new line
 501                                                         fTokenLength++;
 502                                                         fHeredocIdEnd  = "";
 503                                                         fHeredocIdEnd += (char) ch;                                                                     // Add the first character to the (possible) end ID
 504                                                         fLast = ScanState.HEREDOC_ID_END;                                                       // Go for scanning the (possible) end ID
 505                                                         break;
 506
 507                                                 case HEREDOC_ID_END:                                                                                    // We scan the (possible) end ID
 508                                                         if (ch == ';') {                                                                                        // End ID ends with an ';'
 509                                                                 if (fHeredocId.compareTo (fHeredocIdEnd) == 0) {                // If start ID and end ID matches.
 510                                                                         return postFix (PartState.STRING_HEREDOC);                      // It's the end of a heredoc partition
 511                                                                 }
 512                                                                 else {
 513                                                                         consume ();                                                                                     // Wrong end ID, so just eat the character
 514                                                                 }
 515                                                         }
 516                                                         else {
 517                                                                 fTokenLength++;                                                                                 //
 518                                                                 fHeredocIdEnd += (char) ch;                                                             // Add the characther to the possible heredoc end ID
 519                                                         }
 520                                                         break;
 521
 522                                                 default:                                                                                                                // Normally state NONE
 523                                                         consume ();                                                                                                     // Eat the character
 524                                                         break;
 525                                         }
 526                                         break;
 527                         } // end of switch (fState)
 528                 }
 529         }
 530
 531         private static final int getLastLength (ScanState last) {
 532                 switch (last) {
 533                         default:
 534                                 return -1;
 535
 536                         case NONE:
 537                                 return 0;
 538
 539                         case LESS:
 540                         case CARRIAGE_RETURN:
 541                         case BACKSLASH:
 542                         case SLASH:
 543                         case STAR:
 544                                 return 1;
 545
 546                         case LESS_LESS:
 547                         case SLASH_STAR:
 548                                 return 2;
 549
 550                         case SLASH_STAR_STAR:
 551                                 return 3;
 552
 553                         case HEREDOC:
 554                                 return 3;
 555                 }
 556         }
 557
 558         private final void consume() {
 559                 fTokenLength++;                                                 // Count the character
 560                 fLast = ScanState.NONE;                                 // Reset scanner state to nothing special
 561         }
 562
 563         /**
 564          * If we found the end of a partition, return the type of the partition which is currently finished
 565          *
 566          * @param state The type of partition we found the end for
 567          * @return
 568          */
 569         private final IToken postFix (PartState state) {
 570                 fTokenLength++;
 571                 fLast         = ScanState.NONE;                 // Reset the scanner state
 572                 fState        = PartState.PHP;                  // The type of the next partition is just PHP
 573                 fPrefixLength = 0;                                              // and have no prefix length
 574
 575                 return fTokens[state.ordinal()];                // Return the type of partition for which we found the end
 576         }
 577
 578         /**
 579          * If we find the prefix of a new partition, return the type of the previous partition
 580          *
 581          * @param state
 582          * @param newState
 583          * @param last
 584          * @param prefixLength
 585          * @return
 586          */
 587         private final IToken preFix (PartState oldState, PartState newState, ScanState last, int prefixLength) {
 588                 if (fEmulate &&                                                                                 // If we are in emulation run
 589                         (oldState == PartState.PHP) &&
 590                         (fTokenLength - getLastLength (fLast) > 0)) {
 591
 592                         fTokenLength -= getLastLength (fLast);
 593                         fJavaOffset   = fTokenOffset;
 594                         fJavaLength   = fTokenLength;
 595                         fTokenLength  = 1;
 596                         fState        = newState;
 597                         fPrefixLength = prefixLength;
 598                         fLast         = last;
 599
 600                         return fTokens[oldState.ordinal()];
 601                 }
 602                 else {
 603                         fTokenLength -= getLastLength (fLast);                          // Set the length of the last token (partition)
 604                         fLast         = last;                                                           // Remember the type of the type of the last partition
 605                         fPrefixLength = prefixLength;                                           // Remember the length of the currently found start of new partition
 606                         fState        = newState;                                                       // The type of the new partition we found
 607
 608                         IToken token  = fTokens[oldState.ordinal()];            // Return the type of the old partition
 609
 610                         return token;
 611                 }
 612         }
 613
 614         private static PartState getState (String contentType) {
 615                 if (contentType == null)
 616                         return PartState.PHP;
 617
 618                 else if (contentType.equals (PHP_SINGLELINE_COMMENT))
 619                         return PartState.SINGLE_LINE_COMMENT;
 620
 621                 else if (contentType.equals (PHP_MULTILINE_COMMENT))
 622                         return PartState.MULTI_LINE_COMMENT;
 623
 624                 else if (contentType.equals (PHP_PHPDOC_COMMENT))
 625                         return PartState.PHPDOC;
 626
 627                 else if (contentType.equals (PHP_STRING_DQ))
 628                         return PartState.STRING_DQ;
 629
 630                 else if (contentType.equals (PHP_STRING_SQ))
 631                         return PartState.STRING_SQ;
 632
 633                 else if (contentType.equals (PHP_STRING_HEREDOC))
 634                         return PartState.STRING_HEREDOC;
 635
 636                 else
 637                         return PartState.PHP;
 638         }
 639
 640         /**
 641          * @see IPartitionTokenScanner#setPartialRange (IDocument, int, int, String, int)
 642          *
 643          * @note Because of the PHP heredoc syntax we need to parse from the beginning of a heredoc partition,
 644          * and not from anywhere in the middle. When not reading the start of the heredoc (and the correct heredoc start ID,
 645          * we can't recognize the correct heredoc end ID. So we start if possible form the partitionOffset.
 646          *
 647          */
 648         public void setPartialRange (IDocument document, int offset, int length, String contentType, int partitionOffset) {
 649                 if (partitionOffset >= 0) {
 650                         fScanner.setRange (document, partitionOffset, length + (offset - partitionOffset));
 651
 652                         fTokenOffset  = partitionOffset;
 653                         fTokenLength  = 0;
 654                         fPrefixLength = 0;
 655                         fLast         = ScanState.NONE;
 656                         fState        = PartState.PHP;                                          // restart at beginning of partition
 657                 }
 658                 else {
 659                         fScanner.setRange (document, offset, length);
 660
 661                         fTokenOffset  = partitionOffset;
 662                         fTokenLength  = 0;
 663                         fPrefixLength = offset - partitionOffset;
 664                         fLast         = ScanState.NONE;
 665
 666                         if (offset == partitionOffset) {
 667                                 fState = PartState.PHP;                                         // restart at beginning of partition
 668                         }
 669                         else {
 670                                 fState = getState(contentType);
 671                         }
 672                 }
 673                 // emulate JavaPartitionScanner
 674                 if (fEmulate) {
 675                         fJavaOffset = -1;
 676                         fJavaLength = 0;
 677                 }
 678         }
 679
 680         /**
 681          * @see ITokenScanner#setRange(IDocument, int, int)
 682          */
 683         public void setRange (IDocument document, int offset, int length) {
 684                 fScanner.setRange (document, offset, length);
 685
 686                 fTokenOffset  = offset;
 687                 fTokenLength  = 0;
 688                 fPrefixLength = 0;
 689                 fLast         = ScanState.NONE;
 690                 fState        = PartState.PHP;
 691
 692                 // emulate JavaPartitionScanner
 693                 if (fEmulate) {
 694                         fJavaOffset = -1;
 695                         fJavaLength = 0;
 696                 }
 697         }
 698
 699         /*
 700          * @see ITokenScanner#getTokenLength()
 701          */
 702         public int getTokenLength() {
 703                 return fTokenLength;
 704         }
 705
 706         /*
 707          * @see ITokenScanner#getTokenOffset()
 708          */
 709         public int getTokenOffset() {
 710                 if (AbstractPartitioner.DEBUG) {
 711                         Assert.isTrue(fTokenOffset >= 0, Integer.toString(fTokenOffset));
 712                 }
 713                 return fTokenOffset;
 714         }
 715
 716 }