1) Fixed issue #347: Syntax highlight doesn't like apostrophe in heredoc.
[phpeclipse.git] / net.sourceforge.phpeclipse / src / net / sourceforge / phpdt / internal / ui / text / FastJavaPartitionScanner.java
1 /*******************************************************************************
2  * Copyright (c) 2000, 2004 IBM Corporation and others.
3  * All rights reserved. This program and the accompanying materials
4  * are made available under the terms of the Common Public License v1.0
5  * which accompanies this distribution, and is available at
6  * http://www.eclipse.org/legal/cpl-v10.html
7  *
8  * Contributors:
9  *     IBM Corporation - initial API and implementation
10  *******************************************************************************/
11 package net.sourceforge.phpdt.internal.ui.text;
12
13 import net.sourceforge.phpeclipse.ui.text.rules.AbstractPartitioner;
14
15 //incastrix
16 //import org.eclipse.jface.text.Assert;
17 import org.eclipse.core.runtime.Assert;
18 import org.eclipse.jface.text.IDocument;
19 import org.eclipse.jface.text.rules.ICharacterScanner;
20 import org.eclipse.jface.text.rules.IPartitionTokenScanner;
21 import org.eclipse.jface.text.rules.IToken;
22 import org.eclipse.jface.text.rules.Token;
23
24 /**
25  * This scanner recognizes the JavaDoc comments, Java multi line comments, Java
26  * single line comments, Java strings.
27  */
28 public class FastJavaPartitionScanner implements IPartitionTokenScanner,
29                 IPHPPartitions {
30
31         // states
32         
33         private static enum PartState {
34                 PHP,
35                 SINGLE_LINE_COMMENT,
36                 MULTI_LINE_COMMENT,
37                 PHPDOC,
38                 STRING_DQ,
39                 STRING_SQ,
40                 STRING_HEREDOC,
41         };
42
43         private static enum ScanState {
44                 NONE,
45                 BACKSLASH,                                                                      // postfix for STRING_DQ and CHARACTER
46                 SLASH,                                                                          // prefix for SINGLE_LINE or MULTI_LINE or JAVADOC
47                 SLASH_STAR,                                                                     // prefix for MULTI_LINE_COMMENT or JAVADOC 
48                 SLASH_STAR_STAR,                                                        // prefix for MULTI_LINE_COMMENT or JAVADOC
49                 STAR,                                                                           // postfix for MULTI_LINE_COMMENT or JAVADOC
50                 CARRIAGE_RETURN,                                                        // postfix for STRING_DQ, CHARACTER and SINGLE_LINE_COMMENT
51                 LESS,                                                                           // Found a '<'
52                 LESS_LESS,                                                                      // Found a '<<'
53                 LESS_LESS_LESS,                                                         // Found a '<<<'
54                 HEREDOC_ID,                                                                     // Found a '<<<' and scanning ID (till and of id, which is a ' ')
55                 HEREDOC,                                                                        // Found a '<<<' and ID
56                 HEREDOC_ID_END,                                                         // Searching the heredoc end ID
57         };
58
59         /** The heredoc ID string */
60         private String fHeredocId;
61         
62         /** The possible heredoc ID string which is read right after a new line. Ends with a ';' and should
63          * match the heredoc ID string fHeredocId
64          */
65         private String fHeredocIdEnd;
66         
67         /** The scanner. */
68         private final BufferedDocumentScanner fScanner = new BufferedDocumentScanner (1000); // faster implementation
69
70         /** The offset of the last returned token. */
71         private int fTokenOffset;
72
73         /** The length of the last returned token. */
74         private int fTokenLength;
75
76         /** The state of the scanner. */
77         private PartState fState;
78
79         /** The last significant characters read. */
80         private ScanState fLast;
81         
82         /** The amount of characters already read on first call to nextToken(). */
83         private int fPrefixLength;
84
85         // emulate JavaPartitionScanner
86         private boolean fEmulate = false;
87
88         private int fJavaOffset;
89
90         private int fJavaLength;
91
92         private final IToken[] fTokens = new IToken[] { 
93                         new Token (null),
94                         new Token (PHP_SINGLELINE_COMMENT),
95                         new Token (PHP_MULTILINE_COMMENT), 
96                         new Token (PHP_PHPDOC_COMMENT),
97                         new Token (PHP_STRING_DQ), 
98                         new Token (PHP_STRING_SQ),
99                         new Token (PHP_STRING_HEREDOC)};
100
101         public FastJavaPartitionScanner(boolean emulate) {
102                 fEmulate = emulate;
103         }
104
105         public FastJavaPartitionScanner() {
106                 this(false);
107         }
108
109         /**
110          * Emulate JavaPartitionScanner
111          * 
112          * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
113          */
114         public IToken nextToken() { 
115                 if (fEmulate) {
116                         if ((fJavaOffset != -1) && (fTokenOffset + fTokenLength != fJavaOffset + fJavaLength)) {
117                                 fTokenOffset += fTokenLength;
118                                 
119                                 return fTokens[PartState.PHP.ordinal()];
120                         } 
121                         else {
122                                 fJavaOffset = -1;
123                                 fJavaLength = 0;
124                         }
125                 }
126
127                 fTokenOffset += fTokenLength;                           // The new token offset is the offset of the previous partition + length of previous partition 
128                 fTokenLength  = fPrefixLength;                          // The new partition is at least the length of the start of the new partition
129                 
130                 while (true) {
131                         final int ch = fScanner.read();
132
133                         switch (ch) {
134                                 case ICharacterScanner.EOF:
135                                         if (fTokenLength > 0) {
136                                                 fLast = ScanState.NONE; // ignore last
137                                                 return preFix (fState, PartState.PHP, ScanState.NONE, 0);
138                                         } 
139                                         else {
140                                                 fLast = ScanState.NONE;
141                                                 fPrefixLength = 0;
142                                                 return Token.EOF;
143                                         }
144         
145                                 case '\r':                                                                              // Found a carriage return 
146                                         // emulate JavaPartitionScanner
147                                         if (!fEmulate && (fLast != ScanState.CARRIAGE_RETURN)) {
148                                                 fLast = ScanState.CARRIAGE_RETURN;              // Set to what we currently found
149                                                 fTokenLength++;                                                 // and count the partition length
150                                                 
151                                                 continue;                                                               // Go for the next character to read
152                                         } 
153                                         else {
154                                                 switch (fState) {
155                                                         case SINGLE_LINE_COMMENT:
156                                                                 if (fTokenLength > 0) {
157                                                                         IToken token = fTokens[fState.ordinal()];
158                 
159                                                                         // emulate JavaPartitionScanner
160                                                                         if (fEmulate) {
161                                                                                 fTokenLength++;
162                                                                                 fLast = ScanState.NONE;
163                                                                                 fPrefixLength = 0;
164                                                                         } 
165                                                                         else {
166                                                                                 fLast = ScanState.CARRIAGE_RETURN;
167                                                                                 fPrefixLength = 1;
168                                                                         }
169                 
170                                                                         fState = PartState.PHP;
171                                                                         return token;
172                                                                 } 
173                                                                 else {
174                                                                         consume();
175                                                                         continue;
176                                                                 }
177                                                                 
178                                                         default:
179                                                                 consume();
180                                                                 continue;
181                                                 }
182                                         }
183         
184                                 case '\n':                                                                                      // Found a line feed
185                                         switch (fState) {
186                                                 case SINGLE_LINE_COMMENT:                                       // If we running within a single line comment,
187                                                         return postFix (fState);                                // this is the end my friend
188                 
189                                                 case STRING_HEREDOC:                                            // If we running within a heredoc string
190                                                         fTokenLength++;                                                 // Count the character
191                                                         fLast = ScanState.CARRIAGE_RETURN;              // and state is still new line 
192                                                         continue;
193                                                         
194                                                 default:                                                                        // If running anywhere else than on a single line comment
195                                                         consume();                                                              // count the length of the current partition
196                                                         continue;
197                                         }
198         
199                                 case '?':
200                                         if (fState == PartState.SINGLE_LINE_COMMENT) {
201                                                 int nextch = fScanner.read();
202                                                 
203                                                 if (nextch == '>') {
204                                                         // <h1>This is an <?php # echo 'simple' ?> example.</h1>
205                                                         fTokenLength--;
206                                                         fScanner.unread();
207                                                         fScanner.unread();
208                                                         
209                                                         return postFix (fState);
210                                                 } 
211                                                 else {
212                                                         // bug #1404228: Crash on <?php // comment ?>
213                                                         if (nextch != ICharacterScanner.EOF) {
214                                                                 fScanner.unread();
215                                                         }
216                                                 }
217                                         }
218         
219                                 default:
220                                         if (!fEmulate && (fLast == ScanState.CARRIAGE_RETURN)) {
221                                                 switch (fState) {
222                                                         case SINGLE_LINE_COMMENT:
223                                                                 // case CHARACTER:
224                                                                 // case STRING_DQ:
225                                                                 // case STRING_SQ:
226                                                                 ScanState last;
227                                                                 PartState newState;
228                                                                 
229                                                                 switch (ch) {
230                                                                         case '/':
231                                                                                 last = ScanState.SLASH;
232                                                                                 newState = PartState.PHP;
233                                                                                 break;
234                         
235                                                                         case '*':
236                                                                                 last = ScanState.STAR;
237                                                                                 newState = PartState.PHP;
238                                                                                 break;
239                         
240                                                                         case '\'':
241                                                                                 last = ScanState.NONE;
242                                                                                 newState = PartState.STRING_SQ;
243                                                                                 break;
244                         
245                                                                         case '"':
246                                                                                 last = ScanState.NONE;
247                                                                                 newState = PartState.STRING_DQ;
248                                                                                 break;
249                         
250                                                                         case '\r':
251                                                                                 last = ScanState.CARRIAGE_RETURN;
252                                                                                 newState = PartState.PHP;
253                                                                                 break;
254                         
255                                                                         case '\\':
256                                                                                 last = ScanState.BACKSLASH;
257                                                                                 newState = PartState.PHP;
258                                                                                 break;
259                         
260                                                                         default:
261                                                                                 last = ScanState.NONE;
262                                                                                 newState = PartState.PHP;
263                                                                                 break;
264                                                                 }
265                 
266                                                                 fLast = ScanState.NONE; // ignore fLast
267                                                                 return preFix (fState, newState, last, 1);
268                 
269                                                         default:
270                                                                 break;
271                                                 }
272                                         }
273                         }
274
275                         // states
276                         switch (fState) {
277                                 case PHP:
278                                         switch (ch) {
279                                                 case '#':                                                       // Start of a single line comment
280                                                         if (fTokenLength > 0) {
281                                                                 return preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 1);
282                                                         } 
283                                                         else {
284                                                                 preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 1);
285                                                                 fTokenOffset += fTokenLength;
286                                                                 fTokenLength = fPrefixLength;
287                                                         }
288                                                         break;
289                                         
290                                                 case '<':
291                                                         if (fLast == ScanState.LESS) {
292                                                                 fTokenLength++;
293                                                                 fLast = ScanState.LESS_LESS;
294                                                         }
295                                                         else if (fLast == ScanState.LESS_LESS) {
296                                                                 if (fTokenLength - getLastLength(fLast) > 0) {  // this is the start of a single line comment
297                                                                         return preFix (PartState.PHP, PartState.STRING_HEREDOC, ScanState.LESS_LESS_LESS, 3);
298                                                                 } 
299                                                                 else {
300                                                                         preFix (PartState.PHP, PartState.STRING_HEREDOC, ScanState.LESS_LESS_LESS, 3);
301                                                                         fTokenOffset += fTokenLength;
302                                                                         fTokenLength = fPrefixLength;
303                                                                 }
304                                                         }
305                                                         else {
306                                                                 fTokenLength++;
307                                                                 fLast = ScanState.LESS;
308                                                         }
309                                                         break;
310                                                         
311                                                 case '/':                                                                                               // Start of single line comment?
312                                                         if (fLast == ScanState.SLASH) {                                         // If previous character was already a slash,
313                                                                 if (fTokenLength - getLastLength(fLast) > 0) {  // this is the start of a single line comment
314                                                                         return preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 2);
315                                                                 } 
316                                                                 else {
317                                                                         preFix (PartState.PHP, PartState.SINGLE_LINE_COMMENT, ScanState.NONE, 2);
318                                                                         fTokenOffset += fTokenLength;
319                                                                         fTokenLength = fPrefixLength;
320                                                                 }
321                                                         } 
322                                                         else {
323                                                                 fTokenLength++;
324                                                                 fLast = ScanState.SLASH;                                                // We currently found a slash 
325                                                         }
326                                                         break;
327                 
328                                                 case '*':
329                                                         if (fLast == ScanState.SLASH) {                                         // If previous character was a slash
330                                                                 if (fTokenLength - getLastLength (fLast) > 0) { // this is the start of a comment /*
331                                                                         return preFix (PartState.PHP, PartState.MULTI_LINE_COMMENT, ScanState.SLASH_STAR, 2);
332                                                                 }
333                                                                 else {
334                                                                         preFix (PartState.PHP, PartState.MULTI_LINE_COMMENT, ScanState.SLASH_STAR, 2);
335                                                                         fTokenOffset += fTokenLength;
336                                                                         fTokenLength = fPrefixLength;
337                                                                 }
338                                                         } 
339                                                         else {                                                                                          // No slash before the '*', so it's a normal character
340                                                                 consume ();
341                                                         }
342                                                         break;
343                 
344                                                 case '\'':                                                                                              // The start of a single quoted string
345                                                         fLast = ScanState.NONE; // ignore fLast
346                                                         
347                                                         if (fTokenLength > 0) {
348                                                                 return preFix (PartState.PHP, PartState.STRING_SQ, ScanState.NONE, 1);
349                                                         }
350                                                         else {
351                                                                 preFix (PartState.PHP, PartState.STRING_SQ, ScanState.NONE, 1);
352                                                                 fTokenOffset += fTokenLength;
353                                                                 fTokenLength = fPrefixLength;
354                                                         }
355                                                         break;
356                 
357                                                 case '"':                                                                                               // The start of a double quoted string
358                                                         fLast = ScanState.NONE; // ignore fLast
359                                                         
360                                                         if (fTokenLength > 0) {
361                                                                 return preFix (PartState.PHP, PartState.STRING_DQ, ScanState.NONE, 1);
362                                                         }
363                                                         else {
364                                                                 preFix (PartState.PHP, PartState.STRING_DQ, ScanState.NONE, 1);
365                                                                 fTokenOffset += fTokenLength;
366                                                                 fTokenLength = fPrefixLength;
367                                                         }
368                                                         break;
369                 
370                                                 default:                                                                                                // Just a normal character with no special meaning
371                                                         consume ();
372                                                         break;
373                                         }
374                                         break;
375         
376                                 case SINGLE_LINE_COMMENT:                                                                               // We are just running within a single line comment (started with // or #)
377                                         consume();
378                                         break;
379         
380                                 case PHPDOC:                                                                                                    // We are just running within a php doc comment
381                                         switch (ch) {
382                                                 case '/':
383                                                         switch (fLast) {
384                                                                 case SLASH_STAR_STAR:
385                                                                         return postFix (PartState.MULTI_LINE_COMMENT);
386                         
387                                                                 case STAR:
388                                                                         return postFix (PartState.PHPDOC);                      // Found the end of the php doc (multi line) comment
389                         
390                                                                 default:
391                                                                         consume();
392                                                                         break;
393                                                         }
394                                                         break;
395                 
396                                                 case '*':                                                                                               // Found a '*'
397                                                         fTokenLength++;
398                                                         fLast = ScanState.STAR;                                                         // Remember that we found a '*'
399                                                         break;
400                 
401                                                 default:
402                                                         consume();
403                                                         break;
404                                         }
405                                         break;
406         
407                                 case MULTI_LINE_COMMENT:                                                                        // We are currently running through a (possible) multi line comment 
408                                         switch (ch) {
409                                                 case '*':                                                                                       // and we found a '*'                                                                                                                                                                                           
410                                                         if (fLast == ScanState.SLASH_STAR) {                    // If the previous characters have been a /*
411                                                                 fLast = ScanState.SLASH_STAR_STAR;
412                                                                 fTokenLength++;
413                                                                 fState = PartState.PHPDOC;
414                                                         } 
415                                                         else {
416                                                                 fTokenLength++;
417                                                                 fLast = ScanState.STAR;
418                                                         }
419                                                         break;
420                 
421                                                 case '/':
422                                                         if (fLast == ScanState.STAR) {
423                                                                 return postFix (PartState.MULTI_LINE_COMMENT);
424                                                         } 
425                                                         else {
426                                                                 consume();
427                                                                 break;
428                                                         }
429                 
430                                                 default:
431                                                         consume();
432                                                         break;
433                                         }
434                                         break;
435         
436                                 case STRING_DQ:
437                                         switch (ch) {
438                                                 case '\\':
439                                                         fLast = (fLast == ScanState.BACKSLASH) ? ScanState.NONE : ScanState.BACKSLASH;
440                                                         fTokenLength++;
441                                                         break;
442                 
443                                                 case '\"':
444                                                         if (fLast != ScanState.BACKSLASH) {
445                                                                 return postFix (PartState.STRING_DQ);
446                                                         } 
447                                                         else {
448                                                                 consume();
449                                                         }
450                                                         break;
451                 
452                                                 default:
453                                                         consume();
454                                                         break;
455                                         }
456                                         break;
457                                         
458                                 case STRING_SQ:
459                                         switch (ch) {
460                                                 case '\\':
461                                                         fLast = (fLast == ScanState.BACKSLASH) ? ScanState.NONE : ScanState.BACKSLASH;
462                                                         fTokenLength++;
463                                                         break;
464                 
465                                                 case '\'':
466                                                         if (fLast != ScanState.BACKSLASH) {
467                                                                 return postFix (PartState.STRING_SQ);
468                                                         } 
469                                                         else {
470                                                                 consume();
471                                                         }
472                                                         break;
473                 
474                                                 default:
475                                                         consume();
476                                                         break;
477                                         }
478                                         break;
479                                 
480                                 case STRING_HEREDOC:                                                                                                    // We are just running within a heredoc string
481                                         switch (fLast) {
482                                                 case LESS_LESS_LESS:                                                                                    // The first time after we recognized the '<<<'
483                                                         fLast = ScanState.HEREDOC_ID;                                                           // We do a scan of the heredoc id string
484                                                         fHeredocId  = "";
485                                                         fHeredocId += (char) ch;
486                                                         fTokenLength++;
487                                                         break;
488                                                         
489                                                 case HEREDOC_ID:                                                                                                // Scan the starting heredoc ID
490                                                         if (ch == ' ') {                                                                                
491                                                                 fLast = ScanState.HEREDOC;
492                                                                 fTokenLength++;
493                                                         }
494                                                         else {
495                                                                 fHeredocId += (char) ch;
496                                                                 fTokenLength++;
497                                                         }
498                                                         break;
499                                                         
500                                                 case CARRIAGE_RETURN:                                                                                   // We previously found a new line
501                                                         fTokenLength++;
502                                                         fHeredocIdEnd  = "";
503                                                         fHeredocIdEnd += (char) ch;                                                                     // Add the first character to the (possible) end ID
504                                                         fLast = ScanState.HEREDOC_ID_END;                                                       // Go for scanning the (possible) end ID
505                                                         break;
506                                                         
507                                                 case HEREDOC_ID_END:                                                                                    // We scan the (possible) end ID
508                                                         if (ch == ';') {                                                                                        // End ID ends with an ';'
509                                                                 if (fHeredocId.compareTo (fHeredocIdEnd) == 0) {                // If start ID and end ID matches.
510                                                                         return postFix (PartState.STRING_HEREDOC);                      // It's the end of a heredoc partition                          
511                                                                 }
512                                                                 else {
513                                                                         consume ();                                                                                     // Wrong end ID, so just eat the character
514                                                                 }
515                                                         }
516                                                         else {
517                                                                 fTokenLength++;                                                                                 // 
518                                                                 fHeredocIdEnd += (char) ch;                                                             // Add the characther to the possible heredoc end ID
519                                                         }
520                                                         break;
521                                                         
522                                                 default:                                                                                                                // Normally state NONE
523                                                         consume ();                                                                                                     // Eat the character
524                                                         break;
525                                         }
526                                         break;
527                         } // end of switch (fState)
528                 }
529         }
530
531         private static final int getLastLength (ScanState last) {
532                 switch (last) {
533                         default:
534                                 return -1;
535         
536                         case NONE:
537                                 return 0;
538         
539                         case LESS:
540                         case CARRIAGE_RETURN:
541                         case BACKSLASH:
542                         case SLASH:
543                         case STAR:
544                                 return 1;
545         
546                         case LESS_LESS: 
547                         case SLASH_STAR:
548                                 return 2;
549         
550                         case SLASH_STAR_STAR:
551                                 return 3;
552                                 
553                         case HEREDOC:
554                                 return 3;
555                 }
556         }
557
558         private final void consume() {
559                 fTokenLength++;                                                 // Count the character
560                 fLast = ScanState.NONE;                                 // Reset scanner state to nothing special  
561         }
562
563         /**
564          * If we found the end of a partition, return the type of the partition which is currently finished  
565          * 
566          * @param state The type of partition we found the end for
567          * @return
568          */
569         private final IToken postFix (PartState state) {
570                 fTokenLength++;
571                 fLast         = ScanState.NONE;                 // Reset the scanner state
572                 fState        = PartState.PHP;                  // The type of the next partition is just PHP                           
573                 fPrefixLength = 0;                                              // and have no prefix length
574                 
575                 return fTokens[state.ordinal()];                // Return the type of partition for which we found the end
576         }
577
578         /**
579          * If we find the prefix of a new partition, return the type of the previous partition
580          * 
581          * @param state
582          * @param newState
583          * @param last
584          * @param prefixLength
585          * @return
586          */
587         private final IToken preFix (PartState oldState, PartState newState, ScanState last, int prefixLength) {
588                 if (fEmulate &&                                                                                 // If we are in emulation run
589                         (oldState == PartState.PHP) && 
590                         (fTokenLength - getLastLength (fLast) > 0)) {
591                         
592                         fTokenLength -= getLastLength (fLast);
593                         fJavaOffset   = fTokenOffset;
594                         fJavaLength   = fTokenLength;
595                         fTokenLength  = 1;
596                         fState        = newState;
597                         fPrefixLength = prefixLength;
598                         fLast         = last;
599                         
600                         return fTokens[oldState.ordinal()];
601                 } 
602                 else {
603                         fTokenLength -= getLastLength (fLast);                          // Set the length of the last token (partition) 
604                         fLast         = last;                                                           // Remember the type of the type of the last partition 
605                         fPrefixLength = prefixLength;                                           // Remember the length of the currently found start of new partition
606                         fState        = newState;                                                       // The type of the new partition we found
607                         
608                         IToken token  = fTokens[oldState.ordinal()];            // Return the type of the old partition
609                         
610                         return token;
611                 }
612         }
613
614         private static PartState getState (String contentType) {
615                 if (contentType == null)
616                         return PartState.PHP;
617
618                 else if (contentType.equals (PHP_SINGLELINE_COMMENT))
619                         return PartState.SINGLE_LINE_COMMENT;
620
621                 else if (contentType.equals (PHP_MULTILINE_COMMENT))
622                         return PartState.MULTI_LINE_COMMENT;
623
624                 else if (contentType.equals (PHP_PHPDOC_COMMENT))
625                         return PartState.PHPDOC;
626
627                 else if (contentType.equals (PHP_STRING_DQ))
628                         return PartState.STRING_DQ;
629
630                 else if (contentType.equals (PHP_STRING_SQ))
631                         return PartState.STRING_SQ;
632
633                 else if (contentType.equals (PHP_STRING_HEREDOC))
634                         return PartState.STRING_HEREDOC;
635
636                 else
637                         return PartState.PHP;
638         }
639
640         /**
641          * @see IPartitionTokenScanner#setPartialRange (IDocument, int, int, String, int)
642          * 
643          * @note Because of the PHP heredoc syntax we need to parse from the beginning of a heredoc partition,
644          * and not from anywhere in the middle. When not reading the start of the heredoc (and the correct heredoc start ID,
645          * we can't recognize the correct heredoc end ID. So we start if possible form the partitionOffset.  
646          * 
647          */
648         public void setPartialRange (IDocument document, int offset, int length, String contentType, int partitionOffset) {
649                 if (partitionOffset >= 0) {
650                         fScanner.setRange (document, partitionOffset, length + (offset - partitionOffset));
651                         
652                         fTokenOffset  = partitionOffset;
653                         fTokenLength  = 0;
654                         fPrefixLength = 0;
655                         fLast         = ScanState.NONE;
656                         fState        = PartState.PHP;                                          // restart at beginning of partition
657                 }
658                 else {
659                         fScanner.setRange (document, offset, length);
660                         
661                         fTokenOffset  = partitionOffset;
662                         fTokenLength  = 0;
663                         fPrefixLength = offset - partitionOffset;
664                         fLast         = ScanState.NONE;
665         
666                         if (offset == partitionOffset) {
667                                 fState = PartState.PHP;                                         // restart at beginning of partition
668                         } 
669                         else {
670                                 fState = getState(contentType);
671                         }
672                 }
673                 // emulate JavaPartitionScanner
674                 if (fEmulate) {
675                         fJavaOffset = -1;
676                         fJavaLength = 0;
677                 }
678         }
679
680         /**
681          * @see ITokenScanner#setRange(IDocument, int, int)
682          */
683         public void setRange (IDocument document, int offset, int length) {
684                 fScanner.setRange (document, offset, length);
685                 
686                 fTokenOffset  = offset;
687                 fTokenLength  = 0;
688                 fPrefixLength = 0;
689                 fLast         = ScanState.NONE;
690                 fState        = PartState.PHP;
691
692                 // emulate JavaPartitionScanner
693                 if (fEmulate) {
694                         fJavaOffset = -1;
695                         fJavaLength = 0;
696                 }
697         }
698
699         /*
700          * @see ITokenScanner#getTokenLength()
701          */
702         public int getTokenLength() {
703                 return fTokenLength;
704         }
705
706         /*
707          * @see ITokenScanner#getTokenOffset()
708          */
709         public int getTokenOffset() {
710                 if (AbstractPartitioner.DEBUG) {
711                         Assert.isTrue(fTokenOffset >= 0, Integer.toString(fTokenOffset));
712                 }
713                 return fTokenOffset;
714         }
715
716 }