TokenMarker


1   /*
2    * Copyright (C) 2005 - 2006 JasperSoft Corporation.  All rights reserved. 
3    * http://www.jaspersoft.com.
4    *
5    * Unless you have purchased a commercial license agreement from JasperSoft,
6    * the following license terms apply:
7    *
8    * This program is free software; you can redistribute it and/or modify
9    * it under the terms of the GNU General Public License version 2 as published by
10   * the Free Software Foundation.
11   *
12   * This program is distributed WITHOUT ANY WARRANTY; and without the
13   * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
14   * See the GNU General Public License for more details.
15   *
16   * You should have received a copy of the GNU General Public License
17   * along with this program; if not, see http://www.gnu.org/licenses/gpl.txt
18   * or write to:
19   *
20   * Free Software Foundation, Inc.,
21   * 59 Temple Place - Suite 330,
22   * Boston, MA  USA  02111-1307
23   *
24   *
25   *
26   *
27   * TokenMarker.java
28   * 
29   */
30  
31  package org.syntax.jedit.tokenmarker;
32  
33  import org.syntax.jedit.*;
34  
35  import javax.swing.text.Segment  ;
36  import java.util.*;
37  
38  /**
39   * A token marker that splits lines of text into tokens. Each token carries
40   * a length field and an indentification tag that can be mapped to a color
41   * for painting that token.<p>
42   *
43   * For performance reasons, the linked list of tokens is reused after each
44   * line is tokenized. Therefore, the return value of <code>markTokens</code>
45   * should only be used for immediate painting. Notably, it cannot be
46   * cached.
47   *
48   * @author Slava Pestov
49   * @version $Id: TokenMarker.java 932 2006-10-20 09:32:45Z gtoffoli $
50   * @version $Id: TokenMarker.java 932 2006-10-20 09:32:45Z gtoffoli $
51   * @see org.syntax.jedit.Token
52   */
53  public abstract class TokenMarker
54  {
55      /**
56       * A wrapper for the lower-level <code>markTokensImpl</code> method
57       * that is called to split a line up into tokens.
58       * @param line The line
59       * @param lineIndex The line number
60       */
61      public Token markTokens(Segment   line, int lineIndex)
62      {
63          if(lineIndex >= length)
64          {
65              throw new IllegalArgumentException  ("Tokenizing invalid line: "
66                  + lineIndex);
67          }
68  
69          lastToken = null;
70  
71          LineInfo info = lineInfo[lineIndex];
72          LineInfo prev;
73          if(lineIndex == 0)
74              prev = null;
75          else
76              prev = lineInfo[lineIndex - 1];
77  
78          byte oldToken = info.token;
79          byte token = markTokensImpl(prev == null ?
80              Token.NULL : prev.token,line,lineIndex);
81  
82          info.token = token;
83  
84          /*
85           * This is a foul hack. It stops nextLineRequested
86           * from being cleared if the same line is marked twice.
87           *
88           * Why is this necessary? It's all JEditTextArea's fault.
89           * When something is inserted into the text, firing a
90           * document event, the insertUpdate() method shifts the
91           * caret (if necessary) by the amount inserted.
92           *
93           * All caret movement is handled by the select() method,
94           * which eventually pipes the new position to scrollTo()
95           * and calls repaint().
96           *
97           * Note that at this point in time, the new line hasn't
98           * yet been painted; the caret is moved first.
99           *
100          * scrollTo() calls offsetToX(), which tokenizes the line
101          * unless it is being called on the last line painted
102          * (in which case it uses the text area's painter cached
103          * token list). What scrollTo() does next is irrelevant.
104          *
105          * After scrollTo() has done it's job, repaint() is
106          * called, and eventually we end up in paintLine(), whose
107          * job is to paint the changed line. It, too, calls
108          * markTokens().
109          *
110          * The problem was that if the line started a multiline
111          * token, the first markTokens() (done in offsetToX())
112          * would set nextLineRequested (because the line end
113          * token had changed) but the second would clear it
114          * (because the line was the same that time) and therefore
115          * paintLine() would never know that it needed to repaint
116          * subsequent lines.
117          *
118          * This bug took me ages to track down, that's why I wrote
119          * all the relevant info down so that others wouldn't
120          * duplicate it.
121          */
122          if(!(lastLine == lineIndex && nextLineRequested))
123             nextLineRequested = (oldToken != token);
124 
125         lastLine = lineIndex;
126 
127         addToken(0,Token.END);
128 
129         return firstToken;
130     }
131 
132     /**
133      * An abstract method that splits a line up into tokens. It
134      * should parse the line, and call <code>addToken()</code> to
135      * add syntax tokens to the token list. Then, it should return
136      * the initial token type for the next line.<p>
137      *
138      * For example if the current line contains the start of a 
139      * multiline comment that doesn't end on that line, this method
140      * should return the comment token type so that it continues on
141      * the next line.
142      *
143      * @param token The initial token type for this line
144      * @param line The line to be tokenized
145      * @param lineIndex The index of the line in the document,
146      * starting at 0
147      * @return The initial token type for the next line
148      */
149     protected abstract byte markTokensImpl(byte token, Segment   line,
150         int lineIndex);
151 
152     /**
153      * Returns if the token marker supports tokens that span multiple
154      * lines. If this is true, the object using this token marker is
155      * required to pass all lines in the document to the
156      * <code>markTokens()</code> method (in turn).<p>
157      *
158      * The default implementation returns true; it should be overridden
159      * to return false on simpler token markers for increased speed.
160      */
161     public boolean supportsMultilineTokens()
162     {
163         return true;
164     }
165 
166     /**
167      * Informs the token marker that lines have been inserted into
168      * the document. This inserts a gap in the <code>lineInfo</code>
169      * array.
170      * @param index The first line number
171      * @param lines The number of lines 
172      */
173     public void insertLines(int index, int lines)
174     {
175         if(lines <= 0)
176             return;
177         length += lines;
178         ensureCapacity(length);
179         int len = index + lines;
180         System.arraycopy(lineInfo,index,lineInfo,len,
181             lineInfo.length - len);
182 
183         for(int i = index + lines - 1; i >= index; i--)
184         {
185             lineInfo[i] = new LineInfo();
186         }
187     }
188     
189     /**
190      * Informs the token marker that line have been deleted from
191      * the document. This removes the lines in question from the
192      * <code>lineInfo</code> array.
193      * @param index The first line number
194      * @param lines The number of lines
195      */
196     public void deleteLines(int index, int lines)
197     {
198         if (lines <= 0)
199             return;
200         int len = index + lines;
201         length -= lines;
202         System.arraycopy(lineInfo,len,lineInfo,
203             index,lineInfo.length - len);
204     }
205 
206     /**
207      * Returns the number of lines in this token marker.
208      */
209     public int getLineCount()
210     {
211         return length;
212     }
213 
214     /**
215      * Returns true if the next line should be repainted. This
216      * will return true after a line has been tokenized that starts
217      * a multiline token that continues onto the next line.
218      */
219     public boolean isNextLineRequested()
220     {
221         return nextLineRequested;
222     }
223 
224     // protected members
225 
226     /**
227      * The first token in the list. This should be used as the return
228      * value from <code>markTokens()</code>.
229      */
230     protected Token firstToken;
231 
232     /**
233      * The last token in the list. New tokens are added here.
234      * This should be set to null before a new line is to be tokenized.
235      */
236     protected Token lastToken;
237 
238     /**
239      * An array for storing information about lines. It is enlarged and
240      * shrunk automatically by the <code>insertLines()</code> and
241      * <code>deleteLines()</code> methods.
242      */
243     protected LineInfo[] lineInfo;
244 
245     /**
246      * The number of lines in the model being tokenized. This can be
247      * less than the length of the <code>lineInfo</code> array.
248      */
249     protected int length;
250 
251     /**
252      * The last tokenized line.
253      */
254     protected int lastLine;
255 
256     /**
257      * True if the next line should be painted.
258      */
259     protected boolean nextLineRequested;
260 
261         protected KeywordLookupIF   keywordLookup;
262     /**
263      * Creates a new <code>TokenMarker</code>. This DOES NOT create
264      * a lineInfo array; an initial call to <code>insertLines()</code>
265      * does that.
266      */
267     protected TokenMarker()
268     {
269         lastLine = -1;
270     }
271 
272     /**
273      * Ensures that the <code>lineInfo</code> array can contain the
274      * specified index. This enlarges it if necessary. No action is
275      * taken if the array is large enough already.<p>
276      *
277      * It should be unnecessary to call this under normal
278      * circumstances; <code>insertLine()</code> should take care of
279      * enlarging the line info array automatically.
280      *
281      * @param index The array index
282      */
283     protected void ensureCapacity(int index)
284     {
285         if(lineInfo == null)
286             lineInfo = new LineInfo[index + 1];
287         else if(lineInfo.length <= index)
288         {
289             LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
290             System.arraycopy(lineInfo,0,lineInfoN,0,
291                      lineInfo.length);
292             lineInfo = lineInfoN;
293         }
294     }
295 
296     /**
297      * Adds a token to the token list.
298      * @param length The length of the token
299      * @param id The id of the token
300      */
301     protected void addToken(int length, byte id)
302     {
303         if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
304             throw new InternalError  ("Invalid id: " + id);
305 
306         if(length == 0 && id != Token.END)
307             return;
308 
309         if(firstToken == null)
310         {
311             firstToken = new Token(length,id);
312             lastToken = firstToken;
313         }
314         else if(lastToken == null)
315         {
316             lastToken = firstToken;
317             firstToken.length = length;
318             firstToken.id = id;
319         }
320         else if(lastToken.next == null)
321         {
322             lastToken.next = new Token(length,id);
323             lastToken = lastToken.next;
324         }
325         else
326         {
327             lastToken = lastToken.next;
328             lastToken.length = length;
329             lastToken.id = id;
330         }
331     }
332 
333     /**
334      * Inner class for storing information about tokenized lines.
335      */
336     public class LineInfo
337     {
338         /**
339          * Creates a new LineInfo object with token = Token.NULL
340          * and obj = null.
341          */
342         public LineInfo()
343         {
344         }
345 
346         /**
347          * Creates a new LineInfo object with the specified
348          * parameters.
349          */
350         public LineInfo(byte token, Object   obj)
351         {
352             this.token = token;
353             this.obj = obj;
354         }
355 
356         /**
357          * The id of the last token of the line.
358          */
359         public byte token;
360 
361         /**
362          * This is for use by the token marker implementations
363          * themselves. It can be used to store anything that
364          * is an object and that needs to exist on a per-line
365          * basis.
366          */
367         public Object   obj;
368     }
369 
370   /** Add your own keywords.
371    */
372   public void setKeywordLookup(KeywordLookupIF keywordLookup)
373   {
374     this.keywordLookup = keywordLookup;
375   }
376 }
377
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags