TokenMarker


1   package org.jedit.syntax;
2   
3   /*
4    * TokenMarker.java - Generic token marker
5    * Copyright (C) 1998, 1999 Slava Pestov
6    *
7    * You may use and modify this package for any purpose. Redistribution is
8    * permitted, in both source and binary form, provided that this notice
9    * remains intact in all source distributions of this package.
10   */
11  
12  import javax.swing.text.Segment  ;
13  import java.util.*;
14  
15  /**
16   * A token marker that splits lines of text into tokens. Each token carries
17   * a length field and an indentification tag that can be mapped to a color
18   * for painting that token.<p>
19   *
20   * For performance reasons, the linked list of tokens is reused after each
21   * line is tokenized. Therefore, the return value of <code>markTokens</code>
22   * should only be used for immediate painting. Notably, it cannot be
23   * cached.
24   *
25   * @author Slava Pestov
26   * @version $Id: TokenMarker.java,v 1.1 2003/12/14 16:29:49 daggerrz Exp $
27   *
28   * @see org.gjt.sp.jedit.syntax.Token
29   */
30  public abstract class TokenMarker
31  {
32     /**
33      * A wrapper for the lower-level <code>markTokensImpl</code> method
34      * that is called to split a line up into tokens.
35      * @param line The line
36      * @param lineIndex The line number
37      */
38     public Token markTokens(Segment   line, int lineIndex)
39     {
40        if(lineIndex >= length)
41        {
42           throw new IllegalArgumentException  ("Tokenizing invalid line: "
43              + lineIndex);
44        }
45  
46        lastToken = null;
47  
48        LineInfo info = lineInfo[lineIndex];
49        LineInfo prev;
50        if(lineIndex == 0)
51           prev = null;
52        else
53           prev = lineInfo[lineIndex - 1];
54  
55        byte oldToken = info.token;
56        byte token = markTokensImpl(prev == null ?
57           Token.NULL : prev.token,line,lineIndex);
58  
59        info.token = token;
60  
61        /*
62         * This is a foul hack. It stops nextLineRequested
63         * from being cleared if the same line is marked twice.
64         *
65         * Why is this necessary? It's all JEditTextArea's fault.
66         * When something is inserted into the text, firing a
67         * document event, the insertUpdate() method shifts the
68         * caret (if necessary) by the amount inserted.
69         *
70         * All caret movement is handled by the select() method,
71         * which eventually pipes the new position to scrollTo()
72         * and calls repaint().
73         *
74         * Note that at this point in time, the new line hasn't
75         * yet been painted; the caret is moved first.
76         *
77         * scrollTo() calls offsetToX(), which tokenizes the line
78         * unless it is being called on the last line painted
79         * (in which case it uses the text area's painter cached
80         * token list). What scrollTo() does next is irrelevant.
81         *
82         * After scrollTo() has done it's job, repaint() is
83         * called, and eventually we end up in paintLine(), whose
84         * job is to paint the changed line. It, too, calls
85         * markTokens().
86         *
87         * The problem was that if the line started a multiline
88         * token, the first markTokens() (done in offsetToX())
89         * would set nextLineRequested (because the line end
90         * token had changed) but the second would clear it
91         * (because the line was the same that time) and therefore
92         * paintLine() would never know that it needed to repaint
93         * subsequent lines.
94         *
95         * This bug took me ages to track down, that's why I wrote
96         * all the relevant info down so that others wouldn't
97         * duplicate it.
98         */
99         if(!(lastLine == lineIndex && nextLineRequested))
100          nextLineRequested = (oldToken != token);
101 
102       lastLine = lineIndex;
103 
104       addToken(0,Token.END);
105 
106       return firstToken;
107    }
108 
109    /**
110     * An abstract method that splits a line up into tokens. It
111     * should parse the line, and call <code>addToken()</code> to
112     * add syntax tokens to the token list. Then, it should return
113     * the initial token type for the next line.<p>
114     *
115     * For example if the current line contains the start of a
116     * multiline comment that doesn't end on that line, this method
117     * should return the comment token type so that it continues on
118     * the next line.
119     *
120     * @param token The initial token type for this line
121     * @param line The line to be tokenized
122     * @param lineIndex The index of the line in the document,
123     * starting at 0
124     * @return The initial token type for the next line
125     */
126    protected abstract byte markTokensImpl(byte token, Segment   line,
127       int lineIndex);
128 
129    /**
130     * Returns if the token marker supports tokens that span multiple
131     * lines. If this is true, the object using this token marker is
132     * required to pass all lines in the document to the
133     * <code>markTokens()</code> method (in turn).<p>
134     *
135     * The default implementation returns true; it should be overridden
136     * to return false on simpler token markers for increased speed.
137     */
138    public boolean supportsMultilineTokens()
139    {
140       return true;
141    }
142 
143    /**
144     * Informs the token marker that lines have been inserted into
145     * the document. This inserts a gap in the <code>lineInfo</code>
146     * array.
147     * @param index The first line number
148     * @param lines The number of lines
149     */
150    public void insertLines(int index, int lines)
151    {
152       if(lines <= 0)
153          return;
154       length += lines;
155       ensureCapacity(length);
156       int len = index + lines;
157       System.arraycopy(lineInfo,index,lineInfo,len,
158          lineInfo.length - len);
159 
160       for(int i = index + lines - 1; i >= index; i--)
161       {
162          lineInfo[i] = new LineInfo();
163       }
164    }
165    
166    /**
167     * Informs the token marker that line have been deleted from
168     * the document. This removes the lines in question from the
169     * <code>lineInfo</code> array.
170     * @param index The first line number
171     * @param lines The number of lines
172     */
173    public void deleteLines(int index, int lines)
174    {
175       if (lines <= 0)
176          return;
177       int len = index + lines;
178       length -= lines;
179       System.arraycopy(lineInfo,len,lineInfo,
180          index,lineInfo.length - len);
181    }
182 
183    /**
184     * Returns the number of lines in this token marker.
185     */
186    public int getLineCount()
187    {
188       return length;
189    }
190 
191    /**
192     * Returns true if the next line should be repainted. This
193     * will return true after a line has been tokenized that starts
194     * a multiline token that continues onto the next line.
195     */
196    public boolean isNextLineRequested()
197    {
198       return nextLineRequested;
199    }
200 
201    // protected members
202 
203    /**
204     * The first token in the list. This should be used as the return
205     * value from <code>markTokens()</code>.
206     */
207    protected Token firstToken;
208 
209    /**
210     * The last token in the list. New tokens are added here.
211     * This should be set to null before a new line is to be tokenized.
212     */
213    protected Token lastToken;
214 
215    /**
216     * An array for storing information about lines. It is enlarged and
217     * shrunk automatically by the <code>insertLines()</code> and
218     * <code>deleteLines()</code> methods.
219     */
220    protected LineInfo[] lineInfo;
221 
222    /**
223     * The number of lines in the model being tokenized. This can be
224     * less than the length of the <code>lineInfo</code> array.
225     */
226    protected int length;
227 
228    /**
229     * The last tokenized line.
230     */
231    protected int lastLine;
232 
233    /**
234     * True if the next line should be painted.
235     */
236    protected boolean nextLineRequested;
237 
238    /**
239     * Creates a new <code>TokenMarker</code>. This DOES NOT create
240     * a lineInfo array; an initial call to <code>insertLines()</code>
241     * does that.
242     */
243    protected TokenMarker()
244    {
245       lastLine = -1;
246    }
247 
248    /**
249     * Ensures that the <code>lineInfo</code> array can contain the
250     * specified index. This enlarges it if necessary. No action is
251     * taken if the array is large enough already.<p>
252     *
253     * It should be unnecessary to call this under normal
254     * circumstances; <code>insertLine()</code> should take care of
255     * enlarging the line info array automatically.
256     *
257     * @param index The array index
258     */
259    protected void ensureCapacity(int index)
260    {
261       if(lineInfo == null)
262          lineInfo = new LineInfo[index + 1];
263       else if(lineInfo.length <= index)
264       {
265          LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
266          System.arraycopy(lineInfo,0,lineInfoN,0,
267                 lineInfo.length);
268          lineInfo = lineInfoN;
269       }
270    }
271 
272    /**
273     * Adds a token to the token list.
274     * @param length The length of the token
275     * @param id The id of the token
276     */
277    protected void addToken(int length, byte id)
278    {
279       if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
280          throw new InternalError  ("Invalid id: " + id);
281 
282       if(length == 0 && id != Token.END)
283          return;
284 
285       if(firstToken == null)
286       {
287          firstToken = new Token(length,id);
288          lastToken = firstToken;
289       }
290       else if(lastToken == null)
291       {
292          lastToken = firstToken;
293          firstToken.length = length;
294          firstToken.id = id;
295       }
296       else if(lastToken.next == null)
297       {
298          lastToken.next = new Token(length,id);
299          lastToken = lastToken.next;
300       }
301       else
302       {
303          lastToken = lastToken.next;
304          lastToken.length = length;
305          lastToken.id = id;
306       }
307    }
308 
309    /**
310     * Inner class for storing information about tokenized lines.
311     */
312    public class LineInfo
313    {
314       /**
315        * Creates a new LineInfo object with token = Token.NULL
316        * and obj = null.
317        */
318       public LineInfo()
319       {
320       }
321 
322       /**
323        * Creates a new LineInfo object with the specified
324        * parameters.
325        */
326       public LineInfo(byte token, Object   obj)
327       {
328          this.token = token;
329          this.obj = obj;
330       }
331 
332       /**
333        * The id of the last token of the line.
334        */
335       public byte token;
336 
337       /**
338        * This is for use by the token marker implementations
339        * themselves. It can be used to store anything that
340        * is an object and that needs to exist on a per-line
341        * basis.
342        */
343       public Object   obj;
344    }
345 }
346
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags