Preprocessor


1   /*
2    * The contents of this file are subject to the terms of the Common Development
3    * and Distribution License (the License). You may not use this file except in
4    * compliance with the License.
5    *
6    * You can obtain a copy of the License at http://www.netbeans.org/cddl.html
7    * or http://www.netbeans.org/cddl.txt.
8    *
9    * When distributing Covered Code, include this CDDL Header Notice in each file
10   * and include the License file at http://www.netbeans.org/cddl.txt.
11   * If applicable, add the following below the CDDL Header, with the fields
12   * enclosed by brackets [] replaced by your own identifying information:
13   * "Portions Copyrighted [year] [name of copyright owner]"
14   *
15   * The Original Software is NetBeans. The Initial Developer of the Original
16   * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
17   * Microsystems, Inc. All Rights Reserved.
18   */
19  package org.netbeans.mdrshell;
20  
21  import java.util.*;
22  import java.io.*;
23  
24  /** Implements the #define and #include commands well known from C/C++
25   *
26   * @author  Petr Hrebejk
27   * @version
28   */
29  
30  public class Preprocessor extends Object   {
31  
32      /** #deinfe / #include command delimiters */
33      private static final String   DELIMITERS = " ";
34  
35      private static final char HASH ='#';
36      private static final char BACK_QUOTE = '`';
37      private static final String   DEFINE = "#define";
38      private static final String   INCLUDE = "#include";
39      private static final String   UNDEFINE = "#undef";
40      private static final String   PARAMS = "[$]";
41      
42      
43      /** Variables defined in this shell */
44      Hashtable definedVariables;
45      
46      /** Parametrized variables (commands) in this shell */
47      Hashtable definedParamVariables;
48  
49      /** The shell this preprocessor works for */
50      private Shell shell;
51      
52      /** Creates new CommandPerformer 
53      */
54      Preprocessor( Shell shell ) {
55          definedVariables = new Hashtable( 11 );
56          definedParamVariables = new Hashtable( 11 );
57          this.shell = shell;
58      }
59      
60      /** Recognizes the commands and replaces predefined variables
61      */
62      String   processLine( String   line ) throws PreprocessorException {
63          
64          if ( processCommand( line ) ) {
65              return null;
66          }
67          
68          String   processedLine = replaceVariables( line );
69          
70          return processedLine;
71      }
72      
73      // Private methods ---------------------------------------------------------
74      
75      boolean processCommand( String   line ) throws PreprocessorException {
76          
77          if ( line.charAt( 0 ) != HASH ) {
78              return false;
79          }
80          
81          StringTokenizer st = new StringTokenizer( line, DELIMITERS );
82  
83          int tokenCount = st.countTokens();
84  
85          if ( tokenCount == 0 ) {
86              return false;
87          }
88          
89          String   command = st.nextToken();
90              
91          if ( command.equals( DEFINE ) ) {  // Handle the #define command
92              
93              if ( tokenCount < 2 ) {
94                  throw new PreprocessorException( "Variable not specified: " + line );
95              }
96        
97              String   variable = st.nextToken();
98              
99              String   value = tokenCount == 2 ? "" : st.nextToken( "" ).trim();
100             VariableDescriptor vd = new VariableDescriptor( variable, value );
101             definedVariables.put( vd.getName(), vd );
102             return true;  
103         }
104         else if ( command.equals( UNDEFINE ) ) {  // Handle the #undef command
105            
106             if ( tokenCount != 2 ) {
107                 throw new PreprocessorException( "Variable not specified: " + line );
108             }
109             
110             String   variable = st.nextToken();
111             
112             if ( definedVariables.get( variable ) == null ) {
113                 throw new PreprocessorException( "Variable " + variable + " not defined: " + line );
114             }
115                 
116             definedVariables.remove( variable );
117             return true;                
118         }   
119         else if ( command.equals( INCLUDE ) ) { // Handle #include command
120             if ( tokenCount == 1 ) {
121                 throw new PreprocessorException( "File not specified: " + line );
122             }
123             
124             String   fileName = st.nextToken( );
125             
126             File file = new File( fileName );
127             try { 
128                 InputStream is = new FileInputStream( fileName );
129                 shell.pushInput( is );
130             }
131             catch ( java.io.FileNotFoundException   e ) {
132                 throw new PreprocessorException( "File " + fileName + " not found: " + line );
133             }
134             return true;
135         }
136         else {
137             return false;
138         }
139       
140     }
141     
142     
143     
144     /** Replaces the variables in given string. 
145     */
146     private String   replaceVariables( String   line ) throws PreprocessorException {
147         
148         ArrayList tokens = tokenize( line );  // Get tokenized form of the line
149         
150         //printTokens( tokens );
151     
152         while( replaceVariables( tokens) );  // Replace variables until no is found
153        
154         // Now we need to find and remove all tokens which are just BACK_QUOTES 
155         for( Iterator it = tokens.iterator(); it.hasNext(); ) {
156             Token t = (Token)it.next();
157             
158             if ( t.ttype == BACK_QUOTE ) {
159                 it.remove();
160             }
161         }
162        
163         return restoreString(  tokens ); 
164     }
165        
166      /** Replaces variables in tokenized string. Starts at startIndex 
167      */    
168      private boolean replaceVariables( ArrayList tokens ) {   
169         
170         for ( int i = tokens.size() - 1; i >= 0; i-- ) { // For all tokens from right to left
171             
172             Token t = (Token)tokens.get(i);
173             
174             if ( t.ttype != StreamTokenizer.TT_WORD )  // Skip whitespaces and separators
175                 continue; 
176             
177             // Test wether token equals to some variable
178             Collection vars = definedVariables.keySet();
179             for( Iterator it = vars.iterator(); it.hasNext(); ) {  // For all variables
180                 String   variableName = (String  )it.next();
181                 if ( t.sval.equals( variableName ) ) {
182                     // Variable found replace it
183                     replaceVariable( tokens, variableName, i );
184                     return true;
185                 }
186             }      
187         }
188         
189         return false;
190    
191    
192     }
193         
194     /** Replaces the variable in the tokeized string 
195     */
196     private void replaceVariable( ArrayList tokens, String   variable, int index ) {
197         
198         VariableDescriptor vd = (VariableDescriptor)definedVariables.get( variable );
199         
200         tokens.remove( index ); // Removes the token with the variable
201         
202         ArrayList params = vd.extractParameters( tokens, index );  // Extract params and remove the tokens
203        
204         String   variableValue = vd.getValue();
205          
206         if ( vd.isParam() ) { // We have replace the $n with param values in the variable value
207             for( int i = vd.getParamCount(); i > 0 ; i-- ) { // For all params
208                 String   paramName = "$" + i;
209                 while( variableValue.indexOf( paramName ) != -1 ) {  // For all occurences of the param
210                     variableValue = replaceString( variableValue, paramName, (String  )params.get( i - 1 ) );
211                 }
212             }
213         }
214         
215         ArrayList tv = tokenize( variableValue );
216         
217         if ( index >= tokens.size() ) {
218             tokens.addAll( tv );
219         }
220         else {
221             tokens.addAll( index, tv );
222         }
223                 
224     }    
225         
226         
227     /** Parses a string to collection of tokens
228     */
229     private ArrayList tokenize( String   line ) {
230         
231         ArrayList tokens = new ArrayList( 100 );
232         
233         StreamTokenizer stok = new StreamTokenizer( new StringReader( line ) );
234    
235         stok.resetSyntax();
236         
237         // Initialize the Stream tokenizer to standard Java except of parsing numbers
238         stok.wordChars('a', 'z');
239     stok.wordChars('A', 'Z');
240     stok.wordChars(128 + 32, 255);
241     //stok.whitespaceChars(0, ' ');
242     stok.commentChar('/');
243         stok.quoteChar('"');
244     stok.quoteChar('\'');
245         
246         // Changes to standard Java
247         
248         
249         // if ( quotas ) {
250         //    stok.quoteChar('`');           // This quote means that the wohle token should be taken as 
251                                                         // command parameter.
252         //}
253         //stok.quoteChar('~');                  // Tilda means execute command and put result as string into command
254         stok.wordChars('$', '$');               // Command parameters
255         stok.wordChars('_', '_');               // Underlines are part of identifiers
256         stok.wordChars('0', '9');               // Take numbers as part of words
257         stok.ordinaryChars(0, ' ');             // We want to know about spaces 
258         stok.ordinaryChar(BACK_QUOTE); // We want to know about spaces 
259         try {
260             while ( stok.nextToken() != StreamTokenizer.TT_EOF ) {          
261                 tokens.add( new Token( stok.sval, stok.ttype ) );
262             }
263         }
264         catch ( java.io.IOException   e ) {
265         }
266         
267         return tokens;
268         
269     }
270     
271     
272     /** ReCreates a string from collection of tokens
273     */
274     private String   restoreString( ArrayList tokens ) {
275         
276         StringBuffer   sb = new StringBuffer  ( 200 );
277         
278         for ( Iterator it = tokens.iterator(); it.hasNext(); ) {
279             Token token = (Token)it.next();
280             
281             if ( token.sval == null ) {
282                 sb.append( (char)token.ttype );
283             }
284             else if ( token.ttype < 0 ) {
285                 sb.append( token.sval );
286             }
287             else {
288                 sb.append( (char)token.ttype );
289                 sb.append( token.sval );
290                 sb.append( (char)token.ttype );
291             }
292         }
293         
294         return sb.toString();
295     }
296     
297     private String   replaceString( String   in, String   what, String   by ) {
298         
299         StringBuffer   sb = new StringBuffer  ( in );
300         
301         int index = in.indexOf( what );
302         
303         sb.delete( index, index + what.length() );
304         sb.insert( index, by );
305         
306         return sb.toString();
307     }
308     
309     // Innerclasses ------------------------------------------------------------
310     
311     /** Description of a defined variable 
312     */
313     private class VariableDescriptor extends Object   {
314         
315         private String   name;
316         private String   value;
317         private boolean  isParam;
318         private int paramCount; 
319         
320         VariableDescriptor( String   var, String   value ) {
321             
322             if ( var.endsWith( PARAMS ) ) {
323                 name = var.substring(0, var.length() - PARAMS.length() );
324                 isParam = true;
325                
326                 for ( int i = 1; true; i++ ) {
327                     String   pname = "$" + i;
328                     if ( value.indexOf( pname ) == -1 ) {
329                         paramCount = i - 1;
330                         break;
331                     }
332                 }
333             }
334             else {
335                 name = var;
336                 isParam = false;
337             }
338             
339             this.value = value;
340         }
341         
342         String   getName() {
343             return name;
344         }
345         
346         String   getValue() {
347             return value;
348         }
349         
350         boolean isParam() {
351             return isParam;
352         }
353         
354         int getParamCount() {
355             return paramCount;
356         }
357         
358         /** Returns collection of strings representing the command parameters. The original
359         * tokens are removed from the tokens list.
360         */
361         ArrayList extractParameters( ArrayList tokens, int index ) {
362             
363             if ( !isParam() ) {
364                 return null;
365             }
366             
367             ArrayList parameters = new ArrayList( getParamCount() );
368             
369             Token t = (Token)tokens.get( index );
370             
371             for ( int i = 0; i < getParamCount(); i++ ) { // For all parameters
372                 
373                 try {
374                     
375                     while( t.ttype >= 0 && t.ttype <= 32 ) {  // Remove trailing spaces
376                         tokens.remove( index );
377                         t = (Token)tokens.get( index );
378                     }
379                     
380                     StringBuffer   sb = new StringBuffer  ( "" );
381                     boolean isQuoted = false;
382                     
383                     if ( t.ttype == BACK_QUOTE ) {
384                         isQuoted = true;
385                         tokens.remove( index );
386                         t = (Token)tokens.get( index );
387                     }
388                     
389                     while( true ) {
390                        
391                         if ( (t.ttype >= 0 && t.ttype <= 32) || t.ttype == BACK_QUOTE ) {
392                             if ( isQuoted && t.ttype == BACK_QUOTE ) {
393                                 tokens.remove( index );
394                                 break;
395                             }
396                             else if ( !isQuoted ) {
397                                 break;
398                             }
399                         }
400                         
401                         if ( t.sval == null ) {
402                             sb.append( (char)t.ttype );
403                         }
404                         else if ( t.ttype < 0 ) {
405                             sb.append( t.sval );
406                         }
407                         else {
408                             sb.append( (char)t.ttype );
409                             sb.append( t.sval );
410                             sb.append( (char)t.ttype );
411                         }
412                         
413                         tokens.remove( index ); 
414                         
415                         if ( index >= tokens.size()  ) {
416                             break;
417                         }
418                         
419                         t = (Token)tokens.get( index );
420                     }
421                     
422                     parameters.add( sb.toString() );
423                 }
424                 catch( IndexOutOfBoundsException   e ) {
425                     parameters.add( "null" );
426                 }
427                 
428             }
429             
430             return parameters;
431         }
432         
433     }
434     
435     /** Token in a parsed string
436     */
437     private static class Token {
438         
439         String   sval;
440         int ttype;
441 
442         Token( String   value, int type ) {
443             sval = value;
444             ttype = type;
445         }
446         
447         public String   toString() {
448             return "[ " + sval + ", " + ttype + " ]";
449         }
450 
451     }
452     
453     
454     void printTokens( ArrayList tokens ) {
455         
456         System.out.print( "{ ");
457         
458         for( Iterator it = tokens.iterator(); it.hasNext(); ) {
459             System.out.print( it.next() + ", ");
460         }
461         
462         System.out.println(" }");
463     }
464     
465 }
466
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags