Tokenizer


1   /* Copyright (c) 1995-2000, The Hypersonic SQL Group.
2    * All rights reserved.
3    *
4    * Redistribution and use in source and binary forms, with or without
5    * modification, are permitted provided that the following conditions are met:
6    *
7    * Redistributions of source code must retain the above copyright notice, this
8    * list of conditions and the following disclaimer.
9    *
10   * Redistributions in binary form must reproduce the above copyright notice,
11   * this list of conditions and the following disclaimer in the documentation
12   * and/or other materials provided with the distribution.
13   *
14   * Neither the name of the Hypersonic SQL Group nor the names of its
15   * contributors may be used to endorse or promote products derived from this
16   * software without specific prior written permission.
17   *
18   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21   * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
22   * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23   * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24   * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29   *
30   * This software consists of voluntary contributions made by many individuals 
31   * on behalf of the Hypersonic SQL Group.
32   *
33   *
34   * For work added by the HSQL Development Group:
35   *
36   * Copyright (c) 2001-2005, The HSQL Development Group
37   * All rights reserved.
38   *
39   * Redistribution and use in source and binary forms, with or without
40   * modification, are permitted provided that the following conditions are met:
41   *
42   * Redistributions of source code must retain the above copyright notice, this
43   * list of conditions and the following disclaimer.
44   *
45   * Redistributions in binary form must reproduce the above copyright notice,
46   * this list of conditions and the following disclaimer in the documentation
47   * and/or other materials provided with the distribution.
48   *
49   * Neither the name of the HSQL Development Group nor the names of its
50   * contributors may be used to endorse or promote products derived from this
51   * software without specific prior written permission.
52   *
53   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
54   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56   * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
57   * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
58   * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
59   * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
60   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
61   * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
62   * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
63   * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
64   */
65  
66  
67  package org.hsqldb;
68  
69  import java.math.BigDecimal  ;
70  import java.util.Locale  ;
71  
72  import org.hsqldb.lib.IntValueHashMap;
73  import org.hsqldb.store.ValuePool;
74  import org.hsqldb.lib.java.JavaSystem;
75  
76  // fredt@users 20020218 - patch 455785 by hjbusch@users - large DECIMAL inserts
77  // also Long.MIM_VALUE (bug 473388) inserts - applied to different parts
78  // fredt@users 20020408 - patch 1.7.0 by fredt - exact integral types
79  // integral values are cast into the smallest type that can hold them
80  // fredt@users 20020501 - patch 550970 by boucherb@users - fewer StringBuffers
81  // fredt@users 20020611 - patch 1.7.0 by fredt - correct statement logging
82  // changes to the working of getLastPart() to return the correct statement for
83  // logging in the .script file.
84  // also restructuring to reduce use of objects and speed up tokenising of
85  // strings and quoted identifiers
86  // fredt@users 20021112 - patch 1.7.2 by Nitin Chauhan - use of switch
87  // rewrite of the majority of multiple if(){}else{} chains with switch(){}
88  // fredt@users 20030610 - patch 1.7.2 - no StringBuffers
89  
90  /**
91   * Provides the ability to tokenize SQL character sequences.
92   *
93   * Extensively rewritten and extended in successive versions of HSQLDB.
94   *
95   * @author Thomas Mueller (Hypersonic SQL Group)
96   * @version 1.8.0
97   * @since Hypersonic SQL
98   */
99  public class Tokenizer {
100 
101     private static final int NO_TYPE     = 0,
102                              NAME        = 1,
103                              LONG_NAME   = 2,
104                              SPECIAL     = 3,
105                              NUMBER      = 4,
106                              FLOAT       = 5,
107                              STRING      = 6,
108                              LONG        = 7,
109                              DECIMAL     = 8,
110                              BOOLEAN     = 9,
111                              DATE        = 10,
112                              TIME        = 11,
113                              TIMESTAMP   = 12,
114                              NULL        = 13,
115                              NAMED_PARAM = 14;
116 
117     // used only internally
118     private static final int QUOTED_IDENTIFIER = 15,
119                              REMARK_LINE       = 16,
120                              REMARK            = 17;
121     private String             sCommand;
122     private int              iLength;
123     private int              iIndex;
124     private int              tokenIndex;
125     private int              nextTokenIndex;
126     private int              beginIndex;
127     private int              iType;
128     private String             sToken;
129     private String             sLongNameFirst = null;
130     private int              typeLongNameFirst;
131 
132     // getToken() will clear LongNameFirst unless retainFirst is set.
133     private boolean retainFirst = false;
134 
135 //    private String           sLongNameLast;
136     // WAIT.  Don't do anything before popping another Token (because the
137     // state variables aren't set properly due to a call of wait()).
138     private boolean bWait;
139     private boolean lastTokenQuotedID;
140 
141     // literals that are values
142     static IntValueHashMap valueTokens;
143 
144     static {
145         valueTokens = new IntValueHashMap();
146 
147         valueTokens.put(Token.T_NULL, NULL);
148         valueTokens.put(Token.T_TRUE, BOOLEAN);
149         valueTokens.put(Token.T_FALSE, BOOLEAN);
150     }
151 
152     public Tokenizer() {}
153 
154     public Tokenizer(String   s) {
155 
156         sCommand = s;
157         iLength  = s.length();
158         iIndex   = 0;
159     }
160 
161     public void reset(String   s) {
162 
163         sCommand          = s;
164         iLength           = s.length();
165         iIndex            = 0;
166         tokenIndex        = 0;
167         nextTokenIndex    = 0;
168         beginIndex        = 0;
169         iType             = NO_TYPE;
170         typeLongNameFirst = NO_TYPE;
171         sToken            = null;
172         sLongNameFirst    = null;
173 
174 //        sLongNameLast  = null;
175         bWait             = false;
176         lastTokenQuotedID = false;
177         retainFirst       = false;
178     }
179 
180     /**
181      *
182      * @throws HsqlException
183      */
184     void back() throws HsqlException {
185 
186         if (bWait) {
187             Trace.doAssert(false, "Querying state when in Wait mode");
188         }
189 
190         nextTokenIndex = iIndex;
191         iIndex         = tokenIndex;
192         bWait          = true;
193     }
194 
195     /**
196      * get the given token or throw
197      *
198      * for commands and simple unquoted identifiers only
199      *
200      * @param match
201      *
202      * @throws HsqlException
203      */
204     String   getThis(String   match) throws HsqlException {
205 
206         getToken();
207         matchThis(match);
208 
209         return sToken;
210     }
211 
212     /**
213      * for commands and simple unquoted identifiers only
214      */
215     void matchThis(String   match) throws HsqlException {
216 
217         if (bWait) {
218             Trace.doAssert(false, "Querying state when in Wait mode");
219         }
220 
221         if (!sToken.equals(match) || iType == QUOTED_IDENTIFIER
222                 || iType == LONG_NAME) {
223             String   token = iType == LONG_NAME ? sLongNameFirst
224                                               : sToken;
225 
226             throw Trace.error(Trace.UNEXPECTED_TOKEN, Trace.TOKEN_REQUIRED,
227                               new Object  [] {
228                 token, match
229             });
230         }
231     }
232 
233     void throwUnexpected() throws HsqlException {
234 
235         String   token = iType == LONG_NAME ? sLongNameFirst
236                                           : sToken;
237 
238         throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
239     }
240 
241     /**
242      * Used for commands only
243      *
244      *
245      * @param match
246      */
247     public boolean isGetThis(String   match) throws HsqlException {
248 
249         getToken();
250 
251         if (iType != QUOTED_IDENTIFIER && iType != LONG_NAME
252                 && sToken.equals(match)) {
253             return true;
254         }
255 
256         back();
257 
258         return false;
259     }
260 
261     /**
262      * this methode is called before other wasXXX methods and takes
263      * precedence
264      */
265     boolean wasValue() throws HsqlException {
266 
267         if (bWait) {
268             Trace.doAssert(false, "Querying state when in Wait mode");
269         }
270 
271         switch (iType) {
272 
273             case STRING :
274             case NUMBER :
275             case LONG :
276             case FLOAT :
277             case DECIMAL :
278             case BOOLEAN :
279             case NULL :
280                 return true;
281 
282             default :
283                 return false;
284         }
285     }
286 
287     boolean wasQuotedIdentifier() throws HsqlException {
288 
289         if (bWait) {
290             Trace.doAssert(false, "Querying state when in Wait mode");
291         }
292 
293         return lastTokenQuotedID;
294 
295         // iType won't help for LONG_NAMEs.
296         //return iType == QUOTED_IDENTIFIER;
297     }
298 
299     boolean wasFirstQuotedIdentifier() throws HsqlException {
300 
301         if (bWait) {
302             Trace.doAssert(false, "Querying state when in Wait mode");
303         }
304 
305         return (typeLongNameFirst == QUOTED_IDENTIFIER);
306     }
307 
308     /**
309      * Method declaration
310      *
311      *
312      * @return
313      */
314     boolean wasLongName() throws HsqlException {
315 
316         if (bWait) {
317             Trace.doAssert(false, "Querying state when in Wait mode");
318         }
319 
320         return iType == LONG_NAME;
321     }
322 
323     /**
324      * Simple Name means a quoted or unquoted identifier without
325      * qualifiers provided it is not in the hKeyword list.
326      *
327      * @return
328      */
329     boolean wasSimpleName() throws HsqlException {
330 
331         if (bWait) {
332             Trace.doAssert(false, "Querying state when in Wait mode");
333         }
334 
335         if (iType == QUOTED_IDENTIFIER && sToken.length() != 0) {
336             return true;
337         }
338 
339         if (iType != NAME) {
340             return false;
341         }
342 
343         return !Token.isKeyword(sToken);
344     }
345 
346     /**
347      * checks whether the previously obtained token was a (named) parameter
348      *
349      * @return true if the previously obtained token was a (named) parameter
350      */
351     boolean wasParameter() throws HsqlException {
352 
353         Trace.doAssert(!bWait, "Querying state when in Wait mode");
354 
355         return (iType == NAMED_PARAM);
356     }
357 
358     /**
359      * Name means all quoted and unquoted identifiers plus any word not in the
360      * hKeyword list.
361      *
362      * @return true if it's a name
363      */
364     boolean wasName() throws HsqlException {
365 
366         if (bWait) {
367             Trace.doAssert(false, "Querying state when in Wait mode");
368         }
369 
370         if (iType == QUOTED_IDENTIFIER) {
371             return true;
372         }
373 
374         if (iType != NAME && iType != LONG_NAME) {
375             return false;
376         }
377 
378         return !Token.isKeyword(sToken);
379     }
380 
381     String   getLongNamePre() throws HsqlException {
382         return null;
383     }
384 
385     /**
386      * Return first part of long name
387      *
388      *
389      * @return
390      */
391     String   getLongNameFirst() throws HsqlException {
392 
393         if (bWait) {
394             Trace.doAssert(false, "Querying state when in Wait mode");
395         }
396 
397         return sLongNameFirst;
398     }
399 
400     boolean wasSimpleToken() throws HsqlException {
401         return iType != QUOTED_IDENTIFIER && iType != LONG_NAME
402                && iType != STRING && iType != NAMED_PARAM;
403     }
404 
405     String   getSimpleToken() throws HsqlException {
406 
407         getToken();
408 
409         if (!wasSimpleToken()) {
410             String   token = iType == LONG_NAME ? sLongNameFirst
411                                               : sToken;
412 
413             throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
414         }
415 
416         return sToken;
417     }
418 
419     public boolean wasThis(String   match) throws HsqlException {
420 
421         if (sToken.equals(match) && iType != QUOTED_IDENTIFIER
422                 && iType != LONG_NAME) {
423             return true;
424         }
425 
426         return false;
427     }
428 
429     /**
430      * getName() is more broad than getSimpleName() in that it includes
431      * 2-part names as well
432      *
433      * @return popped name
434      * @throws HsqlException if next token is not an AName
435      */
436     String   getName() throws HsqlException {
437 
438         getToken();
439 
440         if (!wasName()) {
441             throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
442         }
443 
444         return sToken;
445     }
446 
447     /**
448      * Returns a single, unqualified name (identifier)
449      *
450      * @return name
451      * @throws HsqlException
452      */
453     public String   getSimpleName() throws HsqlException {
454 
455         getToken();
456 
457         if (!wasSimpleName()) {
458             String   token = iType == LONG_NAME ? sLongNameFirst
459                                               : sToken;
460 
461             throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
462         }
463 
464         return sToken;
465     }
466 
467     /**
468      * Return any token.
469      *
470      *
471      * @return
472      *
473      * @throws HsqlException
474      */
475     public String   getString() throws HsqlException {
476 
477         getToken();
478 
479         return sToken;
480     }
481 
482     int getInt() throws HsqlException {
483 
484         long v = getBigint();
485 
486         if (v > Integer.MAX_VALUE || v < Integer.MIN_VALUE) {
487             throw Trace.error(Trace.WRONG_DATA_TYPE,
488                               Types.getTypeString(getType()));
489         }
490 
491         return (int) v;
492     }
493 
494     static BigDecimal   LONG_MAX_VALUE_INCREMENT =
495         BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.valueOf(1));
496 
497     long getBigint() throws HsqlException {
498 
499         boolean minus = false;
500 
501         getToken();
502 
503         if (sToken.equals("-")) {
504             minus = true;
505 
506             getToken();
507         }
508 
509         Object   o = getAsValue();
510         int    t = getType();
511 
512         switch (t) {
513 
514             case Types.INTEGER :
515             case Types.BIGINT :
516                 break;
517 
518             case Types.DECIMAL :
519 
520                 // only Long.MAX_VALUE + 1 together with minus is acceptable
521                 if (minus && LONG_MAX_VALUE_INCREMENT.equals(o)) {
522                     return Long.MIN_VALUE;
523                 }
524             default :
525                 throw Trace.error(Trace.WRONG_DATA_TYPE,
526                                   Types.getTypeString(t));
527         }
528 
529         long v = ((Number  ) o).longValue();
530 
531         return minus ? -v
532                      : v;
533     }
534 
535     Object   getInType(int type) throws HsqlException {
536 
537         getToken();
538 
539         Object   o = getAsValue();
540         int    t = getType();
541 
542         if (t != type) {
543             throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(t));
544         }
545 
546         return o;
547     }
548 
549     /**
550      *
551      *
552      *
553      * @return
554      */
555     public int getType() throws HsqlException {
556 
557         if (bWait) {
558             Trace.doAssert(false, "Querying state when in Wait mode");
559         }
560 
561         // todo: make sure it's used only for Values!
562         // todo: synchronize iType with hColumn
563         switch (iType) {
564 
565             case STRING :
566                 return Types.VARCHAR;
567 
568             case NUMBER :
569                 return Types.INTEGER;
570 
571             case LONG :
572                 return Types.BIGINT;
573 
574             case FLOAT :
575                 return Types.DOUBLE;
576 
577             case DECIMAL :
578                 return Types.DECIMAL;
579 
580             case BOOLEAN :
581                 return Types.BOOLEAN;
582 
583             case DATE :
584                 return Types.DATE;
585 
586             case TIME :
587                 return Types.TIME;
588 
589             case TIMESTAMP :
590                 return Types.TIMESTAMP;
591 
592             default :
593                 return Types.NULL;
594         }
595     }
596 
597     /**
598      * Method declaration
599      *
600      *
601      * @return
602      *
603      * @throws HsqlException
604      */
605     Object   getAsValue() throws HsqlException {
606 
607         if (!wasValue()) {
608             throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
609         }
610 
611         switch (iType) {
612 
613             case NULL :
614                 return null;
615 
616             case STRING :
617 
618                 //fredt - no longer returning string with a singlequote as last char
619                 return sToken;
620 
621             case LONG :
622                 return ValuePool.getLong(Long.parseLong(sToken));
623 
624             case NUMBER :
625 
626                 // fredt - this returns unsigned values which are later negated.
627                 // as a result Integer.MIN_VALUE or Long.MIN_VALUE are promoted
628                 // to a wider type.
629                 if (sToken.length() < 11) {
630                     try {
631                         return ValuePool.getInt(Integer.parseInt(sToken));
632                     } catch (Exception   e1) {}
633                 }
634 
635                 if (sToken.length() < 20) {
636                     try {
637                         iType = LONG;
638 
639                         return ValuePool.getLong(Long.parseLong(sToken));
640                     } catch (Exception   e2) {}
641                 }
642 
643                 iType = DECIMAL;
644 
645                 return new BigDecimal  (sToken);
646 
647             case FLOAT :
648                 double d = JavaSystem.parseDouble(sToken);
649                 long   l = Double.doubleToLongBits(d);
650 
651                 return ValuePool.getDouble(l);
652 
653             case DECIMAL :
654                 return new BigDecimal  (sToken);
655 
656             case BOOLEAN :
657                 return sToken.equalsIgnoreCase("TRUE") ? Boolean.TRUE
658                                                        : Boolean.FALSE;
659 
660             case DATE :
661                 return HsqlDateTime.dateValue(sToken);
662 
663             case TIME :
664                 return HsqlDateTime.timeValue(sToken);
665 
666             case TIMESTAMP :
667                 return HsqlDateTime.timestampValue(sToken);
668 
669             default :
670                 return sToken;
671         }
672     }
673 
674     /**
675      * return the current position to be used for VIEW processing
676      *
677      * @return
678      */
679     int getPosition() {
680         return iIndex;
681     }
682 
683     /**
684      * mark the current position to be used for future getLastPart() calls
685      *
686      * @return
687      */
688     String   getPart(int begin, int end) {
689         return sCommand.substring(begin, end);
690     }
691 
692     /**
693      * mark the current position to be used for future getLastPart() calls
694      *
695      * @return
696      */
697     int getPartMarker() {
698         return beginIndex;
699     }
700 
701     /**
702      * mark the current position to be used for future getLastPart() calls
703      *
704      */
705     void setPartMarker() {
706         beginIndex = iIndex;
707     }
708 
709     /**
710      * mark the position to be used for future getLastPart() calls
711      *
712      */
713     void setPartMarker(int position) {
714         beginIndex = position;
715     }
716 
717     /**
718      * return part of the command string from the last marked position
719      *
720      * @return
721      */
722     String   getLastPart() {
723         return sCommand.substring(beginIndex, iIndex);
724     }
725 
726 // fredt@users 20020910 - patch 1.7.1 by Nitin Chauhan - rewrite as switch
727 
728     /**
729      * Method declaration
730      *
731      *
732      * @throws HsqlException
733      */
734     private void getToken() throws HsqlException {
735 
736         if (bWait) {
737             bWait  = false;
738             iIndex = nextTokenIndex;
739 
740             return;
741         }
742 
743         if (!retainFirst) {
744             sLongNameFirst    = null;
745             typeLongNameFirst = NO_TYPE;
746         }
747 
748         while (iIndex < iLength
749                 && Character.isWhitespace(sCommand.charAt(iIndex))) {
750             iIndex++;
751         }
752 
753         sToken     = "";
754         tokenIndex = iIndex;
755 
756         if (iIndex >= iLength) {
757             iType = NO_TYPE;
758 
759             return;
760         }
761 
762         char    c        = sCommand.charAt(iIndex);
763         boolean point    = false,
764                 digit    = false,
765                 exp      = false,
766                 afterexp = false;
767         boolean end      = false;
768         char    cfirst   = 0;
769 
770         lastTokenQuotedID = false;
771 
772         if (Character.isJavaIdentifierStart(c)) {
773             iType = NAME;
774         } else if (Character.isDigit(c)) {
775             iType = NUMBER;
776             digit = true;
777         } else {
778             switch (c) {
779 
780                 case '(' :
781                     sToken = Token.T_OPENBRACKET;
782                     iType  = SPECIAL;
783 
784                     iIndex++;
785 
786                     return;
787 
788                 case ')' :
789                     sToken = Token.T_CLOSEBRACKET;
790                     iType  = SPECIAL;
791 
792                     iIndex++;
793 
794                     return;
795 
796                 case ',' :
797                     sToken = Token.T_COMMA;
798                     iType  = SPECIAL;
799 
800                     iIndex++;
801 
802                     return;
803 
804                 case '*' :
805                     sToken = Token.T_MULTIPLY;
806                     iType  = SPECIAL;
807 
808                     iIndex++;
809 
810                     return;
811 
812                 case '=' :
813                     sToken = Token.T_EQUALS;
814                     iType  = SPECIAL;
815 
816                     iIndex++;
817 
818                     return;
819 
820                 case ';' :
821                     sToken = Token.T_SEMICOLON;
822                     iType  = SPECIAL;
823 
824                     iIndex++;
825 
826                     return;
827 
828                 case '+' :
829                     sToken = Token.T_PLUS;
830                     iType  = SPECIAL;
831 
832                     iIndex++;
833 
834                     return;
835 
836                 case '%' :
837                     sToken = Token.T_PERCENT;
838                     iType  = SPECIAL;
839 
840                     iIndex++;
841 
842                     return;
843 
844                 case '?' :
845                     sToken = Token.T_QUESTION;
846                     iType  = SPECIAL;
847 
848                     iIndex++;
849 
850                     return;
851 
852                 case ':' :
853                     Trace.check(++iIndex < iLength,
854                                 Trace.UNEXPECTED_END_OF_COMMAND);
855 
856                     c = sCommand.charAt(iIndex);
857 
858                     Trace.check(Character.isJavaIdentifierStart(c),
859                                 Trace.INVALID_IDENTIFIER, ":" + c);
860 
861                     iType = NAMED_PARAM;
862                     break;
863 
864                 case '\"' :
865                     lastTokenQuotedID = true;
866                     iType             = QUOTED_IDENTIFIER;
867 
868                     iIndex++;
869 
870                     sToken = getString('"');
871 
872                     if (iIndex == sCommand.length()) {
873                         return;
874                     }
875 
876                     c = sCommand.charAt(iIndex);
877 
878                     if (c == '.') {
879                         sLongNameFirst    = sToken;
880                         typeLongNameFirst = iType;
881 
882                         iIndex++;
883 
884                         if (retainFirst) {
885                             throw Trace.error(Trace.THREE_PART_IDENTIFIER);
886                         }
887 
888 // fredt - todo - avoid recursion - this has problems when there is whitespace
889 // after the dot - the same with NAME
890                         retainFirst = true;
891 
892                         getToken();
893 
894                         retainFirst = false;
895                         iType       = LONG_NAME;
896                     }
897 
898                     return;
899 
900                 case '\'' :
901                     iType = STRING;
902 
903                     iIndex++;
904 
905                     sToken = getString('\'');
906 
907                     return;
908 
909                 case '!' :
910                 case '<' :
911                 case '>' :
912                 case '|' :
913                 case '/' :
914                 case '-' :
915                     cfirst = c;
916                     iType  = SPECIAL;
917                     break;
918 
919                 case '.' :
920                     iType = DECIMAL;
921                     point = true;
922                     break;
923 
924                 default :
925                     throw Trace.error(Trace.UNEXPECTED_TOKEN,
926                                       String.valueOf(c));
927             }
928         }
929 
930         int start = iIndex++;
931 
932         while (true) {
933             if (iIndex >= iLength) {
934                 c   = ' ';
935                 end = true;
936 
937                 Trace.check(iType != STRING && iType != QUOTED_IDENTIFIER,
938                             Trace.UNEXPECTED_END_OF_COMMAND);
939             } else {
940                 c = sCommand.charAt(iIndex);
941             }
942 
943             switch (iType) {
944 
945                 case NAMED_PARAM :
946                 case NAME :
947                     if (Character.isJavaIdentifierPart(c)) {
948                         break;
949                     }
950 
951                     // fredt - todo new char[] to back sToken
952                     sToken = sCommand.substring(start, iIndex).toUpperCase(
953                         Locale.ENGLISH);
954 
955                     // the following only for NAME, not for NAMED_PARAM
956                     if (iType == NAMED_PARAM) {
957                         return;
958                     }
959 
960                     if (c == '.') {
961                         typeLongNameFirst = iType;
962                         sLongNameFirst    = sToken;
963 
964                         iIndex++;
965 
966                         if (retainFirst) {
967                             throw Trace.error(Trace.THREE_PART_IDENTIFIER);
968                         }
969 
970                         retainFirst = true;
971 
972                         getToken();    // todo: eliminate recursion
973 
974                         retainFirst = false;
975                         iType       = LONG_NAME;
976                     } else if (c == '(') {
977 
978                         // it is a function call
979                     } else {
980 
981                         // if in value list then it is a value
982                         int type = valueTokens.get(sToken, -1);
983 
984                         if (type != -1) {
985                             iType = type;
986                         }
987                     }
988 
989                     return;
990 
991                 case QUOTED_IDENTIFIER :
992                 case STRING :
993 
994                     // shouldn't get here
995                     break;
996 
997                 case REMARK :
998                     if (end) {
999 
1000                        // unfinished remark
1001                        // maybe print error here
1002                        iType = NO_TYPE;
1003
1004                        return;
1005                    } else if (c == '*') {
1006                        iIndex++;
1007
1008                        if (iIndex < iLength
1009                                && sCommand.charAt(iIndex) == '/') {
1010
1011                            // using recursion here
1012                            iIndex++;
1013
1014                            getToken();
1015
1016                            return;
1017                        }
1018                    }
1019                    break;
1020
1021                case REMARK_LINE :
1022                    if (end) {
1023                        iType = NO_TYPE;
1024
1025                        return;
1026                    } else if (c == '\r' || c == '\n') {
1027
1028                        // using recursion here
1029                        getToken();
1030
1031                        return;
1032                    }
1033                    break;
1034
1035                case SPECIAL :
1036                    if (c == '/' && cfirst == '/') {
1037                        iType = REMARK_LINE;
1038
1039                        break;
1040                    } else if (c == '-' && cfirst == '-') {
1041                        iType = REMARK_LINE;
1042
1043                        break;
1044                    } else if (c == '*' && cfirst == '/') {
1045                        iType = REMARK;
1046
1047                        break;
1048                    } else if (c == '>' || c == '=' || c == '|') {
1049                        break;
1050                    }
1051
1052                    sToken = sCommand.substring(start, iIndex);
1053
1054                    return;
1055
1056                case NUMBER :
1057                case FLOAT :
1058                case DECIMAL :
1059                    if (Character.isDigit(c)) {
1060                        digit = true;
1061                    } else if (c == '.') {
1062                        iType = DECIMAL;
1063
1064                        if (point) {
1065                            throw Trace.error(Trace.UNEXPECTED_TOKEN, ".");
1066                        }
1067
1068                        point = true;
1069                    } else if (c == 'E' || c == 'e') {
1070                        if (exp) {
1071                            throw Trace.error(Trace.UNEXPECTED_TOKEN, "E");
1072                        }
1073
1074                        // HJB-2001-08-2001 - now we are sure it's a float
1075                        iType = FLOAT;
1076
1077                        // first character after exp may be + or -
1078                        afterexp = true;
1079                        point    = true;
1080                        exp      = true;
1081                    } else if (c == '-' && afterexp) {
1082                        afterexp = false;
1083                    } else if (c == '+' && afterexp) {
1084                        afterexp = false;
1085                    } else {
1086                        afterexp = false;
1087
1088                        if (!digit) {
1089                            if (point && start == iIndex - 1) {
1090                                sToken = ".";
1091                                iType  = SPECIAL;
1092
1093                                return;
1094                            }
1095
1096                            throw Trace.error(Trace.UNEXPECTED_TOKEN,
1097                                              String.valueOf(c));
1098                        }
1099
1100                        sToken = sCommand.substring(start, iIndex);
1101
1102                        return;
1103                    }
1104            }
1105
1106            iIndex++;
1107        }
1108    }
1109
1110// fredt - strings are constructed from new char[] objects to avoid slack
1111// because these strings might end up as part of internal data structures
1112// or table elements.
1113// we may consider using pools to avoid recreating the strings
1114    private String   getString(char quoteChar) throws HsqlException {
1115
1116        try {
1117            int     nextIndex   = iIndex;
1118            boolean quoteInside = false;
1119
1120            for (;;) {
1121                nextIndex = sCommand.indexOf(quoteChar, nextIndex);
1122
1123                if (nextIndex < 0) {
1124                    throw Trace.error(Trace.UNEXPECTED_END_OF_COMMAND);
1125                }
1126
1127                if (nextIndex < iLength - 1
1128                        && sCommand.charAt(nextIndex + 1) == quoteChar) {
1129                    quoteInside = true;
1130                    nextIndex   += 2;
1131
1132                    continue;
1133                }
1134
1135                break;
1136            }
1137
1138            char[] chBuffer = new char[nextIndex - iIndex];
1139
1140            sCommand.getChars(iIndex, nextIndex, chBuffer, 0);
1141
1142            int j = chBuffer.length;
1143
1144            if (quoteInside) {
1145                j = 0;
1146
1147                // fredt - loop assumes all occurences of quoteChar are paired
1148                // this has already been checked by the preprocessing loop
1149                for (int i = 0; i < chBuffer.length; i++, j++) {
1150                    if (chBuffer[i] == quoteChar) {
1151                        i++;
1152                    }
1153
1154                    chBuffer[j] = chBuffer[i];
1155                }
1156            }
1157
1158            iIndex = ++nextIndex;
1159
1160            return new String  (chBuffer, 0, j);
1161        } catch (HsqlException e) {
1162            throw e;
1163        } catch (Exception   e) {
1164            e.toString();
1165        }
1166
1167        return null;
1168    }
1169
1170    /**
1171     * Method declaration
1172     *
1173     *
1174     * @return
1175     */
1176    int getLength() {
1177        return iLength;
1178    }
1179}
1180
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags