JdbcStorageManager


1   
2   /*
3    * Copyright (c) 1998 - 2005 Versant Corporation
4    * All rights reserved. This program and the accompanying materials
5    * are made available under the terms of the Eclipse Public License v1.0
6    * which accompanies this distribution, and is available at
7    * http://www.eclipse.org/legal/epl-v10.html
8    *
9    * Contributors:
10   * Versant Corporation - initial API and implementation
11   */
12  package com.versant.core.jdbc;
13  
14  import com.versant.core.common.State;
15  import com.versant.core.common.*;
16  import com.versant.core.util.CharBuf;
17  import com.versant.core.util.IntArray;
18  import com.versant.core.metadata.*;
19  import com.versant.core.server.*;
20  import com.versant.core.logging.LogEventStore;
21  import com.versant.core.jdo.ServerLogEvent;
22  import com.versant.core.jdo.QueryDetails;
23  import com.versant.core.jdo.VersantQueryPlan;
24  import com.versant.core.storagemanager.*;
25  import com.versant.core.jdbc.metadata.*;
26  import com.versant.core.jdbc.sql.SqlDriver;
27  import com.versant.core.jdbc.sql.exp.SelectExp;
28  import com.versant.core.jdbc.sql.exp.SqlExp;
29  import com.versant.core.jdbc.sql.exp.Join;
30  import com.versant.core.jdbc.sql.exp.ColumnExp;
31  import com.versant.core.jdbc.conn.LoggingResultSet;
32  import com.versant.core.jdbc.conn.PooledPreparedStatement;
33  import com.versant.core.jdbc.query.JdbcCompiledQuery;
34  import com.versant.core.jdbc.query.JdbcJDOQLCompiler;
35  
36  import com.versant.core.jdbc.ejbql.JdbcEJBQLCompiler;
37  import com.versant.core.jdbc.ejbql.JdbcQueryResultEJBQL;
38  
39  
40  import java.sql.*;
41  import java.lang.reflect.Modifier  ;
42  import java.util.*;
43  
44  /**
45   * StorageManager implementation for JDBC.
46   */
47  public final class JdbcStorageManager implements StorageManager {
48  
49      private final ModelMetaData jmd;
50      private final StorageCache cache;
51      private final JdbcConnectionSource conSrc;
52      private final SqlDriver sqlDriver;
53      private final CompiledQueryCache compiledQueryCache;
54      private final LogEventStore pes;
55  
56      private int lockPolicy;
57      private int conPolicy;
58  
59      private Connection conx;
60      private Object   cacheTx;
61      private VersantClientJDBCConnection clientCon;
62  
63      private boolean txActive;
64      private boolean optimistic;
65      private boolean pinned;     // do not release conx until end of tx
66      private boolean flushed;    // updates done without commit on conx so do
67                                  // not use the cache even if optimistic
68      private boolean forUpdateField;
69  
70      private JdbcQueryResult queryResultHead;
71      private JdbcQueryResult queryResultTail;
72  
73      // this is a reference to persistGraphFullSort or persistGraphPartialSort
74      private PersistGraph activePersistGraph;
75      private PersistGraphFullSort persistGraphFullSort;
76      private PersistGraph persistGraphPartialSort;
77  
78      private final boolean useBatchInsert;
79      private final boolean useBatchUpdate;
80  
81      private boolean[] changedClassesFlag;
82      private ClassMetaData[] changedClasses;
83      private int changedClassCount;
84  
85      public static final String   STATUS_OPEN_QUERY_RESULT_COUNT =
86              "openQueryResultCount";
87  
88      public JdbcStorageManager(ModelMetaData jmd, JdbcConnectionSource conSrc,
89              SqlDriver sqlDriver, StorageCache cache,
90              CompiledQueryCache compiledQueryCache, LogEventStore pes,
91              JdbcConfig c) {
92          this.jmd = jmd;
93          this.conSrc = conSrc;
94          this.sqlDriver = sqlDriver;
95          this.cache = cache;
96          this.compiledQueryCache = compiledQueryCache;
97          this.pes = pes;
98          useBatchInsert = !c.jdbcDisableStatementBatching
99              && sqlDriver.isInsertBatchingSupported();
100         useBatchUpdate = !c.jdbcDisableStatementBatching
101             && sqlDriver.isUpdateBatchingSupported();
102         conPolicy = CON_POLICY_RELEASE;
103         lockPolicy = LOCK_POLICY_NONE;
104     }
105 
106     public boolean isForUpdate() {
107         return forUpdateField;
108     }
109 
110     public boolean isOptimistic() {
111         return optimistic;
112     }
113 
114     public boolean isActive() {
115         return txActive;
116     }
117 
118     public void begin(boolean optimistic) {
119         if (txActive) {
120             throw BindingSupportImpl.getInstance().internal("tx already active");
121         }
122         this.optimistic = optimistic;
123         txActive = true;
124         setFlagsForLockPolicy();
125     }
126 
127     public void commit() {
128         checkActiveTx();
129         commitAndReleaseCon(flushed || !optimistic
130                 || conPolicy != CON_POLICY_RELEASE);
131         if (cacheTx != null) {
132             cache.endTx(cacheTx);
133             cacheTx = null;
134         }
135         flushed = pinned = false;
136         txActive = false;
137     }
138 
139     public void rollback() {
140         checkActiveTx();
141         rollbackImp(false);
142     }
143 
144     private void rollbackImp(boolean reset) {
145         try {
146             if (conx != null) {
147                 closeAllQueries();
148                 try {
149                     conx.rollback();
150                     if (conPolicy != CON_POLICY_PIN || reset) {
151                         if (clientCon != null) {
152                             clientCon.close();
153                         }
154                         conSrc.returnConnection(conx);
155                         conx = null;
156                     }
157                 } catch (SQLException e) {
158                     throw handleException(e);
159                 }
160             }
161         } finally {
162             if (cacheTx != null) {
163                 cache.endTx(cacheTx);
164                 cacheTx = null;
165             }
166             flushed = pinned = false;
167             txActive = false;
168         }
169     }
170 
171     public void setConnectionPolicy(int policy) {
172         conPolicy = policy;
173     }
174 
175     public int getConnectionPolicy() {
176         return conPolicy;
177     }
178 
179     public void setLockingPolicy(int policy) {
180         lockPolicy = policy;
181         setFlagsForLockPolicy();
182     }
183 
184     public int getLockingPolicy() {
185         return lockPolicy;
186     }
187 
188     public void logEvent(int level, String   description, int ms) {
189         // todo this must move to the event logging and error handling proxy
190         switch (level) {
191             case EVENT_ERRORS:
192                 if (!pes.isSevere()) return;
193                 break;
194             case EVENT_NORMAL:
195                 if (!pes.isFine()) return;
196                 break;
197             case EVENT_VERBOSE:
198                 if (!pes.isFiner()) return;
199                 break;
200             case EVENT_ALL:
201                 if (!pes.isFinest()) return;
202                 break;
203         }
204         ServerLogEvent ev = new ServerLogEvent(ServerLogEvent.USER,
205                 description);
206         ev.setTotalMs(ms);
207         pes.log(ev);
208     }
209 
210     public StorageManager getInnerStorageManager() {
211         return null;
212     }
213 
214     /**
215      * Can data be retrieved from or stored in the cache given our current
216      * state? This is true if no flush has been done and the current tx is
217      * optimistic or there is no active tx.
218      */
219     private boolean canUseCache() {
220         return optimistic && !flushed || !txActive;
221     }
222 
223     public StatesReturned fetch(ApplicationContext context, OID oid, State current,
224             FetchGroup fetchGroup,
225             FieldMetaData triggerField) {
226         try {
227             StatesReturned container = new StatesReturned(context);
228             if (canUseCache()) {
229                 State s = cache.getState(oid, fetchGroup);
230                 if (s == null) {
231                     ClassMetaData base = oid.getBaseClassMetaData();
232                     if (base.cacheStrategy == MDStatics.CACHE_STRATEGY_ALL
233                             && !base.cacheStrategyAllDone) {
234                         base.cacheStrategyAllDone = true;
235                         StatesReturned all = new StatesReturned(
236                                 DummyApplicationContext.INSTANCE);
237                         getAllStates(context, base, fetchGroup, container.next = all);
238                         s = all.get(oid);
239                     }
240                 }
241                 // todo fetch related objects from cache if s came from cache
242                 if (s == null) {
243                     getState(oid, fetchGroup, container);
244                 } else {
245                     container.add(oid, s);
246                 }
247             } else {
248                 getState(oid, fetchGroup, container);
249             }
250             finishRead(container);
251             return container;
252         } catch (Throwable   t) {
253             finishFailedRead();
254             throw handleException(t);
255         }
256     }
257 
258     public StatesReturned fetch(ApplicationContext context, OIDArray oids,
259             FieldMetaData triggerField) {
260         try {
261             StatesReturned container = new StatesReturned(context);
262             int n = oids.size();
263             if (canUseCache()) {
264                 StatesReturned all = null;
265                 for (int i = 0; i < n; i++) {
266                     OID oid = oids.oids[i];
267                     ClassMetaData cmd = oid.getAvailableClassMetaData();
268                     FetchGroup fg = cmd.fetchGroups[0];
269                     State s = cache.getState(oid, fg);
270                     if (s == null) {
271                         ClassMetaData base = oid.getBaseClassMetaData();
272                         if (base.cacheStrategy == MDStatics.CACHE_STRATEGY_ALL
273                                 && !base.cacheStrategyAllDone) {
274                             base.cacheStrategyAllDone = true;
275                             if (all == null) {
276                                 container.next = all = new StatesReturned(
277                                     DummyApplicationContext.INSTANCE);
278                             }
279                             getAllStates(context, base, base.fetchGroups[0], all);
280                             s = all.get(oid);
281                         }
282                     }
283                     // todo fetch related objects from cache if s came from cache
284                     if (s == null) {
285                         getState(oid, fg, container);
286                     } else {
287                         container.add(oid, s);
288                     }
289                 }
290             } else {
291                 for (int i = 0; i < n; i++) {
292                     OID oid = oids.oids[i];
293                     ClassMetaData cmd = oid.getAvailableClassMetaData();
294                     getState(oid, cmd.fetchGroups[0], container);
295                 }
296             }
297             finishRead(container);
298             return container;
299         } catch (Throwable   t) {
300             finishFailedRead();
301             throw handleException(t);
302         }
303     }
304 
305     public StatesReturned store(StatesToStore toStore, DeletePacket toDelete,
306             boolean returnFieldsUpdatedBySM, int storeOption,
307             boolean evictClasses) {
308         checkActiveTx();
309         if (storeOption == STORE_OPTION_FLUSH) {
310             // make sure open queries will not be cached if this is a flush
311             for (JdbcQueryResult qrw = queryResultHead; qrw != null; qrw = qrw.prev) {
312                 qrw.setNonCacheble();
313             }
314             flushed = pinned = true;
315         }
316 
317         // persist changes to the database
318         StatesReturned container = new StatesReturned(
319                 DummyApplicationContext.INSTANCE);
320         boolean updates = toStore != null && !toStore.isEmpty();
321         boolean deletes = toDelete != null && !toDelete.isEmpty();
322         try {
323             if (updates) {
324                 doUpdates(toStore, container, returnFieldsUpdatedBySM);
325             }
326             if (deletes) {
327                 doDeletes(toDelete);
328             }
329             clearNonAutoSetFields(container);
330         } catch (Exception   e) {
331             throw handleException(e);
332         } finally {
333             if (activePersistGraph != null) {
334                 activePersistGraph.clear();
335                 activePersistGraph = null;
336             }
337         }
338 
339         boolean commit = storeOption == STORE_OPTION_COMMIT;
340         switch (storeOption) {
341             case STORE_OPTION_COMMIT:
342                 commit = true;
343                 commitAndReleaseCon(!optimistic || flushed || updates || deletes
344                         || conPolicy != CON_POLICY_RELEASE);
345                 flushed = pinned = false;
346                 txActive = false;
347                 break;
348             case STORE_OPTION_PREPARE:
349                 // make sure conx.commit() is done when commit() is called later
350                 flushed = flushed || updates || deletes;
351                 break;
352         }
353 
354         // evict from cache
355         cacheTx();
356         if (toStore.epcAll) {
357             cache.evictAll(cacheTx);
358         } else if (evictClasses || changedClassCount > 0) {
359             addChangedClasses(toStore, toDelete);
360             cache.evict(cacheTx, changedClasses, changedClassCount);
361             clearChangedClasses();
362         } else {
363             int expected = toStore.size() + toDelete.size() +
364                     (toStore.epcOids == null ? 0 : toStore.epcOids.length);
365             cache.evict(cacheTx, toStore.oids, 0, toStore.size(), expected);
366             cache.evict(cacheTx, toDelete.oids, 0, toDelete.size(), expected);
367             if (toStore.epcClasses != null) {
368                 int n = toStore.epcClasses.length;
369                 ClassMetaData[] a = new ClassMetaData[n];
370                 for (int i = 0; i < n; i++) {
371                     a[i] = jmd.classes[toStore.epcClasses[i]];
372                 }
373                 cache.evict(cacheTx, a, n);
374             }
375             if (toStore.epcOids != null) {
376                 cache.evict(cacheTx, toStore.epcOids, 0, toStore.epcOids.length,
377                         expected);
378             }
379         }
380 
381         // get rid of cache transaction if we have committed
382         if (commit) {
383             cache.endTx(cacheTx);
384             cacheTx = null;
385         }
386 
387         return container;
388     }
389 
390     public OID createOID(ClassMetaData cmd) {
391         checkActiveTx();
392         JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
393         JdbcKeyGenerator keygen = jdbcClass.jdbcKeyGenerator;
394         if (keygen == null) {
395             throw BindingSupportImpl.getInstance().invalidOperation("Class " + cmd.qname +
396                     " has no jdbc-key-generator");
397         }
398         if (keygen.isPostInsertGenerator()) {
399             throw BindingSupportImpl.getInstance().invalidOperation("Class " + cmd.qname +
400                     " is using a post insert jdbc-key-generator");
401         }
402         OID oid = cmd.createOID(true);
403         Object  [] oidData = new Object  [((JdbcMetaData)jmd.jdbcMetaData).maxPkSimpleColumns];
404         try {
405             Connection kgcon = null;
406             boolean rollback = true;
407             try {
408                 boolean needKgcon = keygen.isRequiresOwnConnection();
409                 if (needKgcon) {
410                     kgcon = conSrc.getConnection(true, false);
411                 }
412                 keygen.generatePrimaryKeyPre(cmd.qname,
413                         jdbcClass.table, 1, oidData, needKgcon ? kgcon : con());
414                 oid.copyKeyFields(oidData);
415                 rollback = false;
416             } finally {
417                 if (kgcon != null) {
418                     if (rollback) {
419                         kgcon.rollback();
420                     } else {
421                         kgcon.commit();
422                     }
423                     conSrc.returnConnection(kgcon);
424                 }
425             }
426             // make sure normal con is not released until end of tx
427             if (kgcon == null && conx != null) {
428                 pinned = true;
429             }
430             return oid;
431         } catch (SQLException e) {
432             throw handleException(e);
433         }
434     }
435 
436     public CompiledQuery compileQuery(QueryDetails query) {
437         JdbcCompiledQuery cq = (JdbcCompiledQuery)compiledQueryCache.get(query);
438         if (cq == null) {
439             cq = compile(query);
440             // apply caching override, if any
441             switch (query.getCacheable()) {
442                 case QueryDetails.FALSE:
443                     cq.setCacheable(false);
444                     break;
445                 case QueryDetails.TRUE:
446                     cq.setCacheable(true);
447                     break;
448             }
449             cq = (JdbcCompiledQuery)compiledQueryCache.add(cq);
450         }
451 
452         return cq;
453     }
454 
455     public ExecuteQueryReturn executeQuery(ApplicationContext context, QueryDetails query,
456             CompiledQuery compiledQuery, Object  [] params) {
457         JdbcCompiledQuery cq;
458         if (compiledQuery == null) {
459             cq = (JdbcCompiledQuery)compileQuery(query);
460         } else {
461             cq = (JdbcCompiledQuery)compiledQuery;
462         }
463         JdbcQueryResult res = null;
464         if (cq.isEJBQLHack()) {
465 
466             res = new JdbcQueryResultEJBQL(this, cq, params,
467                     canUseCache());
468 
469         } else {
470             res = new JdbcQueryResult(this, cq, params,
471                     canUseCache());
472         }
473         addQueryResult(res);
474         return res;
475     }
476 
477     public QueryResultContainer executeQueryAll(ApplicationContext context,
478             QueryDetails query, CompiledQuery compiledQuery, Object  [] params) {
479         JdbcCompiledQuery cq;
480         if (compiledQuery == null) {
481             cq = (JdbcCompiledQuery)compileQuery(query);
482         } else {
483             cq = (JdbcCompiledQuery)compiledQuery;
484         }
485         try {
486             QueryResultContainer container = new QueryResultContainer(context, cq);
487             if (cq.isCacheble() && cache.isQueryCacheEnabled()
488                     && (!txActive || optimistic && !flushed)) {
489                 CachedQueryResult res = cache.getQueryResult(cq, params);
490                 if (res == null || !addToContainer(cq, params, res, container)) {
491                     fillContainerWithAll(context, cq, params, container);
492                     //try and add the results to the cache
493                     res = new CachedQueryResult();
494 //                    container.container.addIndirectOIDs(res);
495                     container.addResultsTo(res, cq.isCopyResultsForCache());
496                     finishRead(container.container, cq, params, res, -1);
497                 }
498             } else {
499                 fillContainerWithAll(context, cq, params, container);
500                 finishRead(container.container, cq, params, null, -1);
501             }
502             return container;
503         } catch (Throwable   t) {
504             finishFailedRead();
505             throw handleException(t);
506         }
507     }
508 
509     public int executeQueryCount(QueryDetails query,
510             CompiledQuery compiledQuery, Object  [] params) {
511         JdbcCompiledQuery cq;
512         if (compiledQuery == null) {
513             cq = (JdbcCompiledQuery)compileQuery(query);
514         } else {
515             cq = (JdbcCompiledQuery)compiledQuery;
516         }
517         try {
518             int ans;
519             if (cq.isCacheble() && cache.isQueryCacheEnabled()
520                         && (!txActive || optimistic && !flushed)) {
521                 ans = cache.getQueryResultCount(cq, params);
522                 if (ans < 0) {
523                     ans = executeCount(cq, params);
524                     finishRead(null, cq, params, null, ans);
525                 }
526             } else {
527                 ans = executeCount(cq, params);
528             }
529             return ans;
530         } catch (Throwable   t) {
531             finishFailedRead();
532             throw handleException(t);
533         }
534     }
535 
536     public VersantQueryPlan getQueryPlan(QueryDetails query,
537             CompiledQuery compiledQuery, Object  [] params) {
538         try {
539             if (compiledQuery == null) {
540                 compiledQuery = compileQuery(query);
541             }
542             VersantQueryPlan qp = executePlan((JdbcCompiledQuery)compiledQuery,
543                     params);
544             finishRead();
545             return qp;
546         } catch (Throwable   t) {
547             finishFailedRead();
548             throw handleException(t);
549         }
550     }
551 
552     public QueryResultContainer fetchNextQueryResult(ApplicationContext context,
553             RunningQuery runningQuery, int skipAmount) {
554         JdbcQueryResult res = (JdbcQueryResult)runningQuery;
555         if (res == null || res.isFinished()) {
556             return null;
557         }
558         try {
559             QueryResultContainer container = new QueryResultContainer(context, res.getCompiledQuery());
560             boolean cacheable = canUseCache() && res.isCachedResultsOk();
561             if (cacheable && checkCacheForQuery(res.getJdbcCompiledQuery(),
562                     res.getParams(), container)) {
563                 // we got data from cache
564                 container.qFinished = true;
565                 res.close();
566                 removeQueryResult(res);
567             } else { // get from database
568                 CachedQueryResult queryData = null;
569                 res.updateCacheble();
570 
571                 if (res.nextBatch(context, skipAmount, container)) {
572                     // query has finished
573                     if (cacheable && res.isCacheble()) {
574                         queryData = res.qRCache;
575                     }
576                     res.close();
577                     removeQueryResult(res);
578                 }
579                 finishRead(container.container, res.getJdbcCompiledQuery(),
580                         res.getParams(), queryData, -1);
581             }
582             return container;
583         } catch (Throwable   t) {
584             finishFailedRead();
585             throw handleException(t);
586         }
587     }
588 
589     public QueryResultContainer fetchRandomAccessQueryResult(
590             ApplicationContext context, RunningQuery runningQuery, int index,
591             int fetchAmount) {
592         try {
593             JdbcQueryResult res = (JdbcQueryResult)runningQuery;
594             QueryResultContainer qContainer = new QueryResultContainer(context, res.getCompiledQuery());
595 
596             res.getAbsolute(context, qContainer, index, fetchAmount);
597             finishRead(qContainer.container, res.getJdbcCompiledQuery(),
598                     res.getParams(), null, 0);
599             return qContainer;
600         } catch (Throwable   t) {
601             finishFailedRead();
602             throw handleException(t);
603         }
604     }
605 
606     public int getRandomAccessQueryCount(ApplicationContext context,
607             RunningQuery runningQuery) {
608         return ((JdbcQueryResult)runningQuery).getResultCount();
609     }
610 
611     public void closeQuery(RunningQuery runningQuery) {
612         JdbcQueryResult res = (JdbcQueryResult)runningQuery;
613         if (!res.isClosed()) {
614             res.close();
615             removeQueryResult(res);
616             finishRead();
617         }
618     }
619 
620     public Object   getDatastoreConnection() {
621         if (clientCon == null) {
622             clientCon = new VersantClientJDBCConnection(this, con());
623             pinned = true;
624         }
625         return clientCon;
626     }
627 
628     /**
629      * This is called when a JDBC Connection previously given to a client
630      * is closed.
631      */
632     public void clientConClosed() {
633         clientCon = null;
634     }
635 
636     public boolean isNotifyDirty() {
637         return false;
638     }
639 
640     public void notifyDirty(OID oid) {
641         throw BindingSupportImpl.getInstance().internal("should not be called");
642     }
643 
644     public void reset() {
645         resetImp();
646         forUpdateField = false;
647         lockPolicy = LOCK_POLICY_NONE;
648         conPolicy = CON_POLICY_RELEASE;
649         clearChangedClasses();
650     }
651 
652     public void destroy() {
653         resetImp();
654     }
655 
656     private void resetImp() {
657         try {
658             rollbackImp(true);
659         } catch (Exception   e) {
660             // ignore
661         }
662         finishFailedRead();
663     }
664 
665     /**
666      * Wrap an exception appropriately and return one to be thrown.
667      */
668     public RuntimeException   handleException(Throwable   e) {
669         return handleException(e.toString(), e, false, null);
670     }
671 
672     /**
673      * Wrap an exception appropriately and return one to be thrown.
674      */
675     public RuntimeException   handleException(String   msg, Throwable   e) {
676         return handleException(msg, e, false, null);
677     }
678 
679     /**
680      * Wrap an exception appropriately and return one to be thrown.
681      */
682     public RuntimeException   handleException(String   msg, Throwable   e, 
683                                             boolean convertLockTimeout, Object   failed) {
684         if (convertLockTimeout && isOptimistic() && 
685             sqlDriver.isHandleLockTimeout() &&
686             sqlDriver.isLockTimeout(e)) {
687             
688             throw BindingSupportImpl.getInstance().concurrentUpdate
689                 ("Row is locked: " + msg, failed);
690         }                           
691         return sqlDriver.mapException(e, msg, true);
692     }
693 
694     /**
695      * Get the names of all tables in the database converted to lower case.
696      * The lower case name is mapped to the real case name.
697      */
698     public HashMap getDatabaseTableNames(Connection con) throws SQLException {
699         ArrayList a = sqlDriver.getTableNames(con);
700         int n = a.size();
701         HashMap ans = new HashMap(n * 2);
702         for (int i = 0; i < a.size(); i++) {
703             String   t = (String  )a.get(i);
704             ans.put(t.toLowerCase(), t);
705         }
706         return ans;
707     }
708 
709     /**
710      * Clear our list of changed classes.
711      */
712     private void clearChangedClasses() {
713         changedClasses = null;
714         changedClassesFlag = null;
715         changedClassCount = 0;
716     }
717 
718     /**
719      * Add a class to the list of changed classes that we keep track of.
720      */
721     private void addChangedClass(ClassMetaData cmd) {
722         if (changedClasses == null) {
723             changedClasses = new ClassMetaData[jmd.classes.length];
724             changedClassesFlag = new boolean[jmd.classes.length];
725         }
726         if (changedClassesFlag[cmd.index]) {
727             return;
728         }
729         changedClasses[changedClassCount++] = cmd;
730         changedClassesFlag[cmd.index] = true;
731     }
732 
733     /**
734      * Add all the classes referenced in toStore and toDelete to our list of
735      * changed classes.
736      */
737     private void addChangedClasses(StatesToStore toStore,
738             DeletePacket toDelete) {
739         // add classes for all states.
740         State[] states = toStore.states;
741         int n = toStore.size();
742         for (int i = 0; i < n; i++) {
743             addChangedClass(states[i].getClassMetaData(jmd));
744         }
745         // add classes for all epc classes.
746         if (toStore.epcClasses != null) {
747             int[] a = toStore.epcClasses;
748             for (int i = toStore.epcClassCount - 1; i >= 0; i--) {
749                 addChangedClass(jmd.classes[a[i]]);
750             }
751         }
752         // Make sure the classes for all deleted OIDs are in
753         OID[] oids = toDelete.oids;
754         n = toDelete.size();
755         for (int i = 0; i < n; i++) {
756             OID oid = oids[i];
757             addChangedClass(oid.getClassMetaData());
758         }
759         // Make sure the classes for all epc OIDs are in
760         oids = toStore.epcOids;
761         if (oids != null) {
762             n = oids.length;
763             for (int i = 0; i < n; i++) {
764                 addChangedClass(oids[i].getClassMetaData());
765             }
766         }
767     }
768 
769     private void checkActiveTx() {
770         if (!txActive) {
771             throw BindingSupportImpl.getInstance().internal(
772                     "no active transaction");
773         }
774     }
775 
776     /**
777      * This method must be called at the end of all top level read operations
778      * to maybe commit and maybe release the database connection (depending on
779      * connection policy). It will add the data in container to the cache
780      * and also add the query data. Either container or queryData or both
781      * may be null.
782      */
783     private void finishRead(StatesReturned container, JdbcCompiledQuery cq,
784             Object  [] params, CachedQueryResult queryData, int queryResultCount) {
785         if (conx == null) {
786             // no connection so all data must have come
787             // from the level 2 cache so we have nothing to do
788             return;
789         }
790         boolean commit;
791         boolean release;
792         if (optimistic || !txActive) {
793             if (pinned) {
794                 commit = release = false;
795             } else {
796                 // if there are open queries then we cannot commit or release
797                 commit = release = queryResultHead == null
798                         && conPolicy == CON_POLICY_RELEASE;
799             }
800         } else {
801             commit = release = false;
802         }
803         boolean ok = false;
804         try {
805             if (commit) {
806                 conx.commit();
807             }
808             if (canUseCache()) {
809                 for (StatesReturned c = container; c != null; c = c.next) {
810                     cache.add(cacheTx(), c);
811                 }
812                 if (cq != null && cq.isCacheble()) {
813                     if (queryData != null) {
814                         cache.add(cacheTx(), cq, params, queryData);
815                     } else if (queryResultCount >= 0) {
816                         cache.add(cacheTx(), cq, params, queryResultCount);
817                     }
818                 }
819             }
820             if (commit && cacheTx != null) {
821                 cache.endTx(cacheTx);
822                 cacheTx = null;
823             }
824             if (release) {
825                 conSrc.returnConnection(conx);
826                 conx = null;
827             }
828             ok = true;
829         } catch (SQLException e) {
830             throw BindingSupportImpl.getInstance().datastore(e.toString(), e);
831         } finally {
832             if (!ok) {
833                 if (release && conx != null) {
834                     try {
835                         conx.rollback();
836                     } catch (Exception   e) {
837                         // ignore
838                     }
839                     try {
840                         conSrc.returnConnection(conx);
841                     } catch (SQLException e) {
842                         // ignore
843                     }
844                     conx = null;
845                 }
846             }
847         }
848     }
849 
850     private void finishRead() {
851         finishRead(null, null, null, null, -1);
852     }
853 
854     private void finishRead(StatesReturned container) {
855         finishRead(container, null, null, null, -1);
856     }
857         
858     /**
859      * This must be called for all top level read operations that fail.
860      * It ensures that if the connection should be released it is released.
861      * Any exceptions are discarded as they would likely hide the original
862      * exception that caused the read to fail.
863      */
864     private void finishFailedRead() {
865         try {
866             finishRead(null, null, null, null, -1);
867         } catch (Exception   e) {
868             // ignore
869         }
870     }
871 
872     /**
873      * If we have a connection optionally commit it and then release it if
874      * the conPolicy allows. If the connection is released then any client
875      * connection is closed. All open queries are closed.
876      */
877     private void commitAndReleaseCon(boolean commit) {
878         if (conx == null) return;
879         closeAllQueries();
880         try {
881             if (commit) {
882                 conx.commit();
883             }
884             if (conPolicy != CON_POLICY_PIN) {
885                 if (clientCon != null) {
886                     clientCon.close();
887                 }
888                 conSrc.returnConnection(conx);
889                 conx = null;
890             }
891         } catch (SQLException e) {
892             throw handleException(e);
893         }
894     }
895 
896     /**
897      * Set flags for lockPolicy as required. This must be called
898      * when the policy changes or when a new tx starts.
899      */
900     private void setFlagsForLockPolicy() {
901         forUpdateField = lockPolicy != LOCK_POLICY_NONE && !optimistic;
902     }
903 
904     /**
905      * Get our database connection. This will allocate one if we currently
906      * have none. It will also start a cache transaction if there is none.
907      */
908     public Connection con() {
909         if (conx == null) {
910             cacheTx();
911             try {
912                 conx = conSrc.getConnection(false, false);
913             } catch (RuntimeException   e) {
914                 throw e;
915             } catch (Exception   e) {
916                 throw BindingSupportImpl.getInstance().internal(e.toString(), e);
917             }
918         }
919         return conx;
920     }
921 
922     /**
923      * Get our cache transaction. This will begin one if there is none.
924      */
925     public Object   cacheTx() {
926         if (cacheTx == null) {
927             cacheTx = cache.beginTx();
928         }
929         return cacheTx;
930     }
931 
932     /**
933      * Get the meta data.
934      */
935     public ModelMetaData getJmd() {
936         return jmd;
937     }
938 
939     /**
940      * Return a state for the supplied oid containing at least the fetch
941      * group specified. Additional states may be supplied to the container.
942      * The must oid must be resolved by this call. The state returned is
943      * added to the container.
944      */
945     public State getState(OID oid, FetchGroup fetchGroup,
946             StateContainer container) {
947         ClassMetaData cmd = oid.getBaseClassMetaData();
948         try {
949             boolean forUpdate = forUpdateField;
950             if (forUpdate) {
951                 JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
952                 if (sqlDriver.getSelectForUpdate() == null) {
953                     // lock the instance with an update
954                     lock((JdbcOID)oid, jdbcClass);
955                     forUpdate = false;
956                 }
957             }
958             State s = getStateParColFetch((JdbcOID)oid.getRealOID(),
959                     fetchGroup, forUpdate, container);
960             if (forUpdateField && lockPolicy == LOCK_POLICY_FIRST) {
961                 forUpdateField = false;
962             }
963             container.add(oid, s);
964             return s;
965         } catch (SQLException x) {
966             throw handleException(x);
967         }
968     }
969 
970     /**
971      * Get a state and any prefeched states from queryResult.
972      */
973     public void getState(ApplicationContext context, OID oid,
974             FetchGroup fetchGroup, JdbcQueryResult queryResult,
975             StateContainer container) {
976         if (!context.isStateRequired(oid, fetchGroup)) {
977             // todo old code added state to container here
978             return;
979         }
980         State s;
981         if (canUseCache()) {
982             s = cache.getState(oid, fetchGroup);
983         } else {
984             s = null;
985         }
986         if (s == null) {
987             s = queryResult.getResultState(forUpdateField, container);
988         }
989         container.add(oid, s);
990     }
991 
992     /**
993      * Lock the oid using an update statement.
994      */
995     private void lock(JdbcOID oid, JdbcClass jdbcClass) {
996         PreparedStatement ps = null;
997         try {
998             ps = con().prepareStatement(jdbcClass.getLockRowSql());
999             oid.setParams(ps, 1);
1000            if (ps.executeUpdate() == 0) {
1001                throw BindingSupportImpl.getInstance().objectNotFound(
1002                        oid.toSString());
1003            }
1004        } catch (SQLException e) {
1005            throw handleException(e);
1006        } finally {
1007            cleanup(ps);
1008        }
1009    }
1010
1011    private void cleanup(Statement s) {
1012        if (s != null) {
1013            try {
1014                s.close();
1015            } catch (SQLException x) {
1016                // ignore
1017            }
1018        }
1019    }
1020
1021    private void cleanup(ResultSet rs) {
1022        if (rs != null) {
1023            try {
1024                rs.close();
1025            } catch (SQLException x) {
1026                // ignore
1027            }
1028        }
1029    }
1030
1031    /**
1032     * Get the state for an OID with parColFetch. A parallel fetch is not
1033     * done for the first level down.
1034     */
1035    private State getStateParColFetch(JdbcOID oid, FetchGroup fetchGroup,
1036            boolean forUpdate, StateContainer container)
1037            throws SQLException {
1038        boolean doCrossJoin = true;
1039
1040        fetchGroup = fetchGroup.resolve(oid, jmd);
1041        State state = null;
1042        boolean includeSubclasses = !oid.isResolved();
1043
1044        ParColFetchUtil parColFetchUtil = new ParColFetchUtil(this, forUpdate,
1045                container, oid.getAvailableClassMetaData(), oid);
1046
1047        final FgDs fgDs = ((JdbcFetchGroup)fetchGroup.storeFetchGroup).getFgDs(includeSubclasses, false);
1048        String   sql = getGetStateSql(fetchGroup, includeSubclasses, forUpdate,
1049                fgDs, doCrossJoin);
1050
1051        if (sql.length() == 0) {
1052            doCrossJoin = false;
1053            // no main table columns in group but we still need to call
1054            // populate to pick up the pass 2 fields
1055            try {
1056                state = createStateImp(null, oid, fetchGroup, forUpdate, 1,
1057                        null, includeSubclasses, container,
1058                        fgDs, false, doCrossJoin, null);
1059                container.add(oid, state);
1060                parColFetchUtil.processParallelFetch(fgDs.getJoinStruct(), 1,
1061                        true, null, oid, state, doCrossJoin, null, 0, fgDs);
1062            } finally {
1063                parColFetchUtil.close();
1064            }
1065        } else {
1066            PreparedStatement ps = null;
1067            ResultSet rs = null;
1068            try {
1069                ps = con().prepareStatement(sql);
1070                oid.setParams(ps, 1);
1071                rs = ps.executeQuery();
1072
1073                //todo return a NullState if not to throw exception
1074                if (!rs.next()) {
1075                    Utils.checkToThrowRowNotFound(oid, jmd);
1076                    state = NULLState.NULL_STATE;
1077                    container.add(oid, state);
1078                } else {
1079                    MutableInt nextVal = new MutableInt();
1080                    state = createStateImp(rs, oid, fetchGroup,
1081                            forUpdate, 1, nextVal,
1082                            includeSubclasses, container,
1083                            fgDs, false, doCrossJoin, null);
1084                    container.add(oid, state);
1085                    parColFetchUtil.processParallelFetch(fgDs.getJoinStruct(),
1086                            1, true, null, oid, state, doCrossJoin, rs,
1087                            nextVal.value, fgDs);
1088                }
1089            } finally {
1090                cleanup(rs);
1091                cleanup(ps);
1092                parColFetchUtil.close();
1093            }
1094        }
1095
1096        if (Debug.DEBUG) {
1097            if (state != NULLState.NULL_STATE
1098                    && Modifier.isAbstract(
1099                            state.getClassMetaData(jmd).cls.getModifiers())) {
1100                throw BindingSupportImpl.getInstance().internal(
1101                        "The cmd of this state is abstract");
1102            }
1103        }
1104
1105//        oid.resolve(state);
1106        return state;
1107    }
1108
1109    /**
1110     * Fetch a pass 2 field (e.g. a collection) and put its value(s) in state.
1111     */
1112    public int fetchPass2Field(OID oid, State state,
1113            FetchGroupField field, boolean forUpdate,
1114            StateContainer container,
1115            boolean fetchPass2Fields, ColFieldHolder colFHolder)
1116            throws SQLException {
1117        JdbcField jf = (JdbcField)field.fmd.storeField;
1118        if (jf instanceof JdbcCollectionField) {
1119            JdbcCollectionField jcf = (JdbcCollectionField)jf;
1120            return jcf.fetch(this, oid, state, field, forUpdate, container,
1121                    fetchPass2Fields, colFHolder);
1122        }
1123        return 0;
1124    }
1125
1126    /**
1127     * Get SQL to select the fetch group for the class. This will include
1128     * joins to pickup the next fetch group for referenced fields with useJoin
1129     * set to INNER or OUTER. This returns an empty String if the group does
1130     * not contain any main table fields.
1131     */
1132    private String   getGetStateSql(FetchGroup group, boolean includeSubclasses,
1133            boolean forUpdate, FgDs fgDs, boolean crossJoinFirstPass2Field) {
1134        String   sql = fgDs.getSql(forUpdate);
1135        if (sql != null) {
1136            return sql;
1137        }
1138
1139        // generate a join query to get the group
1140        SelectExp root = new SelectExp();
1141        JdbcClass jdbcClass = (JdbcClass)group.classMetaData.storeClass;
1142        root.table = jdbcClass.table;
1143        addSelectFetchGroup(root, group, includeSubclasses, fgDs,
1144                crossJoinFirstPass2Field);
1145        root.whereExp = jdbcClass.table.createPkEqualsParamExp(root);
1146        if (root.selectList == null) {
1147            fgDs.setSql(sql = "", forUpdate);
1148        } else {
1149            root.forUpdate = forUpdate;
1150            fgDs.setSql(sql = generateSql(root).toString(),
1151                    forUpdate);
1152        }
1153        return sql;
1154    }
1155
1156    /**
1157     * Generate SQL text for the expression.
1158     */
1159    public CharBuf generateSql(SelectExp root) {
1160        int aliasCount = root.createAlias(0);
1161        if (aliasCount == 1) root.alias = null;
1162        CharBuf s = new CharBuf();
1163        root.appendSQL(sqlDriver, s, null);
1164        return s;
1165    }
1166
1167    /**
1168     * Make whatever changes are necessary to root to select group. Root
1169     * must be a SelectExp from the table for the class of group. This will
1170     * append the fields in the group to select list of root and will
1171     * add joins to pick up references where required. It will reuse
1172     * existing joins where possible.<p>
1173     * <p/>
1174     * If includeSubclasses is true then fields from sub fetch groups will be
1175     * included. Outer joins will be done to pickup the fields for subclasses
1176     * stored in different tables if they have useSubclassJoin true.<p>
1177     *
1178     * @see #getGetStateSql
1179     * @see com.versant.core.jdbc.query.JdbcJDOQLCompiler#compile
1180     */
1181    public SqlExp addSelectFetchGroup(SelectExp root, FetchGroup group,
1182            boolean includeSubclasses, FgDs fgDs,
1183            boolean crossJoinFirstPass2Field) {
1184        SqlExp sqlExp = addSelectFetchGroupImp(root, group, true,
1185                includeSubclasses, fgDs,
1186                fgDs.getJoinStruct(), fgDs.isJoinOk(), root, root.table.pk,
1187                new SqlExp(), false, crossJoinFirstPass2Field, null);
1188        if (!fgDs.isFinished()) fgDs.updateCounts();
1189        return sqlExp;
1190    }
1191
1192    public SqlExp addSelectFetchGroup(SelectExp root, FetchGroup group,
1193            boolean includeSubclasses, FgDs fgDs, SelectExp s,
1194            JdbcColumn[] lCols, JdbcField fromField) {
1195        SqlExp sqlExp = addSelectFetchGroupImp(root, group, true,
1196                includeSubclasses, fgDs,
1197                fgDs.getJoinStruct(), fgDs.isJoinOk(), s, lCols, new SqlExp(),
1198                false, false, null);
1199        if (!fgDs.isFinished()) fgDs.updateCounts();
1200        return sqlExp;
1201    }
1202
1203    private SqlExp addSelectFetchGroupImp(SelectExp root, FetchGroup group,
1204            boolean includeSuperGroups, boolean includeSubclasses, FgDs fgDs,
1205            JoinStructure js, boolean joinAllowed, SelectExp joinFromExp,
1206            JdbcColumn[] joinFromCols,
1207            SqlExp subClsIdCols, boolean addIdCols,
1208            boolean crossJoinFirstPass2Field, JdbcField joinFromField) {
1209        SqlExp list = new SqlExp();
1210        SqlExp pos = list;
1211
1212        SqlExp tail = addSelectFetchGroupImp(root, pos, group,
1213                includeSuperGroups,
1214                includeSubclasses, fgDs, fgDs, js, joinAllowed, joinFromExp,
1215                joinFromCols, subClsIdCols, joinFromField);
1216        if (includeSubclasses) {
1217            JdbcClass jdbcClass = (JdbcClass)group.classMetaData.storeClass;
1218            if (jdbcClass.classIdCol != null) {
1219                JdbcColumn classIdCol = jdbcClass.classIdCol;
1220                if (classIdCol != null) {
1221                    SelectExp se;
1222                    if (classIdCol.table != root.table) {
1223                        Join j = joinFromExp.findJoin(classIdCol.table, joinFromField);
1224                        if (j == null) {
1225                            se = new SelectExp();
1226                            se.table = classIdCol.table;
1227                            se.outer = root.outer;
1228                            j = joinFromExp.addJoin(joinFromCols, se.table.pk,
1229                                    se);
1230                        } else {
1231                            se = j.selectExp;
1232                        }
1233                    } else {
1234                        se = root;
1235                    }
1236
1237                    if (fgDs != null && !fgDs.isFinished()) fgDs.addClsIdCount();
1238
1239                    // put the expression at the start of the select list
1240                    SqlExp e = classIdCol.toSqlExp(se);
1241                    SqlExp ee;
1242                    for (ee = e; ee.next != null; ee = ee.next) ;
1243                    ee.next = list.next;
1244                    list.next = e;
1245                }
1246            } else {
1247                if (group.classMetaData.pcSubclasses != null
1248                        && ((JdbcClass)group.classMetaData.storeClass).readAsClass == null) {
1249                    int count = 0;
1250                    SqlExp e = subClsIdCols.next;
1251                    for (; ; e = e.next) {
1252                        count++;
1253                        if (e.next == null) break;
1254                    }
1255                    e.next = list.next;
1256                    list.next = subClsIdCols.next;
1257                    if (fgDs != null && !fgDs.isFinished()) {
1258                        fgDs.addClsIdCount(count);
1259                    }
1260                }
1261            }
1262        }
1263
1264        if (crossJoinFirstPass2Field && group.crossJoinedCollectionField != null) {
1265            ((JdbcCollectionField)group.crossJoinedCollectionField.fmd.storeField).
1266                    getSelectExpFrom(this, root, group.crossJoinedCollectionField,
1267                            fgDs);
1268        }
1269
1270        /**
1271         * Add the pk columns to the select list.
1272         */
1273        if (addIdCols) {
1274            //todo add pk fields for selection. This must also be updated in the fgds for fieldcount
1275            //should probally not do this if there is a constraint to the ref, because this could not be
1276            //null then.
1277
1278            if (fgDs != null) fgDs.addRefIdFields();
1279            JdbcColumn[] cols = root.table.pk;
1280            ColumnExp ans = new ColumnExp(cols[0], root, null);
1281            SqlExp e = ans;
1282            int nc = cols.length;
1283            for (int k = 1; k < nc; k++) {
1284                e = e.next = new ColumnExp(cols[k], root, null);
1285            }
1286
1287            //add to start of list
1288            e.next = list.next;
1289            list.next = ans;
1290        }
1291
1292        SqlExp e = root.selectList;
1293        if (e == null) {
1294            root.selectList = list.next;
1295        } else {
1296            for (; e.next != null; e = e.next) ;
1297            e.next = list.next;
1298        }
1299        return tail;
1300    }
1301
1302    private ClassMetaData extractType(ResultSet rs, FetchGroup fg, int index)
1303            throws SQLException {
1304        FetchGroup[] subFGs = fg.subFetchGroups;
1305        if (subFGs == null) {
1306            return fg.classMetaData;
1307        }
1308
1309        ClassMetaData cmd = null;
1310        for (int i = 0; i < subFGs.length; i++) {
1311            cmd = extractTypeImp(rs, subFGs[i], index);
1312            index += subFGs[i].classMetaData.totalNoOfSubClasses + 1;
1313            if (cmd != null) {
1314                return cmd;
1315            }
1316        }
1317        return fg.classMetaData;
1318    }
1319
1320    private ClassMetaData extractTypeImp(ResultSet rs, FetchGroup fg,
1321            int index) throws SQLException {
1322        rs.getString(index++);
1323        if (!rs.wasNull()) {
1324            FetchGroup[] subFGs = fg.subFetchGroups;
1325            if (subFGs == null) {
1326                return fg.classMetaData;
1327            }
1328
1329            ClassMetaData cmd = null;
1330            for (int i = 0; i < subFGs.length; i++) {
1331                cmd = extractTypeImp(rs, subFGs[i], index);
1332                index += subFGs[i].classMetaData.totalNoOfSubClasses + 1;
1333                if (cmd != null) {
1334                    return cmd;
1335                }
1336            }
1337            return fg.classMetaData;
1338        } else {
1339            return null;
1340        }
1341    }
1342
1343    private SqlExp addSelectFetchGroupImp(SelectExp root, SqlExp pos,
1344            FetchGroup group, boolean includeSuperGroups,
1345            boolean includeSubclasses, FgDs fgDs,
1346            FgDs refFgDs, JoinStructure js, boolean joinAllowed,
1347            SelectExp joinFromExp, JdbcColumn[] joinFromCols,
1348            SqlExp subClsIdCols, JdbcField joinFromField) {
1349
1350        pos = addFetchGroupFields(root, group, pos, refFgDs, js, joinAllowed,
1351                joinFromExp, joinFromCols);
1352        if (includeSuperGroups) {
1353            // include the fields from all the superclasses by following the
1354            // superFetchGroup references doing joins for superclasses
1355            // in a different table
1356            FetchGroup subG = group;
1357            for (FetchGroup supg = group.superFetchGroup; supg != null;
1358                 supg = supg.superFetchGroup) {
1359                JdbcClass sc = (JdbcClass)supg.classMetaData.storeClass;
1360
1361                FgDs nextFgDs = null;
1362                if (fgDs != null && !fgDs.isFinished()) {
1363                    nextFgDs = new FgDs(fgDs, supg, "",
1364                            JdbcFetchGroup.createOpts(false, false));
1365                    fgDs.addSameTable(nextFgDs);
1366                }
1367                if (sc.table == ((JdbcClass)subG.classMetaData.storeClass).table) {
1368                    pos = addFetchGroupFields(root, supg, pos, nextFgDs, js, joinAllowed,
1369                            joinFromExp, joinFromCols);
1370                } else {
1371                    Join j = joinFromExp.findJoin(sc.table, joinFromField);
1372                    SelectExp se = null;
1373                    if (j == null) {
1374                        // different table so do an join
1375                        se = new SelectExp();
1376                        se.outer = root.outer;
1377                        se.table = sc.table;
1378                        joinFromExp.addJoin(joinFromCols, se.table.pk, se);
1379                    } else {
1380                        se = j.selectExp;
1381                    }
1382                    pos = addFetchGroupFields(se, supg, pos, nextFgDs, js, joinAllowed,
1383                            joinFromExp, joinFromCols);
1384                }
1385                subG = supg;
1386            }
1387        }
1388
1389        if (includeSubclasses && group.subFetchGroups != null) {
1390            FetchGroup[] subgroups = group.subFetchGroups;
1391            for (int i = 0; i < subgroups.length; i++) {
1392                FetchGroup subg = subgroups[i];
1393                JdbcClass sc = (JdbcClass)subg.classMetaData.storeClass;
1394
1395                FgDs nextFgDs = null;
1396                if (fgDs != null && !fgDs.isFinished()) {
1397                    nextFgDs = new FgDs(fgDs, subg, "",
1398                            JdbcFetchGroup.createOpts(false, false));
1399                    fgDs.addSameTable(nextFgDs);
1400                }
1401
1402                if (sc.table == ((JdbcClass)subg.classMetaData.pcSuperMetaData.storeClass).table) {
1403                    // same table so just add fields
1404                    pos = addSelectFetchGroupImp(root, pos, subg, false, true,
1405                            fgDs, nextFgDs, js, joinAllowed, joinFromExp,
1406                            joinFromCols, subClsIdCols, joinFromField);
1407                } else {
1408                    /**
1409                     * Must add the pk cols first and then the fg
1410                     */
1411                    SelectExp se = new SelectExp();
1412                    se.table = sc.table;
1413                    se.outer = true;
1414                    joinFromExp.addJoin(joinFromCols, se.table.pk, se);
1415
1416                    if (addIdCol(group.classMetaData)) {
1417                        subClsIdCols = subClsIdCols.next =
1418                                ((JdbcClass)subg.classMetaData.storeClass).table.pk[0].toSqlExp(
1419                                se);
1420                    }
1421
1422                    pos = addSelectFetchGroupImp(se, pos, subg, false,
1423                            true, fgDs, nextFgDs, js, joinAllowed, joinFromExp,
1424                            joinFromCols, subClsIdCols, joinFromField);
1425
1426                    //advance to tail
1427                    for (;
1428                         subClsIdCols.next != null;
1429                         subClsIdCols = subClsIdCols.next) {
1430                        ;
1431                    }
1432                }
1433            }
1434        }
1435        return pos;
1436    }
1437
1438    /**
1439     * Should we add the id col of the table to determine the type. This is intended for vertical inheritance
1440     * were all classes are mapped to seperate tables.
1441     */
1442    private boolean addIdCol(ClassMetaData cmd) {
1443        if (((JdbcClass)cmd.storeClass).classIdCol == null) {
1444            return true;
1445        }
1446        return false;
1447    }
1448
1449    private SqlExp addFetchGroupFields(SelectExp root, FetchGroup group,
1450            SqlExp pos, FgDs fgDs, JoinStructure js, boolean prefetchAllowed,
1451            SelectExp joinFromExp,
1452            JdbcColumn[] lJoinCols) {
1453        boolean joinok = !root.outer;
1454        FetchGroupField[] fields = group.fields;
1455        int n = fields.length;
1456        for (int i = 0; i < n; i++) {
1457            FetchGroupField field = fields[i];
1458            JdbcField jdbcField = (JdbcField)field.fmd.storeField;
1459            if (field.fmd.isEmbeddedRef()) {
1460                continue;
1461            }
1462            SqlExp ce = jdbcField.toColumnExp(SelectExp.createJoinToSuperTable(
1463                    root,
1464                    joinFromExp, lJoinCols, jdbcField), false);
1465            if (ce != null) {
1466                pos.next = ce;
1467                for (; pos.next != null; pos = pos.next) ;
1468            }
1469
1470            if (field.fmd.secondaryField) {
1471                if (fgDs != null && !fgDs.isFinished()) {
1472                    new JoinStructure(js, field);
1473                }
1474            }
1475
1476            if (joinok && jdbcField instanceof JdbcRefField
1477                    && field.jdbcUseJoin != JdbcRefField.USE_JOIN_NO) {
1478                if ((group.name.equals(FetchGroup.DFG_NAME)
1479                        && !field.fmd.isDefaultFetchGroupTrue())) {
1480                    continue;
1481                }
1482                JdbcRefField rf = (JdbcRefField)jdbcField;
1483                FetchGroup ng = field.nextFetchGroup;
1484
1485                FgDs nextFgDs = null;
1486                JoinStructure nJs = null;
1487                if (fgDs != null && !fgDs.isFinished()) {
1488                    nextFgDs = new FgDs(fgDs, ng, "",
1489                            JdbcFetchGroup.createOpts(false, false));
1490                    fgDs.addReference(nextFgDs, field.fmd);
1491                    nJs = new JoinStructure(js, field);
1492                }
1493
1494                // see if we have an existing join to this table
1495                SelectExp se;
1496                Join j = root.findJoin(rf);
1497                if (j == null) {
1498                    se = new SelectExp();
1499                    se.outer = field.jdbcUseJoin == JdbcRefField.USE_JOIN_OUTER;
1500                    se.table = ((JdbcClass)ng.classMetaData.storeClass).table;
1501                    se.jdbcField = rf;
1502                    root.addJoin(rf.cols, se.table.pk, se);
1503                } else {
1504                    if (j.next != null) {
1505                        if (j == root.joinList) {
1506                            root.joinList = j.next;
1507                        } else {
1508                            Join before = JdbcJDOQLCompiler.findJoinBefore(j, root.joinList);
1509                            if (before != null) {
1510                                before.next = j.next;
1511                            }
1512                        }
1513
1514                        //go to the end of the list
1515                        Join lastJoin = getLastJoin(j);
1516                        lastJoin.next = j;
1517                        j.next = null;
1518                    }
1519                    se = j.selectExp;
1520                }
1521                addSelectFetchGroupImp(se, ng, true, true, nextFgDs, nJs, prefetchAllowed,
1522                        root, rf.cols, new SqlExp(), true, false, rf);
1523            }
1524        }
1525        return pos;
1526    }
1527
1528    /**
1529     * Create a state from rs which must contain columns as added by
1530     * addSelectFetchGroup. If there are no main table columns in the group
1531     * then rs may be null. If includeSubclasses is true then fields from sub
1532     * classes will be also be read. The nextCol parameter is set to the
1533     * index of the column after the last column read if it is not null.
1534     * The oid parameter will be resolved from the created state.
1535     */
1536    public State createStateImp(ResultSet rs, OID oid,
1537            FetchGroup group, boolean forUpdate, int firstCol,
1538            MutableInt nextCol, boolean includeSubclasses,
1539            StateContainer container, FgDs fgDs,
1540            boolean fetchPass2Fields,
1541            boolean crossJoinFirstPass2Field,
1542            JoinStructure js) throws SQLException {
1543        ClassMetaData cmd = group.classMetaData;
1544        if (includeSubclasses) {
1545            /**
1546             * This will only work if the hierarchy is mapped in same table
1547             */
1548            if (((JdbcClass)cmd.storeClass).classIdCol != null) {
1549                // read the classId column from rs to decide what the real class is
1550                JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
1551                JdbcColumn classIdCol = jdbcClass.classIdCol;
1552                Object   classId = classIdCol.get(rs, firstCol++);
1553                if (rs.wasNull()) {
1554                    throw BindingSupportImpl.getInstance().objectNotFound("No row for " +
1555                            cmd.storeClass + " " + oid.toSString() + " OR " + classIdCol.name +
1556                            " is null for row");
1557                }
1558                cmd = jdbcClass.findClass(classId);
1559                if (cmd == null) {
1560                    throw BindingSupportImpl.getInstance().fatalDatastore(
1561                            "Row for OID " + oid.toSString() +
1562                            " is not in the heirachy starting at " +
1563                            group.classMetaData.storeClass +
1564                            " (" + classIdCol.name + " for row is " + classId + ")");
1565                }
1566            } else {
1567                if (cmd.pcSubclasses != null) {
1568                    if (((JdbcClass)cmd.storeClass).readAsClass != null) {
1569                        cmd = ((JdbcClass)cmd.storeClass).readAsClass;
1570                    } else {
1571                        //must start from end of rs and work up to determine type
1572                        cmd = extractType(rs, group, firstCol);
1573                        if (cmd == null) {
1574                            throw BindingSupportImpl.getInstance().internal("The instance type for " +
1575                                    "this row could not be determined");
1576                        }
1577                        //must update firstCol with the amount of possible id cols
1578                        firstCol += (group.classMetaData.totalNoOfSubClasses);
1579                    }
1580                }
1581            }
1582        }
1583
1584        State state = cmd.createState();
1585        oid.resolve(state);
1586
1587        state.copyFields(oid);
1588
1589        //this will read the stuff into the state
1590        firstCol = readFetchGroupFields(rs, oid, state, group, firstCol);
1591        //let the same table fetchGroups read their stuff from rs
1592        final FgDs[] sameTableFgDss = fgDs.getSameTableFgDss();
1593        for (int i = 0; i < sameTableFgDss.length; i++) {
1594            FgDs sameTableFgDs = sameTableFgDss[i];
1595            if (cmd.isAncestorOrSelf(sameTableFgDs.fg.classMetaData)) {
1596                firstCol = readFetchGroupFields(rs, oid, state,
1597                        sameTableFgDs.fg, firstCol);
1598            } else {
1599                firstCol = skipFetchGroupFields(sameTableFgDs.fg, firstCol);
1600            }
1601        }
1602
1603        //add the state to the receiver as to avoid recursive calls
1604        container.visited(oid);
1605
1606        /**
1607         * This follows pass2Fields and joins from fg.
1608         */
1609        firstCol = readFetchGroupPass2Fields(rs, oid, state, group, forUpdate,
1610                firstCol, fgDs, container, fetchPass2Fields,
1611                crossJoinFirstPass2Field);
1612        for (int i = 0; i < sameTableFgDss.length; i++) {
1613            FgDs sameTableFgDs = sameTableFgDss[i];
1614            if (cmd.isAncestorOrSelf(sameTableFgDs.fg.classMetaData)) {
1615                firstCol = readFetchGroupPass2Fields(rs, oid, state,
1616                        sameTableFgDs.fg,
1617                        forUpdate, firstCol, sameTableFgDs,
1618                        container, fetchPass2Fields, false);
1619            } else {
1620                firstCol = skipFetchGroupPass2Fields(firstCol, sameTableFgDs);
1621            }
1622        }
1623
1624        if (nextCol != null) nextCol.value = firstCol;
1625
1626        if (Debug.DEBUG) {
1627            if (Modifier.isAbstract(
1628                    state.getClassMetaData(jmd).cls.getModifiers())) {
1629                throw BindingSupportImpl.getInstance().internal(
1630                        "The cmd of this state is abstract");
1631            }
1632        }
1633        return state;
1634    }
1635
1636    /**
1637     * Read all the fields for the fetch group. This does not pickup reference
1638     * fields joined in.
1639     */
1640    private int readFetchGroupFields(ResultSet rs, OID oid,
1641            State state, FetchGroup group, int firstCol) {
1642        // populate the state with all the fields - this will create OIDs for
1643        // references
1644        try {
1645            ((JdbcState)state).copyPass1Fields(rs, group, firstCol);
1646//            System.out.println("\n\n after copy pass1Fields oid = " + oid + " state = " + state + "\n\n\n");
1647        } catch (Exception   e) {
1648            throw handleException("Error reading fields from ResultSet: " +
1649                        oid.toSString() + ": " + e + "\n" + getResultSetInfo(rs),
1650                        e);
1651        }
1652        return firstCol + group.jdbcTotalCols;
1653    }
1654
1655    private Join getLastJoin(Join j) {
1656        Join lastJoin = j.next;
1657        for (;;) {
1658            if (lastJoin.next == null) break;
1659            lastJoin = lastJoin.next;
1660        }
1661        return lastJoin;
1662    }
1663
1664    private static String   getResultSetInfo(ResultSet rs) {
1665        if (rs instanceof LoggingResultSet) {
1666            LoggingResultSet lrs = (LoggingResultSet)rs;
1667            return "Row data read: " + lrs.getRowDataString() + "\n" + lrs.getSql();
1668        } else {
1669            return "(set event logging to all for more info on the data and SQL)";
1670        }
1671    }
1672
1673    private int skipFetchGroupFields(FetchGroup group, int firstCol) {
1674        return firstCol + group.jdbcTotalCols;
1675    }
1676
1677    private int skipFetchGroupPass2Fields(int firstCol, FgDs fgDs) {
1678        return firstCol + fgDs.getChildrenColumnCount();
1679    }
1680
1681    private int readFetchGroupPass2Fields(ResultSet rs, OID oid,
1682            State state, FetchGroup group,
1683            boolean forUpdate, int firstCol,
1684            FgDs fgDs, StateContainer container,
1685            boolean fetchPass2Fields,
1686            boolean crossJoinFirstPass2Field) throws SQLException {
1687        FgDs currentFgDs;
1688        int currentFgIndex = 0;
1689        FetchGroupField[] fields = group.fields;
1690        int n = fields.length;
1691        for (int i = 0; i < n; i++) {
1692            FetchGroupField field = fields[i];
1693            FieldMetaData fmd = field.fmd;
1694
1695            if (fmd.secondaryField) {
1696                if (crossJoinFirstPass2Field
1697                        && group.crossJoinedCollectionField == field) {
1698                } else {
1699                    if (fetchPass2Fields) {
1700                        fetchPass2Field(oid, state, field, forUpdate,
1701                                container, true, null);
1702                    } else {
1703                        ((JdbcCollectionField)field.fmd.storeField).fillStateWithEmpty(
1704                                field, state);
1705                    }
1706                }
1707                continue;
1708            }
1709
1710            if (fmd.category != FieldMetaData.CATEGORY_REF) continue;
1711            if (field.jdbcUseJoin == JdbcRefField.USE_JOIN_NO) continue;
1712
1713            FetchGroup nextFetchGroup = field.nextFetchGroup;
1714            if (fgDs.isEmpty()) {
1715                currentFgDs = null;
1716            } else {
1717                currentFgDs = fgDs.get(currentFgIndex);
1718            }
1719
1720            /**
1721             * If these fg are not the same then the fg was not included by the sql generation
1722             * so we can skip it.
1723             */
1724            if (currentFgDs != null
1725                    && currentFgDs.fg == nextFetchGroup
1726                    && currentFgDs.refFmd == field.fmd) {
1727                JdbcOID roid = (JdbcOID)state.getInternalObjectField(fmd.stateFieldNo);
1728                if (roid == null || !container.isStateRequired(roid,
1729                        nextFetchGroup)) {
1730                    firstCol += currentFgDs.columnSkipCount;
1731                } else {
1732                    State ns = null;
1733                    int lColIndex = firstCol;
1734                    if (currentFgDs.isRefIdFieldsAdded()) {
1735                        if (roid != null && !roid.validateKeyFields(rs,
1736                                firstCol)) {
1737                            state.setInternalObjectField(fmd.stateFieldNo,
1738                                    null);
1739                            Utils.checkToThrowRowNotFound(oid, roid, jmd);
1740                            ns = NULLState.NULL_STATE;
1741                            container.addState(roid, ns);
1742                        }
1743                        //update the skip count
1744                        firstCol += ((JdbcClass)currentFgDs.fg.classMetaData.storeClass).table.pk.length;
1745                    }
1746
1747                    if (ns != NULLState.NULL_STATE) {
1748                        MutableInt mut = new MutableInt();
1749                        ns = createStateImp(rs, roid, nextFetchGroup,
1750                                forUpdate, firstCol,
1751                                mut, true, container, currentFgDs,
1752                                fetchPass2Fields, false, null);
1753                        roid.resolve(ns);
1754                        container.addState(roid, ns);
1755                    }
1756                    firstCol = lColIndex + currentFgDs.columnSkipCount;
1757
1758                    if (Debug.DEBUG) {
1759                        if (Modifier.isAbstract(
1760                                ns.getClassMetaData(jmd).cls.getModifiers())) {
1761                            throw BindingSupportImpl.getInstance().internal(
1762                                    "The cmd of this state is abstract");
1763                        }
1764                    }
1765                }
1766                currentFgIndex++;
1767            } else {
1768                continue;
1769            }
1770        }
1771
1772        if (Debug.DEBUG) {
1773            if (fgDs.getAmountOfRefFgDs() != currentFgIndex) {
1774                throw BindingSupportImpl.getInstance().internal(
1775                        "Not the right amount of refFg was read");
1776            }
1777        }
1778        return firstCol;
1779    }
1780
1781    private void doUpdates(StatesToStore toStore, StateContainer container,
1782            boolean retainValues) throws SQLException {
1783        initPersistGraph(toStore.isFullSortRequired(), toStore.size());
1784
1785        OID oid = null;
1786        int n = toStore.size();
1787        for (int i = 0; i < n; i++) {
1788            oid = toStore.oids[i];
1789
1790            // check if the state needs to be returned to the client
1791            // i.e. it has autoset fields or is new
1792            ClassMetaData cmd = toStore.states[i].getClassMetaData(jmd);
1793            if (retainValues || cmd.hasAutoSetFields) {
1794                container.add(oid, toStore.states[i]);
1795            } else if (oid.isNew()) {
1796                container.add(oid, null);
1797            }
1798
1799            checkReqFieldsOnUpdate(oid, cmd, toStore, i);
1800            activePersistGraph.add(oid, toStore.origStates[i],
1801                    toStore.states[i]);
1802        }
1803        activePersistGraph.doAutoSets();
1804        activePersistGraph.sort();
1805        if (Debug.DEBUG) {
1806            activePersistGraph.dump();
1807        }
1808        persistPass1(activePersistGraph);
1809        persistPass2(activePersistGraph);
1810    }
1811
1812    /**
1813     * Make sure required (e.g. jdoVersion) fields are filled in origstate.
1814     * Fetch the data from the database if needed.
1815     */
1816    private void checkReqFieldsOnUpdate(OID oid, ClassMetaData cmd,
1817            StatesToStore toStore, int i) throws SQLException {
1818        if (!oid.isNew()) {
1819            FetchGroup reqFetchGroup = cmd.reqFetchGroup;
1820            if (reqFetchGroup != null) {
1821                State old = toStore.origStates[i];
1822                if (!old.containsFetchGroup(reqFetchGroup)) {
1823                    State s = cache.getState(oid, reqFetchGroup);
1824                    if (s == null) {
1825                        s = getStateParColFetch((JdbcOID)oid, reqFetchGroup, false,
1826                                DummyStateContainer.INSTANCE);
1827                    }
1828                    old.updateNonFilled(s);
1829                }
1830            }
1831        }
1832    }
1833
1834    /**
1835     * Select the correct PersistGraph instance to use and make sure it is
1836     * big enough. It is cleared for use.
1837     */
1838    private void initPersistGraph(boolean fullSort, int minSize) {
1839        if (fullSort) {
1840            if (persistGraphFullSort == null
1841                    || minSize > persistGraphFullSort.size()) {
1842                persistGraphFullSort = new PersistGraphFullSort(jmd,
1843                        minSize * 2);
1844            } else {
1845                persistGraphFullSort.clear();
1846            }
1847            activePersistGraph = persistGraphFullSort;
1848        } else {
1849            if (persistGraphPartialSort == null
1850                    || minSize > persistGraphPartialSort.size()) {
1851                persistGraphPartialSort = new PersistGraph(jmd,
1852                        minSize * 2);
1853            } else {
1854                persistGraphPartialSort.clear();
1855            }
1856            activePersistGraph = persistGraphPartialSort;
1857        }
1858        activePersistGraph.optimistic = this.optimistic;
1859    }
1860
1861    /**
1862     * Persist main table fields. All the OID's for new objects are replaced
1863     * with real OID's.
1864     *
1865     * @see #persistPass2
1866     */
1867    public void persistPass1(PersistGraph graph) {
1868        try {
1869            int[] fieldNos = new int[jmd.maxFieldsLength];
1870            Object  [] oidData = new Object  [((JdbcMetaData)jmd.jdbcMetaData).maxPkSimpleColumns];
1871
1872            CharBuf s = new CharBuf();
1873            boolean haveNewObjects = false;
1874            int graphSize = graph.size();
1875
1876            // generate primary keys for all new objects with preinsert keygens
1877            // that do not already have keys and all application identity
1878            // instances not using a post insert keygen
1879            for (int si = 0; si < graphSize; si++) {
1880                OID oid = graph.getOID(si);
1881                if (!oid.isNew() || ((NewObjectOID)oid).realOID != null) continue;
1882                haveNewObjects = true;
1883                ClassMetaData cmd = oid.getClassMetaData();
1884                JdbcKeyGenerator keygen = ((JdbcClass)cmd.storeClass).jdbcKeyGenerator;
1885                if (keygen == null) {
1886                    if (cmd.identityType == MDStatics.IDENTITY_TYPE_APPLICATION) {
1887                        State ns = graph.getNewState(si);
1888                        if (Debug.DEBUG) {
1889                            if (!ns.containsApplicationIdentityFields()) {
1890                                throw BindingSupportImpl.getInstance().internal(
1891                                        "pk fields not filled for appid class\n" + ns);
1892                            }
1893                        }
1894                        NewObjectOID noid = (NewObjectOID)oid;
1895                        ns.copyKeyFields(noid.realOID = cmd.createOID(true));
1896                        continue;
1897                    } else {
1898                        throw BindingSupportImpl.getInstance().runtime("Class " + cmd.qname +
1899                                " has identity-type " +
1900                                MDStaticUtils.toIdentityTypeString(
1901                                        cmd.identityType) +
1902                                " but no jdbc-key-generator");
1903                    }
1904                } else {
1905                    //if it is app id and the state does contain the key field's
1906                    //then we do not use the keygen to do it.
1907                    if (cmd.identityType == MDStatics.IDENTITY_TYPE_APPLICATION) {
1908                        State ns = graph.getNewState(si);
1909                        if (ns.containsValidAppIdFields()) {
1910                            ns.copyKeyFields(((NewObjectOID)oid).realOID = cmd.createOID(
1911                                    true));
1912                            continue;
1913                        }
1914                    }
1915                }
1916                if (keygen.isPostInsertGenerator()) continue;
1917
1918                // count how many keys we need
1919                int keyCount = 1;
1920                for (int i = si + 1; i < graphSize; i++) {
1921                    OID nextOid = graph.getOID(i);
1922                    if (!nextOid.isNew() || nextOid.getClassIndex() != cmd.index) break;
1923                    if (((NewObjectOID)nextOid).realOID == null) keyCount++;
1924                }
1925
1926                Connection kgcon = null;
1927                boolean rollback = true;
1928                try {
1929                    OID realOID;
1930                    for (; keyCount > 0;) {
1931                        NewObjectOID noid = (NewObjectOID)graph.getOID(si++);
1932                        if (noid.realOID != null) continue;
1933                        noid.realOID = realOID = cmd.createOID(true);
1934                        boolean needKgcon = keygen.isRequiresOwnConnection();
1935                        if (kgcon == null && needKgcon) {
1936                            kgcon = conSrc.getConnection(true, false);
1937                        }
1938                        keygen.generatePrimaryKeyPre(cmd.qname,
1939                                ((JdbcClass)cmd.storeClass).table, keyCount--, oidData,
1940                                needKgcon ? kgcon : con());
1941                        realOID.copyKeyFields(oidData);
1942                    }
1943                    rollback = false;
1944                } finally {
1945                    if (kgcon != null) {
1946                        if (rollback) {
1947                            kgcon.rollback();
1948                        } else {
1949                            kgcon.commit();
1950                        }
1951                        conSrc.returnConnection(kgcon);
1952                    }
1953                }
1954                si--;
1955            }
1956
1957            // generate the inserts and updates
1958            IntArray toUpdateIndexes = new IntArray();
1959            for (int i = 0; i < graphSize;) {
1960                OID oid = graph.getOID(i);
1961                ClassMetaData cmd = oid.getClassMetaData();
1962                if (oid.isNew()) {
1963                    toUpdateIndexes.clear();
1964                    i = generateInserts((NewObjectOID)oid, i, cmd, graph,
1965                            fieldNos, s, oidData, toUpdateIndexes);
1966                    if (toUpdateIndexes.size() > 0) {
1967                        //must update these
1968                        int updateStartIndex = 0;
1969                        for (;;) {
1970                            updateStartIndex = generateUpdatesOfCircularReferences(
1971                                    oid, cmd, fieldNos, haveNewObjects,
1972                                    s, toUpdateIndexes, updateStartIndex, graph);
1973                            if (updateStartIndex == toUpdateIndexes.size()) break;
1974                        }
1975                    }
1976                } else {
1977                    i = generateUpdates(oid, i, cmd, graph, fieldNos,
1978                            haveNewObjects, s);
1979                }
1980            }
1981        } catch (SQLException e) {
1982            throw handleException(e);
1983        }
1984    }
1985
1986    /**
1987     * Generate insert statement(s) for pass1 changes to one or more new
1988     * objects. The only objects without keys will be those using app identity
1989     * and no keygen and those using postInsert keygens.
1990     *
1991     * @return The index of the last object inserted + 1
1992     */
1993    private int generateInserts(NewObjectOID oid,
1994            int index, ClassMetaData cmd, PersistGraph graph, int[] fieldNos,
1995            CharBuf s, Object  [] oidData, IntArray toUpdateIndexes) throws SQLException {
1996
1997        int identityType = cmd.identityType;
1998        boolean appIdentity = identityType == MDStatics.IDENTITY_TYPE_APPLICATION;
1999        JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
2000        int graphSize = graph.size();
2001
2002        JdbcColumn classIdCol = jdbcClass.classIdCol;
2003        Object   classId = classIdCol == null ? null : jdbcClass.jdbcClassId;
2004
2005        JdbcKeyGenerator keygen = jdbcClass.jdbcKeyGenerator;
2006
2007        // decide if we can use statement batching
2008        boolean batchPossible = (keygen == null || !keygen.isPostInsertGenerator())
2009                && useBatchInsert
2010                && !jdbcClass.noBatching;
2011
2012        // count how many states we have with the same class that are new
2013        int sameClassCount = 1;
2014        for (int i = index + 1; i < graphSize; sameClassCount++, i++) {
2015            OID nextOid = graph.getOID(i);
2016            if (!nextOid.isNew() || nextOid.getClassIndex() != cmd.index) break;
2017        }
2018
2019        Connection con = con();
2020
2021        for (; sameClassCount > 0;) {
2022
2023            State ns = graph.getNewState(index);
2024
2025            // figure out what key generation needs to happen
2026            boolean useKeyGenPre = false;
2027            boolean useKeyGenPost = false;
2028            boolean fillFieldsFromOid = false;
2029            boolean clearAppIdFields = false;
2030            if (appIdentity) {
2031                if (keygen != null) {
2032                    useKeyGenPost = keygen.isPostInsertGenerator();
2033                    useKeyGenPre = !useKeyGenPost;
2034                    fillFieldsFromOid = true;
2035                    clearAppIdFields = true;
2036                    ns.clearApplicationIdentityFields();
2037                }
2038            } else {
2039                useKeyGenPost = keygen.isPostInsertGenerator();
2040                useKeyGenPre = !useKeyGenPost;
2041            }
2042
2043            // count how many states we can insert with the same SQL
2044            int count = 1;
2045            for (int i = index + 1; count < sameClassCount; count++, i++) {
2046                // make sure the next state has the same field numbers
2047                State nextState = graph.getNewState(i);
2048                if (clearAppIdFields) nextState.clearApplicationIdentityFields();
2049                if (nextState.compareToPass1(ns) != 0) break;
2050            }
2051
2052            boolean batch = batchPossible && count > 1;
2053            int numFieldNos = ns.getPass1FieldNos(fieldNos);
2054
2055            // do a stripe of inserts for each table for the class
2056            int startIndex = index;
2057            int startSameClassCount = sameClassCount;
2058            int startCount = count;
2059            for (int tableNo = 0;
2060                 tableNo < jdbcClass.allTables.length; tableNo++) {
2061                JdbcTable table = jdbcClass.allTables[tableNo];
2062                if (tableNo > 0) {   // reset after first table
2063                    classIdCol = null;
2064                    useKeyGenPre = true;
2065                    useKeyGenPost = false;
2066                    fillFieldsFromOid = false;
2067                    ns = graph.getNewState(index = startIndex);
2068                    oid = (NewObjectOID)graph.getOID(index);
2069                    sameClassCount = startSameClassCount;
2070                    count = startCount;
2071                }
2072
2073                // create PreparedStatement and do count inserts for each
2074                PreparedStatement ps = null;
2075                try {
2076
2077                    // create ps for the insert
2078                    boolean lobColFound = createInsertSql(jdbcClass, table,
2079                            useKeyGenPre, classIdCol, fieldNos, numFieldNos, s,
2080                            ns);
2081                    if (useKeyGenPost) {
2082                        String   suffix = keygen.getPostInsertSQLSuffix(table);
2083                        if (suffix != null) s.append(suffix);
2084                    }
2085                    String   sql = s.toString();
2086                    ps = con.prepareStatement(sql);
2087
2088                    for (; ;) {
2089                        boolean alreadyHaveRealOID = oid.realOID != null;
2090                        JdbcOID realOID;
2091                        if (alreadyHaveRealOID) {
2092                            realOID = (JdbcOID)oid.realOID;
2093                        } else {
2094                            realOID = (JdbcOID)(oid.realOID = cmd.createOID(true));
2095                        }
2096
2097                        int pos = useKeyGenPre ? realOID.setParams(ps, 1) : 1;
2098
2099                        if (classIdCol != null) {
2100                            classIdCol.set(ps, pos++, classId);
2101                        }
2102
2103                        // do the insert
2104                        if (tableNo == 0) {
2105                            if (ns.replaceNewObjectOIDs(fieldNos, numFieldNos)) {
2106                                toUpdateIndexes.add(index);
2107                            }
2108                        }
2109                        try {
2110                            ((JdbcState)ns).setParams(ps, fieldNos, 0, numFieldNos, pos,
2111                                    graph, tableNo);
2112                        } catch (Exception   e) {
2113                            throw handleException(
2114                                    "Error setting parameters on PreparedStatement " +
2115                                    "for insert of '" + Utils.toString(realOID) + "':\n" +
2116                                    JdbcUtils.toString(e) + "\n" +
2117                                    JdbcUtils.getPreparedStatementInfo(sql, ps),
2118                                    e);
2119                        }
2120                        if (batch) {
2121                            ps.addBatch();
2122                        } else {
2123                            try {
2124                                ps.execute();
2125                            } catch (Exception   e) {
2126                                throw handleException("Insert of '" +
2127                                        Utils.toString(realOID) + "' failed: " +
2128                                        JdbcUtils.toString(e) + "\n" +
2129                                        JdbcUtils.getPreparedStatementInfo(sql,
2130                                                ps),
2131                                        e);
2132                            }
2133                        }
2134
2135                        // do post insert key generation if required
2136                        if (useKeyGenPost) {
2137                            keygen.generatePrimaryKeyPost(cmd.qname, table,
2138                                    oidData, con,
2139                                    ((PooledPreparedStatement)ps).getStatement());
2140                            realOID.copyKeyFields(oidData);
2141                        }
2142
2143                        if (fillFieldsFromOid) ns.copyFields(realOID);
2144
2145                        ++index;
2146                        if (--sameClassCount == 0) break;
2147                        oid = (NewObjectOID)graph.getOID(index);
2148                        if (--count == 0) break;
2149
2150                        ns = graph.getNewState(index);
2151                    }
2152                    if (batch) {
2153                        try {
2154                            ps.executeBatch();
2155                        } catch (Exception   e) {
2156                            throw handleException(
2157                                    "Batch insert failed: " +
2158                                    JdbcUtils.toString(e) + "\n" +
2159                                    JdbcUtils.getPreparedStatementInfo(sql, ps),
2160                                    e);
2161                        }
2162                    }
2163
2164                    // If there was at least one Oracle style LOB col we have to
2165                    // select all of the non null LOB cols back to give their
2166                    // converters a chance to set the data in the LOB.
2167                    int lobNumFieldNos;
2168                    if (lobColFound &&
2169                            (lobNumFieldNos = removeNullLOBFields(fieldNos,
2170                                    numFieldNos, ns)) > 0) {
2171                        selectAndUpdateOracleLOBCols(s, startIndex, index,
2172                                jdbcClass, table, fieldNos, lobNumFieldNos,
2173                                graph);
2174                    }
2175
2176                } finally {
2177                    cleanup(ps);
2178                }
2179            }
2180        }
2181
2182        return index;
2183    }
2184
2185    /**
2186     * Look at all the negative entries in fieldNos, make any that are for
2187     * not null fields in state positive and copy them to the beginning of
2188     * fieldsNos. Return the number of not-null negative entries found and
2189     * converted. This effectively compresses the array of fieldNos for
2190     * faster LOB processing.
2191     */
2192    private int removeNullLOBFields(int[] fieldNos, int numFieldNos,
2193            State state) {
2194        int pos = 0;
2195        for (int i = 0; i < numFieldNos; i++) {
2196            int fieldNo = fieldNos[i];
2197            if (fieldNo < 0) {
2198                fieldNo = -(fieldNo + 1);
2199                if (!state.isNull(fieldNo)) fieldNos[pos++] = fieldNo;
2200            }
2201        }
2202        return pos;
2203    }
2204
2205    /**
2206     * Generate SQL to insert a row into the table for a class heirachy. Any
2207     * entries in fieldNos for fields that return true to appendInsertValueList
2208     * (i.e. they did not add a replacable param) are made negative. This is
2209     * used to handle Oracle LOB columns. Returns true if there was at least
2210     * one such column or false otherwise.
2211     */
2212    private boolean createInsertSql(JdbcClass jdbcClass, JdbcTable table,
2213            boolean useKeyGenPre, JdbcColumn classIdCol, int[] fieldNos,
2214            int numFieldNos, CharBuf s, State state) {
2215        JdbcField[] stateFields = jdbcClass.stateFields;
2216        s.clear();
2217        s.append("INSERT INTO ");
2218        s.append(table.name);
2219        s.append(" (");
2220        if (useKeyGenPre) table.appendInsertPKColumnList(s);
2221        boolean first = !useKeyGenPre;
2222        if (classIdCol != null) {
2223            if (first) {
2224                first = false;
2225            } else {
2226                s.append(',');
2227                s.append(' ');
2228            }
2229            classIdCol.appendNames(s);
2230        }
2231        for (int i = 0; i < numFieldNos; i++) {
2232            int fieldNo = fieldNos[i];
2233            JdbcField f = stateFields[fieldNo];
2234            if (f.mainTableColsForUpdate == null || f.mainTable != table) continue;
2235            if (first) {
2236                first = false;
2237            } else {
2238                s.append(',');
2239                s.append(' ');
2240            }
2241            f.appendInsertColumnList(s);
2242        }
2243        s.append(") VALUES (");
2244        if (useKeyGenPre) table.appendInsertPKValueList(s);
2245        first = !useKeyGenPre;
2246        if (classIdCol != null) {
2247            if (first) {
2248                first = false;
2249            } else {
2250                s.append(',');
2251                s.append(' ');
2252            }
2253            classIdCol.appendParams(s);
2254        }
2255        boolean lobColFound = false;
2256        for (int i = 0; i < numFieldNos; i++) {
2257            int fieldNo = fieldNos[i];
2258            JdbcField f = stateFields[fieldNo];
2259            if (f.mainTableColsForUpdate == null || f.mainTable != table) continue;
2260            if (first) {
2261                first = false;
2262            } else {
2263                s.append(',');
2264                s.append(' ');
2265            }
2266            if (f.appendInsertValueList(s, state)) {
2267                // no replaceable param so skip field when params are set
2268                fieldNos[i] = -(fieldNo + 1);
2269                lobColFound = true;
2270            }
2271        }
2272        s.append(")");
2273        return lobColFound;
2274    }
2275
2276    /**
2277     * Select all the LOB cols for the OIDs in the graph and update them.
2278     * All the states will have the same LOB fields and only LOB fields
2279     * for table will be in the fieldNos array.
2280     */
2281    private void selectAndUpdateOracleLOBCols(CharBuf s, int startIndex,
2282            int index, JdbcClass jdbcClass, JdbcTable table, int[] fieldNos,
2283            int numFieldNos, PersistGraph graph)
2284            throws SQLException {
2285        Connection con = con();
2286        ResultSet rs = null;
2287        PreparedStatement ps = null;
2288        try {
2289            int oidCount = index - startIndex;
2290            int maxOIDsForIN = jdbcClass.getMaxOIDsForIN(sqlDriver);
2291
2292            if (oidCount > 1 && maxOIDsForIN > 1) {
2293
2294                // process OIDs in blocks using IN (?, .., ?)
2295                Map map = new HashMap(maxOIDsForIN * 2);
2296                JdbcOID key = (JdbcOID)jdbcClass.cmd.createOID(true);
2297
2298                int fullBlocks = oidCount / maxOIDsForIN;
2299                if (fullBlocks > 0) {
2300                    s.clear();
2301                    createSelectLOBsSql(jdbcClass, table, fieldNos,
2302                            numFieldNos, s, maxOIDsForIN);
2303                    ps = con.prepareStatement(s.toString());
2304                    for (int i = 0; i < fullBlocks; i++) {
2305                        for (int j = 0; j < maxOIDsForIN;) {
2306                            OID oid = graph.getOID(startIndex);
2307                            if (oid instanceof NewObjectOID) {
2308                                oid = ((NewObjectOID)oid).realOID;
2309                            }
2310                            map.put(oid, graph.getNewState(startIndex++));
2311                            ((JdbcOID)oid).setParams(ps, ++j);
2312                        }
2313                        rs = ps.executeQuery();
2314                        for (int j = 0; j < maxOIDsForIN; j++) {
2315                            rs.next();
2316                            key.copyKeyFields(rs, 1);
2317                            State ns = (State)map.get(key);
2318                            ((JdbcState)ns).setOracleStyleLOBs(rs, fieldNos, numFieldNos, 2);
2319                        }
2320                        rs.close();
2321                    }
2322                    rs = null;
2323                    ps.close();
2324                    ps = null;
2325                }
2326
2327                oidCount = oidCount % maxOIDsForIN;
2328                if (oidCount > 1) {
2329                    // process partial block
2330                    s.clear();
2331                    createSelectLOBsSql(jdbcClass, table, fieldNos,
2332                            numFieldNos, s, oidCount);
2333                    ps = con.prepareStatement(s.toString());
2334                    map.clear();
2335                    for (int j = 0; j < oidCount;) {
2336                        OID oid = graph.getOID(startIndex);
2337                        if (oid instanceof NewObjectOID) {
2338                            oid = ((NewObjectOID)oid).realOID;
2339                        }
2340                        map.put(oid, graph.getNewState(startIndex++));
2341                        ((JdbcOID)oid).setParams(ps, ++j);
2342                    }
2343                    rs = ps.executeQuery();
2344                    for (int j = 0; j < oidCount; j++) {
2345                        rs.next();
2346                        key.copyKeyFields(rs, 1);
2347                        State ns = (State)map.get(key);
2348                        ((JdbcState)ns).setOracleStyleLOBs(rs, fieldNos, numFieldNos, 2);
2349                    }
2350                    rs.close();
2351                    rs = null;
2352                    ps.close();
2353                    ps = null;
2354                    oidCount = 0;
2355                }
2356            }
2357
2358            if (oidCount == 1 || maxOIDsForIN <= 1) {
2359                // process OIDs one at a time
2360                s.clear();
2361                createSelectLOBsSql(jdbcClass, table, fieldNos, numFieldNos, s,
2362                        1);
2363                ps = con.prepareStatement(s.toString());
2364                for (int i = startIndex; i < index; i++) {
2365                    OID oid = graph.getOID(i);
2366                    if (oid instanceof NewObjectOID) {
2367                        oid = ((NewObjectOID)oid).realOID;
2368                    }
2369                    ((JdbcOID)oid).setParams(ps, 1);
2370                    rs = ps.executeQuery();
2371                    if (!rs.next()) {
2372                        throw BindingSupportImpl.getInstance().fatalDatastore(
2373                                "Row not found: " + oid.toSString());
2374                    }
2375                    State ns = graph.getNewState(i);
2376                    ((JdbcState)ns).setOracleStyleLOBs(rs, fieldNos, numFieldNos, 1);
2377                    rs.close();
2378                }
2379                rs = null;
2380                ps.close();
2381                ps = null;
2382            }
2383        } finally {
2384            cleanup(rs);
2385            cleanup(ps);
2386        }
2387    }
2388
2389    /**
2390     * Generate SQL to select Oracle style LOB columns for a table for a
2391     * class heirachy. The SQL must provide for blocksz OID parameters for
2392     * the query. If there is more than one then an IN must be used. The
2393     * blocksz will always be 1 if the table uses a composite primary key.
2394     *
2395     * @see #createInsertSql
2396     */
2397    private void createSelectLOBsSql(JdbcClass jdbcClass, JdbcTable table,
2398            int[] fieldNos, int numFieldNos, CharBuf s, int blocksz) {
2399        JdbcField[] stateFields = jdbcClass.stateFields;
2400        s.clear();
2401        s.append("SELECT ");
2402        if (blocksz > 1) {
2403            table.appendInsertPKColumnList(s);
2404            s.append(',');
2405            s.append(' ');
2406        }
2407        stateFields[fieldNos[0]].appendInsertColumnList(s);
2408        for (int i = 1; i < numFieldNos; i++) {
2409            s.append(',');
2410            s.append(' ');
2411            stateFields[fieldNos[i]].appendInsertColumnList(s);
2412        }
2413        s.append(" FROM ");
2414        s.append(table.name);
2415        s.append(" WHERE ");
2416        if (blocksz == 1) {
2417            table.appendWherePK(s);
2418        } else {
2419            s.append(table.pk[0].name); // will never be composite pk
2420            s.append(" IN (");
2421            s.append('?');
2422            for (int i = 1; i < blocksz; i++) {
2423                s.append(',');
2424                s.append('?');
2425            }
2426            s.append(')');
2427        }
2428        s.append(" FOR UPDATE");
2429    }
2430
2431    /**
2432     * Generate update statement(s) for pass1 changes to one or more objects.
2433     *
2434     * @return The index of the last object updated + 1
2435     */
2436    private int generateUpdates(OID oid, int index,
2437            ClassMetaData cmd, PersistGraph graph, int[] fieldNos,
2438            boolean haveNewObjects, CharBuf s)
2439            throws SQLException {
2440
2441        State ns = graph.getNewState(index);
2442        if (!ns.containsPass1Fields()) return ++index;
2443
2444        JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
2445        State os = graph.getOldState(index);
2446        boolean usingChanged =
2447                jdbcClass.optimisticLocking == JdbcClass.OPTIMISTIC_LOCKING_CHANGED;
2448        JdbcSimpleField optimisticLockingField = jdbcClass.optimisticLockingField;
2449        boolean usingOLF = optimisticLockingField != null;
2450
2451        Connection con = con();
2452
2453        // count how many states we can update with the same SQL
2454        int graphSize = graph.size();
2455        // the amount of states with the same fields for update
2456        int count = 1;
2457        for (int i = index + 1; i < graphSize; count++, i++) {
2458            // make sure the next object is not new and has the same class
2459            OID nextOid = graph.getOID(i);
2460            if (Debug.DEBUG) {
2461                if (!nextOid.isNew() && !nextOid.isResolved()) {
2462                    throw BindingSupportImpl.getInstance().internal("OID is not resolved: "
2463                            + oid.toSString());
2464                }
2465            }
2466            if (nextOid.isNew() || nextOid.getClassIndex() != cmd.index) break;
2467
2468            // make sure the next state has the same field numbers
2469            State nextState = graph.getNewState(i);
2470            if (nextState.compareToPass1(ns) != 0) break;
2471
2472            // if we are using changed optimistic locking make sure the next
2473            // old state has the same null fields as the current old state
2474            if (usingChanged) {
2475                if (!((JdbcState)os).hasSameNullFields(graph.getOldState(i), ns)) break;
2476            }
2477        }
2478
2479        // decide if we will use statement batching
2480        boolean batch = count > 1 && useBatchUpdate
2481                && !jdbcClass.noBatching;
2482
2483        // check if the OID must be updated from the state after the update
2484        boolean updateOIDFromState = ns.containsApplicationIdentityFields();
2485
2486        // deny update of app identity for inheritance heirachies
2487        if (updateOIDFromState && cmd.isInHeirachy()) {
2488            throw BindingSupportImpl.getInstance().runtime("Application identity change for inheritance heirachies " +
2489                    "is not supported: " + oid.toSString());
2490        }
2491
2492        final int numFieldNos = ns.getPass1FieldNos(fieldNos);
2493
2494        // do a stripe of inserts for each table for the class
2495        int startIndex = index;
2496        int startCount = count;
2497        for (int tableNo = 0; tableNo < jdbcClass.allTables.length; tableNo++) {
2498            JdbcTable table = jdbcClass.allTables[tableNo];
2499            if (tableNo > 0) {   // reset after first table
2500                ns = graph.getNewState(index = startIndex);
2501                os = graph.getOldState(index);
2502                oid = graph.getOID(index);
2503                count = startCount;
2504            }
2505
2506            // create PreparedStatement(s) and do count updates for each
2507            PreparedStatement ps = null;
2508            try {
2509                // generate the SQL for the table
2510                boolean lobColFound = createUpdateSql(jdbcClass, table, numFieldNos,
2511                        fieldNos, (usingOLF && tableNo == 0), optimisticLockingField, usingChanged, os,
2512                        ns, s);
2513                String   sql = s.toString();
2514                if (sql.length() == 0) {
2515                    index = index + startCount;
2516                    continue;
2517                }
2518
2519                ps = con.prepareStatement(sql);
2520
2521                for (; ;) {
2522                    if (haveNewObjects) {
2523                        ns.replaceNewObjectOIDs(fieldNos,
2524                                numFieldNos);
2525                    }
2526
2527                    // set parameters on ps
2528                    try {
2529                        int pos = ((JdbcState)ns).setParams(ps, fieldNos, 0, numFieldNos, 1,
2530                                graph, tableNo);
2531                        pos = ((JdbcOID)oid).setParams(ps, pos);
2532                        if (usingOLF && tableNo == 0) {
2533                            //this is only needed on the base table
2534                            ((JdbcState)os).setOptimisticLockingParams(ps, pos);
2535                        } else if (usingChanged) {
2536                            ((JdbcState)os).setParamsChangedAndNotNull(ps, fieldNos, 0,
2537                                    numFieldNos, pos, graph, tableNo);
2538                        }
2539                    } catch (SQLException e) {
2540                        throw handleException("Error setting parameters on PreparedStatement for " +
2541                                "update of '" + Utils.toString(oid) + "':\n" +
2542                                JdbcUtils.toString(e) + "\n" +
2543                                JdbcUtils.getPreparedStatementInfo(sql, ps), e);
2544                    }
2545
2546                    // do the update
2547                    if (batch) {
2548                        ps.addBatch();
2549                    } else {
2550                        int uc;
2551                        try {
2552                            uc = ps.executeUpdate();
2553                        } catch (Exception   e) {
2554                            throw handleException(
2555                                    "Update failed: " +
2556                                    JdbcUtils.toString(e) + "\n" +
2557                                    "Row: " + oid.toSString() + "\n" +
2558                                    JdbcUtils.getPreparedStatementInfo(sql, ps),
2559                                    e, true, oid);
2560                        }
2561                        if (uc == 0) {
2562                            throw BindingSupportImpl.getInstance().concurrentUpdate(
2563                                    "Row not found: " +
2564                                    oid.toSString() + "\n" +
2565                                    JdbcUtils.getPreparedStatementInfo(sql, ps), oid);
2566                        }
2567                    }
2568                    if (updateOIDFromState) ns.copyKeyFieldsUpdate(oid);
2569
2570                    index = index + 1;
2571                    if (--count == 0) break;
2572                    oid = graph.getOID(index);
2573                    ns = graph.getNewState(index);
2574                    os = graph.getOldState(index);
2575                }
2576
2577                // if batching then exec the batch and check all the update counts
2578                if (batch) {
2579                    int[] a;
2580                    try {
2581                        a = ps.executeBatch();
2582                    } catch (Exception   e) {
2583                        throw handleException("Batch update failed: " + JdbcUtils.toString(
2584                                e) + "\n" +
2585                                "Row: " + oid.toSString() + "\n" +
2586                                JdbcUtils.getPreparedStatementInfo(sql, ps), e, true, oid);
2587                    }
2588                    for (int j = 0; j < count; j++) {
2589                        int c = a[j];
2590                        if (c <= 0) {
2591                            String   psi = JdbcUtils.getPreparedStatementInfo(
2592                                    sql, ps, j);
2593                            oid = graph.getOID(startIndex + j);
2594                            if (c == 0) {
2595                                throw BindingSupportImpl.getInstance().concurrentUpdate(
2596                                        "Row not found: " + oid.toSString() + "\n" + psi, oid);
2597                            }
2598                            throw BindingSupportImpl.getInstance().datastore(
2599                                    "Unexpected update count " +
2600                                    c + " for row: " + oid.toSString() + "\n" + psi);
2601                        }
2602                    }
2603                }
2604
2605                // If there was at least one Oracle style LOB col we have to
2606                // select all of the non null LOB cols back to give their
2607                // converters a chance to set the data in the LOB.
2608                int lobNumFieldNos;
2609                if (lobColFound &&
2610                        (lobNumFieldNos = removeNullLOBFields(fieldNos,
2611                                numFieldNos, ns)) > 0) {
2612                    selectAndUpdateOracleLOBCols(s, startIndex, index,
2613                            jdbcClass, table, fieldNos, lobNumFieldNos,
2614                            graph);
2615                }
2616            } finally {
2617                cleanup(ps);
2618            }
2619        }
2620        return index;
2621    }
2622
2623    /**
2624      * This method is used as part of the insert operation to update all newOid's with null
2625      * realOids at the time of the insert. This will only happen if there is a circular dep on the
2626      * creation of id's (e.g. two new Application id instances that uses autoinc pk columns).
2627      */
2628     private int generateUpdatesOfCircularReferences(OID oid, ClassMetaData cmd,
2629             int[] fieldNos, boolean haveNewObjects, CharBuf s,
2630             IntArray toUpdateIndexes, int indexInIntArray, PersistGraph graph)
2631             throws SQLException {
2632
2633         State ns = graph.getNewState(toUpdateIndexes.get(indexInIntArray));
2634         if (!ns.containsPass1Fields()) return ++indexInIntArray;
2635
2636         JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
2637         State os = graph.getOldState(toUpdateIndexes.get(indexInIntArray));
2638         boolean usingChanged =
2639                 jdbcClass.optimisticLocking == JdbcClass.OPTIMISTIC_LOCKING_CHANGED;
2640         JdbcSimpleField optimisticLockingField = jdbcClass.optimisticLockingField;
2641         boolean usingOLF = optimisticLockingField != null;
2642
2643         // the amount of states with the same fields for update
2644         int count = 1;
2645         for (int i = indexInIntArray + 1; i < toUpdateIndexes.size(); count++, i++) {
2646             // make sure the next object is not new and has the same class
2647             OID nextOid = graph.getOID(toUpdateIndexes.get(i));
2648             if (Debug.DEBUG) {
2649                 if (!nextOid.isNew() && !nextOid.isResolved()) {
2650                     throw BindingSupportImpl.getInstance().internal("OID is not resolved: "
2651                             + oid.toSString());
2652                 }
2653             }
2654             if (nextOid.isNew() || nextOid.getClassIndex() != cmd.index) break;
2655
2656             // make sure the next state has the same field numbers
2657             State nextState = graph.getNewState(toUpdateIndexes.get(i));
2658             if (nextState.compareToPass1(ns) != 0) break;
2659
2660             // if we are using changed optimistic locking make sure the next
2661             // old state has the same null fields as the current old state
2662             if (usingChanged) {
2663                 if (!((JdbcState)os).hasSameNullFields(graph.getOldState(toUpdateIndexes.get(i)), ns)) break;
2664             }
2665         }
2666
2667         // decide if we will use statement batching
2668         boolean batch = count > 1 && useBatchUpdate && !jdbcClass.noBatching;
2669
2670         // check if the OID must be updated from the state after the update
2671//        boolean updateOIDFromState = ns.containsApplicationIdentityFields();
2672
2673         // deny update of app identity for inheritance heirachies
2674//        if (updateOIDFromState && cmd.isInHeirachy()) {
2675//            throw BindingSupportImpl.getInstance().runtime("Application identity change for inheritance heirachies " +
2676//                    "is not supported: " + oid.toSString());
2677//        }
2678
2679         final int numFieldNos = ns.getPass1FieldRefFieldNosWithNewOids(fieldNos);
2680
2681         // do a stripe of inserts for each table for the class
2682         Connection con = con(); 
2683         int startIndex = indexInIntArray;
2684         int startCount = count;
2685         for (int tableNo = 0; tableNo < jdbcClass.allTables.length; tableNo++) {
2686             JdbcTable table = jdbcClass.allTables[tableNo];
2687             if (tableNo > 0) {   // reset after first table
2688                 ns = graph.getNewState(toUpdateIndexes.get(indexInIntArray = startIndex));
2689                 os = graph.getOldState(toUpdateIndexes.get(indexInIntArray));
2690                 oid = graph.getOID(toUpdateIndexes.get(indexInIntArray));
2691                 count = startCount;
2692             }
2693
2694             // create PreparedStatement(s) and do count updates for each
2695             PreparedStatement ps = null;
2696             try {
2697                 // generate the SQL for the table
2698                 createUpdateSql(jdbcClass, table, numFieldNos,
2699                         fieldNos, false, optimisticLockingField, false, os,
2700                         ns, s);
2701                 String   sql = s.toString();
2702                 if (sql.length() == 0) {
2703                     indexInIntArray = indexInIntArray + startCount;
2704                     continue;
2705                 }
2706
2707                 ps = con.prepareStatement(sql);
2708
2709                 for (; ;) {
2710                     if (haveNewObjects) {
2711                         ns.replaceNewObjectOIDs(fieldNos,
2712                                 numFieldNos);
2713                     }
2714
2715                     // set parameters on ps
2716                     try {
2717                         int pos = ((JdbcState)ns).setParams(ps, fieldNos, 0, numFieldNos, 1,
2718                                 graph, tableNo);
2719                         pos = ((JdbcOID)oid).setParams(ps, pos);
2720                         if (os != null&& usingOLF && tableNo == 0) {
2721                             //this is only needed on the base table
2722                             ((JdbcState)os).setOptimisticLockingParams(ps, pos);
2723                         } else if (usingChanged) {
2724                             ((JdbcState)os).setParamsChangedAndNotNull(ps, fieldNos, 0,
2725                                     numFieldNos, pos, graph, tableNo);
2726                         }
2727                     } catch (SQLException e) {
2728                         throw handleException(
2729                                 "Error setting parameters on PreparedStatement for " +
2730                                 "update of '" + Utils.toString(oid) + "':\n" +
2731                                 JdbcUtils.toString(e) + "\n" +
2732                                 JdbcUtils.getPreparedStatementInfo(sql, ps), e);
2733                     }
2734
2735                     // do the update
2736                     if (batch) {
2737                         ps.addBatch();
2738                     } else {
2739                         int uc;
2740                         try {
2741                             uc = ps.executeUpdate();
2742                         } catch (Exception   e) {
2743                             throw handleException(
2744                                     "Update failed: " +
2745                                     JdbcUtils.toString(e) + "\n" +
2746                                     "Row: " + oid.toSString() + "\n" +
2747                                     JdbcUtils.getPreparedStatementInfo(sql, ps), e, true, oid);
2748                         }
2749                         if (uc == 0) {
2750                             throw BindingSupportImpl.getInstance().concurrentUpdate(
2751                                     "Row not found: " +
2752                                     oid.toSString() + "\n" +
2753                                     JdbcUtils.getPreparedStatementInfo(sql, ps), oid);
2754                         }
2755                     }
2756
2757                     indexInIntArray = indexInIntArray + 1;
2758                     if (--count == 0) break;
2759                     oid = graph.getOID(toUpdateIndexes.get(indexInIntArray));
2760                     ns = graph.getNewState(toUpdateIndexes.get(indexInIntArray));
2761                     os = graph.getOldState(toUpdateIndexes.get(indexInIntArray));
2762                 }
2763
2764                 // if batching then exec the batch and check all the update counts
2765                 if (batch) {
2766                     int[] a;
2767                     try {
2768                         a = ps.executeBatch();
2769                     } catch (Exception   e) {
2770                         throw handleException(
2771                              "Batch update failed: " + JdbcUtils.toString(
2772                                  e) + "\n" +
2773                              "Row: " + oid.toSString() + "\n" +
2774                              JdbcUtils.getPreparedStatementInfo(sql, ps), e, true, oid);
2775                     }
2776                     for (int j = 0; j < count; j++) {
2777                         int c = a[j];
2778                         if (c <= 0) {
2779                             String   psi = JdbcUtils.getPreparedStatementInfo(
2780                                     sql, ps, j);
2781                             oid = graph.getOID(startIndex + j);
2782                             if (c == 0) {
2783                                 throw BindingSupportImpl.getInstance().concurrentUpdate(
2784                                         "Row not found: " + oid.toSString() + "\n" + psi, oid);
2785                             }
2786                             throw BindingSupportImpl.getInstance().datastore(
2787                                     "Unexpected update count " +
2788                                     c + " for row: " + oid.toSString() + "\n" + psi);
2789                         }
2790                     }
2791                 }
2792             } finally {
2793                 cleanup(ps);
2794             }
2795         }
2796         return indexInIntArray;
2797     }
2798
2799    /**
2800     * Generate SQL to update a row in the base table for a class heirachy.
2801     * Any entries in fieldNos for fields that return true to appendUpdate
2802     * (i.e. they did not add a replacable param) are made negative. This is
2803     * used to handle Oracle LOB columns. Returns true if there was at least
2804     * one such column or false otherwise.
2805     */
2806    private boolean createUpdateSql(JdbcClass jdbcClass,
2807            JdbcTable table, int numFieldNos, int[] fieldNos, boolean usingOLF,
2808            JdbcSimpleField optimisticLockingField,
2809            boolean usingChanged, State os, State ns, CharBuf s) {
2810        JdbcField[] fields = jdbcClass.stateFields;
2811        boolean lobColFound = false;
2812        s.clear();
2813        s.append("UPDATE ");
2814        s.append(table.name);
2815        s.append(" SET ");
2816        boolean first = true;
2817        for (int i = 0; i < numFieldNos; i++) {
2818            int fieldNo = fieldNos[i];
2819            JdbcField f = fields[fieldNo];
2820            if (f.mainTableColsForUpdate == null || f.mainTable != table) continue;
2821            if (first) {
2822                first = false;
2823            } else {
2824                s.append(',');
2825                s.append(' ');
2826            }
2827            if (f.appendUpdate(s, ns)) {
2828                // no replaceable param so skip field when params are set
2829                fieldNos[i] = -(fieldNo + 1);
2830                lobColFound = true;
2831            }
2832        }
2833        if (first) {    // no columns to update
2834            s.clear();
2835        } else {
2836            s.append(" WHERE ");
2837            table.appendWherePK(s);
2838            if (usingOLF) {
2839                s.append(" AND ");
2840                optimisticLockingField.appendWhere(s, sqlDriver);
2841            } else if (usingChanged) {
2842                for (int i = 0; i < numFieldNos; i++) {
2843                    int fieldNo = fieldNos[i];
2844                    if (fieldNo < 0) continue;
2845                    JdbcField f = fields[fieldNo];
2846                    if (f.mainTableColsForUpdate == null
2847                            || !f.includeForChangedLocking
2848                            || f.mainTable != table) {
2849                        continue;
2850                    }
2851                    s.append(" AND ");
2852                    if (os.isNull(fieldNo)) {
2853                        f.appendWhereIsNull(s, sqlDriver);
2854                    } else {
2855                        f.appendWhere(s, sqlDriver);
2856                    }
2857                }
2858            }
2859        }
2860        return lobColFound;
2861    }
2862
2863    /**
2864     * Perist fields stored in link tables and so on.
2865     *
2866     * @see #persistPass1
2867     */
2868    private void persistPass2(PersistGraph graph) {
2869        try {
2870            int[] fieldNos = new int[jmd.maxFieldsLength];
2871            CharBuf s = new CharBuf();
2872            int graphSize = graph.size();
2873
2874            // process blocks of the same class together
2875            int startIndex = 0;
2876            for (; startIndex < graphSize;) {
2877                State ns = graph.getNewState(startIndex);
2878                if (!ns.containsPass2Fields()) {
2879                    startIndex++;
2880                    continue;
2881                }
2882
2883                // count entries in the graph with the same class
2884                int classIndex = ns.getClassIndex();
2885                int blockEnd = startIndex;
2886                for (;
2887                     ++blockEnd < graphSize
2888                        && graph.getNewState(blockEnd).getClassIndex() == classIndex;) {
2889                    ;
2890                }
2891
2892                // get info common to all the entries in the block
2893                ClassMetaData cmd = ns.getClassMetaData(jmd);
2894                Connection con = con();
2895
2896                // find the fields we need to check for each block entry
2897                int[] fna;
2898                int nf;
2899                if (blockEnd - startIndex == 1) {
2900                    // only one entry so check only its pass 2 fields
2901                    nf = ns.getPass2FieldNos(fieldNos);
2902                    fna = fieldNos;
2903                } else {
2904                    // multiple entries so check all pass 2 fields
2905                    fna = cmd.pass2Fields;
2906                    nf = fna.length;
2907                }
2908
2909                // process block for each fieldNo so we can make best use
2910                // of PreparedStatement's and batching
2911                for (int fpos = 0; fpos < nf; fpos++) {
2912                    int fieldNo = fna[fpos];
2913                    FieldMetaData fmd = cmd.stateFields[fieldNo];
2914                    ((JdbcField)fmd.storeField).persistPass2Block(graph, startIndex,
2915                            blockEnd, s, con, useBatchInsert, useBatchUpdate);
2916                }
2917
2918                startIndex = blockEnd;
2919            }
2920        } catch (SQLException e) {
2921            throw handleException(e);
2922        }
2923    }
2924
2925    private void doDeletes(DeletePacket toDelete) {
2926        if (!toDelete.isKeepStates()) {
2927            toDelete.sortOIDs(new OIDRefGraphIndexComparator());
2928        }
2929        deletePass1(toDelete);
2930        deletePass2(toDelete);
2931    }
2932
2933    /**
2934     * Delete rows from link tables.
2935     *
2936     * @see #deletePass2
2937     */
2938    private void deletePass1(DeletePacket graph) {
2939        try {
2940            CharBuf s = new CharBuf();
2941            int graphSize = graph.size();
2942
2943            for (int startIndex = 0; startIndex < graphSize;) {
2944
2945                OID oid = graph.oids[startIndex];
2946                int classIndex = oid.getClassIndex();
2947                ClassMetaData cmd = jmd.classes[classIndex];
2948
2949                // count entries in the graph with the same class
2950                int blockEnd = startIndex;
2951                for (;
2952                     ++blockEnd < graphSize
2953                        && graph.oids[blockEnd].getClassIndex() == classIndex;) {
2954                    ;
2955                }
2956
2957                // get info common to all the entries in the block
2958                Connection con = con();
2959
2960                // find the fields we need to check for each block entry
2961                int[] fna = cmd.pass2Fields;
2962                int nf = fna.length;
2963
2964                // process block for each fieldNo so we can make best use
2965                // of PreparedStatement's and batching
2966                for (int fpos = 0; fpos < nf; fpos++) {
2967                    int fieldNo = fna[fpos];
2968                    FieldMetaData fmd = cmd.stateFields[fieldNo];
2969                    ((JdbcField)fmd.storeField).deletePass2Block(graph, startIndex,
2970                            blockEnd, s, con, useBatchUpdate);
2971                }
2972
2973                startIndex = blockEnd;
2974            }
2975        } catch (SQLException e) {
2976            throw handleException(e);
2977        }
2978    }
2979
2980    /**
2981     * Delete main table rows.
2982     *
2983     * @see #deletePass1
2984     */
2985    public void deletePass2(DeletePacket graph) {
2986        try {
2987            CharBuf s = new CharBuf();
2988            int graphSize = graph.size();
2989
2990            int count;
2991            for (int startIndex = 0; startIndex < graphSize; startIndex += count) {
2992                OID oid = graph.oids[startIndex];
2993                int classIndex = oid.getClassIndex();
2994                ClassMetaData cmd = jmd.classes[classIndex];
2995                JdbcClass jdbcClass = (JdbcClass)cmd.storeClass;
2996                Connection con = con();
2997
2998                boolean batch = useBatchUpdate;
2999                boolean useInList = jdbcClass.table.pkSimpleCols.length == 1;
3000
3001                count = 1;
3002                for (int index = startIndex + 1; index < graphSize; count++, index++) {
3003                    if (graph.oids[startIndex + count].getClassMetaData() != cmd) break;
3004                }
3005                if (count == 1) {
3006                    useInList = false;
3007                }
3008
3009                PreparedStatement ps = null;
3010                try {
3011                    if (!batch && !useInList) {
3012                        //delete heirarchies one-by-one
3013                        int n = jdbcClass.allTables.length;
3014                        for (int tableNo = n - 1; tableNo >= 0; tableNo--) {
3015                            //must create ps now
3016                            String   sql = getDeleteRowSql(
3017                                    jdbcClass.allTables[tableNo], s);
3018                            ps = con.prepareStatement(sql);
3019
3020                            for (int i = 0; i < count; i++) {
3021                                deleteRow(ps, (JdbcOID)graph.oids[startIndex + i], sql);
3022                            }
3023                        }
3024                    } else if (useInList) {
3025                        //use 'IN' list
3026                        final int maxInOps = sqlDriver.getMaxInOperands();
3027                        final char[] whereParam = sqlDriver.getSqlParamStringChars(
3028                                jdbcClass.table.pkSimpleCols[0].jdbcType);
3029
3030                        if (count <= maxInOps) {
3031                            final char[] totalWhereParams = createInParamArray(
3032                                    whereParam, count);
3033                            int n = jdbcClass.allTables.length;
3034                            for (int tableNo = n - 1; tableNo >= 0; tableNo--) {
3035                                getDeleteRowSqlWithInList(jdbcClass.allTables[tableNo], s);
3036                                s.append(totalWhereParams);
3037                                String   sql = s.toString();
3038                                ps = con.prepareStatement(sql);
3039                                for (int i = 0; i < count; i++) {
3040                                    ((JdbcOID)graph.oids[startIndex + i]).setParams(ps, (i + 1));
3041                                }
3042                                try {
3043                                    ps.executeUpdate();
3044                                } catch (Exception   e) {
3045                                    throw handleException(
3046                                            "Delete with IN list failed: " + JdbcUtils.toString(e) + "\n" +
3047                                            JdbcUtils.getPreparedStatementInfo(sql, ps),
3048                                            e);
3049                                }
3050                            }
3051                        } else {
3052                            int n = jdbcClass.allTables.length;
3053                            int amountLeft = count % sqlDriver.getMaxInOperands();
3054                            int amountOfFullRuns = count/maxInOps;
3055
3056                            char[] totalWhereParams1 = null;
3057                            if (amountLeft > 0) {
3058                                totalWhereParams1 = createInParamArray(whereParam,
3059                                        amountLeft);
3060                            }
3061                            char[] totalWhereParams2 = createInParamArray(whereParam,
3062                                    maxInOps);
3063
3064                            for (int tableNo = n - 1; tableNo >= 0; tableNo--) {
3065                                String   sql = null;
3066                                int pos = startIndex;
3067                                if (amountLeft > 0) {
3068                                    //do the smaller amount first and once
3069                                    getDeleteRowSqlWithInList(jdbcClass.allTables[tableNo], s);
3070                                    s.append(totalWhereParams1);
3071
3072                                    sql = s.toString();
3073                                    ps = con.prepareStatement(sql);
3074                                    for (int i = 0; i < amountLeft; i++) {
3075                                        ((JdbcOID)graph.oids[pos++]).setParams(ps, (i + 1));
3076                                    }
3077                                    try {
3078                                        ps.executeUpdate();
3079                                    } catch (Exception   e) {
3080                                        throw handleException(
3081                                                "Delete with IN list failed: " + JdbcUtils.toString(e) + "\n" +
3082                                                JdbcUtils.getPreparedStatementInfo(sql, ps),
3083                                                e);
3084                                    }
3085                                }
3086
3087                                getDeleteRowSqlWithInList(jdbcClass.allTables[tableNo], s);
3088                                s.append(totalWhereParams2);
3089                                sql = s.toString();
3090                                ps = con.prepareStatement(sql);
3091
3092                                //do the full runs
3093                                for (int i = 0; i < amountOfFullRuns; i++) {
3094                                    for (int j = 0; j < maxInOps; j++) {
3095                                        ((JdbcOID)graph.oids[pos++]).setParams(ps, (j + 1));
3096                                    }
3097                                    try {
3098                                        ps.executeUpdate();
3099                                    } catch (Exception   e) {
3100                                        throw handleException(
3101                                                "Delete with IN list failed: " + JdbcUtils.toString(e) + "\n" +
3102                                                JdbcUtils.getPreparedStatementInfo(sql, ps),
3103                                                e);
3104                                    }
3105                                }
3106                            }
3107                        }
3108                    } else {
3109                        //use batching
3110                        int n = jdbcClass.allTables.length;
3111                        for (int tableNo = n - 1; tableNo >= 0; tableNo--) {
3112                            String   sql = getDeleteRowSql(
3113                                    jdbcClass.allTables[tableNo], s);
3114                            ps = con.prepareStatement(sql);
3115                            for (int i = 0; i < count; i++) {
3116                                ((JdbcOID)graph.oids[startIndex + i]).setParams(ps, 1);
3117                                ps.addBatch();
3118                            }
3119
3120                            int[] a;
3121                            try {
3122                                a = ps.executeBatch();
3123                            } catch (Exception   e) {
3124                                throw handleException(
3125                                        "Batch delete failed: " + JdbcUtils.toString(e) + "\n" +
3126                                        "Row: " + graph.oids[startIndex].toSString() + "\n" +
3127                                        JdbcUtils.getPreparedStatementInfo(sql, ps),
3128                                        e, true, graph.oids[startIndex]);
3129                            }
3130                            for (int j = 0; j < count; j++) {
3131                                int c = a[j];
3132                                if (c <= 0) {
3133                                    String   psi = JdbcUtils.getPreparedStatementInfo(
3134                                            sql, ps, j);
3135                                    if (c == 0) {
3136                                        throw BindingSupportImpl.getInstance().concurrentUpdate(
3137                                                "Row not found: " + graph.oids[startIndex + j].toSString() + "\n" + psi, graph.oids[startIndex + j]);
3138                                    }
3139                                    throw BindingSupportImpl.getInstance().datastore(
3140                                            "Unexpected update count " +
3141                                            c + " for row: " + graph.oids[startIndex + j].toSString() + "\n" + psi);
3142                                }
3143                            }
3144                        }
3145                    }
3146                } finally {
3147                    cleanup(ps);
3148                }
3149            }
3150        } catch (SQLException e) {
3151            throw handleException(e);
3152        }
3153    }
3154
3155    private char[] createInParamArray(final char[] whereParam, int count) {
3156        int pos = 0;
3157        char[] totalWhereParams;
3158        if (count == 1) {
3159            totalWhereParams = new char[whereParam.length * count + 1];
3160        } else {
3161            totalWhereParams = new char[whereParam.length * count + (count - 1) + 1];
3162        }
3163        for (int i = 0; i < count; i++) {
3164            if (i != 0) totalWhereParams[pos++] = ',';
3165            for (int j = 0; j < whereParam.length; j++) {
3166                totalWhereParams[pos++] = whereParam[j];
3167            }
3168        }
3169        totalWhereParams[pos] = ')';
3170        return totalWhereParams;
3171    }
3172
3173    private void deleteRow(PreparedStatement ps, JdbcOID oid, String   sql) {
3174        int uc;
3175        try {
3176            oid.setParams(ps, 1);
3177            uc = ps.executeUpdate();
3178        } catch (Exception   e) {
3179            throw handleException(
3180                    "Delete failed: " + JdbcUtils.toString(e) + "\n" +
3181                    "Row: " + oid.toSString() + "\n" +
3182                    JdbcUtils.getPreparedStatementInfo(sql, ps),
3183                    e, true, oid);
3184        }
3185        if (uc == 0) {
3186            throw BindingSupportImpl.getInstance().concurrentUpdate(
3187                    "Row not found: " + oid.toSString() + "\n" +
3188                    JdbcUtils.getPreparedStatementInfo(sql, ps), oid);
3189        }
3190    }
3191
3192    private String   getDeleteRowSql(JdbcTable table, CharBuf s) {
3193        String   sql = table.deleteRowSql;
3194        if (sql != null) return sql;
3195        s.clear();
3196        s.append("DELETE FROM ");
3197        s.append(table.name);
3198        s.append(" WHERE ");
3199        table.appendWherePK(s);
3200        return table.deleteRowSql = s.toString();
3201    }
3202
3203    private void getDeleteRowSqlWithInList(JdbcTable table, CharBuf s) {
3204        s.clear();
3205        s.append("DELETE FROM ");
3206        s.append(table.name);
3207        s.append(" WHERE ");
3208        s.append(table.pkSimpleCols[0].name);
3209        s.append(" IN (");
3210    }
3211
3212    /**
3213     * Close all open queries.
3214     */
3215    private final void closeAllQueries() {
3216        for (JdbcQueryResult res = queryResultHead; res != null;) {
3217            JdbcQueryResult n = res.prev;
3218            res.close();
3219            res.next = null;
3220            res.prev = null;
3221            if (n == null) break;
3222            res = n;
3223        }
3224        queryResultHead = null;
3225        queryResultTail = null;
3226    }
3227
3228    private JdbcCompiledQuery compile(QueryDetails q) {
3229        JdbcCompiledQuery cq = null;
3230        int language = q.getLanguage();
3231        if (language == QueryDetails.LANGUAGE_EJBQL) {
3232
3233            cq = new JdbcEJBQLCompiler(this).compile(q);
3234
3235        } else if (language == QueryDetails.LANGUAGE_SQL) {
3236            ClassMetaData cmd = null;
3237            if (q.getCandidateClass() != null) {
3238                cmd = jmd.getClassMetaData(q.getCandidateClass());
3239            }
3240
3241            //unset everything not used for sql queries
3242            q.setOrdering(null);
3243            q.setGrouping(null);
3244            q.setVariables(null);
3245            q.setCol(null);
3246            q.setImports(null);
3247            q.setResult(null);
3248
3249            cq = new JdbcCompiledQuery(cmd, q);
3250            cq.setCacheable(false); // SQL queries not cached by default
3251
3252            CmdBitSet bits = new CmdBitSet(jmd);
3253            if (cmd != null) {
3254                bits.addPlus(cmd);
3255            }
3256            int[] a = q.getExtraEvictClasses();
3257            if (a != null) {
3258                for (int i = a.length - 1; i >= 0; i--) {
3259                    bits.add(jmd.classes[a[i]]);
3260                }
3261            }
3262            cq.setFilterClsIndexs(bits.toArray());
3263            cq.setEvictionClassBits(bits.getBits());
3264            cq.setEvictionClassIndexes(bits.getIndexes());
3265        } else {
3266            cq = new JdbcJDOQLCompiler(this).compile(q);
3267        }
3268        return cq;
3269    }
3270
3271    public LogEventStore getPerfEventStore() {
3272        return pes;
3273    }
3274
3275    private void addQueryResult(JdbcQueryResult res) {
3276        if (res.next != null || res.prev != null) {
3277            throw BindingSupportImpl.getInstance().internal(
3278                    "Adding a duplicate queryResult to query linked list");
3279        }
3280
3281        res.prev = queryResultHead;
3282        if (queryResultHead != null) queryResultHead.next = res;
3283        queryResultHead = res;
3284        if (queryResultTail == null) queryResultTail = res;
3285    }
3286
3287    private void removeQueryResult(JdbcQueryResult res) {
3288        if (res.prev != null) {
3289            res.prev.next = res.next;
3290        } else {
3291            queryResultTail = res.next;
3292        }
3293        if (res.next != null) {
3294            res.next.prev = res.prev;
3295        } else {
3296            queryResultHead = res.prev;
3297        }
3298        res.next = null;
3299        res.prev = null;
3300    }
3301
3302    private void fillContainerWithAll(ApplicationContext context,
3303            JdbcCompiledQuery cq, Object  [] params,
3304            QueryResultContainer container) {
3305        JdbcQueryResult res = null;
3306        if (cq.isEJBQLHack()) {
3307
3308            res = new JdbcQueryResultEJBQL(this, cq, params,
3309                    canUseCache());
3310
3311        } else {
3312            res = new JdbcQueryResult(this, cq, params,
3313                canUseCache());
3314        }
3315        res.getAllResults(context, container, forUpdateField);
3316    }
3317
3318    private int executeCount(JdbcCompiledQuery cq, Object  [] params) {
3319        PreparedStatement ps = null;
3320        ResultSet rs = null;
3321        try {
3322            Connection con = con();
3323
3324            // this sync block can be removed when compiledQuery is no longer shared
3325            String   sql;
3326            synchronized (cq) {
3327                cq.updateSql(sqlDriver, params, false, true);
3328                ps = con.prepareStatement(sql = cq.getSql());
3329                cq.setParamsOnPS(jmd, sqlDriver, ps, params, sql);
3330            }
3331            try {
3332                rs = ps.executeQuery();
3333            } catch (Exception   e) {
3334                throw sqlDriver.mapException(e, "Count(*) query failed: " + JdbcUtils.toString(
3335                        e) + "\n" + JdbcUtils.getPreparedStatementInfo(sql, ps), true);
3336            }
3337            if (!rs.next()) {
3338                throw BindingSupportImpl.getInstance().fatalDatastore(
3339                        "No row returned by count(*) query:\n" +
3340                        JdbcUtils.getPreparedStatementInfo(sql, ps));
3341            }
3342            return rs.getInt(1);
3343        } catch (SQLException x) {
3344            handleException(x);
3345            return 0; // keep compiler happy
3346        } finally {
3347            cleanup(rs);
3348            cleanup(ps);
3349        }
3350    }
3351
3352    private VersantQueryPlan executePlan(JdbcCompiledQuery cq,
3353            Object  [] params) {
3354        VersantQueryPlan qp = new VersantQueryPlan();
3355        // this sync block can go when compiledQuery is no longer shared
3356        synchronized (cq) {
3357            cq.updateSql(sqlDriver, params, forUpdateField, false);
3358            qp.setDatastoreQuery(cq.getSql());
3359            if (params == null) { // query plans can only be done when there is no params
3360                try {
3361                    Connection con = con();
3362                    PreparedStatement ps;
3363                    String   sql;
3364                    try {
3365                        sql = sqlDriver.prepareForGetQueryPlan(con, cq.getSql());
3366                        ps = con.prepareStatement(sql);
3367                        qp.setDatastorePlan(sqlDriver.getQueryPlan(con, ps));
3368                    } finally {
3369                        sqlDriver.cleanupForGetQueryPlan(con);
3370                    }
3371                    cq.setParamsOnPS(jmd, sqlDriver, ps, params, sql);
3372                } catch (SQLException e) {
3373                    qp.setDatastorePlan(e.getMessage());
3374                }
3375            } else {
3376                qp.setDatastorePlan(
3377                        "Query plan can only be done when there are no parameters.");
3378            }
3379        }
3380        return qp;
3381    }
3382
3383    /**
3384     * Look for cached query results and add them to the container if there
3385     * are some. Returns true if results were found and false otherwise.
3386     */
3387    private boolean checkCacheForQuery(JdbcCompiledQuery cq,
3388            Object  [] params, QueryResultContainer qContainer) {
3389        CachedQueryResult data = cache.getQueryResult(cq, params);
3390        if (data == null) {
3391            return false;
3392        }
3393        StatesReturned container = qContainer.container;
3394
3395        // add all the results to the qContainer cloning all mutable stuff
3396        if (data.results != null) {
3397            qContainer.fillFrom(data);
3398            // update the container with the oid-state pairs
3399            if (cq.isDefaultResult()) {
3400                // this is a query that only contains oids
3401                ArrayList res = data.results;
3402                int n = res.size();
3403                for (int i = 0; i < n; i++) {
3404                    OID oid = (OID)res.get(i);
3405                    if (oid == null) break;
3406                    State s = cache.getState(oid, null);
3407                    if (s == null) {
3408                        cache.evict(cacheTx(), cq, params);
3409                        qContainer.reset();
3410                        return false;
3411                    }
3412                    container.add(oid, s);
3413                }
3414            } else {
3415                final int firstThisCol = cq.getFirstThisIndex();
3416                if (firstThisCol >= 0 && data.results != null) {
3417                    // first put in all the primary query results in correct order
3418                    ArrayList res = data.results;
3419                    int n = res.size();
3420                    for (int i = 0; i < n; i++) {
3421                        Object  [] row = (Object  [])res.get(i);
3422                        OID oid = (OID)row[firstThisCol];
3423                        State s = cache.getState(oid, null);
3424                        if (s == null) {
3425                            cache.evict(cacheTx(), cq, params);
3426                            qContainer.reset();
3427                            return false;
3428                        }
3429                        container.add(oid, s);
3430                    }
3431                }
3432            }
3433        }
3434
3435        // process indirect oids
3436        if (data.indirectOIDs != null) {
3437            OID[] indirectOIDs = data.indirectOIDs.oids;
3438            int n = indirectOIDs.length;
3439            for (int i = 0; i < n; i++) {
3440                OID oid = indirectOIDs[i];
3441                if (oid == null) break;
3442                State s = cache.getState(oid, null);
3443                if (s == null) {
3444                    cache.evict(cacheTx(), cq, params);
3445                    container.clear();
3446                    qContainer.reset();
3447                    return false;
3448                }
3449                container.addIndirect(oid, s);
3450            }
3451        }
3452
3453        qContainer.allResults = true;
3454        return true;
3455    }
3456
3457    public SqlDriver getSqlDriver() {
3458        return sqlDriver;
3459    }
3460    
3461    /**
3462     * Skip over states not needed from a select. This follows the same
3463     * recursive algorithm as populateStateFromSelect but does not read
3464     * anything. It is usefull if some states have to be skipped but
3465     * others need to be read from other columns in the ResultSet.
3466     */
3467    public int skipState(int firstCol, FgDs fgds) {
3468        return firstCol + fgds.columnSkipCount;
3469    }
3470
3471    public boolean isFlushed() {
3472        return flushed;
3473    }
3474
3475    public StorageCache getCache() {
3476        return cache;
3477    }
3478
3479    public boolean isUseBatchInsert() {
3480        return useBatchInsert;
3481    }
3482
3483    public boolean isUseBatchUpdate() {
3484        return useBatchUpdate;
3485    }
3486
3487    public JdbcMetaData getJdbcMetaData() {
3488        return (JdbcMetaData)jmd.jdbcMetaData;
3489    }
3490
3491    /**
3492     * Get all of the states for the class and others pulled in by the fg.
3493     */
3494    private void getAllStates(ApplicationContext context, ClassMetaData cmd,
3495            FetchGroup fg, StatesReturned all) {
3496        fg = fg.resolve(cmd);
3497        QueryDetails qd = new QueryDetails();
3498        qd.setBounded(true);
3499        qd.setCandidateClass(cmd.cls);
3500        qd.setFilter(null);
3501        qd.setFetchGroupIndex(fg.index);
3502        qd.updateCounts();
3503        JdbcCompiledQuery cq = (JdbcCompiledQuery)compileQuery(qd);
3504
3505        QueryResultContainer qrc = new QueryResultContainer(all);
3506        qrc.init(cq);
3507
3508        JdbcQueryResult res = null;
3509        if (cq.isEJBQLHack()) {
3510
3511            res = new JdbcQueryResultEJBQL(this, cq, new Object  [0],
3512                    canUseCache());
3513
3514        } else {
3515            res = new JdbcQueryResult(this, cq, new Object  [0],
3516                    false);
3517        }
3518
3519        res.getAllResults(context, qrc, forUpdateField);
3520    }
3521
3522    /**
3523     * Add all of the cached query results to the container. This returns
3524     * false if any of the states are no longer in the level 2 cache.
3525     */
3526    private boolean addToContainer(JdbcCompiledQuery cq, Object  [] params,
3527            CachedQueryResult data, QueryResultContainer qContainer) {
3528        if (data.results != null) {
3529            qContainer.fillFrom(data);
3530            // this is to update the container with the oid-state pairs
3531            if (cq.isDefaultResult()) {
3532                //this is a query that only contains oids
3533                ArrayList res = data.results;
3534                int n = res.size();
3535                for (int i = 0; i < n; i++) {
3536                    OID oid = (OID)res.get(i);
3537                    if (oid == null) break;
3538                    State s = cache.getState(oid, null);
3539                    if (s == null) {
3540                        cache.evict(cacheTx, cq, params);
3541                        qContainer.reset();
3542                        return false;
3543                    }
3544                    qContainer.container.add(oid, s);
3545                }
3546            } else {
3547                final int firstThisCol = cq.getFirstThisIndex();
3548                if (firstThisCol >= 0 && data.results != null) {
3549                    // first put in all the primary query results in correct order
3550                    ArrayList res = data.results;
3551                    int n = res.size();
3552                    for (int i = 0; i < n; i++) {
3553                        Object  [] row = (Object  [])res.get(i);
3554                        OID oid = (OID)row[firstThisCol];
3555                        State s = cache.getState(oid, null);
3556                        if (s == null) {
3557                            cache.evict(cacheTx, cq, params);
3558                            qContainer.reset();
3559                            return false;
3560                        }
3561                        qContainer.container.add(oid, s);
3562                    }
3563                }
3564            }
3565        }
3566
3567        // process indirect oids
3568        if (data.indirectOIDs != null) {
3569            OID[] indirectOIDs = data.indirectOIDs.oids;
3570            int n = indirectOIDs.length;
3571            for (int i = 0; i < n; i++) {
3572                OID oid = indirectOIDs[i];
3573                if (oid == null) break;
3574                State s = cache.getState(oid, null);
3575                if (s == null) {
3576                    cache.evict(cacheTx, cq, params);
3577                    qContainer.container.clear();
3578                    qContainer.reset();
3579                    return false;
3580                }
3581                qContainer.container.addIndirect(oid, s);
3582            }
3583        }
3584
3585        qContainer.allResults = true;
3586        return true;
3587    }
3588
3589    public JdbcConnectionSource getJdbcConnectionSource() {
3590        return conSrc;
3591    }
3592
3593    /**
3594     * Count the number of open query results.
3595     */
3596    private int getOpenQueryResultCount() {
3597        int c = 0;
3598        for (JdbcQueryResult i = queryResultTail; i != null; i = i.next, ++c);
3599        return c;
3600    }
3601
3602    /**
3603     * Do we have a JDBC Connection?
3604     */
3605    public boolean hasDatastoreConnection() {
3606        return conx != null;
3607    }
3608
3609    public Map getStatus() {
3610        Map m = new HashMap();
3611        m.put(STATUS_OPEN_QUERY_RESULT_COUNT,
3612                new Integer  (getOpenQueryResultCount()));
3613        return m;
3614    }
3615
3616    /**
3617     * This compares OIDs based on the referenceGraphIndex'es of their base class
3618     * meta data.
3619     */
3620    public class OIDRefGraphIndexComparator implements Comparator {
3621
3622        public int compare(Object   o1, Object   o2) {
3623            ClassMetaData ca = ((OID)o1).getAvailableClassMetaData();
3624            ClassMetaData cb = ((OID)o2).getAvailableClassMetaData();
3625            int diff = ca.referenceGraphIndex - cb.referenceGraphIndex;
3626            if (diff != 0) return diff;
3627            return ca.index - cb.index;
3628        }
3629
3630    }
3631
3632    /**
3633     * This is used to clear all non-auto set field on states that returned to
3634     * the client. This is done to save network traffic.
3635     */
3636    private void clearNonAutoSetFields(StatesReturned container) {
3637        for (Iterator i = container.iterator(); i.hasNext(); ) {
3638            EntrySet.Entry e = (EntrySet.Entry)i.next();
3639            State state = (State)e.getValue();
3640            if (state != null) {
3641                state.clearNonAutoSetFields();
3642            }
3643        }
3644    }
3645
3646    public void setUserObject(Object   o) {
3647        // ignore
3648    }
3649
3650}
3651
3652
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags