FileContainer


1   /*
2   
3      Derby - Class org.apache.derby.impl.store.raw.data.FileContainer
4   
5      Licensed to the Apache Software Foundation (ASF) under one or more
6      contributor license agreements.  See the NOTICE file distributed with
7      this work for additional information regarding copyright ownership.
8      The ASF licenses this file to you under the Apache License, Version 2.0
9      (the "License"); you may not use this file except in compliance with
10     the License.  You may obtain a copy of the License at
11  
12        http://www.apache.org/licenses/LICENSE-2.0
13  
14     Unless required by applicable law or agreed to in writing, software
15     distributed under the License is distributed on an "AS IS" BASIS,
16     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17     See the License for the specific language governing permissions and
18     limitations under the License.
19  
20   */
21  
22  package org.apache.derby.impl.store.raw.data;
23  
24  import org.apache.derby.iapi.reference.Property;
25  
26  import org.apache.derby.iapi.reference.Limits;
27  import org.apache.derby.iapi.reference.SQLState;
28  
29  import org.apache.derby.impl.store.raw.data.BaseContainer;
30  import org.apache.derby.impl.store.raw.data.BaseContainerHandle;
31  import org.apache.derby.impl.store.raw.data.BasePage;
32  import org.apache.derby.impl.store.raw.data.PageVersion;
33  
34  import org.apache.derby.iapi.services.cache.Cacheable;
35  import org.apache.derby.iapi.services.cache.CacheManager;
36  import org.apache.derby.iapi.services.context.ContextService;
37  import org.apache.derby.iapi.services.daemon.DaemonService;
38  import org.apache.derby.iapi.services.daemon.Serviceable;
39  import org.apache.derby.iapi.services.monitor.Monitor;
40  import org.apache.derby.iapi.services.sanity.SanityManager;
41  import org.apache.derby.iapi.services.io.FormatIdUtil;
42  import org.apache.derby.iapi.services.io.FormatIdOutputStream;
43  import org.apache.derby.iapi.services.io.StoredFormatIds;
44  import org.apache.derby.iapi.services.io.TypedFormat;
45  
46  import org.apache.derby.iapi.error.StandardException;
47  import org.apache.derby.iapi.store.raw.ContainerHandle;
48  import org.apache.derby.iapi.store.raw.ContainerKey;
49  import org.apache.derby.iapi.store.raw.LockingPolicy;
50  import org.apache.derby.iapi.store.raw.Loggable;
51  import org.apache.derby.iapi.store.raw.Page;
52  import org.apache.derby.iapi.store.raw.PageKey;
53  import org.apache.derby.iapi.store.raw.PageTimeStamp;
54  import org.apache.derby.iapi.store.raw.RecordHandle;
55  import org.apache.derby.iapi.store.raw.RawStoreFactory;
56  import org.apache.derby.iapi.store.raw.Transaction;
57  
58  import org.apache.derby.iapi.store.raw.log.LogInstant;
59  import org.apache.derby.iapi.store.raw.xact.RawTransaction;
60  
61  import org.apache.derby.iapi.store.access.TransactionController;
62  import org.apache.derby.iapi.store.access.AccessFactory;
63  import org.apache.derby.iapi.store.access.SpaceInfo;
64  
65  import org.apache.derby.iapi.services.io.ArrayInputStream;
66  import org.apache.derby.iapi.services.io.ArrayOutputStream;
67  import org.apache.derby.iapi.services.property.PropertyUtil;
68  import org.apache.derby.iapi.util.ByteArray;
69  
70  import java.io.IOException  ;
71  import java.io.DataInput  ;
72  import java.io.DataOutput  ;
73  
74  import java.util.Properties  ;
75  import java.util.zip.CRC32  ;
76  
77  
78  /**
79      FileContainer is an abstract base class for containers
80      which are based on files.
81  
82      This class extends BaseContainer and implements Cacheable and TypedFormat
83  */
84  
85  abstract class FileContainer 
86      extends BaseContainer implements Cacheable, TypedFormat
87  {
88  
89      /*
90       * typed format
91       */
92  
93      protected static final int formatIdInteger = 
94          StoredFormatIds.RAW_STORE_SINGLE_CONTAINER_FILE; 
95  
96      // format Id must fit in 4 bytes
97  
98      /**
99          Return my format identifier.
100     */
101     public int getTypeFormatId() 
102     {
103         return StoredFormatIds.RAW_STORE_SINGLE_CONTAINER_FILE;
104     }
105 
106     /*
107     ** Immutable fields
108     */
109 
110     protected final CacheManager          pageCache;        // my page's cache
111     protected final CacheManager          containerCache;   // cache I am in.
112     protected final BaseDataFileFactory   dataFactory;      // creating factory
113 
114     /*
115     ** Fields that are mutable only during identity changes
116     */
117 
118     protected int pageSize;             // size of my pages
119     protected int spareSpace;           // % space kept free on page in inserts
120     protected int minimumRecordSize;    // minimum space a record should 
121                                         // occupy on the page.
122 
123     protected short initialPages;       // initial number of pages preallocated
124                                         // to the container when created
125                                         
126     protected boolean canUpdate;        // can I be written to?
127 
128     private int PreAllocThreshold;      // how many pages before preallocation 
129                                         // kicks in, only stored in memory
130     private int PreAllocSize;           // how many pages to preallocate at once
131                                         // only stored in memory
132     private boolean bulkIncreaseContainerSize;// if true, the next addPage will
133                                          // attempt to preallocate a larger
134                                          // than normal number of pages.
135                                          //
136     // preallocation parameters
137     private static final int PRE_ALLOC_THRESHOLD    = 8;
138     private static final int MIN_PRE_ALLOC_SIZE     = 1;
139     private static final int DEFAULT_PRE_ALLOC_SIZE = 8;
140     private static final int MAX_PRE_ALLOC_SIZE     = 1000;
141 
142     /* 
143     ** Mutable fields, only valid when the identity is valid.
144     */
145 
146     // RESOLVE: if we run out of bytes in the container, we can change
147     // containerVersion from a long to an int because this number is only
148     // bumped when the container is dropped (and rolled back), so it is almost
149     // impossible for the containverVersion to get beyond a short, let alone
150     // and int - someone will have to write an application that attempt to drop
151     // the container 2 billion times for that to happen.
152     protected long          firstAllocPageNumber; // first alloc page number
153     protected long          firstAllocPageOffset; // first alloc page offset
154     protected long          containerVersion;     // the logged version number
155     protected long          estimatedRowCount;    // value is changed unlogged
156     protected LogInstant    lastLogInstant;       // last time this container 
157                                                   // object was touched.
158     /** 
159      * The sequence number for reusable recordIds . 
160      * As long as this number does not change, recordIds will be stable within
161      * the container.
162      **/
163     private long reusableRecordIdSequenceNumber;
164 
165 
166     /**
167         The page that was last inserted into.  Use this for getPageForInsert.
168         Remember the last allocated non-overflow page, and remember it in
169         memory only.
170         Use Get/Set method to access this field except when we know it is
171         being single thread access.
172      */
173     private long lastInsertedPage[];
174     private int  lastInsertedPage_index;
175 
176     /** 
177         The last unfilled page found.  Use this for getPageForInsert.
178         Remember the last unfilled page found, and remember it in memory only.
179         Use Get/Set method to access this field except when we know it is
180         being single thread access.
181     */
182     private long lastUnfilledPage;
183 
184     /**
185         The last allocated page.  This global var is access *without*
186         synchronization.  It is used as a hint for page allocation to find the
187         next reusable page.
188      */
189     private long lastAllocatedPage;
190 
191     /**
192         An estimated page count. Use this for getEstimatedPagecount.
193         Remember it in memory only.
194      */
195     private long estimatedPageCount;
196 
197 
198     // The isDirty flag indicates if the container has been modified.  The
199     // preDirty flag indicates that the container is about to be modified.  The
200     // reason for these 2 flags instead of just one is to accomodate
201     // checkpoint.  After a clean container sends a log record to the log
202     // stream but before that conatiner is dirtied by the log operation, a
203     // checkpoint could be taken.  If so, then the redoLWM will be after the
204     // log record but, without preDirty, the cache cleaning will not have
205     // waited for the change.  So the preDirty bit is to stop the cache
206     // cleaning from skipping over this container even though it has not really
207     // been modified yet.
208     protected boolean           preDirty;
209     protected boolean           isDirty;
210 
211     /*
212         allocation information cached by the container object.  
213 
214         <P>MT -
215         Access to the allocation cache MUST be synchronized on the allocCache
216         object.  FileContainer manages all MT issue w/r to AllocationCache.
217         The AllocationCache object itself is not MT-safe.
218         <P>
219         The protocol for accessing both the allocation cache and the alloc page
220         is: get the alloc cache semaphore, then get the alloc page.  Once both
221         are held, they can be released in any order.
222         <BR>
223         It is legal to get one or the other, i.e, it is legal to only get the
224         alloc cache semaphore without latching the alloc page, and it is legal
225         to get the alloc page latch without the alloc cache semaphore.
226         <BR>
227         it is illegal to hold alloc page latch and then get the allocation
228         cache semaphore
229         <PRE>
230         Writer to alloc Page (to invalidate alloc cache)
231         1) synchronized(allocCache)
232         2) invalidate cache
233         3) get latch on alloc Page
234         4) release synchonized(allocCache)
235 
236         Reader:
237         1) synchronized(allocCache)
238         2) if valid, read value and release synchronized(allocCache)
239         3) if cache is invalid, get latch on alloc page
240         4) validate cache
241         5) release alloc page latch
242         6) read value
243         7) release synchonized(allocCache)
244         </PRE>
245     */
246     protected AllocationCache   allocCache;
247 
248     /*
249      * array to store persistently stored fields
250      */
251     byte[] containerInfo;
252 
253     private CRC32         checksum;       // holder for the checksum
254 
255     /*
256     ** buffer for encryption/decryption
257     */
258     private byte[] encryptionBuffer;
259 
260     /*
261      * constants
262      */
263 
264     /** the container format must fit in this many bytes */
265     private static final int CONTAINER_FORMAT_ID_SIZE = 4; 
266 
267     /* the checksum size */
268     protected static final int CHECKSUM_SIZE = 8;
269 
270     /** 
271         The size of the persistently stored container info
272         ContainerHeader contains the following information:
273         4 bytes int FormatId
274         4 bytes int status
275         4 bytes int pageSize
276         4 bytes int spareSpace
277         4 bytes int minimumRecordSize
278         2 bytes short initialPages
279         2 bytes short spare1
280         8 bytes long    first Allocation page number
281         8 bytes long    first Allocation page offset
282         8 bytes long    container version
283         8 bytes long    estimated number of rows
284         8 bytes long    reusable recordId sequence number
285         8 bytes long    spare3
286         8 bytes long    checksum
287         container info size is 80 bytes, with 10 bytes of spare space
288     */
289     protected static final int CONTAINER_INFO_SIZE = 
290         CONTAINER_FORMAT_ID_SIZE+4+4+4+4+2+2+8+8+8+8+CHECKSUM_SIZE+8+8;
291 
292     /**
293         the number of arguments we need to pass to alloc page for create
294     */
295     protected static final int STORED_PAGE_ARG_NUM = 5;
296     protected static final int ALLOC_PAGE_ARG_NUM  = 6;
297 
298     /**
299      * where the first alloc page is located - 
300      * the logical page number and the physical page offset
301      * NOTE if it is not 0 this is not going to work for Stream 
302      * file which doesn't support seek
303      */
304     public static final long FIRST_ALLOC_PAGE_NUMBER = 0L;
305     public static final long FIRST_ALLOC_PAGE_OFFSET = 0L;
306 
307     // file status for persistent storage
308     private static final int FILE_DROPPED        = 0x1;
309     private static final int FILE_COMMITTED_DROP = 0x2;
310 
311     // recordId in this container can be reused when a page is reused.
312     private static final int FILE_REUSABLE_RECORDID = 0x8;
313 
314     protected static final String   SPACE_TRACE = 
315         (SanityManager.DEBUG ? "SpaceTrace" : null);
316 
317     FileContainer(BaseDataFileFactory factory) 
318     {
319         dataFactory = factory;
320         pageCache = factory.getPageCache();
321         containerCache = factory.getContainerCache();
322         
323         initContainerHeader(true);
324     }
325 
326     /**
327     Get information about space used by the container.
328     **/
329     public SpaceInfo getSpaceInfo(BaseContainerHandle handle)
330             throws StandardException
331     {
332         SpaceInformation spaceInfo;
333         synchronized(allocCache)
334         {
335             spaceInfo = 
336                 allocCache.getAllPageCounts(handle,firstAllocPageNumber);
337         }
338         spaceInfo.setPageSize(pageSize);
339         return spaceInfo;
340     }
341 
342     /*
343     ** Methods of Cacheable
344     **
345     ** getIdentity() and clearIdentity() are implemented by BaseContainer
346     */
347 
348     /**
349         Containers
350     */
351 
352     /**
353         Open the container.
354 
355         @return a valid object if the container was successfully opened, null if
356         it does not exist.
357 
358         @exception StandardException Some problem in opening a container.
359 
360         @see Cacheable#setIdentity
361     */
362     public Cacheable setIdentity(Object   key) throws StandardException 
363     {
364         return setIdent((ContainerKey) key);
365     }
366 
367     /**
368      * Open the container.
369      * <p>
370      * Open the container with key "newIdentity".
371      * <p>
372      * should be same name as setIdentity but seems to cause method resolution 
373      * ambiguities
374      *
375      * @exception StandardException Some problem in opening a container.
376      *
377      * @see Cacheable#setIdentity
378      **/
379     protected Cacheable setIdent(ContainerKey newIdentity) 
380         throws StandardException 
381     {
382         boolean ok          = openContainer(newIdentity);
383 
384         initializeLastInsertedPage(1);
385         lastUnfilledPage    = ContainerHandle.INVALID_PAGE_NUMBER;
386         lastAllocatedPage   = ContainerHandle.INVALID_PAGE_NUMBER;
387 
388         estimatedPageCount  = -1;
389 
390         if (ok) 
391         {
392             // set up our identity.
393             // If we raise an exception after this we must clear our identity.
394             fillInIdentity(newIdentity);
395             return this;
396         }
397         else
398         { 
399             return null;
400         }
401     }
402 
403     public Cacheable createIdentity(Object   key, Object   createParameter) 
404         throws StandardException 
405     {
406         if (SanityManager.DEBUG) 
407         {
408             SanityManager.ASSERT(
409                 !(key instanceof PageKey), "PageKey input to create container");
410         }
411 
412         return createIdent((ContainerKey) key, createParameter);
413     }
414 
415 
416     // should be same name as createIdentity but seems to cause method 
417     // resolution ambiguities
418     protected Cacheable createIdent(
419     ContainerKey    newIdentity, 
420     Object            createParameter) 
421          throws StandardException 
422     {
423         // createParameter will be this object if this method is being called 
424         // from itself to re-initialize the container (only for tempRAF)
425         // if createParameter == this, do not reinitialize the header, this
426         // object is not being reused for another container
427         if (createParameter != this) 
428         {
429             initContainerHeader(true /* change to different container */);
430 
431             if (createParameter != null && 
432                 (createParameter instanceof ByteArray))
433             {
434                 // this is called during load tran, the create container
435                 // Operation has a byte array created by logCreateContainerInfo
436                 // which contains all the information necessary to recreate the
437                 // container.  Use that to recreate the container properties.
438 
439                 createInfoFromLog((ByteArray)createParameter);
440             }
441             else
442             {
443                 if (SanityManager.DEBUG)
444                 {
445                     if (createParameter != null &&
446                         !(createParameter instanceof Properties  ))
447                     {
448                         SanityManager.THROWASSERT(
449                             "Expecting a  non-null createParameter to a " +
450                             "Properties instead of " +
451                             createParameter.getClass().getName());
452                     }
453                 }
454 
455                 createInfoFromProp((Properties  )createParameter);
456             }
457         }
458         else
459         {
460             // we don't need to completely re-initialize the header
461             // just re-initialize the relavent fields
462             initContainerHeader(false);
463         }
464 
465         if (initialPages > 1)
466         {
467             PreAllocThreshold           = 0;
468             PreAllocSize                = initialPages;
469             bulkIncreaseContainerSize   = true;
470         }
471         else
472         {
473             PreAllocThreshold           = PRE_ALLOC_THRESHOLD;
474         }
475 
476         createContainer(newIdentity);
477 
478         setDirty(true);
479 
480         // set up our identity.
481         // If we raise an exception after this we must clear our identity.
482         fillInIdentity(newIdentity);
483 
484         return this;
485     }
486 
487     public void clearIdentity() 
488     {
489 
490         closeContainer();
491 
492         initializeLastInsertedPage(1);
493         lastUnfilledPage = ContainerHandle.INVALID_PAGE_NUMBER;
494         lastAllocatedPage = ContainerHandle.INVALID_PAGE_NUMBER;
495 
496         canUpdate = false;
497         super.clearIdentity();
498     }
499 
500     /**
501         We treat this container as dirty if it has the container file open.
502         @see Cacheable#isDirty
503     */
504     public boolean isDirty() 
505     {
506         synchronized (this) 
507         {
508             return isDirty;
509         }
510     }
511 
512     public void preDirty(boolean preDirtyOn) 
513     {
514         synchronized (this) 
515         {
516             if (preDirtyOn)
517             {
518                 // prevent the cleaner from cleaning this container or skipping
519                 // over it until the operation which preDirtied it got a chance
520                 // to do the change.
521                 preDirty = true;
522             }
523             else
524             {
525                 preDirty = false;
526                 // if a cleaner is waiting on the dirty bit, wake it up
527                 notifyAll();
528             }
529         }
530     }
531 
532     protected void setDirty(boolean dirty)
533     {
534         synchronized(this) 
535         {
536             preDirty = false;
537             isDirty  = dirty;
538 
539             // if a cleaner is waiting on the dirty bit, wake it up
540             notifyAll();
541         }
542     }
543 
544     /*
545     ** Container creation, opening, and closing
546     */
547 
548     /**
549      * Create a new container.
550      * <p>
551      * Create a new container, all references to identity must be through the
552      * passed in identity, this object will no identity until after this 
553      * method returns.
554      *
555      * @exception StandardException Cloudscape Standard error policy
556      **/
557     abstract void createContainer(ContainerKey newIdentity) 
558         throws StandardException;
559     
560 
561     /**
562      * Open a container.
563      * <p>
564      * Longer descrption of routine.
565      * <p>
566      * Open a container. Open the file that maps to this container, if the
567      * file does not exist then we assume the container was never created.
568      * If the file exists but we have trouble opening it then we throw some 
569      * exception.
570      *
571      * <BR> MT - single thread required - Enforced by cache manager.
572      *
573      * @exception  StandardException  Standard exception policy.
574      **/
575     abstract boolean openContainer(ContainerKey newIdentity) 
576         throws StandardException;
577 
578     abstract void closeContainer();
579 
580     /**
581      * Drop Container.
582      * <p>
583      *
584      * @see Transaction#dropContainer
585      *
586      **/
587     protected void dropContainer(
588     LogInstant  instant, 
589     boolean     isDropped)
590     {
591         synchronized(this)
592         {
593             setDroppedState(isDropped);
594             setDirty(true);
595             bumpContainerVersion(instant);
596         }
597     }
598 
599 
600     /**
601         increment the version by one and return the new version.
602 
603         <BR> MT - caller must synchronized this in the same sync block that
604         modifies the container header.
605     */
606     protected final void bumpContainerVersion(LogInstant instant) 
607     {   
608         lastLogInstant = instant;
609         ++containerVersion;
610     }
611 
612     protected long getContainerVersion()
613     {
614         // it is not really necessary to synchronized this because the only time the
615         // container version is looked at is during recovery, which is single
616         // threaded at the moment.  Put it in an sync block anyway just in case
617         // some other people want to look at this for some bizarre reasons
618         synchronized(this)
619         {
620             return containerVersion;
621         }
622     }
623 
624     /**
625      * Request the system properties associated with a container. 
626      * <p>
627      * Request the value of properties that are associated with a container. 
628      * The following properties can be requested:
629      *     derby.storage.pageSize 
630      *     derby.storage.pageReservedSpace
631      *     derby.storage.minimumRecordSize
632      *     derby.storage.reusableRecordId
633      *     cloudsacpe.storage.initialPages
634      * <p>
635      * To get the value of a particular property add it to the property list,
636      * and on return the value of the property will be set to it's current 
637      * value.  For example:
638      *
639      * get_prop(ConglomerateController cc)
640      * {
641      *     Properties prop = new Properties();
642      *     prop.put("derby.storage.pageSize", "");
643      *     cc.getContainerProperties(prop);
644      *
645      *     System.out.println(
646      *         "table's page size = " + 
647      *         prop.getProperty("derby.storage.pageSize");
648      * }
649      *
650      * @param prop   Property list to fill in.
651      *
652      * @exception  StandardException  Standard exception policy.
653      **/
654     public void getContainerProperties(Properties   prop)
655         throws StandardException
656     {
657         // derby.storage.pageSize
658         if (prop.getProperty(Property.PAGE_SIZE_PARAMETER) != null)
659         {
660             prop.put(
661                 Property.PAGE_SIZE_PARAMETER, 
662                 Integer.toString(pageSize));
663         }
664 
665         // derby.storage.minimumRecordSize
666         if (prop.getProperty(RawStoreFactory.MINIMUM_RECORD_SIZE_PARAMETER) != 
667                 null)
668         {
669             prop.put(
670                 RawStoreFactory.MINIMUM_RECORD_SIZE_PARAMETER, 
671                 Integer.toString(minimumRecordSize));
672         }
673 
674         // derby.storage.pageReservedSpace
675         if (prop.getProperty(RawStoreFactory.PAGE_RESERVED_SPACE_PARAMETER) != 
676                 null)
677         {
678             prop.put(
679                 RawStoreFactory.PAGE_RESERVED_SPACE_PARAMETER, 
680                 Integer.toString(spareSpace));
681         }
682 
683         // derby.storage.reusableRecordId
684         if (prop.getProperty(RawStoreFactory.PAGE_REUSABLE_RECORD_ID) != null)
685         {
686             Boolean   bool = new Boolean  (isReusableRecordId());
687             prop.put(RawStoreFactory.PAGE_REUSABLE_RECORD_ID,
688                      bool.toString());
689         }
690 
691         // derby.storage.initialPages
692         if (prop.getProperty(RawStoreFactory.CONTAINER_INITIAL_PAGES) != null)
693         {
694             prop.put(RawStoreFactory.CONTAINER_INITIAL_PAGES,
695                      Integer.toString(initialPages));
696         }
697 
698     }
699 
700     /**
701         Read the container's header.  Assumes the input stream (fileData)
702         is positioned at the beginning of the file.
703 
704         Subclass that implements openContainer is expected to manufacture a DataInput 
705         stream which is used here to read the header.
706 
707         <BR> MT - single thread required - Enforced by caller.
708 
709         @exception StandardException Cloudscape Standard error policy
710         @exception IOException error in reading the header from file
711     */
712     protected void readHeader(DataInput   fileData) 
713          throws IOException  , StandardException
714     {
715         // Always read the header from the input stread even if the alloc page may
716         // still be in cache.  This is because a stubbify operation only writes
717         // the stub to disk, it did not get rid of any stale page from the page
718         // cache.  So if it so happen that the stubbified container object is
719         // aged out of the container cache but the first alloc page hasn't,
720         // then when any stale page of this container wants to be written out,
721         // the container needs to be reopened, which is when this routine is
722         // called.  We must not get the alloc page in cache because it may be
723         // stale page and it may still say the container has not been dropped.
724 
725         byte[] epage = getEmbryonicPage(fileData);
726 
727         // read persistent container header into containerInfo
728         AllocPage.ReadContainerInfo(containerInfo, epage);
729 
730         // initialize header from information stored in containerInfo
731         readHeaderFromArray(containerInfo);
732 
733         epage = null;
734     }
735 
736     // initialize header information so this container object can be safely
737     // reused as if this container object has just been new'ed
738     private void initContainerHeader(boolean changeContainer)
739     {
740         if (containerInfo == null)
741             containerInfo = new byte[CONTAINER_INFO_SIZE];
742 
743         if (checksum == null)
744             checksum = new CRC32  ();
745         else
746             checksum.reset();
747 
748         if (allocCache == null)
749             allocCache = new AllocationCache();
750         else
751             allocCache.reset();
752 
753         if (changeContainer)
754         {
755             pageSize = 0;
756             spareSpace = 0;
757             minimumRecordSize = 0;
758         }
759 
760         initialPages = 1;
761         firstAllocPageNumber = ContainerHandle.INVALID_PAGE_NUMBER; 
762         firstAllocPageOffset = -1;
763         containerVersion = 0;
764         estimatedRowCount = 0;
765         reusableRecordIdSequenceNumber = 0;
766 
767         setDroppedState(false);
768         setCommittedDropState(false);
769         setReusableRecordIdState(false);
770 
771         // instance variables that are not stored on disk
772         lastLogInstant = null;
773 
774         initializeLastInsertedPage(1);
775         lastUnfilledPage = ContainerHandle.INVALID_PAGE_NUMBER;
776         lastAllocatedPage = ContainerHandle.INVALID_PAGE_NUMBER;
777         estimatedPageCount = -1;
778 
779         PreAllocThreshold = PRE_ALLOC_THRESHOLD;
780         PreAllocSize = DEFAULT_PRE_ALLOC_SIZE;
781         bulkIncreaseContainerSize = false;
782     }
783 
784 
785     /**
786         Read containerInfo from a byte array
787         The container Header array must be written by or of
788         the same format as put together by writeHeaderFromArray.
789 
790         @exception StandardException Cloudscape Standard error policy
791         @exception IOException error in reading the header from file
792     */
793     private void readHeaderFromArray(byte[] a)
794          throws StandardException, IOException  
795     {
796         ArrayInputStream inStream = new ArrayInputStream(a);
797 
798         inStream.setLimit(0, CONTAINER_INFO_SIZE);
799         int fid = inStream.readInt();
800         if (fid != formatIdInteger)
801         {
802             throw StandardException.newException(
803                 SQLState.DATA_UNKNOWN_CONTAINER_FORMAT, getIdentity(), 
804                 new Long  (fid));
805         }
806 
807         int status = inStream.readInt();
808         pageSize = inStream.readInt();
809         spareSpace = inStream.readInt();
810         minimumRecordSize = inStream.readInt();
811         initialPages = inStream.readShort(); 
812         PreAllocSize = inStream.readShort();
813         firstAllocPageNumber = inStream.readLong();
814         firstAllocPageOffset = inStream.readLong();
815         containerVersion = inStream.readLong();
816         estimatedRowCount = inStream.readLong();
817         reusableRecordIdSequenceNumber = inStream.readLong();
818         lastLogInstant = null;
819 
820         if (PreAllocSize == 0)  // pre 2.0, we don't store this.
821             PreAllocSize = DEFAULT_PRE_ALLOC_SIZE;
822 
823         long spare3 = inStream.readLong();  // read spare long
824 
825         // upgrade - if this is a container that was created before
826         // initialPages was stored, it will have a zero value.  Set it to the
827         // default of 1.
828         if (initialPages == 0)  
829             initialPages = 1;
830 
831         // container read in from disk, reset preAllocation values
832         PreAllocThreshold = PRE_ALLOC_THRESHOLD;
833 
834         // validate checksum
835         long onDiskChecksum = inStream.readLong();
836         checksum.reset();
837         checksum.update(a, 0, CONTAINER_INFO_SIZE - CHECKSUM_SIZE);
838 
839         if (onDiskChecksum != checksum.getValue())
840         {
841             PageKey pk = new PageKey(identity, FIRST_ALLOC_PAGE_NUMBER);
842 
843             throw dataFactory.markCorrupt
844                 (StandardException.newException(
845                     SQLState.FILE_BAD_CHECKSUM, 
846                     pk, 
847                     new Long  (checksum.getValue()), 
848                     new Long  (onDiskChecksum), 
849                     org.apache.derby.iapi.util.StringUtil.hexDump(a)));
850         }
851 
852         allocCache.reset();
853 
854         // set the in memory state
855         setDroppedState((status & FILE_DROPPED) != 0);
856         setCommittedDropState((status & FILE_COMMITTED_DROP) != 0);
857         setReusableRecordIdState((status & FILE_REUSABLE_RECORDID) != 0);
858     }
859 
860 
861     /**
862         Write the container header to a page array (the first allocation page)
863 
864         @exception StandardException Cloudscape Standard error policy
865         @exception IOException error in writing the header to file
866     */
867     protected void writeHeader(byte[] pageData)
868          throws StandardException, IOException  
869     {
870         // write out the current containerInfo in the borrowed space to byte
871         // array containerInfo
872         writeHeaderToArray(containerInfo);
873 
874         AllocPage.WriteContainerInfo(containerInfo, pageData, false);
875     }
876 
877     /**
878         Write the container header directly to output stream (fileData).
879         Assumes the output stream is positioned at the beginning of the file.
880 
881         Subclasses that can writes the container header is expected to
882         manufacture a DataOutput stream which is used here.
883 
884         <BR> MT - single thread required - Enforced by caller
885 
886         @exception StandardException Cloudscape Standard error policy
887         @exception IOException error in writing the header to file
888      */
889     protected void writeHeader(DataOutput   fileData, boolean create, byte[] epage)
890          throws IOException  , StandardException
891     {
892         // write out the current containerInfo in the borrowed space to byte
893         // array containerInfo
894         writeHeaderToArray(containerInfo);
895 
896         // RESOLVE: get no wait on the page cache to see if allocation page is
897         // there, if so, use that instead of making a new array and a static
898         // function.
899 
900         AllocPage.WriteContainerInfo(containerInfo, epage, create);
901         // now epage has the containerInfo written inside it
902 
903         // force WAL - and check to see if database is corrupt or is frozen.
904         dataFactory.flush(lastLogInstant);
905         if (lastLogInstant != null)
906             lastLogInstant = null;
907 
908         // write it out
909         dataFactory.writeInProgress();
910         try
911         {
912             fileData.write(epage);
913         }
914         finally
915         {
916             dataFactory.writeFinished();
917         }
918     }
919 
920     /**
921         Get an embryonic page from the dataInput stream.
922 
923         If fileData is not null, then the embyronic page will be read 
924         in from the input stream (fileData), which is assumed to be 
925         positioned at the beginning of the first allocation page.
926 
927         if fileData is null, then just manufacture an array which
928         is the size of an embryonic page.
929 
930         @exception IOException error in read the embryonic page from file
931     */
932     protected byte[] getEmbryonicPage(DataInput   fileData) throws IOException  
933     {
934         byte[] epage = new byte[AllocPage.MAX_BORROWED_SPACE];
935 
936         if (fileData != null)
937         {
938             fileData.readFully(epage);
939         }
940         return epage;
941     }
942 
943     /**
944         Write containerInfo into a byte array
945         The container Header thus put together can be read by readHeaderFromArray.
946 
947         @exception IOException error in writing the header
948     */
949     private void writeHeaderToArray(byte[] a) throws IOException  
950     {
951         if (SanityManager.DEBUG)
952             SanityManager.ASSERT(a.length >= CONTAINER_INFO_SIZE,
953                                  "header won't fit in array");
954 
955         ArrayOutputStream a_out = new ArrayOutputStream(a);
956         FormatIdOutputStream outStream = new FormatIdOutputStream(a_out);
957 
958         int status = 0;
959         if (getDroppedState()) status |= FILE_DROPPED;
960         if (getCommittedDropState()) status |= FILE_COMMITTED_DROP;
961         if (isReusableRecordId()) status |= FILE_REUSABLE_RECORDID;
962 
963         a_out.setPosition(0);
964         a_out.setLimit(CONTAINER_INFO_SIZE);
965         outStream.writeInt(formatIdInteger);
966         outStream.writeInt(status);
967         outStream.writeInt(pageSize);
968         outStream.writeInt(spareSpace);
969         outStream.writeInt(minimumRecordSize);
970         outStream.writeShort(initialPages);
971         outStream.writeShort(PreAllocSize);     // write spare1
972         outStream.writeLong(firstAllocPageNumber);
973         outStream.writeLong(firstAllocPageOffset);
974         outStream.writeLong(containerVersion);
975         outStream.writeLong(estimatedRowCount);
976         outStream.writeLong(reusableRecordIdSequenceNumber);
977         outStream.writeLong(0);     //Write spare3
978 
979         checksum.reset();
980         checksum.update(a, 0, CONTAINER_INFO_SIZE - CHECKSUM_SIZE);
981 
982         // write the checksum to the array
983         outStream.writeLong(checksum.getValue());
984 
985         a_out.clearLimit();
986     }
987 
988     /**
989         Log all information on the container creation necessary to recreate the
990         container during a load tran.
991 
992         @exception StandardException Cloudscape Standard error policy
993      */
994     protected ByteArray logCreateContainerInfo() 
995          throws  StandardException
996     {
997         // just write out the whole container header
998         byte[] array = new byte[CONTAINER_INFO_SIZE];
999         if (array == null || array.length != CONTAINER_INFO_SIZE)
1000        {
1001            throw StandardException.newException(
1002                SQLState.DATA_OBJECT_ALLOCATION_FAILED, "byte[]");
1003        }
1004
1005        try
1006        {
1007            writeHeaderToArray(array);
1008        }
1009        catch (IOException   ioe)
1010        {
1011            throw StandardException.newException(
1012                SQLState.DATA_UNEXPECTED_EXCEPTION, ioe);
1013        }
1014
1015        return new ByteArray(array);
1016    }
1017
1018    /**
1019        Set container properties from the passed in ByteArray, which is created
1020        by logCreateContainerInfo.  This information is used to recreate the
1021        container during recovery load tran.
1022
1023        The following container properties are set:
1024
1025        pageSize
1026        spareSpace
1027        minimumRecordSize
1028        isReusableRecordId
1029        initialPages
1030
1031     */
1032    private void createInfoFromLog(ByteArray byteArray) 
1033         throws StandardException 
1034    {
1035        if (SanityManager.DEBUG)
1036        {
1037            SanityManager.ASSERT(byteArray != null,
1038                "setCreateContainerInfoFromLog: ByteArray is null");
1039            SanityManager.ASSERT(byteArray.getLength() == 
1040                                 CONTAINER_INFO_SIZE,
1041                "setCreateContainerInfoFromLog: ByteArrays.length() != CONTAINER_INFO_SIZE");
1042        }
1043
1044        byte[] array = byteArray.getArray();
1045        
1046        // now extract the relavent information from array - basically 
1047        // duplicate the code in readHeaderFromArray 
1048        ArrayInputStream inStream = new ArrayInputStream(array);
1049
1050        int status = 0;
1051
1052        try
1053        {           
1054            inStream.setLimit(0, CONTAINER_INFO_SIZE);
1055
1056            int fid = inStream.readInt();
1057            if (fid != formatIdInteger)
1058            {
1059                // RESOLVE: do something about this when we have > 1 container format
1060                throw StandardException.newException(
1061                    SQLState.DATA_UNKNOWN_CONTAINER_FORMAT, 
1062                    getIdentity(), new Long  (fid));
1063            }
1064
1065            status = inStream.readInt();
1066            pageSize = inStream.readInt();
1067            spareSpace = inStream.readInt();
1068            minimumRecordSize = inStream.readInt();
1069            initialPages = inStream.readShort(); 
1070
1071        }
1072        catch (IOException   ioe)
1073        {
1074            throw StandardException.newException(
1075                    SQLState.DATA_UNEXPECTED_EXCEPTION, ioe);
1076        }
1077
1078        // set reusable record id property
1079        setReusableRecordIdState((status & FILE_REUSABLE_RECORDID) != 0);
1080
1081        // sanity check to make sure we are not encoutering any
1082        // dropped Container 
1083        if (SanityManager.DEBUG)
1084        {
1085            SanityManager.ASSERT((status & FILE_DROPPED) == 0 &&
1086                                 (status & FILE_COMMITTED_DROP) == 0,
1087                "cannot load a dropped container");
1088        }
1089    }
1090
1091    /**
1092        Set container properties from the passed in createArgs.  
1093        The following container properties are set:
1094
1095        pageSize
1096        spareSpace
1097        minimumRecordSize
1098        isReusableRecordId
1099        initialPages
1100
1101        RESOLVE - in the future setting parameters should be overridable
1102        by sub-class, e.g. one implementation of Container may require a
1103        minimum page size of 4k.
1104     */
1105    private void createInfoFromProp(Properties   createArgs)
1106         throws StandardException
1107    {
1108        // Need a TransactionController to get database/service wide properties.
1109        AccessFactory af = (AccessFactory)
1110            Monitor.getServiceModule(dataFactory, AccessFactory.MODULE);
1111
1112        // RESOLVE: sku defectid 2014
1113        TransactionController tc = 
1114            (af == null) ? 
1115                null : 
1116                af.getTransaction(
1117                        ContextService.getFactory().getCurrentContextManager());
1118
1119        pageSize = 
1120            PropertyUtil.getServiceInt(tc, createArgs,
1121                Property.PAGE_SIZE_PARAMETER,  
1122                Limits.DB2_MIN_PAGE_SIZE, 
1123                Limits.DB2_MAX_PAGE_SIZE, 
1124                RawStoreFactory.PAGE_SIZE_DEFAULT); 
1125
1126        // rather than throw error, just automatically set page size to 
1127        // default if bad value given.
1128        if ((pageSize != 4096)  &&
1129            (pageSize != 8192)  &&
1130            (pageSize != 16384) &&
1131            (pageSize != 32768))
1132        {
1133            pageSize= RawStoreFactory.PAGE_SIZE_DEFAULT;
1134        }
1135
1136        spareSpace = 
1137            PropertyUtil.getServiceInt(tc, createArgs,
1138                RawStoreFactory.PAGE_RESERVED_SPACE_PARAMETER, 
1139                0, 100, 20);
1140
1141        PreAllocSize = 
1142            PropertyUtil.getServiceInt(tc, createArgs,
1143                    RawStoreFactory.PRE_ALLOCATE_PAGE,
1144                    MIN_PRE_ALLOC_SIZE,
1145                    MAX_PRE_ALLOC_SIZE,                
1146                    DEFAULT_PRE_ALLOC_SIZE /* default */);
1147
1148        // RESOLVE - in the future, we will allow user to set minimumRecordSize
1149        // to be larger than pageSize, when long rows are supported.
1150        if (createArgs == null) {
1151            // if the createArgs is null, then the following method call
1152            // will get the system properties from the appropriete places.
1153            // we want to make sure minimumRecrodSize is set to at least
1154            // the default value MINIMUM_RECORD_SIZE_DEFAULT (12)
1155            // as set in rawStoreFactory.
1156            minimumRecordSize = 
1157                PropertyUtil.getServiceInt(tc,
1158                    RawStoreFactory.MINIMUM_RECORD_SIZE_PARAMETER, 
1159                    RawStoreFactory.MINIMUM_RECORD_SIZE_DEFAULT, // this is different from the next call
1160                    // reserving 100 bytes for record/field headers
1161                    (pageSize * (1 - spareSpace/100) - 100), 
1162                    RawStoreFactory.MINIMUM_RECORD_SIZE_DEFAULT);
1163        } else {
1164            // if the createArgs is not null, then it has already been set
1165            // by upper layer or create statement, then, we allow the minimum
1166            // value of this to be MINIMUM_RECORD_SIZE_MINIMUM (1).
1167            minimumRecordSize = 
1168                PropertyUtil.getServiceInt(tc, createArgs,
1169                    RawStoreFactory.MINIMUM_RECORD_SIZE_PARAMETER, 
1170                    RawStoreFactory.MINIMUM_RECORD_SIZE_MINIMUM,  // this is different from the last call
1171                    // reserving 100 bytes for record/field headers
1172                    (pageSize * (1 - spareSpace/100) - 100), 
1173                    RawStoreFactory.MINIMUM_RECORD_SIZE_DEFAULT);
1174        }
1175
1176        // For the following properties, do not check value set in global
1177        // properties, we only listen to what access has to say about them.
1178        //
1179        // whether or not container's recordIds can be reused
1180        // if container is to be created with a large number of pages
1181        if (createArgs != null)
1182        {
1183            String   reusableRecordIdParameter = 
1184                createArgs.getProperty(RawStoreFactory.PAGE_REUSABLE_RECORD_ID);
1185            if (reusableRecordIdParameter != null)
1186            {   
1187                Boolean   reusableRecordId = new Boolean  (reusableRecordIdParameter);
1188                setReusableRecordIdState(reusableRecordId.booleanValue());
1189            }
1190
1191            String   containerInitialPageParameter =
1192                createArgs.getProperty(RawStoreFactory.CONTAINER_INITIAL_PAGES);
1193            if (containerInitialPageParameter != null)
1194            {
1195                initialPages = 
1196                    Short.parseShort(containerInitialPageParameter);
1197                if (initialPages > 1)
1198                {
1199                    if (initialPages > RawStoreFactory.MAX_CONTAINER_INITIAL_PAGES)
1200                        initialPages = RawStoreFactory.MAX_CONTAINER_INITIAL_PAGES;
1201                }
1202            }
1203        }
1204    }
1205
1206    /**
1207    */
1208    protected boolean canUpdate() {
1209        return canUpdate;
1210    }
1211
1212    /**
1213        Deallocate a page from the container.  
1214
1215        @param handle the container handle doing the deallocation
1216        @param page the page to be deallocated.  It is latched upon entry and
1217        will be unlatched by the caller of this function
1218
1219        @exception StandardException Cloudscape Standard error policy
1220    */
1221    protected void deallocatePage(BaseContainerHandle handle, BasePage page)
1222         throws StandardException
1223    {
1224        if (SanityManager.DEBUG) {
1225            SanityManager.ASSERT(page.isLatched(), "page is not latched");
1226            SanityManager.ASSERT(page.getPageNumber() != FIRST_ALLOC_PAGE_NUMBER, 
1227                                 "cannot deallocate an alloc page");
1228        }
1229
1230        long pnum = page.getPageNumber();
1231
1232        // dealloc the page from the alloc page
1233        deallocatePagenum(handle, pnum);
1234
1235        // mark the page as deallocated.  Page should not be touched after this
1236        // the page latch is released by the BaseContainer upon return of this
1237        // method.  Regardless of whether this operation is successful or not,
1238        // the page will be unlatched by BaseContainer.
1239        page.deallocatePage();
1240        
1241    }
1242
1243    /** deallocate the page from the alloc page */
1244    private void deallocatePagenum(BaseContainerHandle handle, long pnum)
1245         throws StandardException
1246    {
1247        synchronized(allocCache)
1248        {
1249            long allocPageNum = allocCache.getAllocPageNumber(handle, pnum, firstAllocPageNumber);
1250
1251            if (SanityManager.DEBUG)
1252            {
1253                if (allocPageNum == ContainerHandle.INVALID_PAGE_NUMBER)
1254                    allocCache.dumpAllocationCache();
1255
1256                if (allocPageNum == ContainerHandle.INVALID_PAGE_NUMBER)
1257                    SanityManager.THROWASSERT(
1258                                     "can't find alloc page for page number " +
1259                                     pnum);
1260            }
1261            // get the alloc page to deallocate this pnum
1262            AllocPage allocPage = (AllocPage)handle.getAllocPage(allocPageNum);
1263            if (allocPage == null)
1264            {
1265                PageKey pkey = new PageKey(identity, allocPageNum);
1266
1267                throw StandardException.newException(
1268                        SQLState.FILE_NO_ALLOC_PAGE, pkey);
1269            }
1270
1271            try
1272            {
1273                allocCache.invalidate(allocPage, allocPageNum); 
1274
1275                // Unlatch alloc page.  The page is protected by the dealloc
1276                // lock. 
1277                allocPage.deallocatePage(handle, pnum);
1278            }
1279            finally
1280            {
1281                allocPage.unlatch();
1282            }
1283        }
1284        // make sure this page gets looked at when someone needs a new page
1285        if (pnum <= lastAllocatedPage) 
1286        {
1287            lastAllocatedPage = pnum - 1;
1288        }
1289
1290    }
1291
1292    /**
1293      Compress free space from container.
1294
1295      <BR> MT - thread aware - It is assumed that our caller (our super class)
1296      has already arranged a logical lock on page allocation to only allow a
1297      single thread through here.
1298
1299      Compressing free space is done in allocation page units, working
1300      it's way from the end of the container to the beginning.  Each
1301      loop operates on the last allocation page in the container.
1302
1303      Freeing space in the container page involves 2 transactions, an
1304      update to an allocation page, N data pages, and possibly the delete
1305      of the allocation page.
1306      The User Transaction (UT) initiated the compress call.
1307      The Nested Top Transaction (NTT) is the transaction started by RawStore
1308      inside the compress call.  This NTT is committed before compress returns.
1309      The NTT is used to access high traffic data structures such as the 
1310      AllocPage.
1311
1312      This is outline of the algorithm used in compressing the container.
1313
1314      Until a non free page is found loop, in each loop return to the OS
1315         all space at the end of the container occupied by free pages, including
1316         the allocation page itself if all of it's pages are free.  
1317      
1318      1) Find last 2 allocation pages in container (last if there is only one).
1319      2) invalidate the allocation information cached by the container.
1320         Without the cache no page can be gotten from the container.  Pages
1321         already in the page cache are not affected.  Thus by latching the 
1322         allocPage and invalidating the allocation cache, this NTT blocks out 
1323         all page gets from this container until it commits.
1324      3) the allocPage determines which pages can be released to the OS, 
1325         mark that in its data structure (the alloc extent).  Mark the 
1326         contiguous block of nallocated/free pages at the end of the file
1327         as unallocated.  This change is associated with the NTT.
1328      4) The NTT calls the OS to deallocate the space from the file.  Note
1329         that the system can handle being booted and asked to get an allocated
1330         page which is past end of file, it just extends the file automatically.
1331      5) If freeing all space on the alloc page, and there is more than one
1332         alloc page, then free the alloc page - this requires an update to the 
1333         previous alloc page which the loop has kept latched also.
1334      6) if the last alloc page was deleted, restart loop at #1
1335
1336      All NTT latches are released before this routine returns.
1337      If we use an NTT, the caller has to commit the NTT to release the
1338      allocPage latch.  If we don't use an NTT, the allocPage latch is released
1339      as this routine returns.
1340
1341      @param ntt - the nested top transaction for the purpose of freeing space.
1342                        If ntt is null, use the user transaction for allocation.
1343      #param allocHandle - the container handle opened by the ntt, 
1344                        use this to latch the alloc page
1345
1346      @exception StandardException Standard Cloudscape error policy 
1347    */
1348    protected void compressContainer(
1349    RawTransaction      ntt,
1350    BaseContainerHandle allocHandle)
1351         throws StandardException 
1352    {
1353        AllocPage alloc_page      = null;
1354        AllocPage prev_alloc_page = null;
1355
1356        if (firstAllocPageNumber == ContainerHandle.INVALID_PAGE_NUMBER)
1357        {
1358            // no allocation pages in container, no work to do!
1359            return;
1360        }
1361
1362        
1363        // make sure we don't execute redo recovery on any page
1364        // which is getting truncated.  At this point we have an exclusive
1365        // table lock on the table, so after checkpoint no page change
1366        // can happen between checkpoint log record and compress of space.
1367        dataFactory.getRawStoreFactory().checkpoint();
1368
1369        // block the backup, If backup is already in progress wait 
1370        // for the backup to finish. Otherwise restore from the backup
1371        // can start recovery at different checkpoint and possibly
1372        // do redo on pages that are going to get truncated.
1373        ntt.blockBackup(true);
1374
1375        try
1376        {
1377            synchronized(allocCache)
1378            {
1379                // loop until last 2 alloc pages are reached.
1380                alloc_page = (AllocPage) 
1381                    allocHandle.getAllocPage(firstAllocPageNumber);
1382
1383                while (!alloc_page.isLast())
1384                {
1385                    if (prev_alloc_page != null)
1386                    {
1387                        // there are more than 2 alloc pages, unlatch the 
1388                        // earliest one.
1389                        prev_alloc_page.unlatch();
1390                    }
1391                    prev_alloc_page = alloc_page;
1392                    alloc_page      = null;
1393
1394                    long nextAllocPageNumber = 
1395                        prev_alloc_page.getNextAllocPageNumber();
1396                    long nextAllocPageOffset = 
1397                        prev_alloc_page.getNextAllocPageOffset();
1398
1399                    alloc_page = (AllocPage) 
1400                        allocHandle.getAllocPage(nextAllocPageNumber);
1401                }
1402
1403                // invalidate cache before compress changes cached information,
1404                // while holding synchronization on cache and latch on 
1405                // allocation page.  This should guarantee that only new info
1406                // is seen after this operation completes.
1407                allocCache.invalidate(); 
1408
1409                // reset, as pages may not exist after compress
1410                lastUnfilledPage    = ContainerHandle.INVALID_PAGE_NUMBER;
1411                lastAllocatedPage   = ContainerHandle.INVALID_PAGE_NUMBER;
1412
1413
1414                alloc_page.compress(ntt, this);
1415            }
1416
1417        }
1418        finally
1419        {
1420            if (alloc_page != null)
1421            {
1422                alloc_page.unlatch();
1423                alloc_page = null;
1424            }
1425            if (prev_alloc_page != null)
1426            {
1427                prev_alloc_page.unlatch();
1428                prev_alloc_page = null;
1429            }
1430
1431            // flush all changes to this file from cache.
1432            flushAll();
1433
1434            // make sure all truncated pages are removed from the cache,
1435            // as it will get confused in the future if we allocate the same
1436            // page again, but find an existing copy of it in the cache - 
1437            // it expects to not find new pages in the cache.  Could just
1438            // get rid of truncated pages, iterface allows one page or
1439            // all pages.
1440            pageCache.discard(identity);
1441        }
1442    }
1443
1444    /**
1445     * Get the reusable RecordId sequence number for the container.
1446     * @see BaseContainer#getReusableRecordIdSequenceNumber
1447     * @return reusable RecordId sequence number for the container.
1448     */
1449    public final long getReusableRecordIdSequenceNumber() {
1450        synchronized(this) {
1451            return reusableRecordIdSequenceNumber;
1452        }
1453    }
1454    
1455    /**
1456     * Increment the reusable RecordId version sequence number.
1457     */
1458    protected final void incrementReusableRecordIdSequenceNumber()
1459    {
1460        final boolean readOnly = dataFactory.isReadOnly();
1461        
1462        synchronized (this) {
1463            reusableRecordIdSequenceNumber++;
1464            if (!readOnly)
1465            {
1466                isDirty = true;
1467            }
1468        }
1469    }
1470
1471    /**
1472      Create a new page in the container.
1473
1474      <BR> MT - thread aware - It is assumed that our caller (our super class)
1475      has already arranged a logical lock on page allocation to only allow a
1476      single thread through here.
1477
1478      Adding a new page involves 2 transactions and 2 pages.  
1479      The User Transaction (UT) initiated the addPage call and expects a
1480      latched page (owns by the UT) to be returned.
1481      The Nested Top Transaction (NTT) is the transaction started by RawStore
1482      inside an addPage call.  This NTT is committed before the page is
1483      returned.  The NTT is used to accessed high traffic data structure such
1484      as the AllocPage.
1485
1486      This is outline of the algorithm used in adding a page:
1487      1) find or make an allocPage which can handle the addding of a new page.
1488        Latch the allocPage with the NTT.
1489      2) invalidate the allocation information cached by the container.
1490        Without the cache no page can be gotten from the container.  Pages
1491        already in the page cache is not affected.  Thus by latching the 
1492        allocPage and invalidating the allocation cache, this NTT blocks out 
1493        all page gets from this container until it commits.
1494      3) the allocPage determines which page can be allocated, mark that in its
1495        data structure (the alloc extent) and returns the page number of the
1496        new page.  This change is associated with the NTT.
1497      4) the NTT gets or creates the new page in the page cache (bypassing the
1498        lookup of the allocPage since that is already latched by the NTT and
1499        will deadlock).
1500      5) the NTT initializes the page (mark it is being a VALID page).
1501      6) the page latch is transfered to the UT from the NTT.
1502      7) the new page is returned, latched by UT
1503
1504      If we use an NTT, the caller has to commit the NTT to release the
1505      allocPage latch.  If we don't use an NTT, the allocPage latch is released
1506      as this routine returns.
1507
1508      @param userHandle - the container handle opened by the user transaction, 
1509                        use this to latch the new user page
1510      @param ntt - the nested top transaction for the purpose of allocating the new page
1511                        If ntt is null, use the user transaction for allocation.
1512      #param allocHandle - the container handle opened by the ntt, 
1513                        use this to latch the alloc page
1514
1515      @exception StandardException Standard Cloudscape error policy 
1516    */
1517    protected BasePage newPage(BaseContainerHandle userHandle,
1518                               RawTransaction ntt,
1519                               BaseContainerHandle allocHandle,
1520                               boolean isOverflow) 
1521         throws StandardException 
1522    {
1523        // NOTE: we are single threaded thru this method, see MT comment
1524
1525        boolean useNTT = (ntt != null);
1526
1527        // if ntt is null, use user transaction
1528        if (!useNTT)
1529            ntt = userHandle.getTransaction();
1530
1531        long lastPage;          // last allocated page
1532        long lastPreallocPage;  // last pre-allcated page
1533        long pageNumber;        // the page number of the new page
1534        PageKey pkey;           // the identity of the new page
1535        boolean reuse;          // if true, we are trying to reuse a page
1536
1537        /* in case the page recommeded by allocPage is not committed yet, may
1538        /* need to retry a couple of times */
1539        boolean retry;
1540        int numtries = 0;
1541        long startSearch = lastAllocatedPage;
1542
1543        AllocPage allocPage = null; // the alloc page
1544        BasePage page = null;   // the new page
1545
1546        try
1547        {
1548            do
1549            {
1550                retry = false;      // we don't expect we need to retry
1551
1552                synchronized(allocCache)
1553                {
1554                    if (SanityManager.DEBUG)
1555                    {
1556                        SanityManager.ASSERT(
1557                            ntt.getId().equals(
1558                                allocHandle.getTransaction().getId()));
1559
1560                        if (useNTT)
1561                            SanityManager.ASSERT(
1562                                !ntt.getId().equals(
1563                                    userHandle.getTransaction().getId()));
1564                    }
1565
1566                    /* find an allocation page that can handle adding a new 
1567                     * page.
1568                     *
1569                     * allocPage is unlatched when the ntt commits. The new 
1570                     * page is initialized by the ntt but the latch is 
1571                     * transfered to the user transaction before the allocPage 
1572                     * is unlatched.  The allocPage latch prevents almost any 
1573                     * other reader or writer from finding the new page until 
1574                     * the ntt is committed and the new page is latched by the
1575                     * user transaction.
1576                     *
1577                     * (If the page is being reused, it is possible for another
1578                     * xact which kept a handle on the reused page to find the 
1579                     * page during the transfer UT -> NTT. If this unlikely 
1580                     * even occurs and the transfer fails [see code relating 
1581                     * to transfer below], we retry from the beginning.)
1582                     *
1583                     * After the NTT commits a reader (getNextPageNumber) may 
1584                     * get the page number of the newly allocated page and it 
1585                     * will wait for the new page and latch it when the user 
1586                     * transaction commits, aborts or unlatches the new page. 
1587                     * Whether the user transaction commits or aborts, the new 
1588                     * page stay allocated.
1589                     *
1590                     * RESOLVE: before NTT rolls back (or commits) the latch is
1591                     * released.  To repopulate the allocation cache, need to 
1592                     * get either the container lock on add page, or get a per 
1593                     * allocation page lock.
1594                     *
1595                     * This blocks all page read (getPage) from accessing this 
1596                     * alloc page in this container until the alloc page is 
1597                     * unlatched.  Those who already have a page handle into 
1598                     * this container are unaffected.
1599                     *
1600                     * In other words, allocation blocks out reader (of any 
1601                     * page that is managed by this alloc page) by the latch 
1602                     * on the allocation page.
1603                     *
1604                     * Note that write page can proceed as usual.
1605                     */
1606                    allocPage = 
1607                        findAllocPageForAdd(allocHandle, ntt, startSearch);
1608
1609                    allocCache.invalidate(allocPage, allocPage.getPageNumber());
1610                }
1611
1612                if (SanityManager.DEBUG)
1613                {
1614                    if (allocPage == null)
1615                        allocCache.dumpAllocationCache();
1616
1617                    SanityManager.ASSERT(allocPage != null,
1618                         "findAllocPageForAdd returned a null alloc page");
1619                }
1620
1621                //
1622                // get the next free page's number.
1623                // for case 1, page number > lastPreallocPage
1624                // for case 2, page number <= lastPage
1625                // for case 3, lastPage < page number <= lastPreallocPage
1626                //
1627                pageNumber = allocPage.nextFreePageNumber(startSearch);
1628
1629                // need to distinguish between the following 3 cases:
1630                // 1) the page has not been allocate or initalized.
1631                //      Create it in the page cache and sync it to disk.
1632                // 2) the page is being re-allocated.
1633                //      We need to read it in to re-initialize it
1634                // 3) the page has been preallocated.
1635                //      Create it in the page cache and don't sync it to disk
1636                //
1637                // first find out the current last initialized page and
1638                // preallocated page before the new page is added
1639                lastPage         = allocPage.getLastPagenum();
1640                lastPreallocPage = allocPage.getLastPreallocPagenum();
1641
1642                reuse = pageNumber <= lastPage;
1643
1644                // no address translation necessary
1645                pkey = new PageKey(identity, pageNumber);
1646
1647
1648                if (reuse)
1649                {
1650                    // if re-useing a page, make sure the deallocLock on the new
1651                    // page is not held.  We only need a zero duration lock on
1652                    // the new page because the allocPage is latched and this
1653                    // is the only thread which can be looking at this
1654                    // pageNumber.
1655
1656                    RecordHandle deallocLock = BasePage.MakeRecordHandle(pkey,
1657                                 RecordHandle.DEALLOCATE_PROTECTION_HANDLE);
1658
1659                    if (!getDeallocLock(allocHandle, deallocLock,
1660                                        false /* nowait */,
1661                                        true /* zeroDuration */))
1662                    {
1663
1664                        // The transaction which deallocated this page has not
1665                        // committed yet. Try going to some other page.  If
1666                        // this is the first time we fail to get the dealloc
1667                        // lock, try from the beginning of the allocated page.
1668                        // If we already did that and still fail, keep going
1669                        // until we get a brand new page.
1670                        if (numtries == 0)
1671                        {
1672                            startSearch = ContainerHandle.INVALID_PAGE_NUMBER;
1673                            lastAllocatedPage = pageNumber;
1674                        }
1675                        else    // continue from where we were
1676                            startSearch = pageNumber;
1677
1678                        numtries++;
1679
1680                        // We have to unlatch the allocPage so that if that
1681                        // transaction rolls back, it won't deadlock with this
1682                        // transaction.
1683                        allocPage.unlatch();
1684                        allocPage = null;
1685
1686                        retry = true;
1687                    }
1688                    else
1689                    {
1690                        // we got the lock, next time start from there
1691                        lastAllocatedPage = pageNumber;
1692                    }
1693                }
1694                else
1695                {
1696                    // we got a new page, next time, start from beginning of
1697                    // the bit map again if we suspect there are some some
1698                    // deallocated pages
1699                    if (numtries > 0)
1700                        lastAllocatedPage = ContainerHandle.INVALID_PAGE_NUMBER;
1701                    else
1702                        lastAllocatedPage = pageNumber;
1703                }
1704
1705                // Retry from the beginning if necessary.
1706                if (retry)
1707                    continue;
1708
1709                // If we get past here must have (retry == false)
1710                if (SanityManager.DEBUG)
1711                {
1712                    SanityManager.ASSERT(retry == false);
1713                }
1714
1715                // Now we have verified that the allocPage is latched and we 
1716                // can get the zeroDuration deallocLock nowait.  This means the
1717                // transaction which freed the page has committed.  Had that 
1718                // transaction aborted, we would have retried.
1719
1720                if (SanityManager.DEBUG)
1721                {
1722                    // ASSERT lastPage <= lastPreallocPage
1723                    if (lastPage > lastPreallocPage)
1724                    {
1725                        SanityManager.THROWASSERT("last page " +
1726                            lastPage + " > lastPreallocPage " + 
1727                            lastPreallocPage);
1728                    }
1729                }
1730
1731                // No I/O at all if this new page is requested as part of a 
1732                // create and load statement or this new page is in a temporary
1733                // container.
1734                //
1735                // In the former case, BaseContainer will allow the 
1736                // MODE_UNLOGGED bit to go thru to the nested top transaction 
1737                // alloc handle.  In the later case, there is no nested top 
1738                // transaction and the alloc handle is the user handle, which 
1739                // is UNLOGGED.
1740                boolean noIO = 
1741                    (allocHandle.getMode() & ContainerHandle.MODE_UNLOGGED) ==
1742                        ContainerHandle.MODE_UNLOGGED;
1743
1744                // If we do not need the I/O (either because we are in a
1745                // create_unlogged mode or we are dealing with a temp table), 
1746                // don't do any preallocation.  Otherwise, see if we should be
1747                // pre-Allocating page by now.  We don't call it before
1748                // nextFreePageNumber because finding a reusable page may be
1749                // expensive and we don't want to start preAllocation unless 
1750                // there is no more reusable page.  Unless we are called 
1751                // explicitly to bulk increase the container size in a preload 
1752                // or in a create container.
1753                if (!noIO && 
1754                    (bulkIncreaseContainerSize ||
1755                     (pageNumber > lastPreallocPage && 
1756                      pageNumber > PreAllocThreshold)))
1757                {
1758                    allocPage.preAllocatePage(
1759                        this, PreAllocThreshold, PreAllocSize);
1760                }
1761
1762                // update last preAllocated Page, it may have been changed by 
1763                // the preAllocatePage call.  We don't want to do the sync if 
1764                // preAllocatePage already took care of it.
1765                lastPreallocPage = allocPage.getLastPreallocPagenum();
1766                boolean prealloced = pageNumber <= lastPreallocPage;
1767
1768                // Argument to the create is an array of ints.
1769                // The array is only used for new page creation or for creating
1770                // a preallocated page, not for reuse.
1771                // 0'th element is the page format
1772                // 1'st element is whether or not to sync the page to disk
1773                // 2'nd element is pagesize
1774                // 3'rd element is spareSpace
1775
1776                int[] createPageArgs = new int[STORED_PAGE_ARG_NUM];
1777                createPageArgs[0] = StoredPage.FORMAT_NUMBER;
1778                createPageArgs[1] = prealloced ? 
1779                                        0 : (noIO ? 0 : CachedPage.WRITE_SYNC);
1780                createPageArgs[2] = pageSize;
1781                createPageArgs[3] = spareSpace;
1782                createPageArgs[4] = minimumRecordSize;
1783
1784                // RESOLVE: right now, there is no re-mapping of pages, so
1785                // pageOffset = pageNumber*pageSize
1786                long pageOffset = pageNumber * pageSize;
1787
1788                // initialize a new user page
1789                // we first use the NTT to initialize the new page - in case the
1790                // allocation failed, it is rolled back with the NTT.
1791                // Later, we transfer the latch to the userHandle so it won't be
1792                // released when the ntt commits
1793
1794                try
1795                {
1796                page = initPage(allocHandle, pkey, createPageArgs, pageOffset,
1797                                reuse, isOverflow);
1798                }
1799                catch (StandardException se)
1800                {
1801                    if (SanityManager.DEBUG) {
1802                        SanityManager.DEBUG_PRINT("FileContainer",
1803                            "got exception from initPage:"  +
1804                            "\nreuse = " + reuse +
1805                            "\ncreatePageArgs[1] = " + createPageArgs[1] +
1806                            "\nallocPage = " + allocPage
1807                            );
1808                    }
1809                    allocCache.dumpAllocationCache();
1810
1811                    throw se;
1812                }
1813
1814                if (SanityManager.DEBUG)
1815                {
1816                    SanityManager.ASSERT(
1817                        page != null, "initPage returns null page");
1818                    SanityManager.ASSERT(
1819                        page.isLatched(), "initPage returns unlatched page");
1820                }
1821
1822                // allocate the page in the allocation page bit map
1823                allocPage.addPage(this, pageNumber, ntt, userHandle);
1824
1825                if (useNTT)
1826                {
1827                    // transfer the page latch from NTT to UT.
1828                    //
1829                    // after the page is unlatched by NTT, it is still 
1830                    // protected from being found by almost everybody else 
1831                    // because the alloc page is still latched and the alloc 
1832                    // cache is invalidated.
1833                    //
1834                    // However (beetle 3942) it is possible for the page to be 
1835                    // found by threads who specifically ask for this 
1836                    // pagenumber (e.g. HeapPostCommit).
1837                    // We may find that such a thread has latched the page. 
1838                    // We shouldn't wait for it because we have the alloc page 
1839                    // latch, and this could cause deadlock (e.g. 
1840                    // HeapPostCommit might call removePage and this would wait
1841                    // on the alloc page).
1842                    //
1843                    // We may instead find that we can latch the page, but that
1844                    // another thread has managed to get hold of it during the 
1845                    // transfer and either deallocate it or otherwise change it
1846                    // (add rows, delete rows etc.)
1847                    //
1848                    // Since this doesn't happen very often, we retry in these 
1849                    // 2 cases (we give up the alloc page and page and we start
1850                    // this method from scratch).
1851                    //
1852                    // If the lock manager were changed to allow latches to be 
1853                    // transferred between transactions, wouldn't need to 
1854                    // unlatch to do the transfer, and would avoid having to 
1855                    // retry in these cases (beetle 4011).
1856
1857                    page.unlatch();
1858                    page = null;
1859
1860                    // need to find it in the cache again since unlatch also 
1861                    // unkept the page from the cache
1862                    page = (BasePage)pageCache.find(pkey);
1863                    page = latchPage(
1864                                userHandle, page, 
1865                                false /* don't wait, it might deadlock */);
1866
1867                    if (page == null ||
1868                        // recordCount will only return true if there are no 
1869                        // rows (including deleted rows)
1870                        page.recordCount() != 0 ||
1871                        page.getPageStatus() != BasePage.VALID_PAGE)
1872                    {
1873                        retry = true;
1874                        if (page != null)
1875                        {
1876                            page.unlatch();
1877                            page = null;
1878                        }
1879                        allocPage.unlatch();
1880                        allocPage = null;
1881                    }
1882
1883                }
1884                // if ntt is null, no need to transfer.  Page is latched by user
1885                // transaction already.  Will be no need to retry.
1886                // the alloc page is unlatched in the finally block.
1887            }
1888            while (retry == true);
1889
1890            // At this point, should have a page suitable for returning
1891            if (SanityManager.DEBUG)
1892                SanityManager.ASSERT(page.isLatched());
1893        }
1894        catch (StandardException se)
1895        {
1896            if (page != null)
1897                page.unlatch();
1898            page = null;
1899
1900            throw se;           // rethrow error
1901        }
1902        finally
1903        {
1904            if (!useNTT && allocPage != null)
1905            {
1906                allocPage.unlatch();
1907                allocPage = null;
1908            }
1909
1910            // NTT is committed by the caller
1911        }
1912
1913        if (SanityManager.DEBUG)
1914            SanityManager.ASSERT(page.isLatched());
1915
1916
1917        // if bulkIncreaseContainerSize is set, that means this newPage call
1918        // may have greatly expanded the container size due to preallocation.
1919        // Regardless of how many page it actually created, reset preAllocSize
1920        // to the default so we won't attempt to always preallocate 1000 pages
1921        // at a time in the future.
1922        if (bulkIncreaseContainerSize)
1923        {
1924            bulkIncreaseContainerSize = false;
1925            PreAllocSize = DEFAULT_PRE_ALLOC_SIZE;
1926        }
1927
1928        if (!isOverflow && page != null)
1929            setLastInsertedPage(pageNumber);
1930
1931
1932        // increase estimated page count - without any synchronization or
1933        // logging, this is an estimate only
1934        if (estimatedPageCount >= 0)
1935            estimatedPageCount++;
1936
1937        if (!this.identity.equals(page.getPageId().getContainerId())) {
1938
1939            if (SanityManager.DEBUG) {
1940                SanityManager.THROWASSERT(
1941                    "just created a new page from a different container"
1942                    + "\n this.identity = " + this.identity
1943                    + "\n page.getPageId().getContainerId() = " + 
1944                        page.getPageId().getContainerId()
1945                    + "\n userHandle is: " + userHandle
1946                    + "\n allocHandle is: " + allocHandle
1947                    + "\n this container is: " + this);
1948            }
1949
1950            throw StandardException.newException(
1951                    SQLState.DATA_DIFFERENT_CONTAINER, 
1952                    this.identity, page.getPageId().getContainerId());
1953        }
1954
1955        return page;            // return the newly added page
1956    }
1957
1958    protected void clearPreallocThreshold()
1959    {
1960        // start life with preallocated page if possible
1961        PreAllocThreshold = 0;
1962    }
1963
1964    protected void prepareForBulkLoad(BaseContainerHandle handle, int numPage)
1965    {
1966        clearPreallocThreshold();
1967        RawTransaction tran = handle.getTransaction();
1968
1969        // find the last allocation page - do not invalidate the alloc cache,
1970        // we don't want to prevent other people from reading or writing
1971        // pages. 
1972        AllocPage allocPage = findLastAllocPage(handle, tran);
1973
1974        // preallocate numPages.  Do whatever this allocPage can handle, if it
1975        // is full, too bad.  We don't guarentee that we will preallocate this
1976        // many pages, we only promise to try.
1977        if (allocPage != null)
1978        {
1979            allocPage.preAllocatePage(this, 0, numPage); 
1980            allocPage.unlatch();    
1981        }
1982    }
1983
1984    private boolean pageValid(BaseContainerHandle handle, long pagenum)
1985         throws StandardException
1986    {
1987        boolean retval = false;
1988
1989        synchronized(allocCache)
1990        {
1991            if (pagenum <= allocCache.getLastPageNumber(handle, firstAllocPageNumber) && 
1992                allocCache.getPageStatus(handle, pagenum, firstAllocPageNumber) == AllocExtent.ALLOCATED_PAGE)
1993                retval = true;
1994        }
1995
1996        return retval;
1997    }
1998
1999    protected long getLastPageNumber(BaseContainerHandle handle) 
2000        throws StandardException
2001    {
2002        long retval;
2003        synchronized(allocCache)
2004        {
2005            // check if the first alloc page number is valid, it is invalid 
2006            // if some one attempts to access the container info before the 
2007            // first alloc page got created. One such case is online backup. 
2008            // If first alloc page itself is invalid, then there are no pages
2009            // on the disk yet for this container, just return
2010            // ContainerHandle.INVALID_PAGE_NUMBER, caller can decide what to
2011            // do. 
2012            
2013            if (firstAllocPageNumber == ContainerHandle.INVALID_PAGE_NUMBER)    
2014            {
2015                retval = ContainerHandle.INVALID_PAGE_NUMBER;
2016            }
2017            else
2018            {
2019                retval = 
2020                    allocCache.getLastPageNumber(handle, firstAllocPageNumber);
2021            }
2022        }
2023        return retval;
2024    }
2025
2026    /*
2027        Find or allocate an allocation page which can handle adding a new page.
2028        Return a latched allocPage.
2029
2030        <BR> MT - single thread required - called as part of add page
2031    */
2032    private AllocPage findAllocPageForAdd(BaseContainerHandle allocHandle,
2033                                          RawTransaction ntt, long lastAllocatedPage)
2034         throws StandardException
2035    {
2036        AllocPage allocPage = null;
2037        AllocPage oldAllocPage = null; // in case we need to walk the alloc page chain
2038        boolean success = false; // set this for clean up
2039
2040        try
2041        {
2042            if (firstAllocPageNumber == ContainerHandle.INVALID_PAGE_NUMBER)
2043            {
2044                // make and return a latched new allocation page
2045                allocPage = makeAllocPage(ntt, allocHandle, FIRST_ALLOC_PAGE_NUMBER,
2046                                          FIRST_ALLOC_PAGE_OFFSET, CONTAINER_INFO_SIZE);
2047
2048                if (SanityManager.DEBUG)
2049                {
2050                    SanityManager.ASSERT(firstAllocPageNumber == FIRST_ALLOC_PAGE_NUMBER,
2051                                         "first Alloc Page number is still not set");
2052                    SanityManager.ASSERT(firstAllocPageOffset == FIRST_ALLOC_PAGE_OFFSET,
2053                                         "first Alloc Page offset is still not set");
2054                }
2055            }
2056            else
2057            {
2058                // an allocation page already exist, go get it
2059                allocPage = (AllocPage)allocHandle.getAllocPage(firstAllocPageNumber);
2060            }
2061
2062            /* allocPage is latched by allocHandle */
2063
2064            if (!allocPage.canAddFreePage(lastAllocatedPage))
2065            {
2066                // allocPage cannot manage the addition of one more page, walk the
2067                // alloc page chain till we find an allocPage that can
2068                // RESOLVE: always start with the first page for now...
2069
2070                boolean found = false; // found an alloc page that can handle 
2071                                        // adding a new page
2072
2073                while(allocPage.isLast() != true)
2074                {
2075                    long nextAllocPageNumber = allocPage.getNextAllocPageNumber();
2076                    long nextAllocPageOffset = allocPage.getNextAllocPageOffset();
2077
2078                    // RESOLVE (future): chain this info to in memory structure so
2079                    // getAllocPage can find this alloc page
2080
2081                    allocPage.unlatch();
2082                    allocPage = null;
2083
2084                    // the nextAllocPage is stable once set - even though it is
2085                    // save to get the next page latch before releasing this
2086                    // allocPage.
2087                    allocPage = (AllocPage)allocHandle.getAllocPage(nextAllocPageNumber);
2088
2089                    if (allocPage.canAddFreePage(lastAllocatedPage))
2090                    {
2091                        found = true;
2092                        break;
2093                    }
2094                }
2095
2096                if (!found)
2097                {
2098                    // allocPage is last and it is full
2099                    oldAllocPage = allocPage;
2100                    allocPage = null;
2101
2102                    if (SanityManager.DEBUG)
2103                        SanityManager.ASSERT(oldAllocPage.getLastPagenum() ==
2104                                             oldAllocPage.getMaxPagenum(),
2105                                             "expect allocpage to be full but last pagenum != maxpagenum");
2106
2107                    long newAllocPageNum = oldAllocPage.getMaxPagenum() + 1;
2108                    long newAllocPageOffset = newAllocPageNum; // no translation
2109
2110                    allocPage = makeAllocPage(ntt, allocHandle,
2111                                              newAllocPageNum,
2112                                              newAllocPageOffset,
2113                                              0 /* no containerInfo */);
2114
2115                    // this writes out the new alloc page and return a latched page
2116                    // nobody can find the new alloc page until oldAllocPage is unlatched.
2117
2118                    // oldAllocPage is no longer the last alloc page, 
2119                    // it has a pointer to the new last alloc page
2120                    oldAllocPage.chainNewAllocPage(allocHandle, newAllocPageNum, newAllocPageOffset);
2121                    oldAllocPage.unlatch();
2122                    oldAllocPage = null;
2123                }
2124            }
2125
2126            /* no error handling necessary */
2127            success = true;
2128        }
2129        finally                 // unlatch allocation page if any error happened
2130        {
2131            if (!success)
2132            {
2133                if (oldAllocPage != null)
2134                    oldAllocPage.unlatch();
2135
2136                if (allocPage != null)
2137                    allocPage.unlatch();
2138
2139                allocPage = null;
2140            }
2141
2142            // if success drop out of finally block
2143        }
2144
2145        return allocPage;
2146    }
2147
2148    /**
2149        Find the last alloc page, returns null if no alloc page is found
2150     */
2151    private AllocPage findLastAllocPage(BaseContainerHandle handle,
2152                                        RawTransaction tran)
2153    {
2154        AllocPage allocPage = null;
2155        AllocPage oldAllocPage = null;
2156
2157        if (firstAllocPageNumber == ContainerHandle.INVALID_PAGE_NUMBER)
2158            return null;
2159
2160        try
2161        {
2162            allocPage = (AllocPage)handle.getAllocPage(firstAllocPageNumber);
2163            while(!allocPage.isLast())
2164            {
2165                long nextAllocPageNumber = allocPage.getNextAllocPageNumber();
2166                long nextAllocPageOffset = allocPage.getNextAllocPageOffset();
2167
2168                allocPage.unlatch();
2169                allocPage = null;
2170
2171                allocPage = (AllocPage)handle.getAllocPage(nextAllocPageNumber);
2172            }
2173        }
2174        catch (StandardException se)
2175        {
2176            if (allocPage != null)
2177                allocPage.unlatch();
2178            allocPage = null;
2179        }
2180
2181        return allocPage;
2182
2183    }
2184
2185
2186    /*
2187        Make a new alloc page, latch it with the passed in container handle.
2188    */
2189    private AllocPage makeAllocPage(RawTransaction ntt, 
2190                                    BaseContainerHandle handle, 
2191                                    long pageNumber, 
2192                                    long pageOffset,
2193                                    int containerInfoSize)
2194         throws StandardException
2195    {
2196        if (SanityManager.DEBUG)
2197        {
2198            if (containerInfoSize != 0 && 
2199                                 containerInfoSize != CONTAINER_INFO_SIZE)
2200                SanityManager.THROWASSERT(
2201                                 "expect 0 or " + CONTAINER_INFO_SIZE +
2202                                 ", got " + containerInfoSize);
2203
2204            if (pageNumber != FIRST_ALLOC_PAGE_NUMBER &&
2205                                 containerInfoSize != 0)
2206                SanityManager.THROWASSERT(
2207                                 "Not first alloc page but container info size "
2208                                 + containerInfoSize);
2209        }
2210
2211        // argument to the create is an array of ints
2212        // 0'th element is the page format
2213        // 1'st element is whether or not to sync the page to disk
2214        // 2'nd element is the pagesize
2215        // 3'rd element is spareSpace
2216        // 4'th element is number of bytes to reserve for the container header
2217        // 5'th element is the minimumRecordSize
2218        // NOTE: the arg list here must match the one in allocPage
2219
2220        // No I/O at all if this new page is requested as part of a create
2221        // and load statement or this new alloc page is in a temporary 
2222        // container.
2223        // In the former case, BaseContainer will allow the MODE_UNLOGGED
2224        // bit to go thru to the nested top transaction alloc handle.
2225        // In the later case, there is no nested top transaction and the
2226        // alloc handle is the user handle, which is UNLOGGED.
2227
2228        boolean noIO = (handle.getMode() & ContainerHandle.MODE_UNLOGGED) ==
2229            ContainerHandle.MODE_UNLOGGED;
2230
2231        int[] createAllocPageArgs = new int[ALLOC_PAGE_ARG_NUM];
2232        createAllocPageArgs[0] = AllocPage.FORMAT_NUMBER;   
2233        createAllocPageArgs[1] = noIO ? 0 : CachedPage.WRITE_SYNC;
2234        createAllocPageArgs[2] = pageSize;
2235        createAllocPageArgs[3] = 0;     // allocation page has no need for spare
2236        createAllocPageArgs[4] = containerInfoSize;
2237        createAllocPageArgs[5] = minimumRecordSize;
2238
2239        if (SanityManager.DEBUG)
2240        {
2241            if (SanityManager.DEBUG_ON(SPACE_TRACE))
2242            {
2243                SanityManager.DEBUG(
2244                    SPACE_TRACE, "making new allocation page at " + pageNumber);
2245            }
2246        }
2247
2248        if (pageNumber == FIRST_ALLOC_PAGE_NUMBER)
2249        {
2250            // RESOLVE: make sure the following is true
2251            // 
2252            // firstAllocPageNumber and Offset can be set and access without
2253            // synchronization since the first allocation page is
2254            // created as part of the container create, this value is set
2255            // before any other transaction has a chance to open the container.
2256            // Once set, the first allocation page does not move or change
2257            // position 
2258            firstAllocPageNumber = pageNumber;
2259            firstAllocPageOffset = pageOffset;
2260
2261        }
2262
2263        PageKey pkey = new PageKey(identity, pageNumber);
2264
2265        // return a latched new alloc page
2266        return (AllocPage)initPage(handle, pkey, createAllocPageArgs, 
2267                                   pageOffset,
2268                                   false, /* not reuse */
2269                                   false /* not overflow */);
2270    }
2271
2272    /**
2273        Initialize a page 
2274
2275        @return a latched page that has been initialized.
2276
2277        @param allochandle the contianer handle to initialize the page with - the ntt
2278        @param pkey the page number of the page to be initialized
2279        @param createArgs the int array for page creation
2280        @param reuse is true if we are reusing a page that has 
2281                already been initialized once
2282
2283        @exception StandardException Cloudscape Standard error policy
2284    */
2285    protected BasePage initPage(BaseContainerHandle allochandle, 
2286                                PageKey pkey,
2287                                int[] createArgs,
2288                                long pageOffset,
2289                                boolean reuse,
2290                                boolean overflow) throws StandardException
2291    {
2292        BasePage page = null;
2293
2294        boolean releasePage = true;
2295
2296        try
2297        {
2298            if (reuse)              //  read the page in first
2299            {
2300                // Cannot go thru the container handle because all read pages are blocked.  
2301                // do it underneath the handle and directly to the cache.  
2302                // Nobody can get thru becuase getPage will block at getting the alloc page.
2303
2304                if (SanityManager.DEBUG)
2305                {
2306                    if (SanityManager.DEBUG_ON(SPACE_TRACE))
2307                    {
2308                        SanityManager.DEBUG(
2309                            SPACE_TRACE, "reusing page " + pkey);
2310                    }
2311                }
2312
2313                page = (BasePage)pageCache.find(pkey);
2314                if (page == null)   // hmmm?
2315                {
2316                    throw StandardException.newException(
2317                            SQLState.FILE_REUSE_PAGE_NOT_FOUND, pkey);
2318                }
2319            }
2320            else
2321            {
2322                if (SanityManager.DEBUG)
2323                {
2324                    if (SanityManager.DEBUG_ON(SPACE_TRACE))
2325                    {
2326                        SanityManager.DEBUG(
2327                            SPACE_TRACE, "allocation new page " + pkey);
2328                    }
2329                }
2330
2331                // a brand new page, initialize and a new page in cache
2332                page = (BasePage) pageCache.create(pkey, createArgs);
2333
2334                if (SanityManager.DEBUG)
2335                    SanityManager.ASSERT(page != null, "page Cache create return a null page");
2336            }
2337            releasePage = false;
2338            page = latchPage(allochandle, page, true /* may need to wait, track3822 */);
2339
2340            if (page == null)
2341            {
2342                throw StandardException.newException(
2343                        SQLState.FILE_NEW_PAGE_NOT_LATCHED, pkey);
2344            }     
2345
2346            // page is either brand new or is read from disk, in either case,
2347            // it knows how to get itself initialized.
2348            int initPageFlag = 0;
2349            if (reuse) initPageFlag |= BasePage.INIT_PAGE_REUSE;
2350            if (overflow) initPageFlag |= BasePage.INIT_PAGE_OVERFLOW;
2351            if (reuse && isReusableRecordId())
2352                initPageFlag |= BasePage.INIT_PAGE_REUSE_RECORDID;
2353
2354            page.initPage(initPageFlag, pageOffset);
2355            page.setContainerRowCount(estimatedRowCount);
2356
2357        }
2358        finally
2359        {
2360            if (releasePage && page != null)
2361            {
2362                // release the new page from cache if it errors 
2363                // out before the exclusive lock is set
2364                pageCache.release((Cacheable)page);
2365                page = null;
2366            }
2367        }
2368
2369        return page;
2370    }
2371
2372
2373    /**
2374        Get a page in the container.    
2375
2376        Get User page is the generic base routine for all user (client to raw
2377        store) getPage.  This routine coordinate with allocation/deallocation
2378        to ensure that no page can be gotten from the container while page is
2379        in the middle of being allocated or deallocated.
2380        This routine latches the page.
2381
2382        @param handle the container handle
2383        @param pageNumber the page number of the page to get
2384        @param overflowOK if true then an overflow page is OK,
2385                if false, then only non-overflow page is OK
2386        @param wait if true then wait for a latch
2387        @return the latched page
2388
2389        <BR> MT - thread safe
2390
2391        @exception StandardException Standard Cloudscape error policy
2392    */
2393    private BasePage getUserPage(BaseContainerHandle handle, long pageNumber,
2394        boolean overflowOK, boolean wait)
2395         throws StandardException
2396    {
2397
2398        if (SanityManager.DEBUG) 
2399        {
2400            SanityManager.ASSERT(
2401                pageNumber != FIRST_ALLOC_PAGE_NUMBER,
2402                "getUserPage trying to get an alloc page, pageNumber = " + 
2403                pageNumber);
2404
2405            if (pageNumber < ContainerHandle.FIRST_PAGE_NUMBER)
2406                SanityManager.THROWASSERT("pageNumber = " + pageNumber);
2407        }
2408
2409        if (pageNumber < ContainerHandle.FIRST_PAGE_NUMBER)
2410            return null;
2411
2412        if (getCommittedDropState()) // committed and dropped, cannot get a page
2413            return null;
2414
2415        if (!pageValid(handle, pageNumber))
2416        {
2417            return null;
2418        }
2419
2420        // RESOLVE: no translation!
2421
2422        PageKey pageSearch = new PageKey(identity, pageNumber);
2423        BasePage page = (BasePage)pageCache.find(pageSearch);
2424
2425        if (page == null)
2426        {
2427            return page;
2428        }
2429
2430        // latch the page
2431        if (latchPage(handle,page,wait) == null)
2432        {
2433            // page was already released from cache
2434            return null;
2435        }
2436
2437        // double check for overflow and deallocated page
2438        // a page that was valid before maybe invalid by now if it was
2439        // deallocated in the interum.
2440        // a page that is invalid can also become valid in the interim, but
2441        // we do not handle that.  The client must supply other locking
2442        // mechanism to prevent that (an allocatino happenning where there are
2443        // readers) if that is needed
2444        if ((page.isOverflowPage() && !overflowOK) ||
2445            (page.getPageStatus() != BasePage.VALID_PAGE))
2446        {
2447            // unlatch releases page from cache, see StoredPage.releaseExclusive()
2448            page.unlatch();
2449            page = null;
2450        }
2451
2452        return page;
2453    }
2454
2455    protected void trackUnfilledPage(long pagenumber, boolean unfilled)
2456    {
2457        if (!dataFactory.isReadOnly())
2458            allocCache.trackUnfilledPage(pagenumber, unfilled);
2459    }
2460
2461    /**
2462        Get a valid (non-deallocated or free) page in the container.
2463        Overflow page is OK. Resulting page is latched.
2464
2465        <BR> MT - thread safe
2466
2467        @exception StandardException Standard Cloudscape error policy
2468    */
2469    protected BasePage getPage(BaseContainerHandle handle, long pageNumber,
2470        boolean wait)
2471         throws StandardException
2472    {
2473        return getUserPage(handle, pageNumber, true /* overflow page OK */,
2474            wait);
2475    }
2476
2477
2478    /**
2479        Get any old page - turn off all validation
2480
2481        @exception StandardException Cloudscape Standard error policy
2482    */
2483    protected BasePage getAnyPage(BaseContainerHandle handle, long pageNumber) throws StandardException
2484    {
2485        // get AllocPage get a page without any validation (exception a
2486        // committed dropped container)
2487
2488        if (getCommittedDropState()) // committed and dropped, cannot get a page
2489            return null;
2490
2491        // make sure alloc cache has no stale info
2492        synchronized(allocCache)
2493        {
2494            allocCache.invalidate();
2495        }
2496        
2497        PageKey pageSearch = new PageKey(identity, pageNumber);
2498        BasePage page = (BasePage) pageCache.find(pageSearch);
2499
2500        return page;
2501    }
2502
2503    /**
2504     * ReCreate a page for rollforward recovery.  
2505     * <p>
2506     * During redo recovery it is possible for the system to try to redo
2507     * the creation of a page (ie. going from non-existence to version 0).
2508     * It first trys to read the page from disk, but a few different types
2509     * of errors can occur:
2510     *     o the page does not exist at all on disk, this can happen during
2511     *       rollforward recovery applied to a backup where the file was
2512     *       copied and the page was added to the file during the time frame
2513     *       of the backup but after the physical file was copied.
2514     *     o space in the file exists, but it was never initalized.  This
2515     *       can happen if you happen to crash at just the right moment during
2516     *       the allocation process.  Also
2517     *       on some OS's it is possible to read from a part of the file that
2518     *       was not ever written - resulting in garbage from the store's 
2519     *       point of view (often the result is all 0's).  
2520     *
2521     * All these errors are easy to recover from as the system can easily 
2522     * create a version 0 from scratch and write it to disk.
2523     *
2524     * Because the system does not sync allocation of data pages, it is also
2525     * possible at this point that whlie writing the version 0 to disk to 
2526     * create it we may encounter an out of disk space error (caught in this
2527     * routine as a StandardException from the create() call.  We can't 
2528     * recovery from this without help from outside, so the caught exception
2529     * is nested and a new exception thrown which the recovery system will
2530     * output to the user asking them to check their disk for space/errors.
2531     *
2532     * @exception  StandardException  Standard exception policy.
2533     **/
2534    protected BasePage reCreatePageForRedoRecovery(
2535    BaseContainerHandle handle,
2536    int                 pageFormat,
2537    long                pageNumber,
2538    long                pageOffset)
2539         throws StandardException
2540    {
2541        // recreating a page should be done only if are in the middle of 
2542        // rollforward recovery or if derby.storage.patchInitPageRecoverError 
2543        // is set to true.
2544
2545        //check if we are in rollforward recovery
2546        boolean rollForwardRecovery = 
2547            ((RawTransaction)handle.getTransaction()).inRollForwardRecovery();
2548
2549        if (!rollForwardRecovery && !(PropertyUtil.getSystemBoolean(
2550                    RawStoreFactory.PATCH_INITPAGE_RECOVER_ERROR)))
2551        {
2552            return null;
2553        }
2554
2555        // RESOLVE: first need to verify that the page is really NOT in the
2556        // container!
2557
2558        // no address translation necessary
2559        PageKey pkey = new PageKey(identity, pageNumber);
2560
2561        int[] reCreatePageArgs = null;
2562
2563        if (pageFormat == StoredPage.FORMAT_NUMBER)
2564        {
2565            reCreatePageArgs = new int[STORED_PAGE_ARG_NUM];
2566            reCreatePageArgs[0] = pageFormat;
2567            reCreatePageArgs[1] = CachedPage.WRITE_SYNC;
2568            reCreatePageArgs[2] = pageSize;
2569            reCreatePageArgs[3] = spareSpace;
2570            reCreatePageArgs[4] = minimumRecordSize;
2571        }
2572        else if (pageFormat == AllocPage.FORMAT_NUMBER)
2573        {
2574            reCreatePageArgs = new int[ALLOC_PAGE_ARG_NUM];
2575
2576            // only the first allocation page have borrowed space for the
2577            // container info
2578
2579            int containerInfoSize = 0;
2580            if (pageNumber == FIRST_ALLOC_PAGE_NUMBER)
2581            {
2582                containerInfoSize = CONTAINER_INFO_SIZE;
2583                firstAllocPageNumber = pageNumber;
2584                firstAllocPageOffset = pageOffset;
2585            }
2586
2587
2588            reCreatePageArgs[0] = pageFormat;
2589            reCreatePageArgs[1] = CachedPage.WRITE_SYNC;
2590            reCreatePageArgs[2] = pageSize;
2591            reCreatePageArgs[3] = 0; // allocation page has no need for spare
2592            reCreatePageArgs[4] = containerInfoSize;
2593            reCreatePageArgs[5] = minimumRecordSize;
2594        }
2595        else
2596        {
2597            throw StandardException.newException(
2598                    SQLState.DATA_UNKNOWN_PAGE_FORMAT, pkey);
2599        }
2600
2601        if (SanityManager.DEBUG) 
2602        {
2603            if (SanityManager.DEBUG_ON("LoadTran"))
2604                SanityManager.DEBUG_PRINT(
2605                    "Trace", "recreating page " + pkey + " for load tran");
2606        }
2607
2608        // Can't just call initPage because that wants to log an initPage
2609        // operation, whereas we are here because of an initPage operation in
2610        // the log already.
2611        BasePage page = null;
2612        boolean releasePage = true;
2613
2614        try
2615        {
2616            try
2617            {
2618                // a brand new page, initialize a new page in cache
2619                page = (BasePage) pageCache.create(pkey, reCreatePageArgs);
2620            }
2621            catch (StandardException se)
2622            {
2623                throw StandardException.newException(
2624                    SQLState.FILE_NEW_PAGE_DURING_RECOVERY, se, pkey);
2625            }
2626
2627            if (page != null)
2628            {
2629                releasePage = false;
2630                page = latchPage(handle, page, false /* never need to wait */);
2631
2632                if (page == null)
2633                {
2634                    throw StandardException.newException(
2635                            SQLState.FILE_NEW_PAGE_NOT_LATCHED, pkey);
2636                }
2637            }
2638            else
2639            {
2640                throw StandardException.newException(
2641                    SQLState.FILE_NEW_PAGE_DURING_RECOVERY, pkey);
2642            }
2643
2644        }
2645        finally
2646        {
2647            if (releasePage && page != null)
2648            {
2649                // release the new page from cache if it errors out before 
2650                // the exclusive lock is set error in roll forward recovery.
2651                // , we are doomed anyway
2652                pageCache.release((Cacheable)page);
2653                page = null;
2654            }
2655        }
2656
2657        return page;
2658
2659    }
2660
2661
2662    /** 
2663        Get an alloc page - only accessible to the raw store 
2664        (container and recovery)
2665
2666        @exception StandardException Cloudscape Standard error policy
2667     */
2668    protected BasePage getAllocPage(long pageNumber) throws StandardException 
2669    {
2670        if (getCommittedDropState()) // committed and dropped, cannot get a page
2671            return null;
2672
2673        PageKey pageSearch = new PageKey(identity, pageNumber);
2674        BasePage page = (BasePage) pageCache.find(pageSearch);
2675
2676        if (SanityManager.DEBUG)
2677        {
2678            if (page == null)
2679                SanityManager.THROWASSERT(
2680                    "getting a null alloc page page " + 
2681                    getIdentity() + pageNumber);
2682
2683            if ( ! (page instanceof AllocPage))
2684                SanityManager.THROWASSERT(
2685                    "trying to get a user page as an alloc page " + 
2686                    getIdentity() + pageNumber); 
2687        }
2688
2689        // assuming that allocation page lives in the page cache...
2690        return page;
2691    }
2692
2693    /**
2694        Get only a valid, non-overflow page.  If page number is either invalid
2695        or overflow, returns null
2696
2697        @exception StandardException Cloudscape Standard error policy
2698     */
2699    protected BasePage getHeadPage(BaseContainerHandle handle, long pageNumber,
2700        boolean wait)
2701         throws StandardException
2702    {
2703        return getUserPage(handle, pageNumber, false /* overflow not ok */,
2704            wait);
2705    }
2706
2707    /**
2708        Get the first valid page in the container
2709
2710        @exception StandardException Cloudscape Standard error policy
2711     */
2712    protected BasePage getFirstHeadPage(BaseContainerHandle handle, boolean wait)
2713         throws StandardException
2714    {
2715        return getNextHeadPage(handle, ContainerHandle.FIRST_PAGE_NUMBER-1, wait);
2716    }
2717
2718    /**
2719        Get the next page in the container.
2720        @exception StandardException Standard Cloudscape error policy
2721    */
2722    protected BasePage getNextHeadPage(BaseContainerHandle handle,
2723        long pageNumber, boolean wait)
2724         throws StandardException
2725    {
2726        long nextNumber;
2727
2728        while(true)
2729        {
2730            synchronized(allocCache)
2731            {
2732                // ask the cache for the next pagenumber
2733                nextNumber = allocCache.getNextValidPage(handle, pageNumber, firstAllocPageNumber);
2734            }
2735
2736            if (nextNumber == ContainerHandle.INVALID_PAGE_NUMBER)
2737                return null;
2738
2739            // optimistically go for the next page
2740            BasePage p = getUserPage(handle, nextNumber,
2741                false /* no overflow page*/, wait);
2742            if (p != null)
2743                return p;
2744
2745            pageNumber = nextNumber;
2746        }
2747    }
2748
2749
2750    private BasePage getInsertablePage(BaseContainerHandle handle,
2751                                       long pageNumber,
2752                                       boolean wait,
2753                                       boolean overflowOK)
2754         throws StandardException
2755    {
2756        if (pageNumber == ContainerHandle.INVALID_PAGE_NUMBER)
2757            return null;
2758
2759        BasePage p = getUserPage(handle, pageNumber, overflowOK, wait);
2760        if (p != null)
2761        {
2762            // make sure the page is not too full
2763            if (!p.allowInsert())
2764            {
2765                p.unlatch();
2766                p = null;
2767
2768                // it is too full, make sure we are tracking it so we won't
2769                // see it again.
2770                allocCache.trackUnfilledPage(pageNumber, false);
2771            }
2772        }
2773        /*
2774        RESOLVE track 3757
2775        Need to check if this fix resolves the bug.
2776        This is commented out because we can't conclude here that this is not
2777        a user page, it may just be that we failed to get a latch on the page.
2778        In a high contention scenario this could cause alot of relatively empty
2779        pages to not be considered for insert.
2780        TODO
2781        May be a good idea to move the trackUnfilledPage call below to some of
2782        the lines in the getUserPage method.
2783
2784        else
2785        {
2786            // it is not a user page, make sure we are tracking its fillness so
2787            // we won't consider it as a 1/2 filled page ever
2788            allocCache.trackUnfilledPage(pageNumber, false);
2789        }
2790        */
2791        return p;
2792    }
2793
2794    /**
2795     * Get candidate page to move a row for compressing the table.
2796     * <p>
2797     * The caller is moving rows from the end of the table toward the beginning,
2798     * with the goal of freeing up a block of empty pages at the end of the
2799     * container which can be returned to the OS.
2800     * <p>
2801     * On entry pageno will be latched by the caller.  Only return pages with
2802     * numbers below pageno.  Attempting to return pageno will result in a
2803     * latch/latch deadlock on the same thread.
2804     *
2805     * @exception  StandardException  Standard exception policy.
2806     **/
2807    protected BasePage getPageForCompress(
2808    BaseContainerHandle handle,
2809    int                 flag,
2810    long                pageno)
2811         throws StandardException
2812    {
2813        BasePage p = null;
2814        boolean getLastInserted = 
2815            (flag & ContainerHandle.GET_PAGE_UNFILLED) == 0;
2816
2817        if (getLastInserted)
2818        {
2819            // There is nothing protecting lastInsertePage from being changed
2820            // by another thread.  Make a local copy.
2821            long localLastInsertedPage = getLastInsertedPage();
2822
2823            if ((localLastInsertedPage < pageno) &&
2824                (localLastInsertedPage != ContainerHandle.INVALID_PAGE_NUMBER))
2825            {
2826                // First try getting last inserted page.
2827
2828                p = getInsertablePage(
2829                        handle, 
2830                        localLastInsertedPage,
2831                        true, /* wait */
2832                        false /* no overflow page */);
2833
2834                // if localLastInsertedPage is not an insertable page, 
2835                // don't waste time getting it again.
2836                if (p == null)
2837                {
2838                    // There is a slight possibility that lastUnfilledPage and
2839                    // lastInsertedPage will change between the if and the
2840                    // assignment.  The worse that will happen is we lose the
2841                    // optimization.  Don't want to slow down allocation by 
2842                    // adding more synchronization.
2843
2844                    if (localLastInsertedPage == getLastUnfilledPage())
2845                        setLastUnfilledPage(
2846                            ContainerHandle.INVALID_PAGE_NUMBER);
2847
2848                    if (localLastInsertedPage == getLastInsertedPage())
2849                        setLastInsertedPage(
2850                            ContainerHandle.INVALID_PAGE_NUMBER);
2851                }
2852            }
2853        }
2854        else                    
2855        {
2856            // get a relatively unfilled page that is not the last Inserted page
2857
2858            long localLastUnfilledPage = getLastUnfilledPage();
2859
2860            if (localLastUnfilledPage == ContainerHandle.INVALID_PAGE_NUMBER ||
2861                localLastUnfilledPage >= pageno ||
2862                localLastUnfilledPage == getLastInsertedPage())
2863            {
2864                // get an unfilled page, searching from beginning of container.
2865                localLastUnfilledPage = 
2866                    getUnfilledPageNumber(handle, 0);
2867            }
2868
2869            if ((localLastUnfilledPage != 
2870                    ContainerHandle.INVALID_PAGE_NUMBER) &&
2871                (localLastUnfilledPage < pageno))
2872            {
2873                p = getInsertablePage(
2874                        handle, localLastUnfilledPage, true, false);
2875            }
2876
2877            // return this page for insert
2878            if (p != null)
2879            {
2880                setLastUnfilledPage(localLastUnfilledPage);
2881                setLastInsertedPage(localLastUnfilledPage);
2882            }
2883        }
2884
2885        return p;
2886    }
2887
2888    /**
2889        Get a potentially suitable page for insert and latch it.
2890        @exception StandardException Standard Cloudscape error policy
2891     */
2892    protected BasePage getPageForInsert(BaseContainerHandle handle,
2893                                        int flag)
2894         throws StandardException
2895    {
2896        BasePage p = null;
2897        boolean getLastInserted = (flag & ContainerHandle.GET_PAGE_UNFILLED) == 0;
2898
2899        if (getLastInserted)
2900        {
2901            // There is nothing protecting lastInsertePage from being changed
2902            // by another thread.  Make a local copy.
2903            long localLastInsertedPage = getLastInsertedPage();
2904
2905            if (localLastInsertedPage != ContainerHandle.INVALID_PAGE_NUMBER)
2906            {
2907                // First try getting last allocated page, NOWAIT
2908
2909                p = getInsertablePage(handle, localLastInsertedPage,
2910                                      false, /* wait */
2911                                      false /* no overflow page */);
2912
2913                if (p == null)
2914                {
2915                    // most likely we could not get the latch NOWAIT, try again
2916                    // with a new page, and tell the system to switch to 
2917                    // multi-page mode.
2918                    /* switchToMultiInsertPageMode(handle); */
2919
2920                    localLastInsertedPage = getLastInsertedPage();
2921
2922                    p = getInsertablePage(handle, localLastInsertedPage,
2923                                          true, /* wait */
2924                                          false /* no overflow page */);
2925                }
2926            }
2927
2928            // if lastUnfilledPage is not an insertable page, don't waste time
2929            // getting it again.
2930            if (p == null)
2931            {
2932                // There is a slight possibility that lastUnfilledPage and
2933                // lastInsertedPage will change between the if and the
2934                // assignment.  The worse that will happen is we lose the
2935                // optimization.  Don't want to slow down allocation by adding
2936                // more synchronization.
2937
2938                if (localLastInsertedPage == getLastUnfilledPage())
2939                    setLastUnfilledPage(ContainerHandle.INVALID_PAGE_NUMBER);
2940
2941                if (localLastInsertedPage == getLastInsertedPage())
2942                    setLastInsertedPage(ContainerHandle.INVALID_PAGE_NUMBER);
2943            }
2944        }
2945        else                    // get a relatively unfilled page that is not
2946        {                       // the last Inserted page
2947            long localLastUnfilledPage = getLastUnfilledPage();
2948
2949            if (localLastUnfilledPage == ContainerHandle.INVALID_PAGE_NUMBER ||
2950                localLastUnfilledPage == getLastInsertedPage())
2951                localLastUnfilledPage = getUnfilledPageNumber(handle, localLastUnfilledPage);
2952
2953            if (localLastUnfilledPage != ContainerHandle.INVALID_PAGE_NUMBER)
2954            {
2955                // try the last unfilled page we found - this could be
2956                // different from lastInserted if the last unfilled one we
2957                // found does not have enough space for the insert and the
2958                // client wants to get a brand new page.
2959                p = getInsertablePage(handle, localLastUnfilledPage, true, false);
2960
2961                // try again
2962                if (p == null)
2963                {
2964                    localLastUnfilledPage = getUnfilledPageNumber(handle, localLastUnfilledPage);
2965                    if (localLastUnfilledPage != ContainerHandle.INVALID_PAGE_NUMBER)
2966                    {
2967                        p = getInsertablePage(handle, localLastUnfilledPage, true,
2968                                              false);
2969                    }
2970                }
2971            }
2972
2973            // return this page for insert
2974            if (p != null)
2975            {
2976                setLastUnfilledPage(localLastUnfilledPage);
2977                setLastInsertedPage(localLastUnfilledPage);
2978            }
2979        }
2980
2981        return p;
2982
2983    }
2984
2985
2986    /** 
2987     *  Get a latched page. Incase of backup page Latch is necessary to 
2988     *  prevent modification to the page when it is being written to the backup.
2989     *  Backup process relies on latches to get consistent snap
2990     *  shot of the page , user level table/page/row locks are NOT 
2991     *  acquired  by the online backup mechanism.
2992     *
2993     *  @param handle the container handle used to latch the page
2994     *  @param pageNumber the page number of the page to get
2995     *  @return the latched page
2996     *  @exception StandardException Standard Derby error policy
2997     */
2998    protected BasePage getLatchedPage(BaseContainerHandle handle, 
2999                                        long pageNumber) 
3000        throws StandardException 
3001    {
3002        PageKey pageKey = new PageKey(identity, pageNumber);
3003        BasePage page = (BasePage) pageCache.find(pageKey);
3004                
3005        if (SanityManager.DEBUG){
3006            SanityManager.ASSERT(page != null, "page is not found :" + pageKey);
3007        }
3008        
3009        // latch the page
3010        page = latchPage(handle, page, true);
3011        
3012        if (SanityManager.DEBUG){
3013            SanityManager.ASSERT(page.isLatched(), "page is not latched:" + pageKey);
3014        }
3015
3016        return page;
3017    }
3018
3019    
3020
3021    private long getUnfilledPageNumber(BaseContainerHandle handle, long pagenum)
3022         throws StandardException
3023    {
3024        synchronized(allocCache)
3025        {
3026            return allocCache.
3027                getUnfilledPageNumber(handle, firstAllocPageNumber, pagenum);
3028        }
3029    }       
3030
3031    /*
3032        Cost estimates
3033    */
3034    /**
3035        <BR>MT - this routine is NOT MT-safe and clients don't need to provide
3036        synchronization.
3037
3038        @see ContainerHandle#getEstimatedRowCount
3039     */
3040    public long getEstimatedRowCount(int flag)
3041    {
3042        return estimatedRowCount;
3043    }
3044
3045    /**
3046        @see ContainerHandle#setEstimatedRowCount
3047     */
3048    public void setEstimatedRowCount(long count, int flag)
3049    {
3050        boolean readOnly = dataFactory.isReadOnly();
3051
3052        synchronized(this)
3053        {
3054            estimatedRowCount = count;
3055
3056            if (!readOnly)
3057                isDirty = true;
3058        }
3059    }
3060
3061    /**
3062        Update estimated row count by page as it leaves the cache.
3063        The estimated row count is updated without logging!
3064     */
3065    protected void updateEstimatedRowCount(int delta)
3066    {
3067        boolean readOnly = dataFactory.isReadOnly();
3068
3069        synchronized(this)
3070        {
3071            estimatedRowCount += delta;
3072            if (estimatedRowCount < 0)
3073                estimatedRowCount = 0;
3074
3075            // mark the container as dirty without bumping the container
3076            // version because row count changes are not logged.
3077            if (!readOnly)
3078                isDirty = true;
3079        }
3080    }
3081
3082
3083    /**
3084        @see ContainerHandle#getEstimatedPageCount
3085        @exception StandardException Standard Cloudscape error policy
3086     */
3087    public long getEstimatedPageCount(BaseContainerHandle handle, int flag)
3088         throws StandardException 
3089    {
3090        // page count is set once per container materialization in cache
3091
3092        if (estimatedPageCount < 0)
3093        {
3094            synchronized(allocCache)
3095            {
3096                estimatedPageCount = 
3097                    allocCache.getEstimatedPageCount(handle, firstAllocPageNumber);
3098            }
3099        }
3100
3101        if (SanityManager.DEBUG)
3102            SanityManager.ASSERT(estimatedPageCount >= 0,
3103                                 "AllocCache returns negatie estimatedPageCount");
3104
3105        return estimatedPageCount;
3106    }
3107
3108    /*
3109    ** Methods used solely by StoredPage
3110    */
3111
3112    /**
3113        Read a page into the supplied array.
3114
3115        <BR> MT - thread safe
3116        @exception IOException error reading page
3117        @exception StandardException standard cloudscape error message
3118    */
3119    protected abstract void readPage(long pageNumber, byte[] pageData)
3120         throws IOException  , StandardException;
3121    
3122
3123    /**
3124        Write a page from the supplied array.
3125
3126        <BR> MT - thread safe
3127        @exception IOException error writing page
3128        @exception StandardException Standard Cloudscape error policy
3129    */
3130    protected abstract void writePage(long pageNumber, byte[] pageData, boolean syncPage) 
3131        throws IOException  , StandardException;
3132
3133    /*
3134     * Encryption/decryption
3135     */
3136    /**
3137        Decrypts a page
3138
3139        <BR>MT - MT safe.
3140
3141        @exception StandardException Standard Cloudscape error policy
3142     */
3143    protected void decryptPage(byte[] pageData, int pageSize)
3144         throws StandardException
3145    {
3146        // because all our page header looks identical, the 
3147        // checksum is moved to the front so that it will hopefully
3148        // encrypt differently from page to page
3149        synchronized(this)
3150        {
3151            if (encryptionBuffer == null || encryptionBuffer.length < pageSize)
3152                encryptionBuffer = new byte[pageSize];
3153
3154            int len = dataFactory.decrypt(pageData, 0, pageSize,
3155                                          encryptionBuffer, 0);
3156
3157            if (SanityManager.DEBUG)
3158                SanityManager.ASSERT(len == pageSize,
3159                                 "Encrypted page length != page length");
3160
3161            // put the checksum where it belongs
3162            System.arraycopy(encryptionBuffer, 8, pageData, 0, pageSize-8);
3163            System.arraycopy(encryptionBuffer, 0, pageData, pageSize-8, 8);
3164        }
3165    }
3166
3167    /**
3168        Encrypts a page.
3169
3170        <BR> MT - not safe, call within synchronized block and only use the
3171        returned byte array withing synchronized block. 
3172
3173        @exception StandardException Standard Cloudscape error policy
3174     */
3175    protected byte[] encryptPage(byte[] pageData, 
3176                                 int pageSize, 
3177                                 byte[] encryptionBuffer,
3178                                 boolean newEngine)
3179        throws StandardException
3180    {
3181        // because all our page header looks identical, move the
3182        // checksum to the front so that it will hopefully encrypt
3183        // differently from page to page
3184
3185        System.arraycopy(pageData, pageSize-8, encryptionBuffer, 0, 8);
3186        System.arraycopy(pageData, 0, encryptionBuffer, 8, pageSize-8);
3187
3188        int len = dataFactory.encrypt(encryptionBuffer, 0, pageSize,
3189                                      encryptionBuffer, 0, newEngine);
3190
3191        if (SanityManager.DEBUG)
3192            SanityManager.ASSERT(len == pageSize,
3193                             "Encrypted page length != page length");
3194
3195        return encryptionBuffer;
3196    }
3197
3198
3199    /** 
3200     * Get encryption buffer.
3201     *  MT - not safe, call within synchronized block and only use the
3202     *  returned byte array withing synchronized block. 
3203     * @return byte array to be used for encryping a page.
3204     */
3205    protected byte[] getEncryptionBuffer() {
3206
3207        if (encryptionBuffer == null || encryptionBuffer.length < pageSize)
3208            encryptionBuffer = new byte[pageSize];
3209        return encryptionBuffer;
3210    }
3211    
3212    
3213
3214    /*
3215     * page preallocation
3216     */
3217
3218    /**
3219        preAllocate writes out the preallocated pages to disk if necessary.
3220
3221        <BR>Make sure the container is large enough and the
3222        pages are well formatted.  The only reason to do this is to save some
3223        I/O during page initialization.  Once the initPage log record is
3224        written, it is expected that the page really do exist and is well
3225        formed or recovery will fail.  However, we can gain some performance by
3226        writing a bunch of pages at a time rather than one at a time.
3227
3228        <BR>If it doesn't make sense for the the implementation to have 
3229        pre-allocation, just return 0. 
3230
3231        <BR>If the container is not being logged, don't actually do anything,
3232        just return 0.  
3233
3234        @return number of successfully preallocated page, or 0 if
3235                no page has been preallocated
3236
3237        @param lastPreallocPagenum the last preallocated page number as known
3238                by the allocation page
3239        @param preAllocSize try to preallocate this page number of pages.
3240                Since only the container knows how many pages are actually on
3241                disk, it may determine that certain number of pages that the
3242                allocation page thinks need to be preallocated is already
3243                allocated, in those case, act as if the preallocation is
3244                successful.
3245    */
3246    protected abstract int preAllocate(long lastPreallocPagenum, int preAllocSize);
3247
3248    /**
3249        Preallocate the pages - actually doing it, called by subclass only
3250    */
3251    protected int doPreAllocatePages(long lastPreallocPagenum,
3252                                     int preAllocSize)
3253    {
3254        if (SanityManager.DEBUG)
3255            SanityManager.ASSERT(!dataFactory.isReadOnly(), 
3256                                 "how can we be Preallocating pages in a read only database?");
3257
3258        // initialize and a new page in cache
3259        int[] createArgs = new int[5];
3260        createArgs[0] = StoredPage.FORMAT_NUMBER;   // default is a stored page
3261        createArgs[1] = CachedPage.WRITE_NO_SYNC;   // write it but no sync
3262        createArgs[2] = pageSize;
3263        createArgs[3] = spareSpace;
3264        createArgs[4] = minimumRecordSize;
3265
3266        StoredPage page = new StoredPage();
3267        page.setFactory(dataFactory);
3268
3269        boolean error = false;
3270        int count = 0;
3271
3272        while(count < preAllocSize)
3273        {
3274            PageKey pkey = new PageKey(identity, 
3275                                       lastPreallocPagenum+count+1);
3276            try
3277            {
3278                // create Identity will do a writePage
3279                page.createIdentity(pkey, createArgs);
3280
3281                // if create identity somehow failed to do a write page
3282                if (SanityManager.DEBUG)
3283                    SanityManager.ASSERT(!page.isDirty(),
3284                                         "create identity failed to do a write page");
3285
3286                page.clearIdentity(); // ready the page for the next loop 
3287
3288            }
3289            catch (StandardException se)
3290            {
3291                // if something went wrong, stop and return how many we did
3292                // successfully 
3293                error = true;
3294            }
3295
3296            if (error)
3297                break;
3298
3299            count++;
3300        }
3301
3302        return count;
3303    }
3304
3305    protected int getPageSize() {
3306        return pageSize;
3307    }
3308    protected int getSpareSpace() {
3309        return spareSpace;
3310    }
3311    protected int getMinimumRecordSize() {
3312        return minimumRecordSize;
3313    }
3314
3315    private synchronized void switchToMultiInsertPageMode(
3316    BaseContainerHandle handle)
3317        throws StandardException
3318    {
3319        if (lastInsertedPage.length == 1)
3320        {
3321            long last = lastInsertedPage[0];
3322
3323            lastInsertedPage = new long[4];
3324            lastInsertedPage[0] = last;
3325
3326            for (int i = 3; i > 0; i--)
3327            {
3328                Page page = addPage(handle, false);
3329                lastInsertedPage[i] = page.getPageNumber();
3330                page.unlatch();
3331            }
3332        }
3333    }
3334
3335    /*
3336     * Setting and getting lastInserted Page and lastUnfilledPage in a thead
3337     * safe manner. 
3338     */
3339    private synchronized long getLastInsertedPage()
3340    {
3341        if (lastInsertedPage.length == 1)
3342        {
3343            if (SanityManager.DEBUG)
3344                SanityManager.ASSERT(lastInsertedPage_index == 0);
3345
3346            // optimize the usual case where no concurrent insert has kicked us
3347            // into multi-page mode - ie. only ONE last page.  
3348            return(lastInsertedPage[0]);
3349        }
3350        else
3351        {
3352            long ret = lastInsertedPage[lastInsertedPage_index++];
3353
3354            if (lastInsertedPage_index > (lastInsertedPage.length - 1))
3355            {
3356                lastInsertedPage_index = 0;
3357            }
3358
3359            return(ret);
3360        }
3361    }
3362
3363    private synchronized long getLastUnfilledPage()
3364    {
3365        return lastUnfilledPage;
3366    }
3367
3368    private synchronized void initializeLastInsertedPage(int size)
3369    {
3370        lastInsertedPage = new long[size];
3371
3372        for (int i = lastInsertedPage.length - 1; i >= 0; i--)
3373            lastInsertedPage[i] = ContainerHandle.INVALID_PAGE_NUMBER;
3374
3375        lastInsertedPage_index = 0;
3376    }
3377
3378    private synchronized void setLastInsertedPage(long val)
3379    {
3380        lastInsertedPage[lastInsertedPage_index] = val;
3381    }
3382
3383    private synchronized void setLastUnfilledPage(long val)
3384    {
3385        lastUnfilledPage = val;
3386    }
3387
3388
3389
3390    /*
3391    ** Hide our super-classes methods to ensure that cache management
3392    ** is correct when the container is obtained and release.
3393    */
3394
3395    /**
3396        The container is kept by the find() in File.openContainer. 
3397    */
3398    protected void letGo(BaseContainerHandle handle) {
3399        super.letGo(handle);
3400
3401        containerCache.release(this);
3402    }
3403
3404    protected BasePage latchPage(BaseContainerHandle handle, BasePage foundPage, boolean wait)
3405        throws StandardException {
3406
3407        if (foundPage == null)
3408            return null;
3409
3410        BasePage ret = super.latchPage(handle, foundPage, wait);
3411        if (ret == null) {
3412            // page is still cached
3413            pageCache.release((Cacheable) foundPage);
3414        }
3415        return ret;
3416    }
3417    
3418
3419
3420    /**
3421     * backup the container.
3422     * 
3423     * @param handle the container handle.
3424     * @param backupLocation location of the backup container. 
3425     * @exception StandardException Standard Derby error policy 
3426     */
3427    protected abstract void backupContainer(BaseContainerHandle handle, 
3428                                            String   backupLocation)
3429        throws StandardException;
3430}
3431
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags