Java > Open Source Codes > org > apache > lucene > index > TestCompoundFile


1 package org.apache.lucene.index;
2
3 /**
4  * Copyright 2004 The Apache Software Foundation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18
19 import java.io.IOException ;
20 import java.io.File ;
21
22 import junit.framework.TestCase;
23 import junit.framework.TestSuite;
24 import junit.textui.TestRunner;
25 import org.apache.lucene.store.OutputStream;
26 import org.apache.lucene.store.Directory;
27 import org.apache.lucene.store.InputStream;
28 import org.apache.lucene.store.FSDirectory;
29 import org.apache.lucene.store.RAMDirectory;
30 import org.apache.lucene.store._TestHelper;
31
32
33 /**
34  * @author dmitrys@earthlink.net
35  * @version $Id: TestCompoundFile.java,v 1.5 2004/03/29 22:48:06 cutting Exp $
36  */
37 public class TestCompoundFile extends TestCase
38 {
39     /** Main for running test case by itself. */
40     public static void main(String args[]) {
41         TestRunner.run (new TestSuite(TestCompoundFile.class));
42 // TestRunner.run (new TestCompoundFile("testSingleFile"));
43 // TestRunner.run (new TestCompoundFile("testTwoFiles"));
44 // TestRunner.run (new TestCompoundFile("testRandomFiles"));
45 // TestRunner.run (new TestCompoundFile("testClonedStreamsClosing"));
46 // TestRunner.run (new TestCompoundFile("testReadAfterClose"));
47 // TestRunner.run (new TestCompoundFile("testRandomAccess"));
48 // TestRunner.run (new TestCompoundFile("testRandomAccessClones"));
49 // TestRunner.run (new TestCompoundFile("testFileNotFound"));
50 // TestRunner.run (new TestCompoundFile("testReadPastEOF"));
51
52 // TestRunner.run (new TestCompoundFile("testIWCreate"));
53
54     }
55
56
57     private Directory dir;
58
59
60     public void setUp() throws IOException {
61         //dir = new RAMDirectory();
62 dir = FSDirectory.getDirectory(new File (System.getProperty("tempDir"), "testIndex"), true);
63     }
64
65
66     /** Creates a file of the specified size with random data. */
67     private void createRandomFile(Directory dir, String name, int size)
68     throws IOException
69     {
70         OutputStream os = dir.createFile(name);
71         for (int i=0; i<size; i++) {
72             byte b = (byte) (Math.random() * 256);
73             os.writeByte(b);
74         }
75         os.close();
76     }
77
78     /** Creates a file of the specified size with sequential data. The first
79      * byte is written as the start byte provided. All subsequent bytes are
80      * computed as start + offset where offset is the number of the byte.
81      */
82     private void createSequenceFile(Directory dir,
83                                     String name,
84                                     byte start,
85                                     int size)
86     throws IOException
87     {
88         OutputStream os = dir.createFile(name);
89         for (int i=0; i < size; i++) {
90             os.writeByte(start);
91             start ++;
92         }
93         os.close();
94     }
95
96
97     private void assertSameStreams(String msg,
98                                    InputStream expected,
99                                    InputStream test)
100     throws IOException
101     {
102         assertNotNull(msg + " null expected", expected);
103         assertNotNull(msg + " null test", test);
104         assertEquals(msg + " length", expected.length(), test.length());
105         assertEquals(msg + " position", expected.getFilePointer(),
106                                         test.getFilePointer());
107
108         byte expectedBuffer[] = new byte[512];
109         byte testBuffer[] = new byte[expectedBuffer.length];
110
111         long remainder = expected.length() - expected.getFilePointer();
112         while(remainder > 0) {
113             int readLen = (int) Math.min(remainder, expectedBuffer.length);
114             expected.readBytes(expectedBuffer, 0, readLen);
115             test.readBytes(testBuffer, 0, readLen);
116             assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer,
117                 testBuffer, 0, readLen);
118             remainder -= readLen;
119         }
120     }
121
122
123     private void assertSameStreams(String msg,
124                                    InputStream expected,
125                                    InputStream actual,
126                                    long seekTo)
127     throws IOException
128     {
129         if(seekTo >= 0 && seekTo < expected.length())
130         {
131             expected.seek(seekTo);
132             actual.seek(seekTo);
133             assertSameStreams(msg + ", seek(mid)", expected, actual);
134         }
135     }
136
137
138
139     private void assertSameSeekBehavior(String msg,
140                                         InputStream expected,
141                                         InputStream actual)
142     throws IOException
143     {
144         // seek to 0
145 long point = 0;
146         assertSameStreams(msg + ", seek(0)", expected, actual, point);
147
148         // seek to middle
149 point = expected.length() / 2l;
150         assertSameStreams(msg + ", seek(mid)", expected, actual, point);
151
152         // seek to end - 2
153 point = expected.length() - 2;
154         assertSameStreams(msg + ", seek(end-2)", expected, actual, point);
155
156         // seek to end - 1
157 point = expected.length() - 1;
158         assertSameStreams(msg + ", seek(end-1)", expected, actual, point);
159
160         // seek to the end
161 point = expected.length();
162         assertSameStreams(msg + ", seek(end)", expected, actual, point);
163
164         // seek past end
165 point = expected.length() + 1;
166         assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
167     }
168
169
170     private void assertEqualArrays(String msg,
171                                    byte[] expected,
172                                    byte[] test,
173                                    int start,
174                                    int len)
175     {
176         assertNotNull(msg + " null expected", expected);
177         assertNotNull(msg + " null test", test);
178
179         for (int i=start; i<len; i++) {
180             assertEquals(msg + " " + i, expected[i], test[i]);
181         }
182     }
183
184
185     // ===========================================================
186 // Tests of the basic CompoundFile functionality
187 // ===========================================================
188
189
190     /** This test creates compound file based on a single file.
191      * Files of different sizes are tested: 0, 1, 10, 100 bytes.
192      */
193     public void testSingleFile() throws IOException {
194         int data[] = new int[] { 0, 1, 10, 100 };
195         for (int i=0; i<data.length; i++) {
196             String name = "t" + data[i];
197             createSequenceFile(dir, name, (byte) 0, data[i]);
198             CompoundFileWriter csw = new CompoundFileWriter(dir, name + ".cfs");
199             csw.addFile(name);
200             csw.close();
201
202             CompoundFileReader csr = new CompoundFileReader(dir, name + ".cfs");
203             InputStream expected = dir.openFile(name);
204             InputStream actual = csr.openFile(name);
205             assertSameStreams(name, expected, actual);
206             assertSameSeekBehavior(name, expected, actual);
207             expected.close();
208             actual.close();
209             csr.close();
210         }
211     }
212
213
214     /** This test creates compound file based on two files.
215      *
216      */
217     public void testTwoFiles() throws IOException {
218         createSequenceFile(dir, "d1", (byte) 0, 15);
219         createSequenceFile(dir, "d2", (byte) 0, 114);
220
221         CompoundFileWriter csw = new CompoundFileWriter(dir, "d.csf");
222         csw.addFile("d1");
223         csw.addFile("d2");
224         csw.close();
225
226         CompoundFileReader csr = new CompoundFileReader(dir, "d.csf");
227         InputStream expected = dir.openFile("d1");
228         InputStream actual = csr.openFile("d1");
229         assertSameStreams("d1", expected, actual);
230         assertSameSeekBehavior("d1", expected, actual);
231         expected.close();
232         actual.close();
233
234         expected = dir.openFile("d2");
235         actual = csr.openFile("d2");
236         assertSameStreams("d2", expected, actual);
237         assertSameSeekBehavior("d2", expected, actual);
238         expected.close();
239         actual.close();
240         csr.close();
241     }
242
243     /** This test creates a compound file based on a large number of files of
244      * various length. The file content is generated randomly. The sizes range
245      * from 0 to 1Mb. Some of the sizes are selected to test the buffering
246      * logic in the file reading code. For this the chunk variable is set to
247      * the length of the buffer used internally by the compound file logic.
248      */
249     public void testRandomFiles() throws IOException {
250         // Setup the test segment
251 String segment = "test";
252         int chunk = 1024; // internal buffer size used by the stream
253 createRandomFile(dir, segment + ".zero", 0);
254         createRandomFile(dir, segment + ".one", 1);
255         createRandomFile(dir, segment + ".ten", 10);
256         createRandomFile(dir, segment + ".hundred", 100);
257         createRandomFile(dir, segment + ".big1", chunk);
258         createRandomFile(dir, segment + ".big2", chunk - 1);
259         createRandomFile(dir, segment + ".big3", chunk + 1);
260         createRandomFile(dir, segment + ".big4", 3 * chunk);
261         createRandomFile(dir, segment + ".big5", 3 * chunk - 1);
262         createRandomFile(dir, segment + ".big6", 3 * chunk + 1);
263         createRandomFile(dir, segment + ".big7", 1000 * chunk);
264
265         // Setup extraneous files
266 createRandomFile(dir, "onetwothree", 100);
267         createRandomFile(dir, segment + ".notIn", 50);
268         createRandomFile(dir, segment + ".notIn2", 51);
269
270         // Now test
271 CompoundFileWriter csw = new CompoundFileWriter(dir, "test.cfs");
272         final String data[] = new String [] {
273             ".zero", ".one", ".ten", ".hundred", ".big1", ".big2", ".big3",
274             ".big4", ".big5", ".big6", ".big7"
275         };
276         for (int i=0; i<data.length; i++) {
277             csw.addFile(segment + data[i]);
278         }
279         csw.close();
280
281         CompoundFileReader csr = new CompoundFileReader(dir, "test.cfs");
282         for (int i=0; i<data.length; i++) {
283             InputStream check = dir.openFile(segment + data[i]);
284             InputStream test = csr.openFile(segment + data[i]);
285             assertSameStreams(data[i], check, test);
286             assertSameSeekBehavior(data[i], check, test);
287             test.close();
288             check.close();
289         }
290         csr.close();
291     }
292
293
294     /** Setup a larger compound file with a number of components, each of
295      * which is a sequential file (so that we can easily tell that we are
296      * reading in the right byte). The methods sets up 20 files - f0 to f19,
297      * the size of each file is 1000 bytes.
298      */
299     private void setUp_2() throws IOException {
300         CompoundFileWriter cw = new CompoundFileWriter(dir, "f.comp");
301         for (int i=0; i<20; i++) {
302             createSequenceFile(dir, "f" + i, (byte) 0, 2000);
303             cw.addFile("f" + i);
304         }
305         cw.close();
306     }
307
308
309     public void testReadAfterClose() throws IOException {
310         demo_FSInputStreamBug((FSDirectory) dir, "test");
311     }
312
313     private void demo_FSInputStreamBug(FSDirectory fsdir, String file)
314     throws IOException
315     {
316         // Setup the test file - we need more than 1024 bytes
317 OutputStream os = fsdir.createFile(file);
318         for(int i=0; i<2000; i++) {
319             os.writeByte((byte) i);
320         }
321         os.close();
322
323         InputStream in = fsdir.openFile(file);
324
325         // This read primes the buffer in InputStream
326 byte b = in.readByte();
327
328         // Close the file
329 in.close();
330
331         // ERROR: this call should fail, but succeeds because the buffer
332 // is still filled
333 b = in.readByte();
334
335         // ERROR: this call should fail, but succeeds for some reason as well
336 in.seek(1099);
337
338         try {
339             // OK: this call correctly fails. We are now past the 1024 internal
340 // buffer, so an actual IO is attempted, which fails
341 b = in.readByte();
342         } catch (IOException e) {
343         }
344     }
345
346
347     static boolean isCSInputStream(InputStream is) {
348         return is instanceof CompoundFileReader.CSInputStream;
349     }
350
351     static boolean isCSInputStreamOpen(InputStream is) throws IOException {
352         if (isCSInputStream(is)) {
353             CompoundFileReader.CSInputStream cis =
354             (CompoundFileReader.CSInputStream) is;
355
356             return _TestHelper.isFSInputStreamOpen(cis.base);
357         } else {
358             return false;
359         }
360     }
361
362
363     public void testClonedStreamsClosing() throws IOException {
364         setUp_2();
365         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
366
367         // basic clone
368 InputStream expected = dir.openFile("f11");
369         assertTrue(_TestHelper.isFSInputStreamOpen(expected));
370
371         InputStream one = cr.openFile("f11");
372         assertTrue(isCSInputStreamOpen(one));
373
374         InputStream two = (InputStream) one.clone();
375         assertTrue(isCSInputStreamOpen(two));
376
377         assertSameStreams("basic clone one", expected, one);
378         expected.seek(0);
379         assertSameStreams("basic clone two", expected, two);
380
381         // Now close the first stream
382 one.close();
383         assertTrue("Only close when cr is closed", isCSInputStreamOpen(one));
384
385         // The following should really fail since we couldn't expect to
386 // access a file once close has been called on it (regardless of
387 // buffering and/or clone magic)
388 expected.seek(0);
389         two.seek(0);
390         assertSameStreams("basic clone two/2", expected, two);
391
392
393         // Now close the compound reader
394 cr.close();
395         assertFalse("Now closed one", isCSInputStreamOpen(one));
396         assertFalse("Now closed two", isCSInputStreamOpen(two));
397
398         // The following may also fail since the compound stream is closed
399 expected.seek(0);
400         two.seek(0);
401         //assertSameStreams("basic clone two/3", expected, two);
402
403
404         // Now close the second clone
405 two.close();
406         expected.seek(0);
407         two.seek(0);
408         //assertSameStreams("basic clone two/4", expected, two);
409
410         expected.close();
411     }
412
413
414     /** This test opens two files from a compound stream and verifies that
415      * their file positions are independent of each other.
416      */
417     public void testRandomAccess() throws IOException {
418         setUp_2();
419         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
420
421         // Open two files
422 InputStream e1 = dir.openFile("f11");
423         InputStream e2 = dir.openFile("f3");
424
425         InputStream a1 = cr.openFile("f11");
426         InputStream a2 = dir.openFile("f3");
427
428         // Seek the first pair
429 e1.seek(100);
430         a1.seek(100);
431         assertEquals(100, e1.getFilePointer());
432         assertEquals(100, a1.getFilePointer());
433         byte be1 = e1.readByte();
434         byte ba1 = a1.readByte();
435         assertEquals(be1, ba1);
436
437         // Now seek the second pair
438 e2.seek(1027);
439         a2.seek(1027);
440         assertEquals(1027, e2.getFilePointer());
441         assertEquals(1027, a2.getFilePointer());
442         byte be2 = e2.readByte();
443         byte ba2 = a2.readByte();
444         assertEquals(be2, ba2);
445
446         // Now make sure the first one didn't move
447 assertEquals(101, e1.getFilePointer());
448         assertEquals(101, a1.getFilePointer());
449         be1 = e1.readByte();
450         ba1 = a1.readByte();
451         assertEquals(be1, ba1);
452
453         // Now more the first one again, past the buffer length
454 e1.seek(1910);
455         a1.seek(1910);
456         assertEquals(1910, e1.getFilePointer());
457         assertEquals(1910, a1.getFilePointer());
458         be1 = e1.readByte();
459         ba1 = a1.readByte();
460         assertEquals(be1, ba1);
461
462         // Now make sure the second set didn't move
463 assertEquals(1028, e2.getFilePointer());
464         assertEquals(1028, a2.getFilePointer());
465         be2 = e2.readByte();
466         ba2 = a2.readByte();
467         assertEquals(be2, ba2);
468
469         // Move the second set back, again cross the buffer size
470 e2.seek(17);
471         a2.seek(17);
472         assertEquals(17, e2.getFilePointer());
473         assertEquals(17, a2.getFilePointer());
474         be2 = e2.readByte();
475         ba2 = a2.readByte();
476         assertEquals(be2, ba2);
477
478         // Finally, make sure the first set didn't move
479 // Now make sure the first one didn't move
480 assertEquals(1911, e1.getFilePointer());
481         assertEquals(1911, a1.getFilePointer());
482         be1 = e1.readByte();
483         ba1 = a1.readByte();
484         assertEquals(be1, ba1);
485
486         e1.close();
487         e2.close();
488         a1.close();
489         a2.close();
490         cr.close();
491     }
492
493     /** This test opens two files from a compound stream and verifies that
494      * their file positions are independent of each other.
495      */
496     public void testRandomAccessClones() throws IOException {
497         setUp_2();
498         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
499
500         // Open two files
501 InputStream e1 = cr.openFile("f11");
502         InputStream e2 = cr.openFile("f3");
503
504         InputStream a1 = (InputStream) e1.clone();
505         InputStream a2 = (InputStream) e2.clone();
506
507         // Seek the first pair
508 e1.seek(100);
509         a1.seek(100);
510         assertEquals(100, e1.getFilePointer());
511         assertEquals(100, a1.getFilePointer());
512         byte be1 = e1.readByte();
513         byte ba1 = a1.readByte();
514         assertEquals(be1, ba1);
515
516         // Now seek the second pair
517 e2.seek(1027);
518         a2.seek(1027);
519         assertEquals(1027, e2.getFilePointer());
520         assertEquals(1027, a2.getFilePointer());
521         byte be2 = e2.readByte();
522         byte ba2 = a2.readByte();
523         assertEquals(be2, ba2);
524
525         // Now make sure the first one didn't move
526 assertEquals(101, e1.getFilePointer());
527         assertEquals(101, a1.getFilePointer());
528         be1 = e1.readByte();
529         ba1 = a1.readByte();
530         assertEquals(be1, ba1);
531
532         // Now more the first one again, past the buffer length
533 e1.seek(1910);
534         a1.seek(1910);
535         assertEquals(1910, e1.getFilePointer());
536         assertEquals(1910, a1.getFilePointer());
537         be1 = e1.readByte();
538         ba1 = a1.readByte();
539         assertEquals(be1, ba1);
540
541         // Now make sure the second set didn't move
542 assertEquals(1028, e2.getFilePointer());
543         assertEquals(1028, a2.getFilePointer());
544         be2 = e2.readByte();
545         ba2 = a2.readByte();
546         assertEquals(be2, ba2);
547
548         // Move the second set back, again cross the buffer size
549 e2.seek(17);
550         a2.seek(17);
551         assertEquals(17, e2.getFilePointer());
552         assertEquals(17, a2.getFilePointer());
553         be2 = e2.readByte();
554         ba2 = a2.readByte();
555         assertEquals(be2, ba2);
556
557         // Finally, make sure the first set didn't move
558 // Now make sure the first one didn't move
559 assertEquals(1911, e1.getFilePointer());
560         assertEquals(1911, a1.getFilePointer());
561         be1 = e1.readByte();
562         ba1 = a1.readByte();
563         assertEquals(be1, ba1);
564
565         e1.close();
566         e2.close();
567         a1.close();
568         a2.close();
569         cr.close();
570     }
571
572
573     public void testFileNotFound() throws IOException {
574         setUp_2();
575         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
576
577         // Open two files
578 try {
579             InputStream e1 = cr.openFile("bogus");
580             fail("File not found");
581
582         } catch (IOException e) {
583             /* success */
584             //System.out.println("SUCCESS: File Not Found: " + e);
585 }
586
587         cr.close();
588     }
589
590
591     public void testReadPastEOF() throws IOException {
592         setUp_2();
593         CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
594         InputStream is = cr.openFile("f2");
595         is.seek(is.length() - 10);
596         byte b[] = new byte[100];
597         is.readBytes(b, 0, 10);
598
599         try {
600             byte test = is.readByte();
601             fail("Single byte read past end of file");
602         } catch (IOException e) {
603             /* success */
604             //System.out.println("SUCCESS: single byte read past end of file: " + e);
605 }
606
607         is.seek(is.length() - 10);
608         try {
609             is.readBytes(b, 0, 50);
610             fail("Block read past end of file");
611         } catch (IOException e) {
612             /* success */
613             //System.out.println("SUCCESS: block read past end of file: " + e);
614 }
615
616         is.close();
617         cr.close();
618     }
619 }
620

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags