KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > org > apache > lucene > index > TestIndexReader


1 package org.apache.lucene.index;
2
3 /**
4  * Copyright 2004 The Apache Software Foundation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */

18
19
20 import junit.framework.TestCase;
21 import junit.framework.TestSuite;
22 import junit.textui.TestRunner;
23
24 import org.apache.lucene.store.Directory;
25 import org.apache.lucene.store.RAMDirectory;
26 import org.apache.lucene.store.FSDirectory;
27 import org.apache.lucene.analysis.standard.StandardAnalyzer;
28 import org.apache.lucene.analysis.WhitespaceAnalyzer;
29 import org.apache.lucene.document.Document;
30 import org.apache.lucene.document.Field;
31
32 import java.util.Collection JavaDoc;
33 import java.io.IOException JavaDoc;
34 import java.io.File JavaDoc;
35
36 public class TestIndexReader extends TestCase
37 {
38     /** Main for running test case by itself. */
39     public static void main(String JavaDoc args[]) {
40         TestRunner.run (new TestSuite(TestIndexReader.class));
41 // TestRunner.run (new TestIndexReader("testBasicDelete"));
42
// TestRunner.run (new TestIndexReader("testDeleteReaderWriterConflict"));
43
// TestRunner.run (new TestIndexReader("testDeleteReaderReaderConflict"));
44
// TestRunner.run (new TestIndexReader("testFilesOpenClose"));
45
}
46
47     public TestIndexReader(String JavaDoc name) {
48         super(name);
49     }
50
51
52     /**
53      * Tests the IndexReader.getFieldNames implementation
54      * @throws Exception on error
55      */

56     public void testGetFieldNames() throws Exception JavaDoc
57     {
58         RAMDirectory d = new RAMDirectory();
59         // set up writer
60
IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(), true);
61         addDocumentWithFields(writer);
62         writer.close();
63         // set up reader
64
IndexReader reader = IndexReader.open(d);
65         Collection JavaDoc fieldNames = reader.getFieldNames();
66         assertTrue(fieldNames.contains("keyword"));
67         assertTrue(fieldNames.contains("text"));
68         assertTrue(fieldNames.contains("unindexed"));
69         assertTrue(fieldNames.contains("unstored"));
70         // add more documents
71
writer = new IndexWriter(d, new StandardAnalyzer(), false);
72         // want to get some more segments here
73
for (int i = 0; i < 5*writer.mergeFactor; i++)
74         {
75             addDocumentWithFields(writer);
76         }
77         // new fields are in some different segments (we hope)
78
for (int i = 0; i < 5*writer.mergeFactor; i++)
79         {
80             addDocumentWithDifferentFields(writer);
81         }
82         writer.close();
83         // verify fields again
84
reader = IndexReader.open(d);
85         fieldNames = reader.getFieldNames();
86         assertEquals(9, fieldNames.size()); // the following fields + an empty one (bug?!)
87
assertTrue(fieldNames.contains("keyword"));
88         assertTrue(fieldNames.contains("text"));
89         assertTrue(fieldNames.contains("unindexed"));
90         assertTrue(fieldNames.contains("unstored"));
91         assertTrue(fieldNames.contains("keyword2"));
92         assertTrue(fieldNames.contains("text2"));
93         assertTrue(fieldNames.contains("unindexed2"));
94         assertTrue(fieldNames.contains("unstored2"));
95
96         // verify that only indexed fields were returned
97
Collection JavaDoc indexedFieldNames = reader.getFieldNames(true);
98         assertEquals(6, indexedFieldNames.size());
99         assertTrue(indexedFieldNames.contains("keyword"));
100         assertTrue(indexedFieldNames.contains("text"));
101         assertTrue(indexedFieldNames.contains("unstored"));
102         assertTrue(indexedFieldNames.contains("keyword2"));
103         assertTrue(indexedFieldNames.contains("text2"));
104         assertTrue(indexedFieldNames.contains("unstored2"));
105
106         // verify that only unindexed fields were returned
107
Collection JavaDoc unindexedFieldNames = reader.getFieldNames(false);
108         assertEquals(3, unindexedFieldNames.size()); // the following fields + an empty one
109
assertTrue(unindexedFieldNames.contains("unindexed"));
110         assertTrue(unindexedFieldNames.contains("unindexed2"));
111     }
112
113
114     private void assertTermDocsCount(String JavaDoc msg,
115                                      IndexReader reader,
116                                      Term term,
117                                      int expected)
118     throws IOException JavaDoc
119     {
120         TermDocs tdocs = null;
121
122         try {
123             tdocs = reader.termDocs(term);
124             assertNotNull(msg + ", null TermDocs", tdocs);
125             int count = 0;
126             while(tdocs.next()) {
127                 count++;
128             }
129             assertEquals(msg + ", count mismatch", expected, count);
130
131         } finally {
132             if (tdocs != null)
133                 try { tdocs.close(); } catch (Exception JavaDoc e) { }
134         }
135
136     }
137
138
139
140     public void testBasicDelete() throws IOException JavaDoc
141     {
142         Directory dir = new RAMDirectory();
143
144         IndexWriter writer = null;
145         IndexReader reader = null;
146         Term searchTerm = new Term("content", "aaa");
147
148         // add 100 documents with term : aaa
149
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
150         for (int i = 0; i < 100; i++)
151         {
152             addDoc(writer, searchTerm.text());
153         }
154         writer.close();
155
156         // OPEN READER AT THIS POINT - this should fix the view of the
157
// index at the point of having 100 "aaa" documents and 0 "bbb"
158
reader = IndexReader.open(dir);
159         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
160         assertTermDocsCount("first reader", reader, searchTerm, 100);
161
162         // DELETE DOCUMENTS CONTAINING TERM: aaa
163
int deleted = 0;
164         reader = IndexReader.open(dir);
165         deleted = reader.delete(searchTerm);
166         assertEquals("deleted count", 100, deleted);
167         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
168         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
169         reader.close();
170
171         // CREATE A NEW READER and re-test
172
reader = IndexReader.open(dir);
173         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
174         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
175         reader.close();
176     }
177
178
179     public void testDeleteReaderWriterConflictUnoptimized() throws IOException JavaDoc{
180       deleteReaderWriterConflict(false);
181     }
182     
183     public void testDeleteReaderWriterConflictOptimized() throws IOException JavaDoc{
184         deleteReaderWriterConflict(true);
185     }
186
187     private void deleteReaderWriterConflict(boolean optimize) throws IOException JavaDoc
188     {
189         //Directory dir = new RAMDirectory();
190
Directory dir = getDirectory(true);
191
192         Term searchTerm = new Term("content", "aaa");
193         Term searchTerm2 = new Term("content", "bbb");
194
195         // add 100 documents with term : aaa
196
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
197         for (int i = 0; i < 100; i++)
198         {
199             addDoc(writer, searchTerm.text());
200         }
201         writer.close();
202
203         // OPEN READER AT THIS POINT - this should fix the view of the
204
// index at the point of having 100 "aaa" documents and 0 "bbb"
205
IndexReader reader = IndexReader.open(dir);
206         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
207         assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
208         assertTermDocsCount("first reader", reader, searchTerm, 100);
209         assertTermDocsCount("first reader", reader, searchTerm2, 0);
210
211         // add 100 documents with term : bbb
212
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
213         for (int i = 0; i < 100; i++)
214         {
215             addDoc(writer, searchTerm2.text());
216         }
217
218         // REQUEST OPTIMIZATION
219
// This causes a new segment to become current for all subsequent
220
// searchers. Because of this, deletions made via a previously open
221
// reader, which would be applied to that reader's segment, are lost
222
// for subsequent searchers/readers
223
if(optimize)
224           writer.optimize();
225         writer.close();
226
227         // The reader should not see the new data
228
assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
229         assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
230         assertTermDocsCount("first reader", reader, searchTerm, 100);
231         assertTermDocsCount("first reader", reader, searchTerm2, 0);
232
233
234         // DELETE DOCUMENTS CONTAINING TERM: aaa
235
// NOTE: the reader was created when only "aaa" documents were in
236
int deleted = 0;
237         try {
238             deleted = reader.delete(searchTerm);
239             fail("Delete allowed on an index reader with stale segment information");
240         } catch (IOException JavaDoc e) {
241             /* success */
242         }
243
244         // Re-open index reader and try again. This time it should see
245
// the new data.
246
reader.close();
247         reader = IndexReader.open(dir);
248         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
249         assertEquals("first docFreq", 100, reader.docFreq(searchTerm2));
250         assertTermDocsCount("first reader", reader, searchTerm, 100);
251         assertTermDocsCount("first reader", reader, searchTerm2, 100);
252
253         deleted = reader.delete(searchTerm);
254         assertEquals("deleted count", 100, deleted);
255         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
256         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
257         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
258         assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
259         reader.close();
260
261         // CREATE A NEW READER and re-test
262
reader = IndexReader.open(dir);
263         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
264         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
265         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
266         assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
267         reader.close();
268     }
269
270   private Directory getDirectory(boolean create) throws IOException JavaDoc {
271     return FSDirectory.getDirectory(new File JavaDoc(System.getProperty("tempDir"), "testIndex"), create);
272   }
273
274   public void testFilesOpenClose() throws IOException JavaDoc
275     {
276         // Create initial data set
277
Directory dir = getDirectory(true);
278         IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
279         addDoc(writer, "test");
280         writer.close();
281         dir.close();
282
283         // Try to erase the data - this ensures that the writer closed all files
284
dir = getDirectory(true);
285
286         // Now create the data set again, just as before
287
writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
288         addDoc(writer, "test");
289         writer.close();
290         dir.close();
291
292         // Now open existing directory and test that reader closes all files
293
dir = getDirectory(false);
294         IndexReader reader1 = IndexReader.open(dir);
295         reader1.close();
296         dir.close();
297
298         // The following will fail if reader did not close all files
299
dir = getDirectory(true);
300     }
301
302     public void testDeleteReaderReaderConflictUnoptimized() throws IOException JavaDoc{
303       deleteReaderReaderConflict(false);
304     }
305     
306     public void testDeleteReaderReaderConflictOptimized() throws IOException JavaDoc{
307       deleteReaderReaderConflict(true);
308     }
309     
310     private void deleteReaderReaderConflict(boolean optimize) throws IOException JavaDoc
311     {
312         Directory dir = getDirectory(true);
313
314         Term searchTerm1 = new Term("content", "aaa");
315         Term searchTerm2 = new Term("content", "bbb");
316         Term searchTerm3 = new Term("content", "ccc");
317
318         // add 100 documents with term : aaa
319
// add 100 documents with term : bbb
320
// add 100 documents with term : ccc
321
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
322         for (int i = 0; i < 100; i++)
323         {
324             addDoc(writer, searchTerm1.text());
325             addDoc(writer, searchTerm2.text());
326             addDoc(writer, searchTerm3.text());
327         }
328         if(optimize)
329           writer.optimize();
330         writer.close();
331
332         // OPEN TWO READERS
333
// Both readers get segment info as exists at this time
334
IndexReader reader1 = IndexReader.open(dir);
335         assertEquals("first opened", 100, reader1.docFreq(searchTerm1));
336         assertEquals("first opened", 100, reader1.docFreq(searchTerm2));
337         assertEquals("first opened", 100, reader1.docFreq(searchTerm3));
338         assertTermDocsCount("first opened", reader1, searchTerm1, 100);
339         assertTermDocsCount("first opened", reader1, searchTerm2, 100);
340         assertTermDocsCount("first opened", reader1, searchTerm3, 100);
341
342         IndexReader reader2 = IndexReader.open(dir);
343         assertEquals("first opened", 100, reader2.docFreq(searchTerm1));
344         assertEquals("first opened", 100, reader2.docFreq(searchTerm2));
345         assertEquals("first opened", 100, reader2.docFreq(searchTerm3));
346         assertTermDocsCount("first opened", reader2, searchTerm1, 100);
347         assertTermDocsCount("first opened", reader2, searchTerm2, 100);
348         assertTermDocsCount("first opened", reader2, searchTerm3, 100);
349
350         // DELETE DOCS FROM READER 2 and CLOSE IT
351
// delete documents containing term: aaa
352
// when the reader is closed, the segment info is updated and
353
// the first reader is now stale
354
reader2.delete(searchTerm1);
355         assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1));
356         assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2));
357         assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3));
358         assertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
359         assertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
360         assertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
361         reader2.close();
362
363         // Make sure reader 1 is unchanged since it was open earlier
364
assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1));
365         assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2));
366         assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3));
367         assertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
368         assertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
369         assertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
370
371
372         // ATTEMPT TO DELETE FROM STALE READER
373
// delete documents containing term: bbb
374
try {
375             reader1.delete(searchTerm2);
376             fail("Delete allowed from a stale index reader");
377         } catch (IOException JavaDoc e) {
378             /* success */
379         }
380
381         // RECREATE READER AND TRY AGAIN
382
reader1.close();
383         reader1 = IndexReader.open(dir);
384         assertEquals("reopened", 100, reader1.docFreq(searchTerm1));
385         assertEquals("reopened", 100, reader1.docFreq(searchTerm2));
386         assertEquals("reopened", 100, reader1.docFreq(searchTerm3));
387         assertTermDocsCount("reopened", reader1, searchTerm1, 0);
388         assertTermDocsCount("reopened", reader1, searchTerm2, 100);
389         assertTermDocsCount("reopened", reader1, searchTerm3, 100);
390
391         reader1.delete(searchTerm2);
392         assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1));
393         assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2));
394         assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3));
395         assertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
396         assertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
397         assertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
398         reader1.close();
399
400         // Open another reader to confirm that everything is deleted
401
reader2 = IndexReader.open(dir);
402         assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
403         assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
404         assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
405         assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
406         assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
407         assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
408         reader2.close();
409
410         dir.close();
411     }
412
413
414     private void addDocumentWithFields(IndexWriter writer) throws IOException JavaDoc
415     {
416         Document doc = new Document();
417         doc.add(Field.Keyword("keyword","test1"));
418         doc.add(Field.Text("text","test1"));
419         doc.add(Field.UnIndexed("unindexed","test1"));
420         doc.add(Field.UnStored("unstored","test1"));
421         writer.addDocument(doc);
422     }
423
424     private void addDocumentWithDifferentFields(IndexWriter writer) throws IOException JavaDoc
425     {
426         Document doc = new Document();
427         doc.add(Field.Keyword("keyword2","test1"));
428         doc.add(Field.Text("text2","test1"));
429         doc.add(Field.UnIndexed("unindexed2","test1"));
430         doc.add(Field.UnStored("unstored2","test1"));
431         writer.addDocument(doc);
432     }
433
434     private void addDoc(IndexWriter writer, String JavaDoc value)
435     {
436         Document doc = new Document();
437         doc.add(Field.UnStored("content", value));
438
439         try
440         {
441             writer.addDocument(doc);
442         }
443         catch (IOException JavaDoc e)
444         {
445             e.printStackTrace();
446         }
447     }
448 }
449
Popular Tags