TestIndexReader


1   package org.apache.lucene.index;
2   
3   /**
4    * Copyright 2004 The Apache Software Foundation
5    *
6    * Licensed under the Apache License, Version 2.0 (the "License");
7    * you may not use this file except in compliance with the License.
8    * You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  
20  import junit.framework.TestCase;
21  import junit.framework.TestSuite;
22  import junit.textui.TestRunner;
23  
24  import org.apache.lucene.store.Directory;
25  import org.apache.lucene.store.RAMDirectory;
26  import org.apache.lucene.store.FSDirectory;
27  import org.apache.lucene.analysis.standard.StandardAnalyzer;
28  import org.apache.lucene.analysis.WhitespaceAnalyzer;
29  import org.apache.lucene.document.Document;
30  import org.apache.lucene.document.Field;
31  
32  import java.util.Collection  ;
33  import java.io.IOException  ;
34  import java.io.File  ;
35  
36  public class TestIndexReader extends TestCase
37  {
38      /** Main for running test case by itself. */
39      public static void main(String   args[]) {
40          TestRunner.run (new TestSuite(TestIndexReader.class));
41  //        TestRunner.run (new TestIndexReader("testBasicDelete"));
42  //        TestRunner.run (new TestIndexReader("testDeleteReaderWriterConflict"));
43  //        TestRunner.run (new TestIndexReader("testDeleteReaderReaderConflict"));
44  //        TestRunner.run (new TestIndexReader("testFilesOpenClose"));
45      }
46  
47      public TestIndexReader(String   name) {
48          super(name);
49      }
50  
51  
52      /**
53       * Tests the IndexReader.getFieldNames implementation
54       * @throws Exception on error
55       */
56      public void testGetFieldNames() throws Exception  
57      {
58          RAMDirectory d = new RAMDirectory();
59          // set up writer
60          IndexWriter writer = new IndexWriter(d, new StandardAnalyzer(), true);
61          addDocumentWithFields(writer);
62          writer.close();
63          // set up reader
64          IndexReader reader = IndexReader.open(d);
65          Collection   fieldNames = reader.getFieldNames();
66          assertTrue(fieldNames.contains("keyword"));
67          assertTrue(fieldNames.contains("text"));
68          assertTrue(fieldNames.contains("unindexed"));
69          assertTrue(fieldNames.contains("unstored"));
70          // add more documents
71          writer = new IndexWriter(d, new StandardAnalyzer(), false);
72          // want to get some more segments here
73          for (int i = 0; i < 5*writer.mergeFactor; i++)
74          {
75              addDocumentWithFields(writer);
76          }
77          // new fields are in some different segments (we hope)
78          for (int i = 0; i < 5*writer.mergeFactor; i++)
79          {
80              addDocumentWithDifferentFields(writer);
81          }
82          writer.close();
83          // verify fields again
84          reader = IndexReader.open(d);
85          fieldNames = reader.getFieldNames();
86          assertEquals(9, fieldNames.size());    // the following fields + an empty one (bug?!)
87          assertTrue(fieldNames.contains("keyword"));
88          assertTrue(fieldNames.contains("text"));
89          assertTrue(fieldNames.contains("unindexed"));
90          assertTrue(fieldNames.contains("unstored"));
91          assertTrue(fieldNames.contains("keyword2"));
92          assertTrue(fieldNames.contains("text2"));
93          assertTrue(fieldNames.contains("unindexed2"));
94          assertTrue(fieldNames.contains("unstored2"));
95  
96          // verify that only indexed fields were returned
97          Collection   indexedFieldNames = reader.getFieldNames(true);
98          assertEquals(6, indexedFieldNames.size());
99          assertTrue(indexedFieldNames.contains("keyword"));
100         assertTrue(indexedFieldNames.contains("text"));
101         assertTrue(indexedFieldNames.contains("unstored"));
102         assertTrue(indexedFieldNames.contains("keyword2"));
103         assertTrue(indexedFieldNames.contains("text2"));
104         assertTrue(indexedFieldNames.contains("unstored2"));
105 
106         // verify that only unindexed fields were returned
107         Collection   unindexedFieldNames = reader.getFieldNames(false);
108         assertEquals(3, unindexedFieldNames.size());    // the following fields + an empty one
109         assertTrue(unindexedFieldNames.contains("unindexed"));
110         assertTrue(unindexedFieldNames.contains("unindexed2"));
111     }
112 
113 
114     private void assertTermDocsCount(String   msg,
115                                      IndexReader reader,
116                                      Term term,
117                                      int expected)
118     throws IOException  
119     {
120         TermDocs tdocs = null;
121 
122         try {
123             tdocs = reader.termDocs(term);
124             assertNotNull(msg + ", null TermDocs", tdocs);
125             int count = 0;
126             while(tdocs.next()) {
127                 count++;
128             }
129             assertEquals(msg + ", count mismatch", expected, count);
130 
131         } finally {
132             if (tdocs != null)
133                 try { tdocs.close(); } catch (Exception   e) { }
134         }
135 
136     }
137 
138 
139 
140     public void testBasicDelete() throws IOException  
141     {
142         Directory dir = new RAMDirectory();
143 
144         IndexWriter writer = null;
145         IndexReader reader = null;
146         Term searchTerm = new Term("content", "aaa");
147 
148         //  add 100 documents with term : aaa
149         writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
150         for (int i = 0; i < 100; i++)
151         {
152             addDoc(writer, searchTerm.text());
153         }
154         writer.close();
155 
156         // OPEN READER AT THIS POINT - this should fix the view of the
157         // index at the point of having 100 "aaa" documents and 0 "bbb"
158         reader = IndexReader.open(dir);
159         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
160         assertTermDocsCount("first reader", reader, searchTerm, 100);
161 
162         // DELETE DOCUMENTS CONTAINING TERM: aaa
163         int deleted = 0;
164         reader = IndexReader.open(dir);
165         deleted = reader.delete(searchTerm);
166         assertEquals("deleted count", 100, deleted);
167         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
168         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
169         reader.close();
170 
171         // CREATE A NEW READER and re-test
172         reader = IndexReader.open(dir);
173         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
174         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
175         reader.close();
176     }
177 
178 
179     public void testDeleteReaderWriterConflictUnoptimized() throws IOException  {
180       deleteReaderWriterConflict(false);
181     }
182     
183     public void testDeleteReaderWriterConflictOptimized() throws IOException  {
184         deleteReaderWriterConflict(true);
185     }
186 
187     private void deleteReaderWriterConflict(boolean optimize) throws IOException  
188     {
189         //Directory dir = new RAMDirectory();
190         Directory dir = getDirectory(true);
191 
192         Term searchTerm = new Term("content", "aaa");
193         Term searchTerm2 = new Term("content", "bbb");
194 
195         //  add 100 documents with term : aaa
196         IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
197         for (int i = 0; i < 100; i++)
198         {
199             addDoc(writer, searchTerm.text());
200         }
201         writer.close();
202 
203         // OPEN READER AT THIS POINT - this should fix the view of the
204         // index at the point of having 100 "aaa" documents and 0 "bbb"
205         IndexReader reader = IndexReader.open(dir);
206         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
207         assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
208         assertTermDocsCount("first reader", reader, searchTerm, 100);
209         assertTermDocsCount("first reader", reader, searchTerm2, 0);
210 
211         // add 100 documents with term : bbb
212         writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
213         for (int i = 0; i < 100; i++)
214         {
215             addDoc(writer, searchTerm2.text());
216         }
217 
218         // REQUEST OPTIMIZATION
219         // This causes a new segment to become current for all subsequent
220         // searchers. Because of this, deletions made via a previously open
221         // reader, which would be applied to that reader's segment, are lost
222         // for subsequent searchers/readers
223         if(optimize)
224           writer.optimize();
225         writer.close();
226 
227         // The reader should not see the new data
228         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
229         assertEquals("first docFreq", 0, reader.docFreq(searchTerm2));
230         assertTermDocsCount("first reader", reader, searchTerm, 100);
231         assertTermDocsCount("first reader", reader, searchTerm2, 0);
232 
233 
234         // DELETE DOCUMENTS CONTAINING TERM: aaa
235         // NOTE: the reader was created when only "aaa" documents were in
236         int deleted = 0;
237         try {
238             deleted = reader.delete(searchTerm);
239             fail("Delete allowed on an index reader with stale segment information");
240         } catch (IOException   e) {
241             /* success */
242         }
243 
244         // Re-open index reader and try again. This time it should see
245         // the new data.
246         reader.close();
247         reader = IndexReader.open(dir);
248         assertEquals("first docFreq", 100, reader.docFreq(searchTerm));
249         assertEquals("first docFreq", 100, reader.docFreq(searchTerm2));
250         assertTermDocsCount("first reader", reader, searchTerm, 100);
251         assertTermDocsCount("first reader", reader, searchTerm2, 100);
252 
253         deleted = reader.delete(searchTerm);
254         assertEquals("deleted count", 100, deleted);
255         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
256         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
257         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
258         assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
259         reader.close();
260 
261         // CREATE A NEW READER and re-test
262         reader = IndexReader.open(dir);
263         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
264         assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
265         assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
266         assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
267         reader.close();
268     }
269 
270   private Directory getDirectory(boolean create) throws IOException   {
271     return FSDirectory.getDirectory(new File  (System.getProperty("tempDir"), "testIndex"), create);
272   }
273 
274   public void testFilesOpenClose() throws IOException  
275     {
276         // Create initial data set
277         Directory dir = getDirectory(true);
278         IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
279         addDoc(writer, "test");
280         writer.close();
281         dir.close();
282 
283         // Try to erase the data - this ensures that the writer closed all files
284         dir = getDirectory(true);
285 
286         // Now create the data set again, just as before
287         writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
288         addDoc(writer, "test");
289         writer.close();
290         dir.close();
291 
292         // Now open existing directory and test that reader closes all files
293         dir = getDirectory(false);
294         IndexReader reader1 = IndexReader.open(dir);
295         reader1.close();
296         dir.close();
297 
298         // The following will fail if reader did not close all files
299         dir = getDirectory(true);
300     }
301 
302     public void testDeleteReaderReaderConflictUnoptimized() throws IOException  {
303       deleteReaderReaderConflict(false);
304     }
305     
306     public void testDeleteReaderReaderConflictOptimized() throws IOException  {
307       deleteReaderReaderConflict(true);
308     }
309     
310     private void deleteReaderReaderConflict(boolean optimize) throws IOException  
311     {
312         Directory dir = getDirectory(true);
313 
314         Term searchTerm1 = new Term("content", "aaa");
315         Term searchTerm2 = new Term("content", "bbb");
316         Term searchTerm3 = new Term("content", "ccc");
317 
318         //  add 100 documents with term : aaa
319         //  add 100 documents with term : bbb
320         //  add 100 documents with term : ccc
321         IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
322         for (int i = 0; i < 100; i++)
323         {
324             addDoc(writer, searchTerm1.text());
325             addDoc(writer, searchTerm2.text());
326             addDoc(writer, searchTerm3.text());
327         }
328         if(optimize)
329           writer.optimize();
330         writer.close();
331 
332         // OPEN TWO READERS
333         // Both readers get segment info as exists at this time
334         IndexReader reader1 = IndexReader.open(dir);
335         assertEquals("first opened", 100, reader1.docFreq(searchTerm1));
336         assertEquals("first opened", 100, reader1.docFreq(searchTerm2));
337         assertEquals("first opened", 100, reader1.docFreq(searchTerm3));
338         assertTermDocsCount("first opened", reader1, searchTerm1, 100);
339         assertTermDocsCount("first opened", reader1, searchTerm2, 100);
340         assertTermDocsCount("first opened", reader1, searchTerm3, 100);
341 
342         IndexReader reader2 = IndexReader.open(dir);
343         assertEquals("first opened", 100, reader2.docFreq(searchTerm1));
344         assertEquals("first opened", 100, reader2.docFreq(searchTerm2));
345         assertEquals("first opened", 100, reader2.docFreq(searchTerm3));
346         assertTermDocsCount("first opened", reader2, searchTerm1, 100);
347         assertTermDocsCount("first opened", reader2, searchTerm2, 100);
348         assertTermDocsCount("first opened", reader2, searchTerm3, 100);
349 
350         // DELETE DOCS FROM READER 2 and CLOSE IT
351         // delete documents containing term: aaa
352         // when the reader is closed, the segment info is updated and
353         // the first reader is now stale
354         reader2.delete(searchTerm1);
355         assertEquals("after delete 1", 100, reader2.docFreq(searchTerm1));
356         assertEquals("after delete 1", 100, reader2.docFreq(searchTerm2));
357         assertEquals("after delete 1", 100, reader2.docFreq(searchTerm3));
358         assertTermDocsCount("after delete 1", reader2, searchTerm1, 0);
359         assertTermDocsCount("after delete 1", reader2, searchTerm2, 100);
360         assertTermDocsCount("after delete 1", reader2, searchTerm3, 100);
361         reader2.close();
362 
363         // Make sure reader 1 is unchanged since it was open earlier
364         assertEquals("after delete 1", 100, reader1.docFreq(searchTerm1));
365         assertEquals("after delete 1", 100, reader1.docFreq(searchTerm2));
366         assertEquals("after delete 1", 100, reader1.docFreq(searchTerm3));
367         assertTermDocsCount("after delete 1", reader1, searchTerm1, 100);
368         assertTermDocsCount("after delete 1", reader1, searchTerm2, 100);
369         assertTermDocsCount("after delete 1", reader1, searchTerm3, 100);
370 
371 
372         // ATTEMPT TO DELETE FROM STALE READER
373         // delete documents containing term: bbb
374         try {
375             reader1.delete(searchTerm2);
376             fail("Delete allowed from a stale index reader");
377         } catch (IOException   e) {
378             /* success */
379         }
380 
381         // RECREATE READER AND TRY AGAIN
382         reader1.close();
383         reader1 = IndexReader.open(dir);
384         assertEquals("reopened", 100, reader1.docFreq(searchTerm1));
385         assertEquals("reopened", 100, reader1.docFreq(searchTerm2));
386         assertEquals("reopened", 100, reader1.docFreq(searchTerm3));
387         assertTermDocsCount("reopened", reader1, searchTerm1, 0);
388         assertTermDocsCount("reopened", reader1, searchTerm2, 100);
389         assertTermDocsCount("reopened", reader1, searchTerm3, 100);
390 
391         reader1.delete(searchTerm2);
392         assertEquals("deleted 2", 100, reader1.docFreq(searchTerm1));
393         assertEquals("deleted 2", 100, reader1.docFreq(searchTerm2));
394         assertEquals("deleted 2", 100, reader1.docFreq(searchTerm3));
395         assertTermDocsCount("deleted 2", reader1, searchTerm1, 0);
396         assertTermDocsCount("deleted 2", reader1, searchTerm2, 0);
397         assertTermDocsCount("deleted 2", reader1, searchTerm3, 100);
398         reader1.close();
399 
400         // Open another reader to confirm that everything is deleted
401         reader2 = IndexReader.open(dir);
402         assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
403         assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
404         assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
405         assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
406         assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
407         assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
408         reader2.close();
409 
410         dir.close();
411     }
412 
413 
414     private void addDocumentWithFields(IndexWriter writer) throws IOException  
415     {
416         Document doc = new Document();
417         doc.add(Field.Keyword("keyword","test1"));
418         doc.add(Field.Text("text","test1"));
419         doc.add(Field.UnIndexed("unindexed","test1"));
420         doc.add(Field.UnStored("unstored","test1"));
421         writer.addDocument(doc);
422     }
423 
424     private void addDocumentWithDifferentFields(IndexWriter writer) throws IOException  
425     {
426         Document doc = new Document();
427         doc.add(Field.Keyword("keyword2","test1"));
428         doc.add(Field.Text("text2","test1"));
429         doc.add(Field.UnIndexed("unindexed2","test1"));
430         doc.add(Field.UnStored("unstored2","test1"));
431         writer.addDocument(doc);
432     }
433 
434     private void addDoc(IndexWriter writer, String   value)
435     {
436         Document doc = new Document();
437         doc.add(Field.UnStored("content", value));
438 
439         try
440         {
441             writer.addDocument(doc);
442         }
443         catch (IOException   e)
444         {
445             e.printStackTrace();
446         }
447     }
448 }
449
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags