Java > Open Source Codes > com > knowgate > hipermail > MboxFile


1 /*
2  * Original Code:
3  * Copyright (c) 2004, Ben Fortuna
4  * All rights reserved.
5  *
6  * Modified by Sergio Montoro Ten on November 2004 for use with hipergate.
7  *
8  * purge(int[]) method fix up by Heidi on September 2006
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  *
14  * o Redistributions of source code must retain the above copyright
15  * notice, this list of conditions and the following disclaimer.
16  *
17  * o Redistributions in binary form must reproduce the above copyright
18  * notice, this list of conditions and the following disclaimer in the
19  * documentation and/or other materials provided with the distribution.
20  *
21  * o Neither the name of Ben Fortuna nor the names of any other contributors
22  * may be used to endorse or promote products derived from this software
23  * without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
29  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36  */
37 package com.knowgate.hipermail;
38
39 import java.io.ByteArrayInputStream ;
40 import java.io.File ;
41 import java.io.FileNotFoundException ;
42 import java.io.FileOutputStream ;
43 import java.io.IOException ;
44 import java.io.InputStream ;
45 import java.io.RandomAccessFile ;
46
47 import java.nio.ByteBuffer ;
48 import java.nio.MappedByteBuffer ;
49 import java.nio.CharBuffer ;
50 import java.nio.channels.FileChannel ;
51 import java.nio.channels.FileLock ;
52 import java.nio.charset.Charset ;
53 import java.nio.charset.CharsetDecoder ;
54 import java.nio.charset.CharsetEncoder ;
55 import java.nio.charset.CodingErrorAction ;
56 import java.nio.charset.CoderResult ;
57
58 import java.text.DateFormat ;
59 import java.text.SimpleDateFormat ;
60
61 import java.util.ArrayList ;
62 import java.util.Date ;
63 import java.util.Iterator ;
64 import java.util.List ;
65 import java.util.TimeZone ;
66 import java.util.regex.Matcher ;
67 import java.util.regex.Pattern ;
68
69 import com.knowgate.debug.DebugFile;
70
71 /**
72  * Provides access to an mbox-formatted file.
73  * @author Ben Fortuna adapted to hipergate by Sergio Montoro Ten
74  * @version 3.0
75  */
76 public class MboxFile {
77
78     public static final String READ_ONLY = "r";
79
80     public static final String READ_WRITE = "rw";
81
82     private static final String TEMP_FILE_EXTENSION = ".tmp";
83
84     /**
85      * The prefix for all "From_" lines in an mbox file.
86      */
87     private static final String FROM__PREFIX = "From ";
88
89     /**
90      * A pattern representing the format of the "From_" line
91      * for the first message in an mbox file.
92      */
93     private static final String INITIAL_FROM__PATTERN = FROM__PREFIX + ".*";
94
95     /**
96      * A pattern representing the format of all "From_" lines
97      * except for the first message in an mbox file.
98      */
99     private static final String FROM__PATTERN = "\n" + FROM__PREFIX;
100
101     private static final String FROM__DATE_FORMAT = "EEE MMM d HH:mm:ss yyyy";
102
103     private static DateFormat from_DateFormat = new SimpleDateFormat (FROM__DATE_FORMAT);
104
105     static {
106         from_DateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
107     }
108
109     /**
110      * The default "From_" line used if a message doesn't already have one.
111      */
112     private static final String DEFAULT_FROM__LINE = FROM__PREFIX + "- " + from_DateFormat.format(new Date (0)) + "\n";
113
114     // Charset and decoder for ISO-8859-1
115 private static Charset charset = Charset.forName("ISO-8859-1");
116
117     private static CharsetDecoder decoder = charset.newDecoder();
118
119     private static CharsetEncoder encoder = charset.newEncoder();
120
121     static {
122         encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
123     }
124
125     private static DebugFile log = new DebugFile();
126
127     /**
128      * Used primarily to provide information about
129      * the mbox file.
130      */
131     private File file;
132
133     private String mode;
134
135     /**
136      * Used to access the mbox file in a random manner.
137      */
138     private FileChannel channel;
139
140     /**
141      * Used to grant exclusive access to the Mbox file to one thread at a time.
142      */
143     private FileLock lock;
144
145     /**
146      * Tracks all message positions within the mbox file.
147      */
148     private long[] messagePositions;
149
150     /**
151      * Constructor.
152      */
153     public MboxFile(File file) throws FileNotFoundException , IOException {
154         this(file, READ_ONLY);
155     }
156
157     /**
158      * Constructor.
159      * @param file
160      * @param mode Either MboxFile.READ_ONLY or MboxFile.READ_WRITE
161      */
162     public MboxFile(File file, String mode)
163         throws FileNotFoundException , IOException {
164         this.file = file;
165         this.mode = mode;
166         if (mode.equals(READ_WRITE))
167           lock = getChannel().lock();
168     }
169
170     /**
171      * Constructor.
172      * @param filepath
173      * @param mode Either MboxFile.READ_ONLY or MboxFile.READ_WRITE
174      */
175     public MboxFile(String filepath) {
176         this.file = new File (filepath);
177         this.mode = READ_ONLY;
178     }
179
180     /**
181      * Constructor.
182      * @param filepath
183      * @param mode Either MboxFile.READ_ONLY or MboxFile.READ_WRITE
184      */
185     public MboxFile(String filepath, String mode)
186         throws FileNotFoundException , IOException {
187         this.file = new File (filepath);
188         this.mode = mode;
189         if (mode.equals(READ_WRITE))
190           lock = getChannel().lock();
191     }
192
193     /**
194      * Returns a channel for reading and writing to the mbox file.
195      * @return a file channel
196      * @throws FileNotFoundException
197      */
198     private FileChannel getChannel() throws FileNotFoundException {
199
200         if (channel == null) {
201             channel = new RandomAccessFile (file, mode).getChannel();
202         }
203
204         return channel;
205     }
206
207     /**
208      * Return MBox file size in bytes
209      * @return long
210      */
211     public long size() throws IOException {
212       return channel.size();
213     }
214
215     /**
216      * Returns an initialised array of file positions
217      * for all messages in the mbox file.
218      * @return a long array
219      * @throws IOException thrown when unable to read
220      * from the specified file channel
221      */
222     public long[] getMessagePositions() throws IOException {
223         if (messagePositions == null) {
224           final long length = getChannel().size();
225           log.debug("Channel size [" + String.valueOf(length) + "] bytes");
226
227           if (0==length) return new long[0];
228
229           List posList = new ArrayList ();
230
231           final long FRAME = 32000;
232           final long STEPBACK = FROM__PATTERN.length() - 1;
233           long size = (length<FRAME ? length : FRAME);
234
235           long offset = 0;
236           FileChannel chnnl = getChannel();
237
238           // read mbox file to determine the message positions..
239 ByteBuffer buffer = chnnl.map(FileChannel.MapMode.READ_ONLY, 0l, size);
240           CharBuffer cb = decoder.decode(buffer);
241
242           // check that first message is correct..
243 if (Pattern.compile(INITIAL_FROM__PATTERN, Pattern.DOTALL).matcher(cb).matches()) {
244             // debugging..
245 log.debug("Matched first message...");
246
247             posList.add(new Long (0));
248           }
249
250           Pattern fromPattern = Pattern.compile(FROM__PATTERN);
251           Matcher matcher;
252
253           do {
254             log.debug("scanning from " + String.valueOf(offset) + " to " + String.valueOf(offset+size));
255             matcher = fromPattern.matcher(cb);
256             while (matcher.find()) {
257                 log.debug("Found match at [" + String.valueOf(offset+matcher.start()) + "]");
258
259                 // add one (1) to position to account for newline..
260 posList.add(new Long (offset+matcher.start() + 1));
261             } // wend
262
263             if (size<FRAME) break;
264
265             offset += FRAME-STEPBACK;
266             size = (offset+FRAME<length) ? FRAME : length-(offset+1);
267
268             buffer = chnnl.map(FileChannel.MapMode.READ_ONLY, offset, size);
269             cb = decoder.decode(buffer);
270           } while (true);
271
272           log.debug("found " + String.valueOf(posList.size()) + " matches");
273
274           messagePositions = new long[posList.size()];
275
276           int count = 0;
277
278           for (Iterator i = posList.iterator(); i.hasNext(); count++) {
279             messagePositions[count] = ((Long ) i.next()).longValue();
280           } // next
281 } // fi (messagePositions == null)
282 return messagePositions;
283     } // getMessagePositions
284
285     /**
286      * <p>Get byte offset position of a given message inside the mbox file</p>
287      * This method is slow when called for the first time, as it has to parse
288      * the whole Mbox file for finding each message index.
289      * @param index Message Index
290      * @return message byte offset position inside the mbox file
291      * @throws IOException
292      * @throws ArrayIndexOutOfBoundsException
293      */
294     public long getMessagePosition (int index)
295       throws IOException , ArrayIndexOutOfBoundsException {
296       if (messagePositions == null) getMessagePositions();
297       return messagePositions[index];
298     }
299
300     /**
301      * Get size of a message in bytes
302      * @param index Message Index
303      * @throws IOException
304      * @throws ArrayIndexOutOfBoundsException
305      */
306     public int getMessageSize (int index)
307       throws IOException , ArrayIndexOutOfBoundsException {
308       long position = getMessagePosition(index);
309       long size;
310
311       if (index < messagePositions.length - 1)
312         size = messagePositions[index + 1] - position;
313       else
314         size = getChannel().size() - position;
315
316       return (int) size;
317     }
318
319     /**
320      * Returns the total number of messages in the mbox file.
321      * @return an int
322      */
323     public int getMessageCount() throws IOException {
324         return getMessagePositions().length;
325     }
326
327     /**
328      * Returns a CharSequence containing the data for
329      * the message at the specified index.
330      * @param index the index of the message to retrieve
331      * @return a CharSequence
332      */
333     public CharSequence getMessage(final int index) throws IOException {
334         long position = getMessagePosition(index);
335         long size;
336
337         if (index < messagePositions.length - 1) {
338             size = messagePositions[index + 1] - position;
339         }
340         else {
341             size = getChannel().size() - position;
342         }
343
344         return decoder.decode(getChannel().map(FileChannel.MapMode.READ_ONLY, position, size));
345     }
346
347     /**
348      * Get message as stream
349      * @param begin long Byte offset position for message
350      * @param size int Number of bytes to be readed
351      * @return InputStream
352      * @throws IOException
353      */
354     public InputStream getMessageAsStream (final long begin, final int size) throws IOException {
355
356       log.debug("MboxFile.getMessageAsStream("+String.valueOf(begin)+","+String.valueOf(size)+")");
357
358       // Skip From line
359 ByteBuffer byFrom = getChannel().map(FileChannel.MapMode.READ_ONLY, begin, 128);
360       CharBuffer chFrom = decoder.decode(byFrom);
361
362       int start = 0;
363       // Ignore any white spaces and line feed
364 char c = chFrom.charAt(start);
365       while (c==' ' || c=='\r' || c=='\n' || c=='\t') c = chFrom.charAt(++start);
366       // If first line does not start with message preffx then raise an exception
367 if (!chFrom.subSequence(start, start+FROM__PREFIX.length()).toString().equals(FROM__PREFIX))
368         throw new IOException ("MboxFile.getMessageAsStream() starting position " + String.valueOf(start) + " \""+chFrom.subSequence(start, start+FROM__PREFIX.length()).toString()+"\" does not match a begin message token \"" + FROM__PREFIX + "\"");
369       // Skip the From line
370 while (chFrom.charAt(start++)!=(char) 10) ;
371
372       log.debug(" skip = " + String.valueOf(start));
373       log.debug(" start = " + String.valueOf(begin+start));
374
375       MappedByteBuffer byBuffer = getChannel().map(FileChannel.MapMode.READ_ONLY, begin+start, size);
376       byte[] byArray = new byte[size];
377       byBuffer.get(byArray);
378
379       ByteArrayInputStream byStrm = new ByteArrayInputStream (byArray);
380
381       return byStrm;
382     }
383
384     // -------------------------------------------------------------------------
385
386     public InputStream getPartAsStream (final long begin, final long offset, final int size)
387       throws IOException {
388       log.debug("MboxFile.getPartAsStream("+String.valueOf(begin)+","+String.valueOf(offset)+","+String.valueOf(size)+")");
389
390       // Skip From line
391 ByteBuffer byFrom = getChannel().map(FileChannel.MapMode.READ_ONLY, begin, 128);
392       CharBuffer chFrom = decoder.decode(byFrom);
393
394       log.debug("from line decoded");
395
396       int start = 0;
397       // Ignore any white spaces and line feed
398 char c = chFrom.charAt(start);
399       while (c==' ' || c=='\r' || c=='\n' || c=='\t') c = chFrom.charAt(++start);
400       // If first line does not start with message preffx then raise an exception
401 log.debug("first line is " + chFrom.subSequence(start, start+FROM__PREFIX.length()).toString());
402       if (!chFrom.subSequence(start, start+FROM__PREFIX.length()).toString().equals(FROM__PREFIX))
403         throw new IOException ("MboxFile.getPartAsStream() starting position " + String.valueOf(start) + " \""+chFrom.subSequence(start, start+FROM__PREFIX.length()).toString()+"\" does not match a begin message token \"" + FROM__PREFIX + "\"");
404       // Skip the From line
405 while (chFrom.charAt(start++)!=(char) 10) ;
406
407       start += offset;
408
409       log.debug(" skip = " + String.valueOf(start));
410       log.debug(" start = " + String.valueOf(start));
411
412       MappedByteBuffer byBuffer = getChannel().map(FileChannel.MapMode.READ_ONLY, begin+start, size);
413       byte[] byArray = new byte[size];
414       byBuffer.get(byArray);
415
416       ByteArrayInputStream byStrm = new ByteArrayInputStream (byArray);
417
418       return byStrm;
419     }
420
421     /**
422      * Opens an input stream to the specified message
423      * data.
424      * @param index the index of the message to open
425      * a stream to
426      * @return an input stream
427      */
428     public InputStream getMessageAsStream(int index) throws IOException {
429       long position = getMessagePosition(index);
430       int size;
431
432       log.debug("MboxFile.getMessageAsStream("+String.valueOf(position)+")");
433
434       if (index < messagePositions.length - 1) {
435           size = (int) (messagePositions[index + 1] - position);
436       }
437       else {
438           size = (int) (getChannel().size() - position);
439       }
440
441       // Skip From line
442 ByteBuffer byFrom = getChannel().map(FileChannel.MapMode.READ_ONLY, position, 256);
443       CharBuffer chFrom = decoder.decode(byFrom);
444
445       int start = 0;
446       // Ignore any white spaces and line feed
447 char c = chFrom.charAt(start);
448       while (c==' ' || c=='\r' || c=='\n' || c=='\t') c = chFrom.charAt(++start);
449       // If first line does not start with message preffx then raise an exception
450 if (!chFrom.subSequence(start, start+FROM__PREFIX.length()).toString().equals(FROM__PREFIX))
451         throw new IOException ("MboxFile.getMessageAsStream() starting position " + String.valueOf(start) + " \""+chFrom.subSequence(start, start+FROM__PREFIX.length()).toString()+"\" does not match a begin message token \"" + FROM__PREFIX + "\"");
452       // Skip the From line
453 while (chFrom.charAt(start++)!=(char) 10) ;
454
455       log.debug(" skip = " + String.valueOf(start));
456       log.debug(" start = " + String.valueOf(position+start));
457
458       MappedByteBuffer byBuffer = getChannel().map(FileChannel.MapMode.READ_ONLY, position+start, size-start);
459       byte[] byArray = new byte[size-start];
460       byBuffer.get(byArray);
461
462       ByteArrayInputStream byStrm = new ByteArrayInputStream (byArray);
463
464       return byStrm;
465     }
466
467     /**
468      * Appends the specified message from another mbox file
469      * @param source Source mbox file
470      * @param srcpos Byte offset position of message at source mbox file
471      * @param srcsize Size of source message in bytes
472      * @return byte offset position where message is appended on this mbox file
473      * @throws IOException
474      */
475     public final long appendMessage(MboxFile source, long srcpos, int srcsize) throws IOException {
476
477       long position = channel.size();
478
479       // if not first message add required newlines..
480 if (position > 0) {
481         channel.write(encoder.encode(CharBuffer.wrap("\n\n")), channel.size());
482       }
483       channel.write(encoder.encode(CharBuffer.wrap(DEFAULT_FROM__LINE)), channel.size());
484
485       channel.write(source.getChannel().map(FileChannel.MapMode.READ_ONLY, srcpos, srcsize));
486
487       return position;
488     }
489
490     /**
491      * Appends the specified message from another mbox file
492      * @param source Source mbox file
493      * @param index Index of message to be appended at the source file
494      * @return byte offset position where message is appended on this mbox file
495      * @throws IOException
496      */
497     public final long appendMessage(MboxFile source, int index) throws IOException {
498       long srcpos = source.getMessagePosition(index);
499       int srcsize;
500
501       if (index < source.messagePositions.length - 1) {
502           srcsize = (int) (source.messagePositions[index + 1] - srcpos);
503       }
504       else {
505           srcsize = (int) (source.getChannel().size() - srcpos);
506       }
507
508       return appendMessage(source, srcpos, srcsize);
509     }
510
511     /**
512      * Appends the specified message (represented by a CharSequence) to the
513      * mbox file.
514      * @param message
515      */
516     public final long appendMessage(final CharSequence message) throws IOException {
517         return appendMessage(message, getChannel());
518     }
519
520     /**
521      * Appends the specified message (represented by a CharSequence) to the specified channel.
522      * @param message
523      * @param channel
524      * @return long Byte position where message is appended
525      * @throws IOException
526      */
527     private long appendMessage(final CharSequence message, FileChannel channel) throws IOException {
528         long position = channel.size();
529
530         if (!hasFrom_Line(message)) {
531             // if not first message add required newlines..
532 if (position > 0) {
533                 channel.write(encoder.encode(CharBuffer.wrap("\n\n")), channel.size());
534             }
535             channel.write(encoder.encode(CharBuffer.wrap(DEFAULT_FROM__LINE)), channel.size());
536         }
537
538         channel.write(encoder.encode(CharBuffer.wrap(message)), channel.size());
539
540         return position;
541     }
542
543     /**
544      * Purge the specified messages from the file.
545      * @param messageNumbers int[]
546      * @throws IOException
547      * @throws IllegalArgumentException
548      */
549     public void purge(int[] messageNumbers)
550          throws IOException ,IllegalArgumentException {
551
552          if (null==messageNumbers) return;
553          if (0==messageNumbers.length) return;
554
555          getMessagePositions();
556
557          if (null==messagePositions) return;
558          if (0==messagePositions.length) return;
559
560          final int total = messagePositions.length;
561          final int count = messageNumbers.length;
562          int size;
563          long start, next, append;
564          boolean perform;
565          ByteBuffer messageBuffer=null;
566          byte[] byBuffer = null;
567
568          log.debug("MboxFile.purge("+String.valueOf(count)+" of "+String.valueOf(total)+")");
569
570          getChannel();
571          long[] newPositions = null;
572          int newIndex = 0;
573
574          newPositions = new long[total-count];
575
576          append = 0;
577          for (int index=0; index<total; index++) {
578
579            perform = true;
580            for (int d=0; d<count; d++)
581              if (messageNumbers[d]==index) perform = false;
582
583            start = messagePositions[index];
584            if (index < total - 1) {
585              next = messagePositions[index+1];
586              size = (int) (next-messagePositions[index]);
587            }
588            else {
589              next = -1l;
590              size = (int) (channel.size()-messagePositions[index]);
591            }
592
593            if (perform) {
594                log.debug("FileChannel.map(MapMode.READ_WRITE,"+String.valueOf(next)+","+String.valueOf(size)+")");
595                newPositions[newIndex] = append;
596                newIndex ++;
597
598                if (start!=append) {
599                  messageBuffer = channel.map(FileChannel.MapMode.READ_WRITE,start, size);
600                  if (byBuffer == null)
601                    byBuffer = new byte[size];
602                  else if (byBuffer.length < size)
603                    byBuffer = new byte[size];
604                  messageBuffer.get(byBuffer, 0, size);
605                  channel.position(append);
606                  channel.write(ByteBuffer.wrap(byBuffer));
607                  messageBuffer.clear();
608                  messageBuffer = null;
609                } // fi (-1!=next)
610 append+=size;
611            }
612          } // next
613 log.debug("FileChannel.truncate("+String.valueOf(append)+")");
614          messageBuffer = null;
615          try {
616            channel.truncate(append);
617          } catch(IOException e){
618            log.debug("MBoxFile.purge() FileChannel.truncate() failed");
619          }
620
621          messagePositions = null;
622          messagePositions = newPositions;
623      } // purge
624
625     /**
626      * Close the mbox file and release any system resources.
627      * @throws IOException
628      */
629     public void close() throws IOException {
630         if (lock != null) {
631           lock.release();
632           lock = null;
633         }
634
635         if (channel != null) {
636             channel.close();
637             channel = null;
638         }
639     }
640
641     /**
642      * Indicates whether the specified CharSequence representation of
643      * a message contains a "From_" line.
644      * @param message a CharSequence representing a message
645      * @return true if a "From_" line is found, otherwise false
646      */
647     private boolean hasFrom_Line(CharSequence message) {
648         return Pattern.compile(FROM__PREFIX + ".*", Pattern.DOTALL).matcher(message).matches();
649     }
650 }
651

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags