Java > Open Source Codes > org > dspace > content > packager > AbstractMETSIngester


1 /*
2  * AbstractMETSIngester
3  *
4  * Version: $Revision: 1.1 $
5  *
6  * Date: $Date: 2006/03/17 00:04:38 $
7  *
8  * Copyright (c) 2002-2005, Hewlett-Packard Company and Massachusetts
9  * Institute of Technology. All rights reserved.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions are
13  * met:
14  *
15  * - Redistributions of source code must retain the above copyright
16  * notice, this list of conditions and the following disclaimer.
17  *
18  * - Redistributions in binary form must reproduce the above copyright
19  * notice, this list of conditions and the following disclaimer in the
20  * documentation and/or other materials provided with the distribution.
21  *
22  * - Neither the name of the Hewlett-Packard Company nor the name of the
23  * Massachusetts Institute of Technology nor the names of their
24  * contributors may be used to endorse or promote products derived from
25  * this software without specific prior written permission.
26  *
27  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
32  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
33  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
34  * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
35  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
36  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
37  * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
38  * DAMAGE.
39  */
40
41 package org.dspace.content.packager;
42
43 import java.io.IOException ;
44 import java.io.InputStream ;
45 import java.sql.SQLException ;
46 import java.util.HashMap ;
47 import java.util.HashSet ;
48 import java.util.Iterator ;
49 import java.util.List ;
50 import java.util.Map ;
51 import java.util.Set ;
52 import java.util.zip.ZipEntry ;
53 import java.util.zip.ZipInputStream ;
54
55 import org.apache.log4j.Logger;
56 import org.dspace.app.mediafilter.MediaFilter;
57 import org.dspace.authorize.AuthorizeException;
58 import org.dspace.content.Bitstream;
59 import org.dspace.content.BitstreamFormat;
60 import org.dspace.content.Bundle;
61 import org.dspace.content.Collection;
62 import org.dspace.content.FormatIdentifier;
63 import org.dspace.content.Item;
64 import org.dspace.content.WorkspaceItem;
65 import org.dspace.content.crosswalk.CrosswalkException;
66 import org.dspace.content.crosswalk.MetadataValidationException;
67 import org.dspace.core.ConfigurationManager;
68 import org.dspace.core.Constants;
69 import org.dspace.core.Context;
70 import org.dspace.core.LogManager;
71 import org.dspace.core.PluginManager;
72 import org.jdom.Element;
73
74 /**
75  * Base class for package ingester of
76  * METS (Metadata Encoding & Transmission Standard) Package.<br>
77  * See <a HREF="http://www.loc.gov/standards/mets/">http://www.loc.gov/standards/mets/</a>
78  * <p>
79  * This is a generic packager framework intended to be subclassed to create
80  * ingesters for more specific METS "profiles". METS is an
81  * abstract and flexible framework that can encompass many
82  * different kinds of metadata and inner package structures.
83  * <p>
84  * <b>Configuration:</b>
85  * If the property <code>mets.submission.preserveManifest</code> is <em>true</em>,
86  * the METS manifest itself is preserved in a bitstream named
87  * <code>mets.xml</code> in the <code>METADATA</code> bundle. If it is
88  * <em>false</em> (the default), the manifest is discarded after ingestion.
89  *
90  * @author Larry Stone
91  * @version $Revision: 1.1 $
92  * @see org.dspace.content.packager.METSManifest
93  */
94 public abstract class AbstractMETSIngester
95        implements PackageIngester
96 {
97     /** log4j category */
98     private static Logger log = Logger.getLogger(AbstractMETSIngester.class);
99
100     /** Filename of manifest, relative to package toplevel. */
101     public static final String MANIFEST_FILE = "mets.xml";
102
103     // bitstream format name of magic METS SIP format..
104 private static final String MANIFEST_BITSTREAM_FORMAT =
105             "DSpace METS SIP";
106
107     // value of mets.submission.preserveManifest config key
108 private static final boolean preserveManifest =
109         ConfigurationManager.getBooleanProperty("mets.submission.preserveManifest", false);
110
111     /**
112      * An instance of MdrefManager holds the state needed to
113      * retrieve the contents (or bitstream corresponding to) an
114      * external metadata stream referenced by an <code>mdRef</code>
115      * element in the METS manifest.
116      * <p>
117      * Initialize it with the DSpace Bundle containing all of the
118      * metadata bitstreams. Match an mdRef by finding the bitstream
119      * with the same name.
120      */
121     protected class MdrefManager
122         implements METSManifest.Mdref
123     {
124     private Bundle mdBundle = null;
125
126         // constructor initializes metadata bundle.
127 private MdrefManager(Bundle mdBundle)
128         {
129             super();
130             this.mdBundle = mdBundle;
131         }
132
133         /**
134          * Find the local Bitstream referenced in
135          * an <code>mdRef</code> element.
136          * @param mdref the METS mdRef element to locate the bitstream for.
137          * @return bitstream or null if none found.
138          */
139         public Bitstream getBitstreamForMdRef(Element mdref)
140             throws MetadataValidationException, IOException , SQLException , AuthorizeException
141         {
142             String path = METSManifest.getFileName(mdref);
143             if (mdBundle == null)
144                 throw new MetadataValidationException("Failed referencing mdRef element, because there were no metadata files.");
145             return mdBundle.getBitstreamByName(path);
146         }
147          
148         /**
149          * Make the contents of an external resource mentioned in
150          * an <code>mdRef</code> element available as an <code>InputStream</code>.
151          * See the <code>METSManifest.MdRef</code> interface for details.
152          * @param mdref the METS mdRef element to locate the input for.
153          * @return the input stream of its content.
154          */
155         public InputStream getInputStream(Element mdref)
156             throws MetadataValidationException, IOException , SQLException , AuthorizeException
157         {
158             Bitstream mdbs = getBitstreamForMdRef(mdref);
159             if (mdbs == null)
160                 throw new MetadataValidationException("Failed dereferencing bitstream for mdRef element="+mdref.toString());
161             return mdbs.retrieve();
162         }
163     }
164
165     /**
166      * Create a new DSpace item out of a METS content package.
167      * All contents are dictated by the METS manifest.
168      * Package is a ZIP archive, all files relative to top level
169      * and the manifest (as per spec) in mets.xml.
170      *
171      * @param context - DSpace context.
172      * @param collection - collection under which to create new item.
173      * @param pkg - input stream containing package to ingest.
174      * @param license - may be null, which takes default license.
175      * @return workspace item created by ingest.
176      * @throws PackageValidationException if package is unacceptable or there is
177      * a fatal error turning it into an Item.
178      */
179     public WorkspaceItem ingest(Context context, Collection collection,
180                                 InputStream pkg, PackageParameters params,
181                                 String license)
182         throws PackageValidationException, CrosswalkException,
183                AuthorizeException, SQLException , IOException
184     {
185         ZipInputStream zip = new ZipInputStream (pkg);
186         HashMap fileIdToBitstream = new HashMap ();
187         WorkspaceItem wi = null;
188         boolean success = false;
189         HashSet packageFiles = new HashSet ();
190
191         boolean validate = params.getBooleanProperty("validate", true);
192
193         try
194         {
195             /* 1. Read all the files in the Zip into bitstreams first,
196              * because we only get to take one pass through a Zip input
197              * stream. Give them temporary bitstream names corresponding
198              * to the same names they had in the Zip, since those MUST
199              * match the URL references in <Flocat> and <mdRef> elements.
200              */
201             METSManifest manifest = null;
202             wi = WorkspaceItem.create(context, collection, false);
203             Item item = wi.getItem();
204             Bundle contentBundle = item.createBundle(Constants.CONTENT_BUNDLE_NAME);
205             Bundle mdBundle = null;
206             ZipEntry ze;
207             while ((ze = zip.getNextEntry()) != null)
208             {
209                 if (ze.isDirectory())
210                     continue;
211                 Bitstream bs = null;
212                 String fname = ze.getName();
213                 if (fname.equals(MANIFEST_FILE))
214                 {
215                     if (preserveManifest)
216                     {
217                         mdBundle = item.createBundle(Constants.METADATA_BUNDLE_NAME);
218                         bs = mdBundle.createBitstream(new PackageUtils.UnclosableInputStream(zip));
219                         bs.setName(fname);
220                         bs.setSource(fname);
221
222                         // Get magic bitstream format to identify manifest.
223 BitstreamFormat manifestFormat = null;
224                         manifestFormat = PackageUtils.findOrCreateBitstreamFormat(context,
225                              MANIFEST_BITSTREAM_FORMAT, "application/xml",
226                              MANIFEST_BITSTREAM_FORMAT+" package manifest");
227                         bs.setFormat(manifestFormat);
228
229                         manifest = METSManifest.create(bs.retrieve(), validate);
230                     }
231                     else
232                     {
233                         manifest = METSManifest.create(new PackageUtils.UnclosableInputStream(zip), validate);
234                         continue;
235                     }
236                 }
237                 else
238                 {
239                     bs = contentBundle.createBitstream(new PackageUtils.UnclosableInputStream(zip));
240                     bs.setSource(fname);
241                     bs.setName(fname);
242                 }
243                 packageFiles.add(fname);
244                 bs.setSource(fname);
245                 bs.update();
246             }
247             zip.close();
248
249             if (manifest == null)
250                 throw new PackageValidationException("No METS Manifest found (filename="+MANIFEST_FILE+"). Package is unacceptable.");
251
252             // initial sanity checks on manifest (in subclass)
253 checkManifest(manifest);
254
255             /* 2. Grovel a file list out of METS Manifest and compare
256              * it to the files in package, as an integrity test.
257              */
258             List manifestContentFiles = manifest.getContentFiles();
259
260             // Compare manifest files with the ones found in package:
261 // a. Start with content files (mentioned in <fileGrp>s)
262 HashSet missingFiles = new HashSet ();
263             for (Iterator mi = manifestContentFiles.iterator(); mi.hasNext(); )
264             {
265                 // First locate corresponding Bitstream and make
266 // map of Bitstream to <file> ID.
267 Element mfile = (Element)mi.next();
268                 String mfileId = mfile.getAttributeValue("ID");
269                 if (mfileId == null)
270                     throw new PackageValidationException("Invalid METS Manifest: file element without ID attribute.");
271                 String path = METSManifest.getFileName(mfile);
272                 Bitstream bs = contentBundle.getBitstreamByName(path);
273                 if (bs == null)
274                 {
275                     log.warn("Cannot find bitstream for filename=\""+path+
276                              "\", skipping it..may cause problems later.");
277                     missingFiles.add(path);
278                 }
279                 else
280                 {
281                     fileIdToBitstream.put(mfileId, bs);
282
283                     // Now that we're done using Name to match to <file>,
284 // set default bitstream Name to last path element;
285 // Zip entries all have '/' pathname separators
286 // NOTE: set default here, hopefully crosswalk of
287 // a bitstream techMD section will override it.
288 String fname = bs.getName();
289                     int lastSlash = fname.lastIndexOf('/');
290                     if (lastSlash >= 0 && lastSlash+1 < fname.length())
291                         bs.setName(fname.substring(lastSlash+1));
292
293                     // Set Default bitstream format:
294 // 1. attempt to guess from MIME type
295 // 2. if that fails, guess from "name" extension.
296 String mimeType = mfile.getAttributeValue("MIMETYPE");
297                     BitstreamFormat bf = (mimeType == null) ? null :
298                             BitstreamFormat.findByMIMEType(context, mimeType);
299                     if (bf == null)
300                         bf = FormatIdentifier.guessFormat(context, bs);
301                     bs.setFormat(bf);
302
303                     // if this bitstream belongs in another Bundle, move it:
304 String bundleName = manifest.getBundleName(mfile);
305                     if (!bundleName.equals(Constants.CONTENT_BUNDLE_NAME))
306                     {
307                         Bundle bn;
308                         Bundle bns[] = item.getBundles(bundleName);
309                         if (bns != null && bns.length > 0)
310                             bn = bns[0];
311                         else
312                             bn = item.createBundle(bundleName);
313                         bn.addBitstream(bs);
314                         contentBundle.removeBitstream(bs);
315                     }
316
317                     // finally, build compare lists by deleting matches.
318 if (packageFiles.contains(path))
319                         packageFiles.remove(path);
320                     else
321                         missingFiles.add(path);
322                 }
323             }
324
325             // b. Process files mentioned in <mdRef>s - check and move
326 // to METADATA bundle.
327 for (Iterator mi = manifest.getMdFiles().iterator(); mi.hasNext(); )
328             {
329                 Element mdref = (Element)mi.next();
330                 String path = METSManifest.getFileName(mdref);
331
332                 // finally, build compare lists by deleting matches.
333 if (packageFiles.contains(path))
334                     packageFiles.remove(path);
335                 else
336                     missingFiles.add(path);
337
338                 // if there is a bitstream with that name in Content, move
339 // it to the Metadata bundle:
340 Bitstream mdbs = contentBundle.getBitstreamByName(path);
341                 if (mdbs != null)
342                 {
343                     if (mdBundle == null)
344                         mdBundle = item.createBundle(Constants.METADATA_BUNDLE_NAME);
345                     mdBundle.addBitstream(mdbs);
346                     contentBundle.removeBitstream(mdbs);
347                 }
348             }
349
350             // KLUDGE: make sure Manifest file doesn't get flagged as missing
351 // or extra, since it won't be mentioned in the manifest.
352 if (packageFiles.contains(MANIFEST_FILE))
353                 packageFiles.remove(MANIFEST_FILE);
354
355             // Give subclass a chance to refine the lists of in-package
356 // and missing files, delete extraneous files, etc.
357 checkPackageFiles(packageFiles, missingFiles, manifest);
358
359             // Any discrepency in file lists is a fatal error:
360 if (!(packageFiles.isEmpty() && missingFiles.isEmpty()))
361             {
362                 StringBuffer msg = new StringBuffer ("Package is unacceptable: contents do not match manifest.");
363                 if (!missingFiles.isEmpty())
364                 {
365                     msg.append("\nPackage is missing these files listed in Manifest:");
366                     for (Iterator mi = missingFiles.iterator(); mi.hasNext(); )
367                         msg.append("\n\t"+(String )mi.next());
368                 }
369                 if (!packageFiles.isEmpty())
370                 {
371                     msg.append("\nPackage contains extra files NOT in manifest:");
372                     for (Iterator mi = packageFiles.iterator(); mi.hasNext(); )
373                         msg.append("\n\t"+(String )mi.next());
374                 }
375                 throw new PackageValidationException(msg.toString());
376             }
377
378             /* 3. crosswalk the metadata
379              */
380             // get mdref'd streams from "callback" object.
381 MdrefManager callback = new MdrefManager(mdBundle);
382
383             chooseItemDmd(context, item, manifest, callback, manifest.getItemDmds());
384
385             // crosswalk content bitstreams too.
386 for (Iterator ei = fileIdToBitstream.entrySet().iterator();
387                  ei.hasNext();)
388             {
389                 Map.Entry ee = (Map.Entry )ei.next();
390                 manifest.crosswalkBitstream(context, (Bitstream)ee.getValue(),
391                                         (String )ee.getKey(), callback);
392             }
393
394             // Take a second pass over files to correct names of derived files
395 // (e.g. thumbnails, extracted text) to what DSpace expects:
396 for (Iterator mi = manifestContentFiles.iterator(); mi.hasNext(); )
397             {
398                 Element mfile = (Element)mi.next();
399                 String bundleName = manifest.getBundleName(mfile);
400                 if (!bundleName.equals(Constants.CONTENT_BUNDLE_NAME))
401                 {
402                     Element origFile = manifest.getOriginalFile(mfile);
403                     if (origFile != null)
404                     {
405                         String ofileId = origFile.getAttributeValue("ID");
406                         Bitstream obs = (Bitstream)fileIdToBitstream.get(ofileId);
407                         String newName = makeDerivedFilename(bundleName, obs.getName());
408                         if (newName != null)
409                         {
410                             String mfileId = mfile.getAttributeValue("ID");
411                             Bitstream bs = (Bitstream)fileIdToBitstream.get(mfileId);
412                             bs.setName(newName);
413                             bs.update();
414                         }
415                     }
416                 }
417             }
418
419             // Sanity-check the resulting metadata on the Item:
420 PackageUtils.checkMetadata(item);
421
422             /* 4. Set primary bitstream; same Bundle
423              */
424             Element pbsFile = manifest.getPrimaryBitstream();
425             if (pbsFile != null)
426             {
427                 Bitstream pbs = (Bitstream)fileIdToBitstream.get(pbsFile.getAttributeValue("ID"));
428                 if (pbs == null)
429                     log.error("Got Primary Bitstream file ID="+pbsFile.getAttributeValue("ID")+
430                              ", but found no corresponding bitstream.");
431                 else
432                 {
433                     Bundle bn[] = pbs.getBundles();
434                     if (bn.length > 0)
435                         bn[0].setPrimaryBitstreamID(pbs.getID());
436                     else
437                         log.error("Sanity check, got primary bitstream without any parent bundle.");
438                 }
439             }
440
441             // have subclass manage license since it may be extra package file.
442 addLicense(context, collection, item, manifest, callback, license );
443
444             // subclass hook for final checks and rearrangements
445 finishItem(context, item);
446
447             // commit any changes to bundles
448 Bundle allBn[] = item.getBundles();
449             for (int i = 0; i < allBn.length; ++i)
450             {
451                 allBn[i].update();
452             }
453
454             wi.update();
455             success = true;
456             log.info(LogManager.getHeader(context, "ingest",
457                 "Created new Item, db ID="+String.valueOf(item.getID())+
458                 ", WorkspaceItem ID="+String.valueOf(wi.getID())));
459             return wi;
460         }
461         catch (SQLException se)
462         {
463             // disable attempt to delete the workspace object, since
464 // database may have suffered a fatal error and the
465 // transaction rollback will get rid of it anyway.
466 wi = null;
467
468             // Pass this exception on to the next handler.
469 throw se;
470         }
471         finally
472         {
473             // kill item (which also deletes bundles, bitstreams) if ingest fails
474 if (!success && wi != null)
475                 wi.deleteAll();
476         }
477     }
478
479     /**
480      * XXX FIXME Replace is not implemented yet.
481      */
482     public Item replace(Context ctx, Item item, InputStream pckage, PackageParameters params)
483         throws PackageException, UnsupportedOperationException ,
484                CrosswalkException, AuthorizeException,
485                SQLException , IOException
486     {
487         throw new UnsupportedOperationException ("The replace operation is not implemented.");
488     }
489
490     // return name of derived file as if MediaFilter created it, or null
491 private String makeDerivedFilename(String bundleName, String origName)
492     {
493         // get the MediaFilter that would create this bundle:
494 String mfNames[] = PluginManager.getAllPluginNames(MediaFilter.class);
495
496         for (int i = 0; i < mfNames.length; ++i)
497         {
498             MediaFilter mf = (MediaFilter)PluginManager.getNamedPlugin(MediaFilter.class, mfNames[i]);
499             if (bundleName.equals(mf.getBundleName()))
500                 return mf.getFilteredName(origName);
501         }
502         return null;
503     }
504
505     /**
506      * Profile-specific tests to validate manifest. The implementation
507      * can access the METS document through the <code>manifest</code>
508      * variable, an instance of <code>METSManifest</code>.
509      * @throws MetadataValidationException if there is a fatal problem with the METS document's conformance to the expected profile.
510      */
511     abstract void checkManifest(METSManifest manifest)
512         throws MetadataValidationException;
513
514     /**
515      * Hook for subclass to modify the test of the package's
516      * integrity, and add other tests. E.g. evaluate a PGP signature of
517      * the manifest in a separate file.
518      * <p>
519      * The <code>packageFiles</code> contains "extra" files that were in
520      * the package but were not referenced by the METS manifest (either as
521      * content or metadata (mdRefs)).
522      * The implementation of this method should look for any "extra" files
523      * uses (e.g. a checksum or cryptographic signature for the manifest
524      * itself) and remove them from the Set.
525      * <p>
526      * The <code>missingFiles</code> set is for
527      * any files
528      * referenced by the manifest but not found in the package.
529      * The implementation can check it for "false positives", or add
530      * other missing files it knows of.
531      * <p>
532      * If either of the Sets <code>missingFiles</code>
533      * or <code>packageFiles</code>
534      * is not empty, the ingest will fail.
535      *
536      * @param packageFiles files in package but not referenced by METS
537      * @param missingFiles files referenced by manifest but not in package
538      *
539      */
540     abstract public void checkPackageFiles(Set packageFiles, Set missingFiles,
541                                            METSManifest manifest)
542         throws PackageValidationException, CrosswalkException;
543
544     /**
545      * Select the <code>dmdSec</code> element(s) to apply to the
546      * Item. The implementation is responsible for choosing which
547      * (if any) of the metadata sections to crosswalk to get the
548      * descriptive metadata for the item being ingested. It is
549      * responsible for calling the crosswalk, using the manifest's helper
550      * i.e. <code>manifest.crosswalkItem(context,item,dmdElement,callback);</code>
551      * (The final argument is a reference to itself since the
552      * class also implements the <code>METSManifest.MdRef</code> interface
553      * to fetch package files referenced by mdRef elements.)
554      * <p>
555      * Note that <code>item</code> and <code>manifest</code> are available
556      * as protected fields from the superclass.
557      *
558      * @param context the DSpace context
559      * @param dmds array of Elements, each a METS dmdSec that applies to the Item as a whole.
560      *
561      */
562     abstract public void chooseItemDmd(Context context, Item item,
563                                        METSManifest manifest, MdrefManager cb,
564                                        Element dmds[])
565         throws CrosswalkException,
566                AuthorizeException, SQLException , IOException ;
567
568     /**
569      * Add license(s) to Item based on contents of METS and other policies.
570      * The implementation of this method controls exactly what licenses
571      * are added to the new item, including the DSpace deposit license.
572      * It is given the collection (which is the source of a default deposit
573      * license), an optional user-supplied deposit license (in the form of
574      * a String), and the METS manifest. It should invoke
575      * <code>manifest.getItemRightsMD()</code> to get an array of
576      * <code>rightsMd</code> elements which might contain other license
577      * information of interest, e.g. a Creative Commons license.
578      * <p>
579      * This framework does not add any licenses by default.
580      *
581      * @param context the DSpace context
582      * @param collection DSpace Collection to which the item is being submitted.
583      * @param license optional user-supplied Deposit License text (may be null)
584      */
585     abstract public void addLicense(Context context, Collection collection,
586                                     Item item, METSManifest manifest,
587                                     MdrefManager callback, String license)
588         throws PackageValidationException, CrosswalkException,
589                AuthorizeException, SQLException , IOException ;
590
591     /**
592      * Hook for final "finishing" operations on the new Item.
593      * This method is called when the new Item is otherwise complete and
594      * ready to be returned. The implementation should use this
595      * opportunity to make whatever final checks and modifications are
596      * necessary.
597      *
598      * @param context the DSpace context
599      */
600     abstract public void finishItem(Context context, Item item)
601         throws PackageValidationException, CrosswalkException,
602          AuthorizeException, SQLException , IOException ;
603
604 }
605

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags