1 4 package com.openedit.archive.cumulus; 5 6 import java.awt.image.BufferedImage ; 7 import java.io.ByteArrayInputStream ; 8 import java.io.File ; 9 import java.io.FileOutputStream ; 10 import java.text.DateFormat ; 11 import java.text.SimpleDateFormat ; 12 import java.util.Date ; 13 import java.util.HashMap ; 14 import java.util.Iterator ; 15 import java.util.List ; 16 import java.util.Map ; 17 18 import javax.imageio.ImageIO ; 19 20 import org.apache.commons.collections.map.ListOrderedMap; 21 import org.apache.commons.logging.Log; 22 import org.apache.commons.logging.LogFactory; 23 24 import com.canto.cumulus.AssetReferencePart; 25 import com.canto.cumulus.CatalogCollection; 26 import com.canto.cumulus.Categories; 27 import com.canto.cumulus.Cumulus; 28 import com.canto.cumulus.CumulusException; 29 import com.canto.cumulus.Field; 30 import com.canto.cumulus.Fields; 31 import com.canto.cumulus.GUID; 32 import com.canto.cumulus.Pixmap; 33 import com.canto.cumulus.Record; 34 import com.canto.cumulus.Records; 35 import com.canto.cumulus.ServerCatalog; 36 import com.openedit.OpenEditException; 37 import com.openedit.archive.Archive; 38 import com.openedit.config.Configuration; 39 import com.openedit.page.Page; 40 import com.openedit.page.manage.PageManager; 41 import com.openedit.store.Category; 42 import com.openedit.store.Product; 43 import com.openedit.store.ProductPathFinder; 44 import com.openedit.store.Store; 45 import com.openedit.store.StoreException; 46 import com.openedit.store.products.Detail; 47 import com.openedit.store.products.PropertyDetails; 48 import com.openedit.util.FileUtils; 49 import com.openedit.util.TaskRunner; 50 51 52 public class CumulusSyncConverter extends BaseCumulusConvert 53 { 54 private static final Log log = LogFactory.getLog(CumulusSyncConverter.class); 55 56 protected SimpleDateFormat fieldCumulusFormat = new SimpleDateFormat ("M/d/yyyy H:mm:ss a"); 57 protected SimpleDateFormat fieldCumulusSyncFormat = new SimpleDateFormat ("EEE MMM dd HH:mm:ss z yyyy"); 60 protected TaskRunner fieldOddRunner; 62 protected TaskRunner fieldEvenRunner; 63 64 public CumulusSyncConverter() 65 { 66 67 } 68 public synchronized boolean convert(Store inStore, List inStatusLog) throws Exception 69 { 70 Archive archive = new Archive(); 71 archive.setStore( inStore ); 72 archive.setPageManager(getPageManager()); 73 74 Page server = getPageManager().getPage("/WEB-INF/cumulus.xml"); 75 if ( server.exists()) 76 { 77 log.info("sync to cumulus " + server); 78 ConvertStatus status = new ConvertStatus(); 79 status.setLog(inStatusLog); 80 syncToCumulus(archive, status); 81 return status.isReindex(); 82 } 83 else 84 { 85 log.error("No such file /WEB-INF/cumulus.xml" ); 86 } 87 return false; 88 } 89 protected void syncToCumulus(final Archive inArchive, ConvertStatus inStatus) throws Exception 90 { 91 for (int i = 0; i < getServerCatalogs().countServerCatalogs(); i++) 92 { 93 ServerCatalog cat = getServerCatalogs().getServerCatalog(i); 95 if ( cat.getIsPublishedToInternet() ) 96 { 97 ConvertTask task = new ConvertTask(); 98 task.setArchive(inArchive); 99 task.setStatus(inStatus); 100 task.setConverter(this); 101 task.setServerCatalog(cat); 102 if( i % 2 == 0) 103 { 104 getEvenRunner().add(task); 105 } 106 else 107 { 108 getOddRunner().add(task); 109 } 110 } 111 else 112 { 113 log.info("Skip unpublished catalog: " + cat.getName() ); 114 inStatus.add("Skipped unpublished catalog " + cat.getName()); 115 } 116 } 117 118 try 120 { 121 waitOn(getEvenRunner(), inStatus); 122 waitOn(getOddRunner(), inStatus); 123 } 124 catch ( Exception ex) 125 { 126 getEvenRunner().getQueue().cancel(); 127 getOddRunner().getQueue().cancel(); 128 getEvenRunner().setQueue(null); 129 getOddRunner().setQueue(null); 130 throw ex; 131 } 132 log.info("Completed Cumulus Sync"); 133 } 134 protected void waitOn(TaskRunner inOddRunner, ConvertStatus inStatus) throws Exception 135 { 136 while( inOddRunner.getCount() > 0 ) 137 { 138 if( inStatus.getException() != null) 139 { 140 throw inStatus.getException(); 141 } 142 try 143 { 144 synchronized (inOddRunner.getNotify()) 145 { 146 inOddRunner.getNotify().wait(60000); 147 } 148 } 149 catch (InterruptedException e) 150 { 151 } 153 } 154 155 } 156 protected void syncOneCatalog(Configuration setting, ServerCatalog cat, Archive inArchive, ConvertStatus inStatusLog) throws Exception 157 { 158 long total = 0; 159 long errortotal = 0; 160 long maxerrors = 0; 161 String errorLimit = inArchive.getSettings().getChildValue("errorlimit"); 162 if (errorLimit != null) 163 { 164 maxerrors = Long.parseLong( errorLimit ); 165 } 166 long start = System.currentTimeMillis(); 167 168 Date newdate = new Date (); 169 170 CatalogCollection col = cat.open(); 171 172 filterResults( col, setting); 173 Records records = col.getRecords(); 175 int c = records.countRecords(); 176 if (c > 100) { 178 inStatusLog.setReindex(true); 179 } 180 181 String colId = extractId(cat.getName()); 182 if ( c > 0 ) 183 { 184 Category parent = new Category( colId, cat.getName() ); 185 186 com.canto.cumulus.Category startCat = null; 187 188 String treeroot = inArchive.getSettings().getChildValue("treeroot"); 189 if( treeroot != null ) 190 { 191 com.canto.cumulus.Category root = col.getCategories().getCategory(treeroot); 192 startCat = findFirstValidCategory(root, inArchive); 193 } 194 else 195 { 196 startCat = findFirstValidCategory(col.getCatalogRootCategory(), inArchive); 197 } 198 if( startCat == null) 199 { 200 log.info("Skipping catalog " + cat.getName()); 201 inStatusLog.add("Skipping catalog " + cat.getName()); 202 return; 203 } 204 log.info("Adding " + cat.getName() + " to " + inArchive.getCatalogHome() + "/categories/categories.xml"); 205 inStatusLog.getLog().add("Adding " + cat.getName() + " to " + inArchive.getCatalogHome() + "/categories/categories.xml"); 206 syncCatalogTree( startCat, parent , inArchive); 207 inArchive.getStore().getCatalogArchive().getRootCatalog().addChild(parent); 208 inArchive.getStore().getCatalogArchive().saveCatalogs(); 209 } 210 int count = 0; 211 log.info("Begin processing " + c + " records in " + cat.getName() ); 212 inStatusLog.add("Begin processing " + c + " records in " + cat.getName() ); 213 214 Map guidToname = ListOrderedMap.decorate(new HashMap ()); 215 for (int j = 0; j < c; j++) 216 { 217 try 218 { 219 updateProduct(cat.getName(), records.getRecord(j), guidToname, inArchive, inStatusLog); 220 } 221 catch (Exception e) 222 { 223 errortotal++; 224 if (errortotal > maxerrors) 225 { 226 if( !(e instanceof OpenEditException)) 227 { 228 throw new OpenEditException(e); 229 } 230 else 231 { 232 throw (OpenEditException)e; 233 } 234 } 235 else 236 { 237 log.info("Exception thrown from updateProduct: " + e); 238 inStatusLog.add("Exception thrown from updateProduct: " + e); 239 } 240 241 } 242 total++; 243 count++; 244 if( count == 50) 245 { 246 double time = (double)(System.currentTimeMillis() - start)/1000D/60D; 247 log.info("Populated " + total + " records from " + cat.getName() + " in " + total/time + " records/minute"); 248 count = 0; 249 } 250 } 251 col.close(); 252 setting.setAttribute("lastchecked",SimpleDateFormat.getDateTimeInstance().format(newdate)); 253 inArchive.saveSettings(); 254 double time = (double)(System.currentTimeMillis() - start)/1000D/60D; 255 log.info("Finished " + c + " products and catalog:"+ cat.getName() + " in " + time + " minutes" ); 256 inStatusLog.add("Finished " + c + " products and catalog:"+ cat.getName() + " in " + time + " minutes" ); 257 } 258 protected void syncCatalogTree( com.canto.cumulus.Category inCategory, Category inParent, Archive inArchive ) throws Exception 259 { 260 copyOldData(inCategory, inParent, inArchive); 261 262 Categories subs = inCategory.getSubCategories(); 263 for (int i = 0; i < subs.countCategories(); i++) 264 { 265 com.canto.cumulus.Category sub = subs.getCategory(i); 266 if( i == 0) 268 { 269 log.debug("reading in: " + inCategory.getName() ); 270 } 271 272 if( inArchive.ignoreCategory(sub.getName()) ) 273 { 274 continue; 276 } 277 if( inArchive.hideCategory(sub.getName()) ) 278 { 279 syncCatalogTree(sub, inParent , inArchive); 280 continue; 281 } 282 String id = extractId( sub.getName() ); 283 String fullId = inParent.getId() + "_" + id; 284 Category child = inParent.getChild(fullId); 285 if ( child == null) 286 { 287 child = inParent.addChild(new Category(fullId, sub.getName() )); 288 } 289 syncCatalogTree(sub, child, inArchive ); 292 } 293 } 294 protected com.canto.cumulus.Category findFirstValidCategory(com.canto.cumulus.Category inCategory,Archive inArchive ) 295 { 296 if( inArchive.ignoreCategory(inCategory.getName()) ) 297 { 298 return null; 299 } 300 if( inArchive.hideCategory(inCategory.getName()) ) 301 { 302 Categories subs = inCategory.getSubCategories(); 303 for (int i = 0; i < subs.countCategories(); i++) 304 { 305 com.canto.cumulus.Category sub = subs.getCategory(i); 306 com.canto.cumulus.Category cat = findFirstValidCategory(sub, inArchive); 307 if (cat != null) 308 { 309 return cat; 310 } 311 } 312 return null; } 314 return inCategory; 315 } 316 317 protected void copyOldData(com.canto.cumulus.Category inCat, Category inParent, Archive inArchive) throws Exception 318 { 319 PropertyDetails details = inArchive.getStore().getCatalogArchive().getCatalogDetails(); 320 if( details.getDetails().size() > 0) 321 { 322 Fields all = inCat.getFields(); 325 if( all == null ) 326 { 327 return; 328 } 329 if( details.getDetails().size() > 3) 330 { 331 int count = 0; 332 int total = all.countFields(); 333 for (int i = 0; i < total; i++) 334 { 335 Field field = all.getField(i); 336 Detail detail = details.getDetailByExternalId(field.getFieldDefinition().getName()); 337 if( detail != null) 338 { 339 count++; 340 if( detail.isStored() ) 341 { 342 copyField(inParent, inArchive, field, detail); 343 if( count > details.getDetails().size()) 344 { 345 break; } 347 } 348 } 349 } 350 } 351 else 352 { 353 for (Iterator iter = details.getDetails().iterator(); iter.hasNext();) 354 { 355 Detail detail = (Detail) iter.next(); 356 if( detail.isStored() ) 357 { 358 Field field = all.getField(detail.getExternalId()); 359 copyField(inParent, inArchive, field, detail); 360 } 361 } 362 } 363 } 366 Category oldCopy = inArchive.getStore().getCatalog(inParent.getId()); 367 if( oldCopy != null) 368 { 369 inParent.setProperties(oldCopy.getProperties()); } 371 } 372 private void copyField(Category inParent, Archive inArchive, Field field, Detail detail) throws CumulusException, Exception , StoreException 373 { 374 375 if( detail.isImage() ) 376 { 377 File img = new File ( inArchive.getStore().getRootDirectory(), inArchive.getStore().getStoreHome() + "/categories/images/thumb/" + inParent.getId() + "-100.jpg"); 378 if( !img.exists() && detail.getExternalId() != null ) 379 { 380 if( field.hasValue()) 381 { 382 saveThumbnail( field, img); 383 } 384 } 385 } 386 else 387 { 388 String id = detail.getExternalId(); 389 if( id != null) 390 { 391 try 392 { 393 if( field.hasValue() && detail.getId().equalsIgnoreCase("description") ) 394 { 395 inParent.setShortDescription( getValue(field) ); 396 } 397 } 398 catch ( Exception ex) 399 { 400 throw new StoreException(ex); 401 } 402 } 403 } 404 } 405 protected void saveThumbnail(Field inField, File inPath) throws Exception 406 { 407 if ( inField.hasValue()) 408 { 409 byte[] raw = (byte[]) inField.getValue(); 410 Pixmap nuPixmap = new Pixmap(raw); 411 byte[] jpegData = nuPixmap.getAsJPEG(1500); 412 inPath.getParentFile().mkdirs(); 413 414 ByteArrayInputStream in = new ByteArrayInputStream (jpegData); 415 BufferedImage image = ImageIO.read(in); 416 417 double change = 1; 418 419 int big = Math.max(image.getWidth(), image.getHeight()); 420 if( big > 150) 421 { 422 change = (double)150/(double)big; 423 int width = (int)Math.round( (double)image.getWidth() * change ); 424 int height = (int)Math.round( (double)image.getHeight() * change ); 425 426 ImageToConvert convert = new ImageToConvert(); 427 convert.setWidth(width); 428 convert.setHeight(height); 429 convert.setOutput(inPath); 430 convert.setImage(image); 431 convert.run(); 433 } 434 else 435 { 436 FileOutputStream out = null; 437 try 438 { 439 out = new FileOutputStream (inPath); 440 out.write(jpegData); 441 } 442 finally 443 { 444 FileUtils.safeClose(out); 445 } 446 447 } 448 } 449 } 450 protected void filterResults(CatalogCollection inCol, Configuration catconfig ) throws Exception 451 { 452 String lastChecked = catconfig.getAttribute("lastchecked"); 459 Date last = null; 460 if ( lastChecked != null) 461 { 462 last = DateFormat.getDateTimeInstance().parse(lastChecked); 463 String sQuery = "Record Modification Date after " + getCumulusFormat().format(last); 464 log.info(sQuery); 465 inCol.find(sQuery, Cumulus.FindNew); 466 } 467 } 468 469 470 protected void updateProduct(String inCollId,Record inRecord , Map inGuidToName, Archive inArchive, ConvertStatus inStatus) throws Exception 471 { 472 Fields fields = inRecord.getFields(); 473 Product newproduct = new Product(); 475 PropertyDetails details = inArchive.getStore().getProductArchive().getPropertyDetails(); 477 478 if( inGuidToName.size() == 0) 479 { 480 int total = fields.countFields(); 483 for (int i = 0; i < total; i++) { 486 Field field = fields.getField(i); 487 String name = field.getFieldDefinition().getName(); 488 inGuidToName.put(name,new Integer (i)); 489 inGuidToName.put( field.getFieldUID().toString(), new Integer (i)); 490 } 491 } 492 Integer loc = (Integer )inGuidToName.get(GUID.UID_REC_ASSET_NAME.toString()); 493 String aname = getValue( fields.getField(loc.intValue()) ); 494 newproduct.setName(aname); 495 loc = (Integer )inGuidToName.get(GUID.UID_REC_ASSET_IDENTIFIER.toString()); 496 String assid = getValue(fields.getField(loc.intValue()) ); 497 for (Iterator iterator = details.getDetails().iterator(); iterator.hasNext();) 498 { 499 Detail detail = (Detail) iterator.next(); 500 String external = detail.getExternalId(); 501 if( external == null) 502 { 503 continue; 504 } 505 if( detail.isStored() || detail.isIndex() || detail.isKeyword() ) 506 { 507 Field field = null; 508 Integer locin = (Integer )inGuidToName.get(external); 509 if( locin == null) 510 { 511 log.error("No such field in Cumulus " + external); 512 continue; 513 } 514 field = fields.getField(locin.intValue()); 515 copyProductField(newproduct, detail, field); 517 } 518 } 519 520 584 String id = extractProductId( newproduct.getName(), assid ); 585 newproduct.setId(id); 586 587 convertCategories( inRecord, newproduct, inArchive); 588 589 newproduct.addProperty("assetidentifier", assid); 590 newproduct.addProperty("cumulusid", inCollId + "_" + inRecord.getID() ); 591 for (int i = 0; i < inRecord.getAssetReference().countParts(); i++) 592 { 593 AssetReferencePart part = inRecord.getAssetReference().getPart(i); 594 if ( part.getName().equals("Windows")) 595 { 596 String path = part.getDisplayString(); 597 path = scrubChars(path); 598 newproduct.addProperty("originalpath",path); 599 newproduct.addKeyword(path); 600 601 boolean checkOriginalExists = Boolean.parseBoolean( inArchive.getSettings().getChildValue("checkoriginalexists") ); 602 if( checkOriginalExists ) 603 { 604 File originalFile = new File (path); 605 if (!originalFile.exists()) 606 { 607 throw new OpenEditException("Original file does not exist"); 608 } 609 } 610 break; 611 } 612 } 613 createImages(fields, newproduct, inGuidToName, inArchive); 615 inArchive.getStore().getProductArchive().saveProduct( newproduct ); 616 if( !inStatus.isReindex()) 617 { 618 inArchive.getStore().getStoreSearcher().updateIndex(newproduct); 620 } 621 inArchive.getStore().getProductArchive().clearProduct(newproduct); 622 623 } 624 private void copyProductField(Product newproduct, Detail detail, Field field) throws CumulusException 625 { 626 if( detail.isStored() || detail.isIndex() || detail.isKeyword() ) 627 { 628 if( !field.hasValue() ) 629 { 630 return; 631 } 633 String val = null; 634 if( detail.isDate() ) 635 { 636 Object date = (Object )field.getValue(); 637 if( date instanceof Date ) 638 { 639 try 640 { 641 val = detail.getDateFormat().format( date ); 642 } 643 catch ( Exception ex) 644 { 645 log.error("Invalid date format " + detail.getId() + " " + date + " in format " + detail.getDateFormat() ); 646 ex.printStackTrace(); 647 } 648 } 649 else 650 { 651 val = date.toString(); 652 } 653 } 654 else 655 { 656 val = getValue( field ); 657 } 658 if( val != null ) 659 { 660 newproduct.addProperty(detail.getId(),val); 661 } 662 } 663 } 664 665 protected void createImages(Fields inFields, Product inNewproduct, Map inIndex, Archive inArchive) throws Exception 666 { 667 ProductPathFinder finder = inArchive.getStore().getProductPathFinder(); 668 String loc = finder.idToPath(inNewproduct.getId()); 669 670 File thumb = new File ( inArchive.getStore().getRootDirectory(),inArchive.getStore().getStoreHome() + "/products/images/thumb/" + loc + ".jpg"); 671 if (!thumb.exists() ) 672 { 673 Integer indexloc = (Integer )inIndex.get(GUID.UID_REC_THUMBNAIL.toString() ); Field nuField = inFields.getField(indexloc.intValue()); 676 saveThumbnail(nuField, thumb); 677 } 678 File med = new File ( inArchive.getStore().getRootDirectory(),inArchive.getStore().getStoreHome() + "/products/images/medium/" + loc + ".jpg"); 679 if (!med.exists() ) 680 { 681 687 Integer indexloc = (Integer )inIndex.get(GUID.UID_REC_THUMBNAIL.toString() ); 703 Field nuField = inFields.getField(indexloc.intValue()); 704 if( nuField.hasValue() ) 705 { 706 Pixmap nuPixmap = new Pixmap((byte[]) nuField.getValue()); 707 byte[] jpegData = nuPixmap.getAsJPEG(); 708 med.getParentFile().mkdirs(); 709 FileOutputStream out = new FileOutputStream (med); 710 try 711 { 712 out.write(jpegData); 713 } 714 finally 715 { 716 FileUtils.safeClose(out); 717 } 718 } 719 } 720 } 721 protected void convertCategories(Record inRecord, Product inNewproduct, Archive inArchive ) throws Exception 722 { 723 Categories cats = inRecord.getCategories(); 724 726 for (int i = 0; i < cats.countCategories(); i++) 727 { 728 com.canto.cumulus.Category c = cats.getCategory(i); 729 730 StringBuffer fullId = new StringBuffer (); 732 boolean ignore = false; 733 while ( c != null ) 734 { 735 String name = c.getName(); 736 738 if( inArchive.ignoreCategory(name)) 740 { 741 ignore =true; 742 break; 743 } 744 if( !inArchive.hideCategory(name) ) 745 { 746 String fid = extractId(name); 747 if( fullId.length() > 0) 748 { 749 fullId.insert(0,"_"); 750 } 751 fullId.insert(0,fid); 752 } 753 c = c.getParentCategory(); 754 } 755 if ( ignore ) 756 { 757 continue; 758 } 759 String catId = fullId.toString(); 760 761 Category cat = inArchive.getStore().getCatalog( catId ); 762 if( cat == null) 763 { 764 log.error("Skipping catalog " + catId ); 765 throw new OpenEditException("Invalid category specified " + catId); 766 } 767 else 768 { 769 inNewproduct.addCatalog(cat); 770 inNewproduct.addKeyword(cat.getName()); 771 } 772 } 773 } 774 775 public SimpleDateFormat getCumulusFormat() 776 { 777 return fieldCumulusFormat; 778 } 779 780 public void setCumulusFormat(SimpleDateFormat inCumulusFormat) 781 { 782 fieldCumulusFormat = inCumulusFormat; 783 } 784 785 protected boolean hasParent( com.canto.cumulus.Category inCat, String parentName) 786 { 787 com.canto.cumulus.Category cat = inCat; 788 while ( cat != null ) 789 { 790 String name = cat.getName(); 791 if (name.equals(parentName)) 792 { 793 return true; 794 } 795 796 cat = cat.getParentCategory(); 797 } 798 return false; 799 } 800 public TaskRunner getOddRunner() 813 { 814 if( fieldOddRunner == null) 815 { 816 fieldOddRunner = new TaskRunner(); 817 } 818 return fieldOddRunner; 819 } 820 public TaskRunner getEvenRunner() 821 { 822 if( fieldEvenRunner == null) 823 { 824 fieldEvenRunner = new TaskRunner(); 825 } 826 return fieldEvenRunner; 827 } 828 } 829 | Popular Tags |