1 32 33 package websphinx.workbench; 34 35 import java.awt.*; 36 import java.io.*; 37 import java.net.URL ; 38 import websphinx.*; 39 import rcm.awt.Constrain; 40 import rcm.awt.PopupDialog; 41 import rcm.awt.TabPanel; 42 import rcm.awt.BorderPanel; 43 import rcm.awt.ClosableFrame; 44 45 public class Workbench extends Panel implements CrawlListener { 46 47 Crawler crawler; 48 49 String currentFilename = ""; 50 51 Panel workbenchPanel; GridBagConstraints workbenchConstraints; 54 WorkbenchVizPanel vizPanel; GridBagConstraints vizConstraints; 56 57 WebGraph graph; 59 WebOutline outline; 60 Statistics statistics; 61 EventLog logger; 62 63 MenuBar menubar; 65 Menu fileMenu; 66 MenuItem newCrawlerItem; 67 MenuItem openCrawlerItem; 68 MenuItem saveCrawlerItem; 69 MenuItem createCrawlerItem; 70 MenuItem exitItem; 71 72 Panel menuPanel; 74 Button newCrawlerButton; 75 Button openCrawlerButton; 76 Button saveCrawlerButton; 77 Button createCrawlerButton; 78 79 WorkbenchTabPanel configPanel; 80 Panel simplePanel; 81 Panel crawlPanel; 82 Panel limitsPanel; 83 Panel classifiersPanel; 84 Panel linksPanel; 85 Panel actionPanel; 86 87 CrawlerEditor crawlerEditor; 88 ClassifierListEditor classifierListEditor; 89 DownloadParametersEditor downloadParametersEditor; 90 LinkPredicateEditor linkPredicateEditor; 91 PagePredicateEditor pagePredicateEditor; 92 ActionEditor actionEditor; 93 SimpleCrawlerEditor simpleCrawlerEditor; 94 95 boolean advancedMode = false; 96 boolean tornOff = false; 97 98 Button startButton, pauseButton, stopButton, clearButton; 99 100 boolean allowExit; 101 102 Frame workbenchFrame; 104 Frame vizFrame; 105 106 static final int MARGIN = 8; 108 public Workbench () { 109 this (makeDefaultCrawler ()); 110 return; 111 } 112 113 private static Crawler makeDefaultCrawler () { 114 Crawler c = new Crawler (); 115 c.setDomain (Crawler.SUBTREE); 116 return c; 117 } 118 119 public Workbench (String filename) throws Exception { 120 this (loadCrawler (new FileInputStream (filename))); 122 126 } 127 128 public Workbench (URL url) throws Exception { 129 this (loadCrawler (url.openStream ())); 135 } 136 137 public Workbench (Crawler _crawler) { 138 Browser browser = Context.getBrowser (); 139 140 setLayout (new BorderLayout ()); 141 setBackground (Color.lightGray); 142 143 setLayout (new GridLayout (2, 1)); 144 145 add (workbenchPanel = new Panel ()); 146 workbenchPanel.setLayout (new GridBagLayout ()); 147 148 makeMenus (); 150 Constrain.add (workbenchPanel, menuPanel, Constrain.labelLike (0, 0)); 151 152 configPanel = new WorkbenchTabPanel (); 154 Constrain.add (workbenchPanel, configPanel, Constrain.areaLike (0, 1)); 155 simplePanel = makeSimplePanel(); 156 crawlPanel = makeCrawlPanel(); 157 linksPanel = makeLinksPanel(); 158 actionPanel = makeActionPanel(); 159 classifiersPanel = makeClassifiersPanel (); 160 limitsPanel = makeLimitsPanel (); 161 162 Constrain.add (workbenchPanel, makeButtonPanel(), Constrain.fieldLike (0, 2)); 164 165 add (vizPanel = new WorkbenchVizPanel (this)); 167 168 graph = new WebGraph (); 170 graph.setBackground (Color.white); 171 if (browser != null) 172 graph.addLinkViewListener (browser); 173 vizPanel.addTabPanel ("Graph", true, graph); 174 175 outline = new WebOutline (); 177 outline.setBackground (Color.white); 178 if (browser != null) 179 outline.addLinkViewListener (browser); 180 vizPanel.addTabPanel ("Outline", true, outline); 181 182 statistics = new Statistics (); 184 Panel p = new Panel (); 185 p.setLayout (new FlowLayout()); 186 p.add (statistics); 187 vizPanel.addTabPanel ("Statistics", true, p); 188 189 logger = new EventLog (); 191 192 setCrawler (_crawler); 195 } 196 197 public Frame makeFrame () { 198 if (workbenchFrame == null) { 199 Dimension screen = Toolkit.getDefaultToolkit().getScreenSize(); 200 workbenchFrame = new WorkbenchFrame (this); 201 workbenchFrame.setForeground (getForeground()); 202 workbenchFrame.setBackground (getBackground()); 203 workbenchFrame.setFont (getFont()); 204 workbenchFrame.setTitle ("Crawler Workbench: " + 205 (crawler != null ? crawler.getName() : "")); 206 workbenchFrame.setLayout (new GridLayout (1,1)); 207 workbenchFrame.add (this); 208 workbenchPanel.remove (menuPanel); 209 workbenchFrame.setMenuBar (menubar); 210 workbenchFrame.reshape (0, 0, 211 Math.min (550, screen.width), 212 screen.height - 50); 213 } 214 return workbenchFrame; 215 } 216 217 public void setAllowExit (boolean yes) { 218 allowExit = yes; 219 } 220 221 public boolean getAllowExit () { 222 return allowExit; 223 } 224 225 public synchronized void setAdvancedMode (boolean adv) { 226 if (advancedMode == adv) 227 return; 228 229 configureCrawler (); advancedMode = adv; 231 setCrawler (crawler); 233 configPanel.advancedButton.setLabel (advancedMode 234 ? "<< Simple" : ">> Advanced"); 235 validate (); 236 } 237 238 public boolean getAdvancedMode () { 239 return advancedMode; 240 } 241 242 245 246 static void setVisible (Component comp, boolean visible) { 247 if (visible) 248 comp.show(); 249 else 250 comp.hide (); 251 } 252 253 static void setEnabled (Component comp, boolean enabled) { 254 if (enabled) 255 comp.enable(); 256 else 257 comp.disable(); 258 } 259 260 static void setEnabled (MenuItem item, boolean enabled) { 261 if (enabled) 262 item.enable(); 263 else 264 item.disable(); 265 } 266 267 Panel makeMenus () { 268 menubar = new MenuBar (); 270 menuPanel = new Panel (); 271 menuPanel.setLayout (new FlowLayout()); 273 menubar.add (fileMenu = new Menu ("File")); 275 276 fileMenu.add (newCrawlerItem = new MenuItem ("New Crawler")); 278 menuPanel.add (newCrawlerButton = new Button ("New")); 279 280 fileMenu.add (openCrawlerItem = new MenuItem ("Open Crawler...")); 282 285 menuPanel.add (openCrawlerButton = new Button ("Open...")); 286 289 290 291 fileMenu.add (saveCrawlerItem = new MenuItem ("Save Crawler...")); 293 296 menuPanel.add (saveCrawlerButton = new Button ("Save...")); 297 300 301 fileMenu.add (createCrawlerItem = new MenuItem ("Create Crawler From Class...")); 303 menuPanel.add (createCrawlerButton = new Button ("Create...")); 304 305 306 fileMenu.add (exitItem = new MenuItem ("Exit")); 308 309 return menuPanel; 310 } 311 312 private Panel makeSimplePanel () { 313 return simpleCrawlerEditor = new SimpleCrawlerEditor (); 314 } 315 316 private Panel makeCrawlPanel () { 318 return crawlerEditor = new CrawlerEditor (); 319 } 320 321 private Panel makeLinksPanel () { 322 Panel panel = new Panel (); 323 panel.setLayout (new GridBagLayout ()); 324 325 Constrain.add (panel, new Label("Follow:"), Constrain.labelLike (0, 0)); 326 Constrain.add (panel, linkPredicateEditor = new LinkPredicateEditor (), 327 Constrain.areaLike (1, 0)); 328 329 return panel; 330 } 331 332 private Panel makeActionPanel () { 333 Panel panel = new Panel (); 334 panel.setLayout (new GridBagLayout ()); 335 336 Constrain.add (panel, new Label ("Action:"), Constrain.labelLike (0, 0)); 337 Constrain.add (panel, actionEditor = new ActionEditor (), Constrain.areaLike (1, 0)); 338 339 Constrain.add (panel, new Label("on pages:"), Constrain.labelLike (0, 1)); 340 Constrain.add (panel, pagePredicateEditor = new PagePredicateEditor (), 341 Constrain.areaLike (1, 1)); 342 return panel; 343 } 344 345 private Panel makeClassifiersPanel () { 346 classifierListEditor = new ClassifierListEditor (); 347 return classifierListEditor; 348 } 349 350 private Panel makeLimitsPanel () { 351 downloadParametersEditor = new DownloadParametersEditor (); 352 return downloadParametersEditor; 353 } 354 355 private Panel makeButtonPanel () { 356 Panel panel = new Panel (); 357 panel.setLayout (new FlowLayout ()); 358 359 panel.add (startButton = new Button ("Start")); 360 panel.add (pauseButton = new Button ("Pause")); 361 panel.add (stopButton = new Button ("Stop")); 362 panel.add (clearButton = new Button ("Clear")); 363 enableButtons (true, false, false, false); 364 return panel; 365 } 366 367 String getCrawlerClassName (String label) { 368 String className = label; 369 if (className != null) { 370 if (className.equals ("Crawler")) 371 className = "websphinx.Crawler"; 372 else if (className.equals ("Load Class...")) 373 className = null; 374 } 375 return className; 376 } 377 378 public boolean handleEvent (Event event) { 379 if (doEvent (event)) 380 return true; 381 else 382 return super.handleEvent (event); 383 } 384 385 boolean doEvent (Event event) { 386 if (event.id == Event.ACTION_EVENT) { 387 if (event.target instanceof MenuItem) { 388 MenuItem item = (MenuItem)event.target; 389 390 if (item == newCrawlerItem) 391 newCrawler (); 392 else if (item == openCrawlerItem) 394 openCrawler (); 395 else if (item == saveCrawlerItem) 396 saveCrawler (); 397 else if (item == createCrawlerItem) 399 createCrawler (null); 400 else if (item == exitItem) 401 close (); 402 else 403 return false; 404 } 405 else if (event.target == newCrawlerButton) 406 newCrawler (); 407 else if (event.target == openCrawlerButton) 409 openCrawler (); 410 else if (event.target == saveCrawlerButton) 411 saveCrawler (); 412 else if (event.target == createCrawlerButton) 414 createCrawler (null); 415 else if (event.target == configPanel.advancedButton) 416 setAdvancedMode (!advancedMode); 417 else if (event.target == vizPanel.optionsButton) 418 new WorkbenchControlPanel (graph, outline).show (); 419 else if (event.target == vizPanel.tearoffButton) 420 if (tornOff) 421 dockVisualizations (); 422 else 423 tearoffVisualizations (); 424 else if (event.target == startButton) 425 start (); 426 else if (event.target == pauseButton) 427 pause (); 428 else if (event.target == stopButton) 429 stop (); 430 else if (event.target == clearButton) 431 clear (); 432 else 433 return false; 434 } 435 else 436 return false; 437 438 return true; 439 } 440 441 442 445 446 protected void finalize () { 447 } 449 450 void close () { 451 if (!allowExit) 452 return; 453 454 456 if (Context.isApplication()) { 457 Runtime.runFinalizersOnExit (true); 459 System.exit (0); 461 } 462 } 463 464 public void refresh () { 465 graph.updateClosure (crawler.getCrawledRoots ()); 466 outline.updateClosure (crawler.getCrawledRoots ()); 467 } 468 469 void connectVisualization (Crawler crawler, Object viz, boolean linksToo) { 470 if (viz instanceof CrawlListener) 471 crawler.addCrawlListener ((CrawlListener)viz); 472 if (linksToo && viz instanceof LinkListener) 473 crawler.addLinkListener ((LinkListener)viz); 474 } 475 476 void disconnectVisualization (Crawler crawler, Object viz, boolean linksToo) { 477 if (viz instanceof CrawlListener) 478 crawler.removeCrawlListener ((CrawlListener)viz); 479 if (linksToo && viz instanceof LinkListener) 480 crawler.removeLinkListener ((LinkListener)viz); 481 } 482 483 484 void showVisualization (Object viz) { 485 if (viz == graph) 486 graph.start (); 487 } 488 489 void hideVisualization (Object viz) { 490 if (viz == graph) 491 graph.stop (); 492 } 493 494 void tearoffVisualizations () { 495 if (tornOff) 496 return; 497 498 if (vizFrame == null) { 499 Dimension screen = Toolkit.getDefaultToolkit().getScreenSize(); 500 vizFrame = new WorkbenchVizFrame (this); 501 vizFrame.setForeground (getForeground()); 502 vizFrame.setBackground (getBackground()); 503 vizFrame.setFont (getFont()); 504 vizFrame.setTitle ("Visualization: " + 505 (crawler != null ? crawler.getName() : "")); 506 vizFrame.setLayout (new GridLayout (1,1)); 507 vizFrame.reshape (0, 0, 508 Math.min (550, screen.width), 509 screen.height/2); 510 } 511 512 remove (vizPanel); 513 setLayout (new GridLayout (1,1)); 514 validate (); 515 516 vizFrame.add (vizPanel); 517 setVisible (vizFrame, true); 518 519 vizPanel.tearoffButton.setLabel ("Glue Back"); 520 521 tornOff = true; 522 } 523 524 void dockVisualizations () { 525 if (!tornOff) 526 return; 527 528 setVisible (vizFrame, false); 529 vizFrame.remove (vizPanel); 530 531 setLayout (new GridLayout (2, 1)); 532 add (vizPanel); 533 validate (); 534 535 vizPanel.tearoffButton.setLabel ("Tear Off"); 536 537 tornOff = false; 538 } 539 540 void newCrawler () { 541 setCrawler (makeDefaultCrawler ()); 542 currentFilename = ""; 543 } 544 545 void createCrawler (String className) { 546 if (className == null || className.length() == 0) { 547 className = PopupDialog.ask (workbenchPanel, 548 "New Crawler", 549 "Create a Crawler of class:", 550 crawler.getClass().getName() 551 ); 552 if (className == null) 553 return; 554 } 555 556 try { 557 Class crawlerClass = (Class )Class.forName (className); 558 Crawler newCrawler = (Crawler)crawlerClass.newInstance (); 559 560 setCrawler (newCrawler); 561 currentFilename = ""; 562 } catch (Exception e) { 563 PopupDialog.warn (workbenchPanel, 564 "Error", 565 e.toString()); 566 return; 567 } 568 } 569 570 571 void openCrawler () { 573 String fn = PopupDialog.askFilename (workbenchPanel, "Open Crawler", "", true); 574 if (fn != null) 575 openCrawler (fn); 576 } 577 578 void openCrawler (String filename) { 579 try { 580 setCrawler (loadCrawler (Access.getAccess ().readFile (new File (filename)))); 581 currentFilename = filename; 582 } catch (Exception e) { 583 PopupDialog.warn (workbenchPanel, 584 "Error", 585 e.toString()); 586 } 587 } 588 589 void openCrawler (URL url) { 590 try { 591 setCrawler (loadCrawler (Access.getAccess ().openConnection (url).getInputStream ())); 592 currentFilename = ""; 593 } catch (Exception e) { 594 PopupDialog.warn (workbenchPanel, 595 "Error", 596 e.toString()); 597 } 598 } 599 600 static Crawler loadCrawler (InputStream stream) throws Exception { 601 ObjectInputStream in = new ObjectInputStream (stream); 602 Crawler loadedCrawler = (Crawler)in.readObject (); 603 in.close (); 604 return loadedCrawler; 605 } 606 607 void saveCrawler () { 608 String fn = PopupDialog.askFilename (workbenchPanel, "Save Crawler As", currentFilename, true); 609 if (fn != null) 610 saveCrawler (fn); 611 } 612 613 void saveCrawler (String filename) { 614 configureCrawler (); 615 616 try { 617 ObjectOutputStream out = 618 new ObjectOutputStream (Access.getAccess ().writeFile (new File (filename), false)); 619 out.writeObject ((Object )crawler); 620 out.close (); 621 622 currentFilename = filename; 623 } catch (Exception e) { 624 PopupDialog.warn (workbenchPanel, 625 "Error", 626 e.toString()); 627 } 628 } 629 631 void configureCrawler () { 632 if (advancedMode) { 633 crawlerEditor.getCrawler (); 634 classifierListEditor.getCrawler (); 635 crawler.setDownloadParameters (downloadParametersEditor.getDownloadParameters ()); 636 if (advancedMode) { 637 crawler.setLinkPredicate (linkPredicateEditor.getLinkPredicate ()); 638 crawler.setPagePredicate (pagePredicateEditor.getPagePredicate ()); 639 crawler.setAction (actionEditor.getAction()); 640 } 641 } 642 else 643 simpleCrawlerEditor.getCrawler (); 644 } 645 646 void enableButtons (boolean fStart, boolean fPause, boolean fStop, boolean fClear) { 647 setEnabled (startButton, fStart); 648 setEnabled (pauseButton, fPause); 649 setEnabled (stopButton, fStop); 650 setEnabled (clearButton, fClear); 651 } 652 653 656 657 public void setCrawler (Crawler _crawler) { 658 if (crawler != _crawler) { 659 if (crawler != null) { 660 clear (); 661 disconnectVisualization (crawler, this, false); 662 disconnectVisualization (crawler, graph, true); 663 disconnectVisualization (crawler, outline, true); 664 disconnectVisualization (crawler, statistics, false); 665 disconnectVisualization (crawler, logger, true); 666 } 667 668 connectVisualization (_crawler, this, false); 669 connectVisualization (_crawler, graph, true); 670 connectVisualization (_crawler, outline, true); 671 connectVisualization (_crawler, statistics, false); 672 connectVisualization (_crawler, logger, true); 673 } 674 675 crawler = _crawler; 676 677 String name = crawler.getName (); 679 if (workbenchFrame != null) 680 workbenchFrame.setTitle ("Crawler Workbench: " + name); 681 if (vizFrame != null) 682 vizFrame.setTitle ("Visualization: " + name); 683 684 686 if (advancedMode) { 687 crawlerEditor.setCrawler (crawler); 688 classifierListEditor.setCrawler (crawler); 689 downloadParametersEditor.setDownloadParameters (crawler.getDownloadParameters ()); 690 if (advancedMode) { 691 linkPredicateEditor.setLinkPredicate (crawler.getLinkPredicate ()); 692 pagePredicateEditor.setPagePredicate (crawler.getPagePredicate ()); 693 actionEditor.setAction (crawler.getAction ()); 694 } 695 } 696 else 697 simpleCrawlerEditor.setCrawler (crawler); 698 699 if (advancedMode) 700 showAdvancedTabs (); 701 else 702 showSimpleTabs (); 703 } 704 705 public Crawler getCrawler () { 706 return crawler; 707 } 708 709 710 private void showAdvancedTabs () { 711 if (configPanel.countTabs () != 5) { 712 configPanel.removeAllTabPanels (); 713 configPanel.addTabPanel ("Crawl", true, crawlPanel); 714 configPanel.addTabPanel ("Links", true, linksPanel); 715 configPanel.addTabPanel ("Pages", true, actionPanel); 716 configPanel.addTabPanel ("Classifiers", true, classifiersPanel); 717 configPanel.addTabPanel ("Limits", true, limitsPanel); 718 } 719 } 720 721 private void showSimpleTabs () { 722 if (configPanel.countTabs () != 1) { 723 configPanel.removeAllTabPanels (); 724 configPanel.addTabPanel ("Crawl", true, simplePanel); 725 } 726 } 727 728 731 732 public void start () { 733 configureCrawler (); 734 735 if (crawler.getState () == CrawlEvent.STOPPED) 736 crawler.clear (); 737 738 Thread thread = new Thread (crawler, crawler.getName ()); 739 thread.setDaemon (true); 740 thread.start (); 741 } 742 743 public void stop () { 744 crawler.stop (); 745 } 746 747 public void pause () { 748 crawler.pause (); 749 } 750 751 public void clear () { 752 crawler.clear (); 753 } 754 755 758 public void started (CrawlEvent event) { 759 enableButtons (false, true, true, false); 760 } 761 762 765 public void stopped (CrawlEvent event) { 766 enableButtons (true, false, false, true); 767 } 768 769 772 public void cleared (CrawlEvent event) { 773 enableButtons (true, false, false, false); 774 } 775 776 779 public void timedOut (CrawlEvent event) { 780 enableButtons (true, false, false, true); 781 } 782 783 786 public void paused (CrawlEvent event) { 787 enableButtons (true, false, true, true); 788 } 789 790 public static void main (String [] args) throws Exception { 791 Workbench w = (args.length == 0) 792 ? new Workbench () 793 : new Workbench (args[0]); 794 w.setAllowExit (true); 795 796 Frame f = w.makeFrame (); 797 f.show (); 798 } 799 } 800 801 class WorkbenchFrame extends ClosableFrame 802 { 803 Workbench workbench; 804 805 public WorkbenchFrame (Workbench workbench) { 806 super (); 807 this.workbench = workbench; 808 } 809 810 public void close () { 811 workbench.close (); 812 } 813 814 public boolean handleEvent (Event event) { 815 if (workbench.doEvent (event)) 816 return true; 817 else 818 return super.handleEvent (event); 819 } 820 } 821 822 class WorkbenchVizFrame extends ClosableFrame 823 { 824 Workbench workbench; 825 826 public WorkbenchVizFrame (Workbench workbench) { 827 super (true); 828 this.workbench = workbench; 829 } 830 831 public void close () { 832 workbench.dockVisualizations (); 833 super.close (); 834 } 835 836 public boolean handleEvent (Event event) { 837 if (workbench.doEvent (event)) 838 return true; 839 else 840 return super.handleEvent (event); 841 } 842 } 843 844 class WorkbenchTabPanel extends TabPanel { 845 Button advancedButton; 846 847 public WorkbenchTabPanel () { 848 super (); 849 add (advancedButton = new Button ("Advanced >>")); 850 } 851 } 852 853 class WorkbenchVizPanel extends TabPanel { 854 Workbench workbench; 855 Button optionsButton; 856 Button tearoffButton; 857 858 public WorkbenchVizPanel (Workbench workbench) { 859 this.workbench = workbench; 860 add (optionsButton = new Button ("Options...")); 861 add (tearoffButton = new Button ("Tear Off")); 862 } 863 864 public void select (int num) { 865 Component prior = getSelectedComponent (); 866 867 super.select (num); 868 869 Component now = getSelectedComponent (); 870 871 if (prior == now) 872 return; 873 874 if (prior != null) 875 workbench.hideVisualization (prior); 876 877 if (now != null) { 878 workbench.showVisualization (now); 879 now.requestFocus (); 880 } 881 } 882 } 883 884 | Popular Tags |