Java > Open Source Codes > net > sourceforge > cruisecontrol > builders > DistributedMasterBuilder


1 /****************************************************************************
2 * CruiseControl, a Continuous Integration Toolkit
3 * Copyright (c) 2001, ThoughtWorks, Inc.
4 * 651 W Washington Ave. Suite 600
5 * Chicago, IL 60661 USA
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 *
12 * + Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * + Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials provided
18 * with the distribution.
19 *
20 * + Neither the name of ThoughtWorks, Inc., CruiseControl, nor the
21 * names of its contributors may be used to endorse or promote
22 * products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
29 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 ****************************************************************************/
37
38 package net.sourceforge.cruisecontrol.builders;
39
40 import java.io.File ;
41 import java.io.IOException ;
42 import java.rmi.RemoteException ;
43 import java.util.Properties ;
44 import java.util.List ;
45 import java.util.Map ;
46 import java.util.Set ;
47 import java.util.Iterator ;
48 import java.util.HashMap ;
49 import java.util.Collections ;
50
51 import net.jini.core.lookup.ServiceItem;
52 import net.jini.core.entry.Entry;
53 import net.sourceforge.cruisecontrol.Builder;
54 import net.sourceforge.cruisecontrol.SelfConfiguringPlugin;
55 import net.sourceforge.cruisecontrol.CruiseControlException;
56 import net.sourceforge.cruisecontrol.PluginRegistry;
57 import net.sourceforge.cruisecontrol.distributed.BuildAgentService;
58 import net.sourceforge.cruisecontrol.distributed.util.MulticastDiscovery;
59 import net.sourceforge.cruisecontrol.distributed.util.PropertiesHelper;
60 import net.sourceforge.cruisecontrol.distributed.util.ReggieUtil;
61 import net.sourceforge.cruisecontrol.distributed.util.ZipUtil;
62 import net.sourceforge.cruisecontrol.util.FileUtil;
63 import net.sourceforge.cruisecontrol.util.Util;
64
65 import org.apache.log4j.Logger;
66 import org.jdom.Attribute;
67 import org.jdom.Element;
68
69 public class DistributedMasterBuilder extends Builder implements SelfConfiguringPlugin {
70
71     private static final Logger LOG = Logger.getLogger(DistributedMasterBuilder.class);
72
73     private static final String CRUISE_PROPERTIES = "cruise.properties";
74     private static final String CRUISE_RUN_DIR = "cruise.run.dir";
75
76     // TODO: Change to property?
77 private static final long DEFAULT_CACHE_MISS_WAIT = 30000;
78     private boolean isFailFast;
79
80     // TODO: Can we get the module from the projectProperties instead of setting
81 // it via an attribute?
82 // Could be set in ModificationSet...
83 private String entries;
84     private String module;
85
86     private String agentLogDir;
87     private String agentOutputDir;
88
89     private String masterLogDir;
90     private String masterOutputDir;
91
92     private Element thisElement;
93     private Element childBuilderElement;
94     private String overrideTarget;
95     private MulticastDiscovery discovery;
96     private Properties cruiseProperties;
97     private File rootDir;
98
99     protected void overrideTarget(final String target) {
100         overrideTarget = target;
101     }
102
103     Element getChildBuilderElement() {
104         return childBuilderElement;
105     }
106
107     /** If true, available agent lookup will not block until an agent is found,
108      * but will return null immediately. */
109     public synchronized void setFailFast(final boolean isFailFast) {
110         this.isFailFast = isFailFast;
111     }
112     private synchronized boolean isFailFast() {
113         return isFailFast;
114     }
115
116     /** Intended only for use by unit tests. **/
117     void setDiscovery(final MulticastDiscovery multicastDiscovery) {
118         discovery = multicastDiscovery;
119     }
120     MulticastDiscovery getDiscovery() {
121         if (discovery == null) {
122             final Entry[] arrEntries = ReggieUtil.convertStringEntries(entries);
123             discovery = new MulticastDiscovery(arrEntries);
124         }
125
126         return discovery;
127     }
128
129     /**
130      *
131      * @param element
132      * @throws net.sourceforge.cruisecontrol.CruiseControlException
133      */
134     public void configure(final Element element) throws CruiseControlException {
135         thisElement = element;
136         final List children = element.getChildren();
137         if (children.size() > 1) {
138             final String message = "DistributedMasterBuilder can only have one nested builder";
139             LOG.error(message);
140             throw new CruiseControlException(message);
141         } else if (children.size() == 0) {
142             // @todo Clarify when configure() can be called...
143 final String message = "Nested Builder required by DistributedMasterBuilder, "
144                     + "ignoring and assuming this call is during plugin-preconfig";
145             LOG.warn(message);
146             return;
147         }
148         childBuilderElement = (Element) children.get(0);
149         // Add default/preconfigured props to builder element
150 addMissingPluginDefaults(childBuilderElement);
151
152         // Add default/preconfigured props to distributed element
153 addMissingPluginDefaults(element);
154
155         Attribute tempAttribute = thisElement.getAttribute("entries");
156         entries = tempAttribute != null ? tempAttribute.getValue() : "";
157
158         tempAttribute = thisElement.getAttribute("module");
159         if (tempAttribute != null) {
160             module = tempAttribute.getValue();
161         } else {
162             // try to use project name as default value
163 final Element elmProj = getElementProject(thisElement);
164             if (elmProj != null) {
165                 module = elmProj.getAttributeValue("name");
166             } else {
167                 module = null;
168             }
169         }
170
171         // optional attributes
172 tempAttribute = thisElement.getAttribute("agentlogdir");
173         setAgentLogDir(tempAttribute != null ? tempAttribute.getValue() : null);
174
175         tempAttribute = thisElement.getAttribute("agentoutputdir");
176         setAgentOutputDir(tempAttribute != null ? tempAttribute.getValue() : null);
177
178         tempAttribute = thisElement.getAttribute("masterlogdir");
179         setMasterLogDir(tempAttribute != null ? tempAttribute.getValue() : null);
180
181         tempAttribute = thisElement.getAttribute("masteroutputdir");
182         setMasterOutputDir(tempAttribute != null ? tempAttribute.getValue() : null);
183
184         try {
185             cruiseProperties = (Properties ) PropertiesHelper.loadRequiredProperties(CRUISE_PROPERTIES);
186         } catch (RuntimeException e) {
187             LOG.error(e.getMessage(), e);
188             System.err.println(e.getMessage());
189             throw new CruiseControlException(e.getMessage(), e);
190         }
191         rootDir = new File (cruiseProperties.getProperty(CRUISE_RUN_DIR));
192         LOG.debug("CRUISE_RUN_DIR: " + rootDir);
193         if (!rootDir.exists()
194                 // Don't think non-existant rootDir matters if agent/master log/output dirs are set
195 && (getAgentLogDir() == null && getMasterLogDir() == null)
196                 && (getAgentOutputDir() == null && getMasterOutputDir() == null)
197         ) {
198             final String message = "Could not get property " + CRUISE_RUN_DIR + " from " + CRUISE_PROPERTIES
199                     + ", or run dir does not exist: " + rootDir;
200             LOG.error(message);
201             System.err.println(message);
202             throw new CruiseControlException(message);
203         }
204
205         validate();
206     }
207
208     /** Package visisble since also used by unit tests to apply plugin default values. */
209     static void addMissingPluginDefaults(final Element elementToAlter) {
210         LOG.debug("Adding missing defaults for plugin: " + elementToAlter.getName());
211         final Map pluginDefaults = getPluginDefaults(elementToAlter);
212         applyPluginDefaults(pluginDefaults, elementToAlter);
213     }
214
215     private static void applyPluginDefaults(final Map pluginDefaults, final Element elementToAlter) {
216         final String pluginName = elementToAlter.getName();
217         // to preserve precedence, only add default attribute if it is not also defined in the tag directly
218 final Set defaultAttribMapKeys = pluginDefaults.keySet();
219         for (Iterator itrKeys = defaultAttribMapKeys.iterator(); itrKeys.hasNext();) {
220             final String attribName = (String ) itrKeys.next();
221             final String attribValueExisting = elementToAlter.getAttributeValue(attribName);
222             if (attribValueExisting == null) { // skip existing attribs
223 final String attribValue = (String ) pluginDefaults.get(attribName);
224                 elementToAlter.setAttribute(attribName, attribValue);
225                 LOG.debug("Added plugin " + pluginName + " default attribute: " + attribName + "=" + attribValue);
226             } else {
227                 LOG.debug("Skipping plugin " + pluginName + " overidden attribute: " + attribName
228                         + "=" + attribValueExisting);
229             }
230         }
231     }
232
233     private static Map getPluginDefaults(final Element elementToAlter) {
234
235         final PluginRegistry pluginsRegistry = PluginRegistry.createRegistry();
236         final Map pluginDefaults = new HashMap ();
237         // note: the map returned here is "unmodifiable"
238 pluginDefaults.putAll(pluginsRegistry.getDefaultProperties(elementToAlter.getName()));
239
240         if (pluginDefaults.size() == 0) { // maybe we're in a unit test
241 // @todo Remove this kludge when we figure out how to make PluginRegistry work in unit test
242 LOG.warn("Unit Test kludge for plugin default values. "
243                     + "Should happen only if no default plugin settings exist OR during unit tests.");
244             final Element elemCC = getElementCruiseControl(elementToAlter);
245             // bail out if we can't find CruiseControl element, since there may actually
246 // be no defaults for this element
247 if (elemCC == null) {
248                 return pluginDefaults;
249             }
250
251             final List plugins = elemCC.getChildren("plugin");
252             final Map pluginDefaultsHack = new HashMap ();
253             for (int i = 0; i < plugins.size(); i++) {
254                 final Element plugin = (Element) plugins.get(i);
255                 if (elementToAlter.getName().equals(plugin.getAttributeValue("name"))) {
256                     // iterate attribs
257 final List attribs = plugin.getAttributes();
258                     for (int j = 0; j < attribs.size(); j++) {
259                         final Attribute attribute = (Attribute) attribs.get(j);
260                         final String attribName = attribute.getName();
261                         // skip certain attribs
262 if (!"name".equals(attribName)) { // ignore "name" attrib of default plugin declaration
263 pluginDefaultsHack.put(attribName, attribute.getValue());
264                         }
265                     }
266                 }
267             }
268             // put kludge results into returned map
269 pluginDefaults.putAll(pluginDefaultsHack);
270         }
271
272         return Collections.unmodifiableMap(pluginDefaults);
273     }
274
275     private static Element getElementCruiseControl(Element element) {
276         LOG.debug("Searching for CC root element, starting at: " + element.toString());
277         while (!"cruisecontrol".equals(element.getName().toLowerCase())) {
278             element = element.getParentElement();
279             LOG.debug("Searching for CC root element, moved up to: "
280                     + (element != null ? element.toString() : "Augh! parent element is null"));
281             if (element == null) {
282                 LOG.warn("Searching for CC root element, not found.");
283                 break;
284             }
285         }
286         return element;
287     }
288
289     /** Used to get default value for "module" attribute if not given. */
290     private static Element getElementProject(Element element) {
291         LOG.debug("Searching for Project element, starting at: " + element.toString());
292         while (!"project".equals(element.getName().toLowerCase())) {
293             element = element.getParentElement();
294             LOG.debug("Searching for Project element, moved up to: "
295                     + (element != null ? element.toString() : "Augh! parent element is null"));
296             if (element == null) {
297                 LOG.warn("Searching for Project element, not found.");
298                 break;
299             }
300         }
301         return element;
302     }
303
304     public void validate() throws CruiseControlException {
305         super.validate();
306         final Element elmChildBuilder = getChildBuilderElement();
307         if (elmChildBuilder == null) {
308             final String message = "A nested Builder is required for DistributedMasterBuilder";
309             LOG.warn(message);
310             throw new CruiseControlException(message);
311         }
312
313         // Add default/preconfigured props to builder element
314 addMissingPluginDefaults(elmChildBuilder);
315
316         /* @todo Should we call validate() on the child builder?
317         // If so, figure out how to do in unit tests.
318         // One problem is config file properties, like: "anthome="${env.ANT_HOME}" don't get expanded...
319         final PluginXMLHelper pluginXMLHelper = PropertiesHelper.createPluginXMLHelper(overrideTarget);
320         PluginRegistry plugins = PluginRegistry.createRegistry();
321         Class pluginClass = plugins.getPluginClass(elmChildBuilder.getName());
322         // this dies due to "anthome="${env.ANT_HOME}" in config file not being expanded...
323         final Builder builder = (Builder) pluginXMLHelper.configure(elmChildBuilder, pluginClass, false);
324         builder.validate();
325         //*/
326
327         if (module == null) {
328             final String message = "The 'module' attribute is required for DistributedMasterBuilder";
329             LOG.warn(message);
330             throw new CruiseControlException(message);
331         }
332     }
333
334     /* Override base schedule methods to expose child-builder values. Otherwise, schedules are not honored.*/
335     public int getDay() {
336         // @todo Replace with real Builder object if possible
337 final String value = childBuilderElement.getAttributeValue("day");
338         final int retVal;
339         if (value == null) {
340             retVal = NOT_SET;
341         } else {
342             retVal = Integer.parseInt(value);
343         }
344         return retVal;
345     }
346     /* Override base schedule methods to expose child-builder values. Otherwise, schedules are not honored.*/
347     public int getTime() {
348         // @todo Replace with real Builder object if possible
349 final String value = childBuilderElement.getAttributeValue("time");
350         final int retVal;
351         if (value == null) {
352             retVal = NOT_SET;
353         } else {
354             retVal = Integer.parseInt(value);
355         }
356         return retVal;
357     }
358     /* Override base schedule methods to expose child-builder values. Otherwise, schedules are not honored.*/
359     public int getMultiple() {
360         // @todo Replace with real Builder object if possible
361 final String value = childBuilderElement.getAttributeValue("multiple");
362         final int retVal;
363         if (getTime() != NOT_SET) {
364             // can't use both time and multiple
365 retVal = NOT_SET;
366         } else if (value == null) {
367             // no multiple attribute is set
368 // use default multiple value
369 retVal = 1;
370         } else {
371             retVal = Integer.parseInt(value);
372         }
373         return retVal;
374     }
375
376     public Element buildWithTarget(Map properties, String target) throws CruiseControlException {
377         String oldOverideTarget = overrideTarget;
378         overrideTarget(target);
379         try {
380             return build(properties);
381         } finally {
382             overrideTarget(oldOverideTarget);
383         }
384     }
385
386     public Element build(final Map projectProperties) throws CruiseControlException {
387         try {
388             final BuildAgentService agent = pickAgent();
389             // agent is now marked as claimed
390
391             String agentMachine = "unknown";
392             try {
393                 agentMachine = agent.getMachineName();
394             } catch (RemoteException e1) {
395                 ; // ignored
396 }
397             
398             final Element buildResults;
399             try {
400                 final Map distributedAgentProps = new HashMap ();
401                 distributedAgentProps.put(PropertiesHelper.DISTRIBUTED_OVERRIDE_TARGET, overrideTarget);
402                 distributedAgentProps.put(PropertiesHelper.DISTRIBUTED_MODULE, module);
403                 distributedAgentProps.put(PropertiesHelper.DISTRIBUTED_AGENT_LOGDIR, getAgentLogDir());
404                 distributedAgentProps.put(PropertiesHelper.DISTRIBUTED_AGENT_OUTPUTDIR, getAgentOutputDir());
405                 LOG.debug("Distributed Agent Props: " + distributedAgentProps.toString());
406                 
407                 LOG.debug("Project Props: " + projectProperties.toString());
408                 
409                 LOG.info("Starting remote build on agent: " + agent.getMachineName() + " of module: " + module);
410                 buildResults = agent.doBuild(getChildBuilderElement(), projectProperties, distributedAgentProps);
411
412                 final String rootDirPath;
413                 try {
414                     // watch out on Windoze, problems if root dir is c: instead of c:/
415 LOG.debug("rootDir: " + rootDir + "; rootDir.cp: " + rootDir.getCanonicalPath());
416                     rootDirPath = rootDir.getCanonicalPath();
417                 } catch (IOException e) {
418                     final String message = "Error getting canonical path for: " + rootDir;
419                     LOG.error(message);
420                     System.err.println(message);
421                     throw new CruiseControlException(message, e);
422                 }
423
424
425                 String masterDir;
426                 if (getMasterLogDir() == null || "".equals(getMasterLogDir())) {
427                     masterDir = rootDirPath + File.separator + PropertiesHelper.RESULT_TYPE_LOGS;
428                 } else {
429                     masterDir = getMasterLogDir();
430                 }
431                 getResultsFiles(agent, PropertiesHelper.RESULT_TYPE_LOGS, rootDirPath, masterDir);
432
433
434                 if (getMasterOutputDir() == null || "".equals(getMasterOutputDir())) {
435                     masterDir = rootDirPath + File.separator + PropertiesHelper.RESULT_TYPE_OUTPUT;
436                 } else {
437                     masterDir = getMasterOutputDir();
438                 }
439                 getResultsFiles(agent, PropertiesHelper.RESULT_TYPE_OUTPUT, rootDirPath, masterDir);
440
441                 agent.clearOutputFiles();
442             } catch (RemoteException e) {
443                 final String message = "RemoteException from"
444                         + "\nagent on: " + agentMachine
445                         + "\nwhile building module: " + module;
446                 LOG.error(message, e);
447                 System.err.println(message + " - " + e.getMessage());
448                 try {
449                     agent.clearOutputFiles();
450                 } catch (RemoteException re) {
451                     LOG.error("Exception after prior exception while clearing agent output files (to set busy false).",
452                             re);
453                 }
454                 throw new CruiseControlException(message, e);
455             }
456             return buildResults;
457         } catch (RuntimeException e) {
458             final String message = "Distributed build runtime exception";
459             LOG.error(message, e);
460             System.err.println(message + " - " + e.getMessage());
461             throw new CruiseControlException(message, e);
462         }
463     }
464
465     public static void getResultsFiles(final BuildAgentService agent, final String resultsType,
466                                        final String rootDirPath, final String masterDir)
467             throws RemoteException {
468
469         if (agent.resultsExist(resultsType)) {
470             final String zipFilePath = FileUtil.bytesToFile(agent.retrieveResultsAsZip(resultsType), rootDirPath,
471                     resultsType + ".zip");
472             try {
473                 LOG.info("unzip " + resultsType + " (" + zipFilePath + ") to: " + masterDir);
474                 ZipUtil.unzipFileToLocation(zipFilePath, masterDir);
475                 Util.deleteFile(new File (zipFilePath));
476             } catch (IOException e) {
477                 // Empty zip for log results--ignore
478 LOG.debug("Ignored retrieve " + resultsType + " results error:", e);
479             }
480         } else {
481             final String message = "No results returned for " + resultsType;
482             LOG.info(message);
483         }
484     }
485
486     protected BuildAgentService pickAgent() throws CruiseControlException {
487         BuildAgentService agent = null;
488
489         while (agent == null) {
490             final ServiceItem serviceItem;
491             try {
492                 serviceItem = getDiscovery().findMatchingService();
493             } catch (RemoteException e) {
494                 throw new CruiseControlException("Error finding matching agent.", e);
495             }
496             if (serviceItem != null) {
497                 agent = (BuildAgentService) serviceItem.service;
498                 try {
499                     LOG.info("Found available agent on: " + agent.getMachineName());
500                 } catch (RemoteException e) {
501                     throw new CruiseControlException("Error calling agent method.", e);
502                 }
503             } else if (isFailFast()) {
504                 break;
505             } else {
506                 // wait a bit and try again
507 LOG.info("Couldn't find available agent. Waiting "
508                         + (DEFAULT_CACHE_MISS_WAIT / 1000) + " seconds before retry.");
509                 try {
510                     Thread.sleep(DEFAULT_CACHE_MISS_WAIT);
511                 } catch (InterruptedException e) {
512                     LOG.error("Lookup Cache Miss Wait was interrupted");
513                     break;
514                 }
515             }
516         }
517
518         return agent;
519     }
520
521     public String getAgentLogDir() {
522         return agentLogDir;
523     }
524
525     public void setAgentLogDir(final String agentLogDir) {
526         this.agentLogDir = agentLogDir;
527     }
528
529     public String getAgentOutputDir() {
530         return agentOutputDir;
531     }
532
533     public void setAgentOutputDir(final String agentOutputDir) {
534         this.agentOutputDir = agentOutputDir;
535     }
536
537     public String getMasterLogDir() {
538         return masterLogDir;
539     }
540
541     public void setMasterLogDir(final String masterLogDir) {
542         this.masterLogDir = masterLogDir;
543     }
544
545     public String getMasterOutputDir() {
546         return masterOutputDir;
547     }
548
549     public void setMasterOutputDir(final String masterOutputDir) {
550         this.masterOutputDir = masterOutputDir;
551     }
552 }
553

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags