Java > Open Source Codes > org > continuent > sequoia > controller > backup > backupers > AbstractPostgreSQLBackuper


1 /**
2  * Sequoia: Database clustering technology.
3  * Copyright (C) 2005 Emic Networks
4  * Contact: sequoia@continuent.org
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *
18  * Initial developer(s): Emmanuel Cecchet.
19  * Contributor(s): Dylan Hansen, Mathieu Peltier.
20  */
21
22 package org.continuent.sequoia.controller.backup.backupers;
23
24 import java.io.File ;
25 import java.io.IOException ;
26 import java.util.ArrayList ;
27 import java.util.Iterator ;
28 import java.util.regex.Matcher ;
29 import java.util.regex.Pattern ;
30
31 import org.continuent.sequoia.common.exceptions.BackupException;
32 import org.continuent.sequoia.common.log.Trace;
33 import org.continuent.sequoia.controller.backup.Backuper;
34
35 /**
36  * This abstract class provides base methods for PostgreSQL backupers.
37  * <p>
38  * Currently the Backupers takes 5 parameters (all are optional):
39  * <ul>
40  * <li>bindir: path to PostgreSQL binaries (if not set, the commands are
41  * searched in the path)</li>
42  * <li>encoding: the encoding of the database that is created upon restore</li>
43  * <li>authentication: flag to use PostgreSQL authentication (default false)</li>
44  * <li>dumpServer: address of interface to offer dumps</li>
45  * <li>preRestoreScript: location of script to run before restoring</li>
46  * <li>postRestoreScript: location of script to run after restoring</li>
47  * <li>dumpTimeout: Timeout period (seconds) while performing DB dump.</li>
48  * <li>restoreTimeout: Timeout period (seconds) while performin DB restore.</li>
49  * <li>pgDumpFlags: Extra pg_dump command-line options to use while performing
50  * DB dump.</li>
51  * </ul>
52  * More options can be easily added. This class makes calls to the pg_dump,
53  * createdb, dropdb, psql and pg_restore commands.
54  *
55  * @author <a HREF="mailto:dhansen@h2st.com">Dylan Hansen</a>
56  * @author <a HREF="mailto:mathieu.peltier@emicnetworks.com">Mathieu Peltier</a>
57  * @version 1.1
58  */
59 public abstract class AbstractPostgreSQLBackuper extends AbstractBackuper
60 {
61   private static final String DEFAULT_POSTGRESQL_PORT = "5432";
62   private static final String DEFAULT_POSTGRESQL_HOST = "localhost";
63
64   // Logger
65 private static Trace logger = Trace
66                                                           .getLogger(AbstractPostgreSQLBackuper.class
67                                                               .getName());
68
69   /** end user logger */
70   static Trace endUserLogger = Trace
71                                                           .getLogger("org.continuent.sequoia.enduser");
72
73   // CommandExec instance for running native commands.
74 protected NativeCommandExec nativeCmdExec = new NativeCommandExec();
75
76   // Static variables for option values
77 protected static String binDir = null;
78   protected static String encoding = null;
79   protected static boolean useAuthentication = false;
80   protected static String dumpServer = null;
81   protected static String preRestoreScript = null;
82   protected static String postRestoreScript = null;
83   protected static String pgDumpFlags = null;
84   protected static String dumpTimeout = null;
85   protected static String restoreTimeout = null;
86   protected static String splitSize = "1000m";
87
88   /**
89    * Creates a new <code>AbstractPostgreSQLBackuper</code> object
90    */
91   public AbstractPostgreSQLBackuper()
92   {
93   }
94
95   /**
96    * @see Backuper#getOptions()
97    */
98   public String getOptions()
99   {
100     return optionsString;
101   }
102
103   /**
104    * @see Backuper#setOptions(java.lang.String)
105    */
106   public void setOptions(String options)
107   {
108     super.setOptions(options);
109
110     // Check the HashMap for options. Only set once, easier than
111 // checking HashMap every time a value is needed.
112 if (optionsMap.containsKey("bindir"))
113       binDir = (String ) optionsMap.get("bindir");
114     if (optionsMap.containsKey("encoding"))
115       encoding = (String ) optionsMap.get("encoding");
116     if (optionsMap.containsKey("authentication")
117         && ((String ) optionsMap.get("authentication")).equalsIgnoreCase("true"))
118       useAuthentication = true;
119     if (optionsMap.containsKey("dumpServer"))
120       dumpServer = (String ) optionsMap.get("dumpServer");
121     if (optionsMap.containsKey("preRestoreScript"))
122       preRestoreScript = (String ) optionsMap.get("preRestoreScript");
123     if (optionsMap.containsKey("postRestoreScript"))
124       postRestoreScript = (String ) optionsMap.get("postRestoreScript");
125     if (optionsMap.containsKey("dumpTimeout"))
126       dumpTimeout = (String ) optionsMap.get("dumpTimeout");
127     if (optionsMap.containsKey("restoreTimeout"))
128       restoreTimeout = (String ) optionsMap.get("restoreTimeout");
129     if (optionsMap.containsKey("pgDumpFlags"))
130       pgDumpFlags = (String ) optionsMap.get("pgDumpFlags");
131     if (optionsMap.containsKey("splitSize"))
132       splitSize = (String ) optionsMap.get("splitSize");
133
134   }
135
136   /**
137    * @see org.continuent.sequoia.controller.backup.Backuper#deleteDump(java.lang.String,
138    * java.lang.String)
139    */
140   public void deleteDump(String path, String dumpName) throws BackupException
141   {
142     File toRemove = new File (getDumpPhysicalPath(path, dumpName));
143     if (logger.isDebugEnabled())
144       logger.debug("Deleting dump " + toRemove);
145     toRemove.delete();
146   }
147
148   /**
149    * Get the dump physical path from its logical name
150    *
151    * @param path the path where the dump is stored
152    * @param dumpName dump logical name
153    * @return path to dump file
154    */
155   protected String getDumpPhysicalPath(String path, String dumpName)
156   {
157     String fullPath = null;
158
159     if (path.endsWith(File.separator))
160       fullPath = path + dumpName;
161     else
162       fullPath = path + File.separator + dumpName;
163
164     return fullPath;
165   }
166
167   /**
168    * Creates a command string based on given info. Will append binDir if
169    * supplied.
170    *
171    * @param command Command to execute
172    * @param info URL info to parse and create parameters
173    * @param options Additional parameters
174    * @param login User login
175    * @return String containing full command to run
176    */
177   protected String makeCommand(String command, PostgreSQLUrlInfo info,
178       String options, String login)
179   {
180     String prefix = binDir != null ? binDir + File.separator : "";
181     return prefix + command + " " + info.getHostParametersString() + " -U "
182         + login + " " + options + " " + info.dbName;
183   }
184
185   /**
186    * Creates a command String array used my exec(). This method uses the
187    * "expect" command to pass the password parameter to the command being run,
188    * thus allowing PostgreSQL to allow authentication. Also uses binDir if
189    * supplied. Note the "psql" command has different output needed by "expect"
190    * than other commands.
191    *
192    * @param command Command to execute
193    * @param info URL info to parse adn add parameters
194    * @param options Additional parameters
195    * @param login User login
196    * @param password User password
197    * @param isPsql Is the command being run a psql command?
198    * @return String array containing full command to run
199    */
200   protected String [] makeCommandWithAuthentication(String command,
201       PostgreSQLUrlInfo info, String options, String login, String password,
202       boolean isPsql)
203   {
204     // Build params for "spawn" command used by expect
205 StringBuffer cmdBuff = new StringBuffer ("spawn ");
206
207     if (binDir != null)
208       cmdBuff.append(binDir + File.separator);
209
210     cmdBuff.append(command);
211     cmdBuff.append(" ");
212     cmdBuff.append(info.getHostParametersString());
213     cmdBuff.append(" -U ");
214     cmdBuff.append(login);
215     cmdBuff.append(" ");
216     cmdBuff.append(options);
217     cmdBuff.append(" ");
218     cmdBuff.append(info.dbName);
219     cmdBuff.append("; ");
220     // "psql" command has different message for password input
221 if (isPsql)
222       cmdBuff.append("expect \"Password:\"; ");
223     else
224       cmdBuff.append("expect \"Password for user " + login + ":\"; ");
225     cmdBuff.append("send \"");
226     cmdBuff.append(password);
227     cmdBuff.append("\"; ");
228     cmdBuff.append("send \"\\r\"; ");
229     cmdBuff.append("expect eof;");
230
231     String [] commands = {"expect", "-c", cmdBuff.toString()};
232     return commands;
233   }
234
235   /**
236    * Creates a Splitted command string based on given info. Will append BINDIR
237    * if supplied.
238    *
239    * @param command Command to execute
240    * @param info URL info to parse and create parameters
241    * @param options Additional parameters
242    * @param login User login
243    * @return String containing full command to run
244    */
245   protected String [] makeSplitCommand(String command, PostgreSQLUrlInfo info,
246       String options, String login)
247   {
248     String prefix = binDir != null ? binDir + File.separator : "";
249     String cmd = prefix + command + " " + info.dbName + " "
250         + info.getHostParametersString() + " -U " + login + " " + options;
251     String [] cmdArray = {"bash", "-c", cmd};
252     return cmdArray;
253     // return prefix + command + " " + info.dbName + " " +
254 // info.getHostParametersString() + " -U " + login + " " + options;
255 }
256
257   /**
258    * Creates an expect dialogue String array to be used as stadard input to a
259    * "expect" command, used to pass the password parameter to the command being
260    * run, thus allowing PostgreSQL to allow authentication. Also uses binDir if
261    * supplied. Note that the "psql" command has different password promting than
262    * other Postgres utilities. This expect-dialogue do a much better job at
263    * detecting errors than the detection you get when using
264    * makeCommandWithAuthentication() and executeNativeCommand(). Above all, it
265    * will report failure if the PG- command never promts for a password.
266    * Secondly, it will report failure if the invoked command exits with a
267    * non-zero exit-status.
268    *
269    * @param command Command to execute
270    * @param info URL info to parse adn add parameters
271    * @param options Additional parameters
272    * @param login User login
273    * @param password User password
274    * @param timeout Is the command being run a psql command?
275    * @return String array containing full Expect dialogue
276    */
277   protected String [] makeExpectDialogueWithAuthentication(String command,
278       PostgreSQLUrlInfo info, String options, String login, String password,
279       int timeout)
280   {
281     String [] cmdBuff = new String [27];
282
283     if (binDir != null)
284       cmdBuff[0] = new String ("spawn " + binDir + File.separator + command
285           + " " + info.getHostParametersString() + " -U " + login + " -W "
286           + options + " " + info.getDbName());
287     else
288       cmdBuff[0] = new String ("spawn " + command + " "
289           + info.getHostParametersString() + " -U " + login + " -W " + options
290           + " " + info.getDbName());
291     cmdBuff[1] = new String ("proc abort {} {");
292     cmdBuff[2] = new String (" system kill [exp_pid]");
293     cmdBuff[3] = new String (" exit 1");
294     cmdBuff[4] = new String ("}");
295     cmdBuff[5] = new String ("set exitcode 1");
296     cmdBuff[6] = new String ("expect {");
297     cmdBuff[7] = new String (" \"Password for user " + login
298         + ":\" { set exitcode 0 }");
299     cmdBuff[8] = new String (" \"assword:\" { set exitcode 0 }");
300     cmdBuff[9] = new String (" timeout { abort }");
301     cmdBuff[10] = new String ("}");
302     cmdBuff[11] = new String ("send \"" + password + "\"");
303     cmdBuff[12] = new String ("send \"\r\"");
304     cmdBuff[13] = new String ("set timeout " + timeout);
305     cmdBuff[14] = new String ("expect {");
306     cmdBuff[15] = new String (
307         " \"ERROR:\" { set exitcode 1; exp_continue -continue_timer }");
308     cmdBuff[16] = new String (
309         " \"FATAL:\" { set exitcode 1; exp_continue -continue_timer }");
310     cmdBuff[17] = new String (" timeout { abort }");
311     cmdBuff[18] = new String (" eof { if {$exitcode != 0} { abort }}");
312     cmdBuff[19] = new String ("}");
313     cmdBuff[20] = new String ("set rc [wait]");
314     cmdBuff[21] = new String ("set os_error [lindex $rc 2]");
315     cmdBuff[22] = new String ("set status [lindex $rc 3]");
316     cmdBuff[23] = new String ("if {$os_error != 0 || $status != 0} {");
317     cmdBuff[24] = new String (" exit 1");
318     cmdBuff[25] = new String ("}");
319     cmdBuff[26] = new String ("exit 0");
320     return cmdBuff;
321   }
322
323   /**
324    * Creates a Splitted command String array used my exec(). This method uses
325    * the "expect" command to pass the password parameter to the command being
326    * run, thus allowing PostgreSQL to allow authentication. Also uses BINDIR if
327    * supplied. Note the "psql" command has different output needed by "expect"
328    * than other commands.
329    *
330    * @param command Command to execute
331    * @param info URL info to parse adn add parameters
332    * @param options Additional parameters
333    * @param login User login
334    * @param password User password
335    * @param isPsql Is the command being run a psql command?
336    * @return String array containing full command to run
337    */
338   protected String [] makeSplitCommandWithAuthentication(String command,
339       PostgreSQLUrlInfo info, String options, String login, String password,
340       boolean isPsql)
341   {
342     // Build params for "spawn" command used by expect
343 StringBuffer cmdBuff = new StringBuffer ("spawn ");
344
345     if (binDir != null)
346       cmdBuff.append(binDir + File.separator);
347
348     cmdBuff.append(command);
349     cmdBuff.append(" ");
350     cmdBuff.append(info.dbName); // gurkan
351 cmdBuff.append(" "); // gurkan
352 cmdBuff.append(info.getHostParametersString());
353     cmdBuff.append(" -U ");
354     cmdBuff.append(login);
355     cmdBuff.append(" ");
356     cmdBuff.append(options);
357     // cmdBuff.append(" "); //gurkan
358 // cmdBuff.append(info.dbName); //gurkan
359 cmdBuff.append("; ");
360     // "psql" command has different message for password input
361 if (isPsql)
362       cmdBuff.append("expect \"Password:\"; ");
363     else
364       cmdBuff.append("expect \"Password for user " + login + ":\"; ");
365     cmdBuff.append("send \"");
366     cmdBuff.append(password);
367     cmdBuff.append("\"; ");
368     cmdBuff.append("send \"\\r\"; ");
369     cmdBuff.append("expect eof;");
370
371     // may need to be bash not expect //not sure ; gurkan
372 String [] commands = {"expect", "-c", "bash", "-c", cmdBuff.toString()}; // gurkan
373 // String[] commands = {"expect", "-c", cmdBuff.toString()};
374 return commands;
375   }
376
377   /**
378    * Executes a native operating system command with a standard-input feed. The
379    * commands are supposed to be one of the Postgres-utilities psql - dropdb -
380    * createdb - pg_dump - pg_restore. Output of the command is captured and
381    * logged.
382    *
383    * @param stdinFeed Array of input strings
384    * @param commands Array of strings (command + args) to execute
385    * @return 0 if successful, any number otherwise
386    * @throws IOException
387    * @throws InterruptedException
388    */
389   protected int executeNativeCommand(String [] stdinFeed, String [] commands)
390       throws IOException , InterruptedException
391   {
392     NativeCommandInputSource input = NativeCommandInputSource
393         .createArrayInputSource(stdinFeed);
394     return nativeCmdExec.executeNativeCommand(commands, input, null, 0,
395         getIgnoreStdErrOutput());
396   }
397
398   /**
399    * Creates a expect command reading the expect-script from standard-input.
400    * Hours of experiments has shown that the only reliable way of
401    * programatically invoking expect with an expect-script, is via stdin or
402    * explicit script-file. The expect "-c command" option simply does not work
403    * the same way, and do cause a lot of problems.
404    *
405    * @return String array containing full command to run
406    */
407   protected String [] makeExpectCommandReadingStdin()
408   {
409     String [] commands = {"expect", "-"};
410     // String[] commands = {"expect", "-d", "-"}; // Diagnostic output
411 // String[] commands = {"cat"}; // Echoes the expect-script instead of
412 // running it
413 return commands;
414   }
415
416   /**
417    * Executes a native operating system command, which currently are one of: -
418    * psql - dropdb - createdb - pg_dump - pg_restore Output of these commands is
419    * captured and logged.
420    *
421    * @param command String of command to execute
422    * @return 0 if successful, any number otherwise
423    * @throws IOException
424    * @throws InterruptedException
425    */
426   protected int executeNativeCommand(String command) throws IOException ,
427       InterruptedException
428   {
429     return nativeCmdExec.executeNativeCommand(command, null, null, 0,
430         getIgnoreStdErrOutput());
431   }
432
433   /**
434    * Executes a native operating system command, which currently are one of:
435    *
436    * <pre>
437    * - psql
438    * - dropdb
439    * - createdb
440    * - pg_dump
441    * - pg_restore
442    * </pre>
443    *
444    * Output of these commands is captured and logged.
445    *
446    * @param commands String array of command to execute
447    * @return 0 if successful, any number otherwise
448    * @throws IOException
449    * @throws InterruptedException
450    */
451   protected int executeNativeCommand(String [] commands) throws IOException ,
452       InterruptedException
453   {
454     return nativeCmdExec.executeNativeCommand(commands, null, null, 0,
455         getIgnoreStdErrOutput());
456   }
457
458   /**
459    * Execute a native command with careful logging of output and errors.
460    *
461    * @param cmd Command to execute
462    * @param inputArray Array of input lines
463    * @param timeout Timeout or 0 for no timeout
464    * @return True if command is successful
465    */
466   protected boolean safelyExecNativeCommand(String cmd, String [] inputArray,
467       int timeout)
468   {
469     NativeCommandInputSource input = NativeCommandInputSource
470         .createArrayInputSource(inputArray);
471     return nativeCmdExec.safelyExecNativeCommand(cmd, input, null, timeout,
472         getIgnoreStdErrOutput());
473   }
474
475   /**
476    * Execute a native command with careful logging of output and errors.
477    *
478    * @param cmds Command array to execute
479    * @param inputArray Array of input lines
480    * @param timeout Timeout or 0 for no timeout
481    * @return True if command is successful
482    */
483   protected boolean safelyExecNativeCommand(String [] cmds, String [] inputArray,
484       int timeout)
485   {
486     NativeCommandInputSource input = NativeCommandInputSource
487         .createArrayInputSource(inputArray);
488     return nativeCmdExec.safelyExecNativeCommand(cmds, input, null, timeout,
489         getIgnoreStdErrOutput());
490   }
491
492   /**
493    * Prints contents of error output (stderr).
494    */
495   protected void printErrors()
496   {
497     ArrayList errors = nativeCmdExec.getStderr();
498     Iterator it = errors.iterator();
499     while (it.hasNext())
500     {
501       String msg = (String ) it.next();
502       logger.info(msg);
503       endUserLogger.error(msg);
504     }
505   }
506
507   /**
508    * Prints contents of regular output (stdout).
509    */
510   protected void printOutput()
511   {
512     ArrayList errors = nativeCmdExec.getStderr();
513     Iterator it = errors.iterator();
514     while (it.hasNext())
515     {
516       String msg = (String ) it.next();
517       logger.info(msg);
518       endUserLogger.error(msg);
519     }
520   }
521
522   /**
523    * Allow to parse PostgreSQL URL.
524    */
525   protected class PostgreSQLUrlInfo
526   {
527     private boolean isLocal;
528     private String host;
529     private String port;
530     private String dbName;
531
532     // Used to parse url
533 // private Pattern pattern =
534 // Pattern.compile("jdbc:postgresql:((//([a-zA-Z0-9_\\-.]+|\\[[a-fA-F0-9:]+])((:(\\d+))|))/|)([a-zA-Z][a-zA-Z0-9_]*)");
535 private Pattern pattern = Pattern
536                                 .compile("jdbc:postgresql:((//([a-zA-Z0-9_\\-.]+|\\[[a-fA-F0-9:]+])((:(\\d+))|))/|)([^\\s?]*).*$");
537     Matcher matcher;
538
539     /**
540      * Creates a new <code>PostgreSQLUrlInfo</code> object, used to parse the
541      * postgresql jdbc options. If host and/or port aren't specified, will
542      * default to localhost:5432. Note that database name must be specified.
543      *
544      * @param url the Postgresql jdbc url to parse
545      */
546     public PostgreSQLUrlInfo(String url)
547     {
548       matcher = pattern.matcher(url);
549
550       if (matcher.matches())
551       {
552         if (matcher.group(3) != null)
553           host = matcher.group(3);
554         else
555           host = DEFAULT_POSTGRESQL_HOST;
556
557         if (matcher.group(6) != null)
558           port = matcher.group(6);
559         else
560           port = DEFAULT_POSTGRESQL_PORT;
561
562         dbName = matcher.group(7);
563       }
564     }
565
566     /**
567      * Gets the HostParameters of this postgresql jdbc url as a String that can
568      * be used to pass into cmd line/shell calls.
569      *
570      * @return a string that can be used to pass into a cmd line/shell call.
571      */
572     public String getHostParametersString()
573     {
574       if (isLocal)
575       {
576         return "";
577       }
578       else
579       {
580         return "-h " + host + " -p " + port;
581       }
582     }
583
584     /**
585      * Gets the database name part of this postgresql jdbc url.
586      *
587      * @return the database name part of this postgresql jdbc url.
588      */
589     public String getDbName()
590     {
591       return dbName;
592     }
593
594     /**
595      * Gets the host part of this postgresql jdbc url.
596      *
597      * @return the host part of this postgresql jdbc url.
598      */
599     public String getHost()
600     {
601       return host;
602     }
603
604     /**
605      * Gets the port part of this postgresql jdbc url.
606      *
607      * @return the port part of this postgresql jdbc url.
608      */
609     public String getPort()
610     {
611       return port;
612     }
613
614     /**
615      * Checks whether this postgresql jdbc url refers to a local db or not, i.e.
616      * has no host specified, e.g. jdbc:postgresql:myDb.
617      *
618      * @return true if this postgresql jdbc url has no host specified, i.e.
619      * refers to a local db.
620      */
621     public boolean isLocal()
622     {
623       return isLocal;
624     }
625
626   }
627 }
628

A to Z: JavaDoc & Examples

---

Daily Java News & Articles

---

Open Source Projects

---

Open Source Codes

---

Free Computer Books

---

Remove Frame

---

Popular Tags