|
1
8
9 package com.sleepycat.je.util;
10
11 import java.io.BufferedReader  ;
12 import java.io.File ;
13 import java.io.FileInputStream ;
14 import java.io.IOException ;
15 import java.io.InputStream ;
16 import java.io.InputStreamReader ;
17 import java.util.Date ;
18 import java.util.logging.Level ;
19
20 import com.sleepycat.je.Database;
21 import com.sleepycat.je.DatabaseConfig;
22 import com.sleepycat.je.DatabaseEntry;
23 import com.sleepycat.je.DatabaseException;
24 import com.sleepycat.je.DbInternal;
25 import com.sleepycat.je.Environment;
26 import com.sleepycat.je.EnvironmentConfig;
27 import com.sleepycat.je.JEVersion;
28 import com.sleepycat.je.OperationStatus;
29 import com.sleepycat.je.utilint.CmdUtil;
30 import com.sleepycat.je.utilint.Tracer;
31
32 public class DbLoad {
33 private static final boolean DEBUG = false;
34
35 protected Environment env;
36 private boolean formatUsingPrintable;
37 private String dbName;
38 private BufferedReader reader;
39 private boolean noOverwrite;
40 private boolean textFileMode;
41 private boolean dupSort;
42 private boolean ignoreUnknownConfig;
43 private boolean commandLine;
44 private long progressInterval;
45 private long totalLoadBytes;
46
47 private static final String usageString =
48 "usage: " + CmdUtil.getJavaCommand(DbLoad.class) + "\n" +
49 " -h <dir> # environment home directory\n" +
50 " [-f <fileName>] # input file\n" +
51 " [-n ] # no overwrite mode\n" +
52 " [-T] # input file is in text mode\n" +
53 " [-I] # ignore unknown parameters\n" +
54 " [-c name=value] # config values\n" +
55 " [-s <databaseName> ] # database to load\n" +
56 " [-v] # show progress\n" +
57 " [-V] # print JE version number";
58
59 static public void main(String argv[])
60 throws DatabaseException, IOException {
61
62 DbLoad loader = parseArgs(argv);
63
64 try {
65 loader.load();
66 } catch (Throwable e) {
67 e.printStackTrace();
68 }
69
70 loader.env.close();
71 }
72
73 static private void printUsage(String msg) {
74 System.err.println(msg);
75 System.err.println(usageString);
76 System.exit(-1);
77 }
78
79 static private DbLoad parseArgs(String argv[])
80 throws IOException , DatabaseException {
81
82 boolean noOverwrite = false;
83 boolean textFileMode = false;
84 boolean ignoreUnknownConfig = false;
85 boolean showProgressInterval = false;
86
87 int argc = 0;
88 int nArgs = argv.length;
89 String inputFileName = null;
90 File envHome = null;
91 String dbName = null;
92 long progressInterval = 0;
93 DbLoad ret = new DbLoad();
94 ret.setCommandLine(true);
95
96 while (argc < nArgs) {
97 String thisArg = argv[argc++].trim();
98 if (thisArg.equals("-n")) {
99 noOverwrite = true;
100 } else if (thisArg.equals("-T")) {
101 textFileMode = true;
102 } else if (thisArg.equals("-I")) {
103 ignoreUnknownConfig = true;
104 } else if (thisArg.equals("-V")) {
105 System.out.println(JEVersion.CURRENT_VERSION);
106 System.exit(0);
107 } else if (thisArg.equals("-f")) {
108 if (argc < nArgs) {
109 inputFileName = argv[argc++];
110 } else {
111 printUsage("-f requires an argument");
112 }
113 } else if (thisArg.equals("-h")) {
114 if (argc < nArgs) {
115 envHome = new File (argv[argc++]);
116 } else {
117 printUsage("-h requires an argument");
118 }
119 } else if (thisArg.equals("-s")) {
120 if (argc < nArgs) {
121 dbName = argv[argc++];
122 } else {
123 printUsage("-s requires an argument");
124 }
125 } else if (thisArg.equals("-c")) {
126 if (argc < nArgs) {
127 try {
128 ret.loadConfigLine(argv[argc++]);
129 } catch (IllegalArgumentException e) {
130 printUsage("-c: " + e.getMessage());
131 }
132 } else {
133 printUsage("-c requires an argument");
134 }
135 } else if (thisArg.equals("-v")) {
136 showProgressInterval = true;
137 }
138 }
139
140 if (envHome == null) {
141 printUsage("-h is a required argument");
142 }
143
144 long totalLoadBytes = 0;
145 InputStream is;
146 if (inputFileName == null) {
147 is = System.in;
148 if (showProgressInterval) {
149
150
154 printUsage("-v requires -f");
155 }
156 } else {
157 is = new FileInputStream (inputFileName);
158 if (showProgressInterval) {
159 totalLoadBytes = ((FileInputStream ) is).getChannel().size();
160
161 progressInterval = totalLoadBytes / 20;
162 }
163 }
164 BufferedReader reader = new BufferedReader (new InputStreamReader (is));
165
166 EnvironmentConfig envConfig = new EnvironmentConfig();
167 envConfig.setAllowCreate(true);
168 Environment env = new Environment(envHome, envConfig);
169 ret.setEnv(env);
170 ret.setDbName(dbName);
171 ret.setInputReader(reader);
172 ret.setNoOverwrite(noOverwrite);
173 ret.setTextFileMode(textFileMode);
174 ret.setIgnoreUnknownConfig(ignoreUnknownConfig);
175 ret.setProgressInterval(progressInterval);
176 ret.setTotalLoadBytes(totalLoadBytes);
177 return ret;
178 }
179
180
184
185 public DbLoad() {
186 }
187
188
192 private void setCommandLine(boolean commandLine) {
193 this.commandLine = commandLine;
194 }
195
196 public void setEnv(Environment env) {
197 this.env = env;
198 }
199
200 public void setDbName(String dbName) {
201 this.dbName = dbName;
202 }
203
204 public void setInputReader(BufferedReader reader) {
205 this.reader = reader;
206 }
207
208 public void setNoOverwrite(boolean noOverwrite) {
209 this.noOverwrite = noOverwrite;
210 }
211
212 public void setTextFileMode(boolean textFileMode) {
213 this.textFileMode = textFileMode;
214 }
215
216 public void setIgnoreUnknownConfig(boolean ignoreUnknownConfigMode) {
217 this.ignoreUnknownConfig = ignoreUnknownConfigMode;
218 }
219
220 public void setProgressInterval(long progressInterval) {
221 this.progressInterval = progressInterval;
222 }
223
224 public void setTotalLoadBytes(long totalLoadBytes) {
225 this.totalLoadBytes = totalLoadBytes;
226 }
227
228 public boolean load()
229 throws IOException , DatabaseException {
230
231 Tracer.trace(Level.INFO, DbInternal.envGetEnvironmentImpl(env),
232 "DbLoad.load of " + dbName + " starting");
233
234 if (progressInterval > 0) {
235 System.out.println("Load start: " + new Date ());
236 }
237
238 if (textFileMode) {
239 formatUsingPrintable = true;
240 } else {
241 loadHeader();
242 }
243
244 if (dbName == null) {
245 throw new IllegalArgumentException
246 ("Must supply a database name if -l not supplied.");
247 }
248
249 DatabaseConfig dbConfig = new DatabaseConfig();
250 dbConfig.setSortedDuplicates(dupSort);
251 dbConfig.setAllowCreate(true);
252 Database db = env.openDatabase(null, dbName, dbConfig);
253
254 loadData(db);
255
256 db.close();
257
258 Tracer.trace(Level.INFO, DbInternal.envGetEnvironmentImpl(env),
259 "DbLoad.load of " + dbName + " ending.");
260
261 if (progressInterval > 0) {
262 System.out.println("Load end: " + new Date ());
263 }
264
265 return true;
266 }
267
268 private void loadConfigLine(String line)
269 throws DatabaseException {
270
271 int equalsIdx = line.indexOf('=');
272 if (equalsIdx < 0) {
273 throw new IllegalArgumentException
274 ("Invalid header parameter: " + line);
275 }
276
277 String keyword = line.substring(0, equalsIdx).trim().toLowerCase();
278 String value = line.substring(equalsIdx + 1).trim();
279
280 if (keyword.equals("version")) {
281 if (DEBUG) {
282 System.out.println("Found version: " + line);
283 }
284 if (!value.equals("3")) {
285 throw new IllegalArgumentException
286 ("Version " + value + " is not supported.");
287 }
288 } else if (keyword.equals("format")) {
289 value = value.toLowerCase();
290 if (value.equals("print")) {
291 formatUsingPrintable = true;
292 } else if (value.equals("bytevalue")) {
293 formatUsingPrintable = false;
294 } else {
295 throw new IllegalArgumentException
296 (value + " is an unknown value for the format keyword");
297 }
298 if (DEBUG) {
299 System.out.println("Found format: " + formatUsingPrintable);
300 }
301 } else if (keyword.equals("dupsort")) {
302 value = value.toLowerCase();
303 if (value.equals("true") ||
304 value.equals("1")) {
305 dupSort = true;
306 } else if (value.equals("false") ||
307 value.equals("0")) {
308 dupSort = false;
309 } else {
310 throw new IllegalArgumentException
311 (value + " is an unknown value for the dupsort keyword");
312 }
313 if (DEBUG) {
314 System.out.println("Found dupsort: " + dupSort);
315 }
316 } else if (keyword.equals("type")) {
317 value = value.toLowerCase();
318 if (!value.equals("btree")) {
319 throw new IllegalArgumentException
320 (value + " is not a supported database type.");
321 }
322 if (DEBUG) {
323 System.out.println("Found type: " + line);
324 }
325 } else if (keyword.equals("database")) {
326 if (dbName == null) {
327 dbName = value;
328 }
329 if (DEBUG) {
330 System.out.println("DatabaseImpl: " + dbName);
331 }
332 } else if (!ignoreUnknownConfig) {
333 throw new IllegalArgumentException
334 ("'" + line + "' is not understood.");
335 }
336 }
337
338 private void loadHeader()
339 throws IOException , DatabaseException {
340
341 if (DEBUG) {
342 System.out.println("loading header");
343 }
344 String line = reader.readLine();
345 while (line != null &&
346 !line.equals("HEADER=END")) {
347 loadConfigLine(line);
348 line = reader.readLine();
349 }
350 }
351
352 private void loadData(Database db)
353 throws DatabaseException, IOException {
354
355 String keyLine = reader.readLine();
356 String dataLine = null;
357 int count = 0;
358 long totalBytesRead = 0;
359 long lastTime = System.currentTimeMillis();
360 long bytesReadThisInterval = 0;
361
362 while (keyLine != null &&
363 !keyLine.equals("DATA=END")) {
364 dataLine = reader.readLine();
365 if (dataLine == null) {
366 throw new DatabaseException("No data to match key " +
367 keyLine);
368 }
369
370 bytesReadThisInterval += dataLine.length() + 1;
371 byte[] keyBytes = loadLine(keyLine.trim());
372 byte[] dataBytes = loadLine(dataLine.trim());
373
374 DatabaseEntry key = new DatabaseEntry(keyBytes);
375 DatabaseEntry data = new DatabaseEntry(dataBytes);
376
377 if (noOverwrite) {
378 if (db.putNoOverwrite(null, key, data) ==
379 OperationStatus.KEYEXIST) {
380
381 if (commandLine) {
382 System.err.println("Key exists: " + key);
383 }
384 }
385 } else {
386 db.put(null, key, data);
387 }
388
389 count++;
390 if ((progressInterval > 0) &&
391 (bytesReadThisInterval > progressInterval)) {
392 totalBytesRead += bytesReadThisInterval;
393 bytesReadThisInterval -= progressInterval;
394 long now = System.currentTimeMillis();
395 System.out.println("loaded " + count + " records " +
396 (now - lastTime) + " ms - % completed: " +
397 ((100 * totalBytesRead) / totalLoadBytes));
398 lastTime = now;
399 }
400
401 keyLine = reader.readLine();
402 if (keyLine == null) {
403 throw new DatabaseException("No \"DATA=END\"");
404 }
405 bytesReadThisInterval += keyLine.length() + 1;
406 }
407 }
408
409 private byte[] loadLine(String line)
410 throws DatabaseException {
411
412 if (formatUsingPrintable) {
413 return readPrintableLine(line);
414 }
415 int nBytes = line.length() / 2;
416 byte[] ret = new byte[nBytes];
417 int charIdx = 0;
418 for (int i = 0; i < nBytes; i++, charIdx += 2) {
419 int b2 = Character.digit(line.charAt(charIdx), 16);
420 b2 <<= 4;
421 b2 += Character.digit(line.charAt(charIdx + 1), 16);
422 ret[i] = (byte) b2;
423 }
424 return ret;
425 }
426
427 static private byte backSlashValue =
428 (byte) (new Character ('\\').charValue() & 0xff);
429
430 private byte[] readPrintableLine(String line)
431 throws DatabaseException {
432
433
434 int maxNBytes = line.length();
435 byte[] ba = new byte[maxNBytes];
436 int actualNBytes = 0;
437
438 for (int charIdx = 0; charIdx < maxNBytes; charIdx++) {
439 char c = line.charAt(charIdx);
440 if (c == '\\') {
441 if (++charIdx < maxNBytes) {
442 char c1 = line.charAt(charIdx);
443 if (c1 == '\\') {
444 ba[actualNBytes++] = backSlashValue;
445 } else {
446 if (++charIdx < maxNBytes) {
447 char c2 = line.charAt(charIdx);
448 int b = Character.digit(c1, 16);
449 b <<= 4;
450 b += Character.digit(c2, 16);
451 ba[actualNBytes++] = (byte) b;
452 } else {
453 throw new DatabaseException("Corrupted file");
454 }
455 }
456 } else {
457 throw new DatabaseException("Corrupted file");
458 }
459 } else {
460 ba[actualNBytes++] = (byte) (c & 0xff);
461 }
462 }
463
464 if (maxNBytes == actualNBytes) {
465 return ba;
466 } else {
467 byte[] ret = new byte[actualNBytes];
468 System.arraycopy(ba, 0, ret, 0, actualNBytes);
469 return ret;
470 }
471 }
472 }
473 |
Popular Tags |
Java API By Example, From Geeks To Geeks.