SQOOP-931: Integrate HCatalog with Sqoop
[sqoop.git] / src / java / org / apache / sqoop / tool / BaseSqoopTool.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.tool;
20
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.sql.SQLException;
26 import java.util.Arrays;
27 import java.util.Properties;
28
29 import org.apache.commons.cli.CommandLine;
30 import org.apache.commons.cli.Option;
31 import org.apache.commons.cli.OptionBuilder;
32 import org.apache.commons.cli.OptionGroup;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.util.StringUtils;
36
37 import com.cloudera.sqoop.ConnFactory;
38 import com.cloudera.sqoop.Sqoop;
39 import com.cloudera.sqoop.SqoopOptions;
40 import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
41 import com.cloudera.sqoop.cli.RelatedOptions;
42 import com.cloudera.sqoop.cli.ToolOptions;
43 import com.cloudera.sqoop.lib.DelimiterSet;
44 import com.cloudera.sqoop.manager.ConnManager;
45 import com.cloudera.sqoop.metastore.JobData;
46 import org.apache.sqoop.util.CredentialsUtil;
47 import org.apache.sqoop.util.LoggingUtils;
48
49 /**
50 * Layer on top of SqoopTool that provides some basic common code
51 * that most SqoopTool implementations will use.
52 *
53 * Subclasses should call init() at the top of their run() method,
54 * and call destroy() at the end in a finally block.
55 */
56 public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
57
58 public static final Log LOG = LogFactory.getLog(
59 BaseSqoopTool.class.getName());
60
61 public static final String HELP_STR = "\nTry --help for usage instructions.";
62
63 // Here are all the arguments that are used by the standard sqoop tools.
64 // Their names are recorded here so that tools can share them and their
65 // use consistently. The argument parser applies the leading '--' to each
66 // string.
67 public static final String CONNECT_STRING_ARG = "connect";
68 public static final String CONN_MANAGER_CLASS_NAME =
69 "connection-manager";
70 public static final String CONNECT_PARAM_FILE = "connection-param-file";
71 public static final String DRIVER_ARG = "driver";
72 public static final String USERNAME_ARG = "username";
73 public static final String PASSWORD_ARG = "password";
74 public static final String PASSWORD_PROMPT_ARG = "P";
75 public static final String PASSWORD_PATH_ARG = "password-file";
76 public static final String DIRECT_ARG = "direct";
77 public static final String BATCH_ARG = "batch";
78 public static final String TABLE_ARG = "table";
79 public static final String STAGING_TABLE_ARG = "staging-table";
80 public static final String CLEAR_STAGING_TABLE_ARG = "clear-staging-table";
81 public static final String COLUMNS_ARG = "columns";
82 public static final String SPLIT_BY_ARG = "split-by";
83 public static final String WHERE_ARG = "where";
84 public static final String HADOOP_HOME_ARG = "hadoop-home";
85 public static final String HADOOP_MAPRED_HOME_ARG = "hadoop-mapred-home";
86 public static final String HIVE_HOME_ARG = "hive-home";
87 public static final String WAREHOUSE_DIR_ARG = "warehouse-dir";
88 public static final String TARGET_DIR_ARG = "target-dir";
89 public static final String APPEND_ARG = "append";
90 public static final String NULL_STRING = "null-string";
91 public static final String INPUT_NULL_STRING = "input-null-string";
92 public static final String NULL_NON_STRING = "null-non-string";
93 public static final String INPUT_NULL_NON_STRING = "input-null-non-string";
94 public static final String MAP_COLUMN_JAVA = "map-column-java";
95 public static final String MAP_COLUMN_HIVE = "map-column-hive";
96
97 public static final String FMT_SEQUENCEFILE_ARG = "as-sequencefile";
98 public static final String FMT_TEXTFILE_ARG = "as-textfile";
99 public static final String FMT_AVRODATAFILE_ARG = "as-avrodatafile";
100 public static final String HIVE_IMPORT_ARG = "hive-import";
101 public static final String HIVE_TABLE_ARG = "hive-table";
102 public static final String HIVE_DATABASE_ARG = "hive-database";
103 public static final String HIVE_OVERWRITE_ARG = "hive-overwrite";
104 public static final String HIVE_DROP_DELIMS_ARG = "hive-drop-import-delims";
105 public static final String HIVE_DELIMS_REPLACEMENT_ARG =
106 "hive-delims-replacement";
107 public static final String HIVE_PARTITION_KEY_ARG = "hive-partition-key";
108 public static final String HIVE_PARTITION_VALUE_ARG = "hive-partition-value";
109 public static final String CREATE_HIVE_TABLE_ARG =
110 "create-hive-table";
111 public static final String HCATALOG_TABLE_ARG = "hcatalog-table";
112 public static final String HCATALOG_DATABASE_ARG = "hcatalog-database";
113 public static final String CREATE_HCATALOG_TABLE_ARG =
114 "create-hcatalog-table";
115 public static final String HCATALOG_STORAGE_STANZA_ARG =
116 "hcatalog-storage-stanza";
117 public static final String HCATALOG_HOME_ARG = "hcatalog-home";
118 public static final String MAPREDUCE_JOB_NAME = "mapreduce-job-name";
119 public static final String NUM_MAPPERS_ARG = "num-mappers";
120 public static final String NUM_MAPPERS_SHORT_ARG = "m";
121 public static final String COMPRESS_ARG = "compress";
122 public static final String COMPRESSION_CODEC_ARG = "compression-codec";
123 public static final String COMPRESS_SHORT_ARG = "z";
124 public static final String DIRECT_SPLIT_SIZE_ARG = "direct-split-size";
125 public static final String INLINE_LOB_LIMIT_ARG = "inline-lob-limit";
126 public static final String FETCH_SIZE_ARG = "fetch-size";
127 public static final String EXPORT_PATH_ARG = "export-dir";
128 public static final String FIELDS_TERMINATED_BY_ARG = "fields-terminated-by";
129 public static final String LINES_TERMINATED_BY_ARG = "lines-terminated-by";
130 public static final String OPTIONALLY_ENCLOSED_BY_ARG =
131 "optionally-enclosed-by";
132 public static final String ENCLOSED_BY_ARG = "enclosed-by";
133 public static final String ESCAPED_BY_ARG = "escaped-by";
134 public static final String MYSQL_DELIMITERS_ARG = "mysql-delimiters";
135 public static final String INPUT_FIELDS_TERMINATED_BY_ARG =
136 "input-fields-terminated-by";
137 public static final String INPUT_LINES_TERMINATED_BY_ARG =
138 "input-lines-terminated-by";
139 public static final String INPUT_OPTIONALLY_ENCLOSED_BY_ARG =
140 "input-optionally-enclosed-by";
141 public static final String INPUT_ENCLOSED_BY_ARG = "input-enclosed-by";
142 public static final String INPUT_ESCAPED_BY_ARG = "input-escaped-by";
143 public static final String CODE_OUT_DIR_ARG = "outdir";
144 public static final String BIN_OUT_DIR_ARG = "bindir";
145 public static final String PACKAGE_NAME_ARG = "package-name";
146 public static final String CLASS_NAME_ARG = "class-name";
147 public static final String JAR_FILE_NAME_ARG = "jar-file";
148 public static final String SQL_QUERY_ARG = "query";
149 public static final String SQL_QUERY_BOUNDARY = "boundary-query";
150 public static final String SQL_QUERY_SHORT_ARG = "e";
151 public static final String VERBOSE_ARG = "verbose";
152 public static final String HELP_ARG = "help";
153 public static final String UPDATE_KEY_ARG = "update-key";
154 public static final String UPDATE_MODE_ARG = "update-mode";
155 public static final String CALL_ARG = "call";
156
157 // Arguments for validation.
158 public static final String VALIDATE_ARG = "validate";
159 public static final String VALIDATOR_CLASS_ARG = "validator";
160 public static final String VALIDATION_THRESHOLD_CLASS_ARG =
161 "validation-threshold";
162 public static final String VALIDATION_FAILURE_HANDLER_CLASS_ARG =
163 "validation-failurehandler";
164
165 // Arguments for incremental imports.
166 public static final String INCREMENT_TYPE_ARG = "incremental";
167 public static final String INCREMENT_COL_ARG = "check-column";
168 public static final String INCREMENT_LAST_VAL_ARG = "last-value";
169
170 // Arguments for all table imports.
171 public static final String ALL_TABLE_EXCLUDES_ARG = "exclude-tables";
172
173 // HBase arguments.
174 public static final String HBASE_TABLE_ARG = "hbase-table";
175 public static final String HBASE_COL_FAM_ARG = "column-family";
176 public static final String HBASE_ROW_KEY_ARG = "hbase-row-key";
177 public static final String HBASE_CREATE_TABLE_ARG = "hbase-create-table";
178
179
180 // Arguments for the saved job management system.
181 public static final String STORAGE_METASTORE_ARG = "meta-connect";
182 public static final String JOB_CMD_CREATE_ARG = "create";
183 public static final String JOB_CMD_DELETE_ARG = "delete";
184 public static final String JOB_CMD_EXEC_ARG = "exec";
185 public static final String JOB_CMD_LIST_ARG = "list";
186 public static final String JOB_CMD_SHOW_ARG = "show";
187
188 // Arguments for the metastore.
189 public static final String METASTORE_SHUTDOWN_ARG = "shutdown";
190
191
192 // Arguments for merging datasets.
193 public static final String NEW_DATASET_ARG = "new-data";
194 public static final String OLD_DATASET_ARG = "onto";
195 public static final String MERGE_KEY_ARG = "merge-key";
196
197 public BaseSqoopTool() {
198 }
199
200 public BaseSqoopTool(String toolName) {
201 super(toolName);
202 }
203
204 protected ConnManager manager;
205
206 public ConnManager getManager() {
207 return manager;
208 }
209
210 public void setManager(ConnManager mgr) {
211 this.manager = mgr;
212 }
213
214 /**
215 * Should be called at the beginning of the run() method to initialize
216 * the connection manager, etc. If this succeeds (returns true), it should
217 * be paired with a call to destroy().
218 * @return true on success, false on failure.
219 */
220 protected boolean init(SqoopOptions sqoopOpts) {
221 // Get the connection to the database.
222 try {
223 JobData data = new JobData(sqoopOpts, this);
224 this.manager = new ConnFactory(sqoopOpts.getConf()).getManager(data);
225 return true;
226 } catch (Exception e) {
227 LOG.error("Got error creating database manager: "
228 + StringUtils.stringifyException(e));
229 if (System.getProperty(Sqoop.SQOOP_RETHROW_PROPERTY) != null) {
230 throw new RuntimeException(e);
231 }
232 }
233
234 return false;
235 }
236
237 /**
238 * Should be called in a 'finally' block at the end of the run() method.
239 */
240 protected void destroy(SqoopOptions sqoopOpts) {
241 if (null != manager) {
242 try {
243 manager.close();
244 } catch (SQLException sqlE) {
245 LOG.warn("Error while closing connection: " + sqlE);
246 }
247 }
248 }
249
250 /**
251 * Examines a subset of the arrray presented, and determines if it
252 * contains any non-empty arguments. If so, logs the arguments
253 * and returns true.
254 *
255 * @param argv an array of strings to check.
256 * @param offset the first element of the array to check
257 * @param len the number of elements to check
258 * @return true if there are any non-null, non-empty argument strings
259 * present.
260 */
261 protected boolean hasUnrecognizedArgs(String [] argv, int offset, int len) {
262 if (argv == null) {
263 return false;
264 }
265
266 boolean unrecognized = false;
267 boolean printedBanner = false;
268 for (int i = offset; i < Math.min(argv.length, offset + len); i++) {
269 if (argv[i] != null && argv[i].length() > 0) {
270 if (!printedBanner) {
271 LOG.error("Error parsing arguments for " + getToolName() + ":");
272 printedBanner = true;
273 }
274 LOG.error("Unrecognized argument: " + argv[i]);
275 unrecognized = true;
276 }
277 }
278
279 return unrecognized;
280 }
281
282 protected boolean hasUnrecognizedArgs(String [] argv) {
283 if (null == argv) {
284 return false;
285 }
286 return hasUnrecognizedArgs(argv, 0, argv.length);
287 }
288
289
290 /**
291 * If argv contains an entry "--", return an array containing all elements
292 * after the "--" separator. Otherwise, return null.
293 * @param argv a set of arguments to scan for the subcommand arguments.
294 */
295 protected String [] getSubcommandArgs(String [] argv) {
296 if (null == argv) {
297 return null;
298 }
299
300 for (int i = 0; i < argv.length; i++) {
301 if (argv[i].equals("--")) {
302 return Arrays.copyOfRange(argv, i + 1, argv.length);
303 }
304 }
305
306 return null;
307 }
308
309 /**
310 * @return RelatedOptions used by job management tools.
311 */
312 protected RelatedOptions getJobOptions() {
313 RelatedOptions relatedOpts = new RelatedOptions(
314 "Job management arguments");
315 relatedOpts.addOption(OptionBuilder.withArgName("jdbc-uri")
316 .hasArg()
317 .withDescription("Specify JDBC connect string for the metastore")
318 .withLongOpt(STORAGE_METASTORE_ARG)
319 .create());
320
321 // Create an option-group surrounding the operations a user
322 // can perform on jobs.
323 OptionGroup group = new OptionGroup();
324 group.addOption(OptionBuilder.withArgName("job-id")
325 .hasArg()
326 .withDescription("Create a new saved job")
327 .withLongOpt(JOB_CMD_CREATE_ARG)
328 .create());
329 group.addOption(OptionBuilder.withArgName("job-id")
330 .hasArg()
331 .withDescription("Delete a saved job")
332 .withLongOpt(JOB_CMD_DELETE_ARG)
333 .create());
334 group.addOption(OptionBuilder.withArgName("job-id")
335 .hasArg()
336 .withDescription("Show the parameters for a saved job")
337 .withLongOpt(JOB_CMD_SHOW_ARG)
338 .create());
339
340 Option execOption = OptionBuilder.withArgName("job-id")
341 .hasArg()
342 .withDescription("Run a saved job")
343 .withLongOpt(JOB_CMD_EXEC_ARG)
344 .create();
345 group.addOption(execOption);
346
347 group.addOption(OptionBuilder
348 .withDescription("List saved jobs")
349 .withLongOpt(JOB_CMD_LIST_ARG)
350 .create());
351
352 relatedOpts.addOptionGroup(group);
353
354 // Since the "common" options aren't used in the job tool,
355 // add these settings here.
356 relatedOpts.addOption(OptionBuilder
357 .withDescription("Print more information while working")
358 .withLongOpt(VERBOSE_ARG)
359 .create());
360 relatedOpts.addOption(OptionBuilder
361 .withDescription("Print usage instructions")
362 .withLongOpt(HELP_ARG)
363 .create());
364
365 return relatedOpts;
366 }
367
368 /**
369 * @return RelatedOptions used by most/all Sqoop tools.
370 */
371 protected RelatedOptions getCommonOptions() {
372 // Connection args (common)
373 RelatedOptions commonOpts = new RelatedOptions("Common arguments");
374 commonOpts.addOption(OptionBuilder.withArgName("jdbc-uri")
375 .hasArg().withDescription("Specify JDBC connect string")
376 .withLongOpt(CONNECT_STRING_ARG)
377 .create());
378 commonOpts.addOption(OptionBuilder.withArgName("class-name")
379 .hasArg().withDescription("Specify connection manager class name")
380 .withLongOpt(CONN_MANAGER_CLASS_NAME)
381 .create());
382 commonOpts.addOption(OptionBuilder.withArgName("properties-file")
383 .hasArg().withDescription("Specify connection parameters file")
384 .withLongOpt(CONNECT_PARAM_FILE)
385 .create());
386 commonOpts.addOption(OptionBuilder.withArgName("class-name")
387 .hasArg().withDescription("Manually specify JDBC driver class to use")
388 .withLongOpt(DRIVER_ARG)
389 .create());
390 commonOpts.addOption(OptionBuilder.withArgName("username")
391 .hasArg().withDescription("Set authentication username")
392 .withLongOpt(USERNAME_ARG)
393 .create());
394 commonOpts.addOption(OptionBuilder.withArgName("password")
395 .hasArg().withDescription("Set authentication password")
396 .withLongOpt(PASSWORD_ARG)
397 .create());
398 commonOpts.addOption(OptionBuilder.withArgName(PASSWORD_PATH_ARG)
399 .hasArg().withDescription("Set authentication password file path")
400 .withLongOpt(PASSWORD_PATH_ARG)
401 .create());
402 commonOpts.addOption(OptionBuilder
403 .withDescription("Read password from console")
404 .create(PASSWORD_PROMPT_ARG));
405
406 commonOpts.addOption(OptionBuilder.withArgName("dir")
407 .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG")
408 .withLongOpt(HADOOP_MAPRED_HOME_ARG)
409 .create());
410
411 commonOpts.addOption(OptionBuilder.withArgName("hdir")
412 .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG")
413 .withLongOpt(HADOOP_HOME_ARG)
414 .create());
415
416 // misc (common)
417 commonOpts.addOption(OptionBuilder
418 .withDescription("Print more information while working")
419 .withLongOpt(VERBOSE_ARG)
420 .create());
421 commonOpts.addOption(OptionBuilder
422 .withDescription("Print usage instructions")
423 .withLongOpt(HELP_ARG)
424 .create());
425
426 return commonOpts;
427 }
428
429 /**
430 * @param explicitHiveImport true if the user has an explicit --hive-import
431 * available, or false if this is implied by the tool.
432 * @return options governing interaction with Hive
433 */
434 protected RelatedOptions getHiveOptions(boolean explicitHiveImport) {
435 RelatedOptions hiveOpts = new RelatedOptions("Hive arguments");
436 if (explicitHiveImport) {
437 hiveOpts.addOption(OptionBuilder
438 .withDescription("Import tables into Hive "
439 + "(Uses Hive's default delimiters if none are set.)")
440 .withLongOpt(HIVE_IMPORT_ARG)
441 .create());
442 }
443
444 hiveOpts.addOption(OptionBuilder.withArgName("dir")
445 .hasArg().withDescription("Override $HIVE_HOME")
446 .withLongOpt(HIVE_HOME_ARG)
447 .create());
448 hiveOpts.addOption(OptionBuilder
449 .withDescription("Overwrite existing data in the Hive table")
450 .withLongOpt(HIVE_OVERWRITE_ARG)
451 .create());
452 hiveOpts.addOption(OptionBuilder
453 .withDescription("Fail if the target hive table exists")
454 .withLongOpt(CREATE_HIVE_TABLE_ARG)
455 .create());
456 hiveOpts.addOption(OptionBuilder.withArgName("table-name")
457 .hasArg()
458 .withDescription("Sets the table name to use when importing to hive")
459 .withLongOpt(HIVE_TABLE_ARG)
460 .create());
461 hiveOpts.addOption(OptionBuilder.withArgName("database-name")
462 .hasArg()
463 .withDescription("Sets the database name to use when importing to hive")
464 .withLongOpt(HIVE_DATABASE_ARG)
465 .create());
466 hiveOpts.addOption(OptionBuilder
467 .withDescription("Drop Hive record \\0x01 and row delimiters "
468 + "(\\n\\r) from imported string fields")
469 .withLongOpt(HIVE_DROP_DELIMS_ARG)
470 .create());
471 hiveOpts.addOption(OptionBuilder
472 .hasArg()
473 .withDescription("Replace Hive record \\0x01 and row delimiters "
474 + "(\\n\\r) from imported string fields with user-defined string")
475 .withLongOpt(HIVE_DELIMS_REPLACEMENT_ARG)
476 .create());
477 hiveOpts.addOption(OptionBuilder.withArgName("partition-key")
478 .hasArg()
479 .withDescription("Sets the partition key to use when importing to hive")
480 .withLongOpt(HIVE_PARTITION_KEY_ARG)
481 .create());
482 hiveOpts.addOption(OptionBuilder.withArgName("partition-value")
483 .hasArg()
484 .withDescription("Sets the partition value to use when importing "
485 + "to hive")
486 .withLongOpt(HIVE_PARTITION_VALUE_ARG)
487 .create());
488 hiveOpts.addOption(OptionBuilder
489 .hasArg()
490 .withDescription("Override mapping for specific column to hive"
491 + " types.")
492 .withLongOpt(MAP_COLUMN_HIVE)
493 .create());
494
495 return hiveOpts;
496 }
497
498 /**
499 * @return options governing interaction with HCatalog.
500 */
501 protected RelatedOptions getHCatalogOptions() {
502 RelatedOptions hCatOptions = new RelatedOptions("HCatalog arguments");
503 hCatOptions.addOption(OptionBuilder
504 .hasArg()
505 .withDescription("HCatalog table name")
506 .withLongOpt(HCATALOG_TABLE_ARG)
507 .create());
508 hCatOptions.addOption(OptionBuilder
509 .hasArg()
510 .withDescription("HCatalog database name")
511 .withLongOpt(HCATALOG_DATABASE_ARG)
512 .create());
513
514 hCatOptions.addOption(OptionBuilder.withArgName("dir")
515 .hasArg().withDescription("Override $HIVE_HOME")
516 .withLongOpt(HIVE_HOME_ARG)
517 .create());
518 hCatOptions.addOption(OptionBuilder.withArgName("hdir")
519 .hasArg().withDescription("Override $HCAT_HOME")
520 .withLongOpt(HCATALOG_HOME_ARG)
521 .create());
522 hCatOptions.addOption(OptionBuilder.withArgName("partition-key")
523 .hasArg()
524 .withDescription("Sets the partition key to use when importing to hive")
525 .withLongOpt(HIVE_PARTITION_KEY_ARG)
526 .create());
527 hCatOptions.addOption(OptionBuilder.withArgName("partition-value")
528 .hasArg()
529 .withDescription("Sets the partition value to use when importing "
530 + "to hive")
531 .withLongOpt(HIVE_PARTITION_VALUE_ARG)
532 .create());
533 hCatOptions.addOption(OptionBuilder
534 .hasArg()
535 .withDescription("Override mapping for specific column to hive"
536 + " types.")
537 .withLongOpt(MAP_COLUMN_HIVE)
538 .create());
539
540 return hCatOptions;
541 }
542
543 protected RelatedOptions getHCatImportOnlyOptions() {
544 RelatedOptions hCatOptions = new RelatedOptions(
545 "HCatalog import specific options");
546 hCatOptions.addOption(OptionBuilder
547 .withDescription("Create HCatalog before import")
548 .withLongOpt(CREATE_HCATALOG_TABLE_ARG)
549 .create());
550 hCatOptions.addOption(OptionBuilder
551 .hasArg()
552 .withDescription("HCatalog storage stanza for table creation")
553 .withLongOpt(HCATALOG_STORAGE_STANZA_ARG)
554 .create());
555 return hCatOptions;
556 }
557
558 /**
559 * @return options governing output format delimiters
560 */
561 protected RelatedOptions getOutputFormatOptions() {
562 RelatedOptions formatOpts = new RelatedOptions(
563 "Output line formatting arguments");
564 formatOpts.addOption(OptionBuilder.withArgName("char")
565 .hasArg()
566 .withDescription("Sets the field separator character")
567 .withLongOpt(FIELDS_TERMINATED_BY_ARG)
568 .create());
569 formatOpts.addOption(OptionBuilder.withArgName("char")
570 .hasArg()
571 .withDescription("Sets the end-of-line character")
572 .withLongOpt(LINES_TERMINATED_BY_ARG)
573 .create());
574 formatOpts.addOption(OptionBuilder.withArgName("char")
575 .hasArg()
576 .withDescription("Sets a field enclosing character")
577 .withLongOpt(OPTIONALLY_ENCLOSED_BY_ARG)
578 .create());
579 formatOpts.addOption(OptionBuilder.withArgName("char")
580 .hasArg()
581 .withDescription("Sets a required field enclosing character")
582 .withLongOpt(ENCLOSED_BY_ARG)
583 .create());
584 formatOpts.addOption(OptionBuilder.withArgName("char")
585 .hasArg()
586 .withDescription("Sets the escape character")
587 .withLongOpt(ESCAPED_BY_ARG)
588 .create());
589 formatOpts.addOption(OptionBuilder
590 .withDescription("Uses MySQL's default delimiter set: "
591 + "fields: , lines: \\n escaped-by: \\ optionally-enclosed-by: '")
592 .withLongOpt(MYSQL_DELIMITERS_ARG)
593 .create());
594
595 return formatOpts;
596 }
597
598 /**
599 * @return options governing input format delimiters.
600 */
601 protected RelatedOptions getInputFormatOptions() {
602 RelatedOptions inputFormatOpts =
603 new RelatedOptions("Input parsing arguments");
604 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
605 .hasArg()
606 .withDescription("Sets the input field separator")
607 .withLongOpt(INPUT_FIELDS_TERMINATED_BY_ARG)
608 .create());
609 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
610 .hasArg()
611 .withDescription("Sets the input end-of-line char")
612 .withLongOpt(INPUT_LINES_TERMINATED_BY_ARG)
613 .create());
614 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
615 .hasArg()
616 .withDescription("Sets a field enclosing character")
617 .withLongOpt(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)
618 .create());
619 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
620 .hasArg()
621 .withDescription("Sets a required field encloser")
622 .withLongOpt(INPUT_ENCLOSED_BY_ARG)
623 .create());
624 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
625 .hasArg()
626 .withDescription("Sets the input escape character")
627 .withLongOpt(INPUT_ESCAPED_BY_ARG)
628 .create());
629
630 return inputFormatOpts;
631 }
632
633 /**
634 * @param multiTable true if these options will be used for bulk code-gen.
635 * @return options related to code generation.
636 */
637 protected RelatedOptions getCodeGenOpts(boolean multiTable) {
638 RelatedOptions codeGenOpts =
639 new RelatedOptions("Code generation arguments");
640 codeGenOpts.addOption(OptionBuilder.withArgName("dir")
641 .hasArg()
642 .withDescription("Output directory for generated code")
643 .withLongOpt(CODE_OUT_DIR_ARG)
644 .create());
645 codeGenOpts.addOption(OptionBuilder.withArgName("dir")
646 .hasArg()
647 .withDescription("Output directory for compiled objects")
648 .withLongOpt(BIN_OUT_DIR_ARG)
649 .create());
650 codeGenOpts.addOption(OptionBuilder.withArgName("name")
651 .hasArg()
652 .withDescription("Put auto-generated classes in this package")
653 .withLongOpt(PACKAGE_NAME_ARG)
654 .create());
655 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
656 .hasArg()
657 .withDescription("Null string representation")
658 .withLongOpt(NULL_STRING)
659 .create());
660 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
661 .hasArg()
662 .withDescription("Input null string representation")
663 .withLongOpt(INPUT_NULL_STRING)
664 .create());
665 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
666 .hasArg()
667 .withDescription("Null non-string representation")
668 .withLongOpt(NULL_NON_STRING)
669 .create());
670 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
671 .hasArg()
672 .withDescription("Input null non-string representation")
673 .withLongOpt(INPUT_NULL_NON_STRING)
674 .create());
675 codeGenOpts.addOption(OptionBuilder
676 .hasArg()
677 .withDescription("Override mapping for specific columns to java types")
678 .withLongOpt(MAP_COLUMN_JAVA)
679 .create());
680
681 if (!multiTable) {
682 codeGenOpts.addOption(OptionBuilder.withArgName("name")
683 .hasArg()
684 .withDescription("Sets the generated class name. "
685 + "This overrides --" + PACKAGE_NAME_ARG + ". When combined "
686 + "with --" + JAR_FILE_NAME_ARG + ", sets the input class.")
687 .withLongOpt(CLASS_NAME_ARG)
688 .create());
689 }
690 return codeGenOpts;
691 }
692
693 protected RelatedOptions getHBaseOptions() {
694 RelatedOptions hbaseOpts =
695 new RelatedOptions("HBase arguments");
696 hbaseOpts.addOption(OptionBuilder.withArgName("table")
697 .hasArg()
698 .withDescription("Import to <table> in HBase")
699 .withLongOpt(HBASE_TABLE_ARG)
700 .create());
701 hbaseOpts.addOption(OptionBuilder.withArgName("family")
702 .hasArg()
703 .withDescription("Sets the target column family for the import")
704 .withLongOpt(HBASE_COL_FAM_ARG)
705 .create());
706 hbaseOpts.addOption(OptionBuilder.withArgName("col")
707 .hasArg()
708 .withDescription("Specifies which input column to use as the row key")
709 .withLongOpt(HBASE_ROW_KEY_ARG)
710 .create());
711 hbaseOpts.addOption(OptionBuilder
712 .withDescription("If specified, create missing HBase tables")
713 .withLongOpt(HBASE_CREATE_TABLE_ARG)
714 .create());
715
716 return hbaseOpts;
717 }
718
719 @SuppressWarnings("static-access")
720 protected void addValidationOpts(RelatedOptions validationOptions) {
721 validationOptions.addOption(OptionBuilder
722 .withDescription("Validate the copy using the configured validator")
723 .withLongOpt(VALIDATE_ARG)
724 .create());
725 validationOptions.addOption(OptionBuilder
726 .withArgName(VALIDATOR_CLASS_ARG).hasArg()
727 .withDescription("Fully qualified class name for the Validator")
728 .withLongOpt(VALIDATOR_CLASS_ARG)
729 .create());
730 validationOptions.addOption(OptionBuilder
731 .withArgName(VALIDATION_THRESHOLD_CLASS_ARG).hasArg()
732 .withDescription("Fully qualified class name for ValidationThreshold")
733 .withLongOpt(VALIDATION_THRESHOLD_CLASS_ARG)
734 .create());
735 validationOptions.addOption(OptionBuilder
736 .withArgName(VALIDATION_FAILURE_HANDLER_CLASS_ARG).hasArg()
737 .withDescription("Fully qualified class name for "
738 + "ValidationFailureHandler")
739 .withLongOpt(VALIDATION_FAILURE_HANDLER_CLASS_ARG)
740 .create());
741 }
742
743 /**
744 * Apply common command-line to the state.
745 */
746 protected void applyCommonOptions(CommandLine in, SqoopOptions out)
747 throws InvalidOptionsException {
748
749 // common options.
750 if (in.hasOption(VERBOSE_ARG)) {
751 // Immediately switch into DEBUG logging.
752 out.setVerbose(true);
753 LoggingUtils.setDebugLevel();
754 LOG.debug("Enabled debug logging.");
755 }
756
757 if (in.hasOption(HELP_ARG)) {
758 ToolOptions toolOpts = new ToolOptions();
759 configureOptions(toolOpts);
760 printHelp(toolOpts);
761 throw new InvalidOptionsException("");
762 }
763
764 if (in.hasOption(CONNECT_STRING_ARG)) {
765 out.setConnectString(in.getOptionValue(CONNECT_STRING_ARG));
766 }
767
768 if (in.hasOption(CONN_MANAGER_CLASS_NAME)) {
769 out.setConnManagerClassName(in.getOptionValue(CONN_MANAGER_CLASS_NAME));
770 }
771
772 if (in.hasOption(CONNECT_PARAM_FILE)) {
773 File paramFile = new File(in.getOptionValue(CONNECT_PARAM_FILE));
774 if (!paramFile.exists()) {
775 throw new InvalidOptionsException(
776 "Specified connection parameter file not found: " + paramFile);
777 }
778 InputStream inStream = null;
779 Properties connectionParams = new Properties();
780 try {
781 inStream = new FileInputStream(
782 new File(in.getOptionValue(CONNECT_PARAM_FILE)));
783 connectionParams.load(inStream);
784 } catch (IOException ex) {
785 LOG.warn("Failed to load connection parameter file", ex);
786 throw new InvalidOptionsException(
787 "Error while loading connection parameter file: "
788 + ex.getMessage());
789 } finally {
790 if (inStream != null) {
791 try {
792 inStream.close();
793 } catch (IOException ex) {
794 LOG.warn("Failed to close input stream", ex);
795 }
796 }
797 }
798 LOG.debug("Loaded connection parameters: " + connectionParams);
799 out.setConnectionParams(connectionParams);
800 }
801
802 if (in.hasOption(NULL_STRING)) {
803 out.setNullStringValue(in.getOptionValue(NULL_STRING));
804 }
805
806 if (in.hasOption(INPUT_NULL_STRING)) {
807 out.setInNullStringValue(in.getOptionValue(INPUT_NULL_STRING));
808 }
809
810 if (in.hasOption(NULL_NON_STRING)) {
811 out.setNullNonStringValue(in.getOptionValue(NULL_NON_STRING));
812 }
813
814 if (in.hasOption(INPUT_NULL_NON_STRING)) {
815 out.setInNullNonStringValue(in.getOptionValue(INPUT_NULL_NON_STRING));
816 }
817
818 if (in.hasOption(DRIVER_ARG)) {
819 out.setDriverClassName(in.getOptionValue(DRIVER_ARG));
820 }
821
822 applyCredentialsOptions(in, out);
823
824
825 if (in.hasOption(HADOOP_MAPRED_HOME_ARG)) {
826 out.setHadoopMapRedHome(in.getOptionValue(HADOOP_MAPRED_HOME_ARG));
827 // Only consider HADOOP_HOME if HADOOP_MAPRED_HOME is not set
828 } else if (in.hasOption(HADOOP_HOME_ARG)) {
829 out.setHadoopMapRedHome(in.getOptionValue(HADOOP_HOME_ARG));
830 }
831 }
832
833 private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
834 throws InvalidOptionsException {
835 if (in.hasOption(USERNAME_ARG)) {
836 out.setUsername(in.getOptionValue(USERNAME_ARG));
837 if (null == out.getPassword()) {
838 // Set password to empty if the username is set first,
839 // to ensure that they're either both null or neither is.
840 out.setPassword("");
841 }
842 }
843
844 if (in.hasOption(PASSWORD_ARG)) {
845 LOG.warn("Setting your password on the command-line is insecure. "
846 + "Consider using -" + PASSWORD_PROMPT_ARG + " instead.");
847 out.setPassword(in.getOptionValue(PASSWORD_ARG));
848 }
849
850 if (in.hasOption(PASSWORD_PROMPT_ARG)) {
851 out.setPasswordFromConsole();
852 }
853
854 if (in.hasOption(PASSWORD_PATH_ARG)) {
855 if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)) {
856 throw new InvalidOptionsException("Either password or path to a "
857 + "password file must be specified but not both.");
858 }
859
860 try {
861 out.setPasswordFilePath(in.getOptionValue(PASSWORD_PATH_ARG));
862 // apply password from file into password in options
863 out.setPassword(CredentialsUtil.fetchPasswordFromFile(out));
864 } catch (IOException ex) {
865 LOG.warn("Failed to load connection parameter file", ex);
866 throw new InvalidOptionsException(
867 "Error while loading connection parameter file: "
868 + ex.getMessage());
869 }
870 }
871 }
872
873 protected void applyHiveOptions(CommandLine in, SqoopOptions out)
874 throws InvalidOptionsException {
875
876 if (in.hasOption(HIVE_HOME_ARG)) {
877 out.setHiveHome(in.getOptionValue(HIVE_HOME_ARG));
878 }
879
880 if (in.hasOption(HIVE_IMPORT_ARG)) {
881 out.setHiveImport(true);
882 }
883
884 if (in.hasOption(HIVE_OVERWRITE_ARG)) {
885 out.setOverwriteHiveTable(true);
886 }
887
888 if (in.hasOption(CREATE_HIVE_TABLE_ARG)) {
889 out.setFailIfHiveTableExists(true);
890 }
891
892 if (in.hasOption(HIVE_TABLE_ARG)) {
893 out.setHiveTableName(in.getOptionValue(HIVE_TABLE_ARG));
894 }
895
896 if (in.hasOption(HIVE_DATABASE_ARG)) {
897 out.setHiveDatabaseName(in.getOptionValue(HIVE_DATABASE_ARG));
898 }
899
900 if (in.hasOption(HIVE_DROP_DELIMS_ARG)) {
901 out.setHiveDropDelims(true);
902 }
903
904 if (in.hasOption(HIVE_DELIMS_REPLACEMENT_ARG)) {
905 out.setHiveDelimsReplacement(
906 in.getOptionValue(HIVE_DELIMS_REPLACEMENT_ARG));
907 }
908
909 if (in.hasOption(HIVE_PARTITION_KEY_ARG)) {
910 out.setHivePartitionKey(in.getOptionValue(HIVE_PARTITION_KEY_ARG));
911 }
912
913 if (in.hasOption(HIVE_PARTITION_VALUE_ARG)) {
914 out.setHivePartitionValue(in.getOptionValue(HIVE_PARTITION_VALUE_ARG));
915 }
916
917 if (in.hasOption(MAP_COLUMN_HIVE)) {
918 out.setMapColumnHive(in.getOptionValue(MAP_COLUMN_HIVE));
919 }
920 }
921
922 protected void applyHCatOptions(CommandLine in, SqoopOptions out) {
923 if (in.hasOption(HCATALOG_TABLE_ARG)) {
924 out.setHCatTableName(in.getOptionValue(HCATALOG_TABLE_ARG));
925 }
926
927 if (in.hasOption(HCATALOG_DATABASE_ARG)) {
928 out.setHCatDatabaseName(in.getOptionValue(HCATALOG_DATABASE_ARG));
929 }
930
931 if (in.hasOption(HCATALOG_STORAGE_STANZA_ARG)) {
932 out.setHCatStorageStanza(in.getOptionValue(HCATALOG_STORAGE_STANZA_ARG));
933 }
934
935 if (in.hasOption(CREATE_HCATALOG_TABLE_ARG)) {
936 out.setCreateHCatalogTable(true);
937 }
938
939 if (in.hasOption(HCATALOG_HOME_ARG)) {
940 out.setHCatHome(in.getOptionValue(HCATALOG_HOME_ARG));
941 }
942
943 // Allow some of the hive options also
944
945 if (in.hasOption(HIVE_HOME_ARG)) {
946 out.setHiveHome(in.getOptionValue(HIVE_HOME_ARG));
947 }
948
949 if (in.hasOption(HIVE_PARTITION_KEY_ARG)) {
950 out.setHivePartitionKey(in.getOptionValue(HIVE_PARTITION_KEY_ARG));
951 }
952
953 if (in.hasOption(HIVE_PARTITION_VALUE_ARG)) {
954 out.setHivePartitionValue(in.getOptionValue(HIVE_PARTITION_VALUE_ARG));
955 }
956
957 if (in.hasOption(MAP_COLUMN_HIVE)) {
958 out.setMapColumnHive(in.getOptionValue(MAP_COLUMN_HIVE));
959 }
960 }
961
962
963 protected void applyOutputFormatOptions(CommandLine in, SqoopOptions out)
964 throws InvalidOptionsException {
965 if (in.hasOption(FIELDS_TERMINATED_BY_ARG)) {
966 out.setFieldsTerminatedBy(SqoopOptions.toChar(
967 in.getOptionValue(FIELDS_TERMINATED_BY_ARG)));
968 out.setExplicitOutputDelims(true);
969 }
970
971 if (in.hasOption(LINES_TERMINATED_BY_ARG)) {
972 out.setLinesTerminatedBy(SqoopOptions.toChar(
973 in.getOptionValue(LINES_TERMINATED_BY_ARG)));
974 out.setExplicitOutputDelims(true);
975 }
976
977 if (in.hasOption(OPTIONALLY_ENCLOSED_BY_ARG)) {
978 out.setEnclosedBy(SqoopOptions.toChar(
979 in.getOptionValue(OPTIONALLY_ENCLOSED_BY_ARG)));
980 out.setOutputEncloseRequired(false);
981 out.setExplicitOutputDelims(true);
982 }
983
984 if (in.hasOption(ENCLOSED_BY_ARG)) {
985 out.setEnclosedBy(SqoopOptions.toChar(
986 in.getOptionValue(ENCLOSED_BY_ARG)));
987 out.setOutputEncloseRequired(true);
988 out.setExplicitOutputDelims(true);
989 }
990
991 if (in.hasOption(ESCAPED_BY_ARG)) {
992 out.setEscapedBy(SqoopOptions.toChar(
993 in.getOptionValue(ESCAPED_BY_ARG)));
994 out.setExplicitOutputDelims(true);
995 }
996
997 if (in.hasOption(MYSQL_DELIMITERS_ARG)) {
998 out.setOutputEncloseRequired(false);
999 out.setFieldsTerminatedBy(',');
1000 out.setLinesTerminatedBy('\n');
1001 out.setEscapedBy('\\');
1002 out.setEnclosedBy('\'');
1003 out.setExplicitOutputDelims(true);
1004 }
1005 }
1006
1007 protected void applyInputFormatOptions(CommandLine in, SqoopOptions out)
1008 throws InvalidOptionsException {
1009 if (in.hasOption(INPUT_FIELDS_TERMINATED_BY_ARG)) {
1010 out.setInputFieldsTerminatedBy(SqoopOptions.toChar(
1011 in.getOptionValue(INPUT_FIELDS_TERMINATED_BY_ARG)));
1012 out.setExplicitInputDelims(true);
1013 }
1014
1015 if (in.hasOption(INPUT_LINES_TERMINATED_BY_ARG)) {
1016 out.setInputLinesTerminatedBy(SqoopOptions.toChar(
1017 in.getOptionValue(INPUT_LINES_TERMINATED_BY_ARG)));
1018 out.setExplicitInputDelims(true);
1019 }
1020
1021 if (in.hasOption(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)) {
1022 out.setInputEnclosedBy(SqoopOptions.toChar(
1023 in.getOptionValue(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)));
1024 out.setInputEncloseRequired(false);
1025 out.setExplicitInputDelims(true);
1026 }
1027
1028 if (in.hasOption(INPUT_ENCLOSED_BY_ARG)) {
1029 out.setInputEnclosedBy(SqoopOptions.toChar(
1030 in.getOptionValue(INPUT_ENCLOSED_BY_ARG)));
1031 out.setInputEncloseRequired(true);
1032 out.setExplicitInputDelims(true);
1033 }
1034
1035 if (in.hasOption(INPUT_ESCAPED_BY_ARG)) {
1036 out.setInputEscapedBy(SqoopOptions.toChar(
1037 in.getOptionValue(INPUT_ESCAPED_BY_ARG)));
1038 out.setExplicitInputDelims(true);
1039 }
1040 }
1041
1042 protected void applyCodeGenOptions(CommandLine in, SqoopOptions out,
1043 boolean multiTable) throws InvalidOptionsException {
1044 if (in.hasOption(CODE_OUT_DIR_ARG)) {
1045 out.setCodeOutputDir(in.getOptionValue(CODE_OUT_DIR_ARG));
1046 }
1047
1048 if (in.hasOption(BIN_OUT_DIR_ARG)) {
1049 out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG));
1050 }
1051
1052 if (in.hasOption(PACKAGE_NAME_ARG)) {
1053 out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
1054 }
1055
1056 if (in.hasOption(MAP_COLUMN_JAVA)) {
1057 out.setMapColumnJava(in.getOptionValue(MAP_COLUMN_JAVA));
1058 }
1059
1060 if (!multiTable && in.hasOption(CLASS_NAME_ARG)) {
1061 out.setClassName(in.getOptionValue(CLASS_NAME_ARG));
1062 }
1063 }
1064
1065 protected void applyHBaseOptions(CommandLine in, SqoopOptions out) {
1066 if (in.hasOption(HBASE_TABLE_ARG)) {
1067 out.setHBaseTable(in.getOptionValue(HBASE_TABLE_ARG));
1068 }
1069
1070 if (in.hasOption(HBASE_COL_FAM_ARG)) {
1071 out.setHBaseColFamily(in.getOptionValue(HBASE_COL_FAM_ARG));
1072 }
1073
1074 if (in.hasOption(HBASE_ROW_KEY_ARG)) {
1075 out.setHBaseRowKeyColumn(in.getOptionValue(HBASE_ROW_KEY_ARG));
1076 }
1077
1078 if (in.hasOption(HBASE_CREATE_TABLE_ARG)) {
1079 out.setCreateHBaseTable(true);
1080 }
1081 }
1082
1083 protected void applyValidationOptions(CommandLine in, SqoopOptions out)
1084 throws InvalidOptionsException {
1085 if (in.hasOption(VALIDATE_ARG)) {
1086 out.setValidationEnabled(true);
1087 }
1088
1089 // Class Names are converted to Class in light of failing early
1090 if (in.hasOption(VALIDATOR_CLASS_ARG)) {
1091 out.setValidatorClass(
1092 getClassByName(in.getOptionValue(VALIDATOR_CLASS_ARG)));
1093 }
1094
1095 if (in.hasOption(VALIDATION_THRESHOLD_CLASS_ARG)) {
1096 out.setValidationThresholdClass(
1097 getClassByName(in.getOptionValue(VALIDATION_THRESHOLD_CLASS_ARG)));
1098 }
1099
1100 if (in.hasOption(VALIDATION_FAILURE_HANDLER_CLASS_ARG)) {
1101 out.setValidationFailureHandlerClass(getClassByName(
1102 in.getOptionValue(VALIDATION_FAILURE_HANDLER_CLASS_ARG)));
1103 }
1104 }
1105
1106 protected Class<?> getClassByName(String className)
1107 throws InvalidOptionsException {
1108 try {
1109 return Class.forName(className, true,
1110 Thread.currentThread().getContextClassLoader());
1111 } catch (ClassNotFoundException e) {
1112 throw new InvalidOptionsException(e.getMessage());
1113 }
1114 }
1115
1116 protected void validateCommonOptions(SqoopOptions options)
1117 throws InvalidOptionsException {
1118 if (options.getConnectString() == null) {
1119 throw new InvalidOptionsException(
1120 "Error: Required argument --connect is missing."
1121 + HELP_STR);
1122 }
1123 }
1124
1125 protected void validateCodeGenOptions(SqoopOptions options)
1126 throws InvalidOptionsException {
1127 if (options.getClassName() != null && options.getPackageName() != null) {
1128 throw new InvalidOptionsException(
1129 "--class-name overrides --package-name. You cannot use both."
1130 + HELP_STR);
1131 }
1132 }
1133
1134 protected void validateOutputFormatOptions(SqoopOptions options)
1135 throws InvalidOptionsException {
1136 if (options.doHiveImport()) {
1137 if (!options.explicitOutputDelims()) {
1138 // user hasn't manually specified delimiters, and wants to import
1139 // straight to Hive. Use Hive-style delimiters.
1140 LOG.info("Using Hive-specific delimiters for output. You can override");
1141 LOG.info("delimiters with --fields-terminated-by, etc.");
1142 options.setOutputDelimiters(DelimiterSet.HIVE_DELIMITERS);
1143 }
1144
1145 if (options.getOutputEscapedBy() != DelimiterSet.NULL_CHAR) {
1146 LOG.warn("Hive does not support escape characters in fields;");
1147 LOG.warn("parse errors in Hive may result from using --escaped-by.");
1148 }
1149
1150 if (options.getOutputEnclosedBy() != DelimiterSet.NULL_CHAR) {
1151 LOG.warn("Hive does not support quoted strings; parse errors");
1152 LOG.warn("in Hive may result from using --enclosed-by.");
1153 }
1154 }
1155 }
1156
1157 protected void validateHiveOptions(SqoopOptions options)
1158 throws InvalidOptionsException {
1159 if (options.getHiveDelimsReplacement() != null
1160 && options.doHiveDropDelims()) {
1161 throw new InvalidOptionsException("The " + HIVE_DROP_DELIMS_ARG
1162 + " option conflicts with the " + HIVE_DELIMS_REPLACEMENT_ARG
1163 + " option." + HELP_STR);
1164 }
1165
1166 // Make sure that one of hCatalog or hive jobs are used
1167 String hCatTable = options.getHCatTableName();
1168 if (hCatTable != null && options.doHiveImport()) {
1169 throw new InvalidOptionsException("The " + HCATALOG_TABLE_ARG
1170 + " option conflicts with the " + HIVE_IMPORT_ARG
1171 + " option." + HELP_STR);
1172 }
1173
1174 if(options.doHiveImport()
1175 && options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
1176 throw new InvalidOptionsException("Hive import is not compatible with "
1177 + "importing into AVRO format.");
1178 }
1179
1180 if(options.doHiveImport()
1181 && options.getFileLayout() == SqoopOptions.FileLayout.SequenceFile) {
1182 throw new InvalidOptionsException("Hive import is not compatible with "
1183 + "importing into SequenceFile format.");
1184 }
1185
1186 // Many users are reporting issues when they are trying to import data
1187 // directly into hive warehouse. This should prevent users from doing
1188 // so in case of a default location.
1189 String defaultHiveWarehouse = "/user/hive/warehouse";
1190 if (options.doHiveImport()
1191 && ((
1192 options.getWarehouseDir() != null
1193 && options.getWarehouseDir().startsWith(defaultHiveWarehouse)
1194 ) || (
1195 options.getTargetDir() != null
1196 && options.getTargetDir().startsWith(defaultHiveWarehouse)
1197 ))) {
1198 LOG.warn("It seems that you're doing hive import directly into default");
1199 LOG.warn("hive warehouse directory which is not supported. Sqoop is");
1200 LOG.warn("firstly importing data into separate directory and then");
1201 LOG.warn("inserting data into hive. Please consider removing");
1202 LOG.warn("--target-dir or --warehouse-dir into /user/hive/warehouse in");
1203 LOG.warn("case that you will detect any issues.");
1204 }
1205
1206 // Warn about using hive specific arguments without hive import itself
1207 // In HCatalog support some of the Hive options are reused
1208 if (!options.doHiveImport()
1209 && ((options.getHiveHome() != null
1210 && !options.getHiveHome().
1211 equals(SqoopOptions.getHiveHomeDefault())
1212 && hCatTable == null))
1213 || options.doOverwriteHiveTable()
1214 || options.doFailIfHiveTableExists()
1215 || (options.getHiveTableName() != null
1216 && !options.getHiveTableName().equals(options.getTableName()))
1217 || (options.getHivePartitionKey() != null && hCatTable == null)
1218 || (options.getHivePartitionValue() != null && hCatTable == null)
1219 || (options.getMapColumnHive().size() > 0 && hCatTable == null)) {
1220 LOG.warn("It seems that you've specified at least one of following:");
1221 LOG.warn("\t--hive-home");
1222 LOG.warn("\t--hive-overwrite");
1223 LOG.warn("\t--create-hive-table");
1224 LOG.warn("\t--hive-table");
1225 LOG.warn("\t--hive-partition-key");
1226 LOG.warn("\t--hive-partition-value");
1227 LOG.warn("\t--map-column-hive");
1228 LOG.warn("Without specifying parameter --hive-import. Please note that");
1229 LOG.warn("those arguments will not be used in this session. Either");
1230 LOG.warn("specify --hive-import to apply them correctly or remove them");
1231 LOG.warn("from command line to remove this warning.");
1232 LOG.info("Please note that --hive-home, --hive-partition-key, ");
1233 LOG.info("\t hive-partition-value and --map-column-hive options are ");
1234 LOG.info("\t are also valid for HCatalog imports and exports");
1235 }
1236 }
1237
1238 protected void validateHCatalogOptions(SqoopOptions options)
1239 throws InvalidOptionsException {
1240 // Make sure that one of hCatalog or hive jobs are used
1241 String hCatTable = options.getHCatTableName();
1242 if (hCatTable == null) {
1243 if (options.getHCatHome() != null && !options.getHCatHome().
1244 equals(SqoopOptions.getHCatHomeDefault())) {
1245 LOG.warn("--hcatalog-home option will be ignored in "
1246 + "non-HCatalog jobs");
1247 }
1248 if (options.getHCatDatabaseName() != null) {
1249 LOG.warn("--hcatalog-database option will be ignored "
1250 + "without --hcatalog-table");
1251 }
1252
1253 if (options.getHCatStorageStanza() != null) {
1254 LOG.warn("--hcatalog-storage-stanza option will be ignored "
1255 + "without --hatalog-table");
1256 }
1257 return;
1258 }
1259
1260 if (options.explicitInputDelims()) {
1261 LOG.warn("Input field/record delimiter options are not "
1262 + "used in HCatalog jobs unless the format is text. It is better "
1263 + "to use --hive-import in those cases. For text formats");
1264 }
1265 if (options.explicitOutputDelims()
1266 || options.getHiveDelimsReplacement() != null
1267 || options.doHiveDropDelims()) {
1268 LOG.warn("Output field/record delimiter options are not useful"
1269 + " in HCatalog jobs for most of the output types except text based "
1270 + " formats is text. It is better "
1271 + "to use --hive-import in those cases. For non text formats, ");
1272 }
1273 if (options.doHiveImport()) {
1274 throw new InvalidOptionsException("The " + HCATALOG_TABLE_ARG
1275 + " option conflicts with the " + HIVE_IMPORT_ARG
1276 + " option." + HELP_STR);
1277 }
1278 if (options.getTargetDir() != null) {
1279 throw new InvalidOptionsException("The " + TARGET_DIR_ARG
1280 + " option conflicts with the " + HCATALOG_TABLE_ARG
1281 + " option." + HELP_STR);
1282 }
1283 if (options.getWarehouseDir() != null) {
1284 throw new InvalidOptionsException("The " + WAREHOUSE_DIR_ARG
1285 + " option conflicts with the " + HCATALOG_TABLE_ARG
1286 + " option." + HELP_STR);
1287 }
1288 if (options.isDirect()) {
1289 throw new InvalidOptionsException("Direct import is incompatible with "
1290 + "HCatalog. Please remove the parameter --direct");
1291 }
1292 if (options.isAppendMode()) {
1293 throw new InvalidOptionsException("Append mode for imports is not "
1294 + " compatible with HCatalog. Please remove the parameter"
1295 + "--append-mode");
1296 }
1297 if (options.getExportDir() != null) {
1298 throw new InvalidOptionsException("The " + EXPORT_PATH_ARG
1299 + " option conflicts with the " + HCATALOG_TABLE_ARG
1300 + " option." + HELP_STR);
1301 }
1302
1303 if (options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
1304 throw new InvalidOptionsException("HCatalog job is not compatible with "
1305 + " AVRO format option " + FMT_AVRODATAFILE_ARG
1306 + " option." + HELP_STR);
1307
1308 }
1309
1310 if (options.getFileLayout() == SqoopOptions.FileLayout.SequenceFile) {
1311 throw new InvalidOptionsException("HCatalog job is not compatible with "
1312 + "SequenceFile format option " + FMT_SEQUENCEFILE_ARG
1313 + " option." + HELP_STR);
1314
1315 }
1316 }
1317
1318 protected void validateHBaseOptions(SqoopOptions options)
1319 throws InvalidOptionsException {
1320 if ((options.getHBaseColFamily() != null && options.getHBaseTable() == null)
1321 || (options.getHBaseColFamily() == null
1322 && options.getHBaseTable() != null)) {
1323 throw new InvalidOptionsException(
1324 "Both --hbase-table and --column-family must be set together."
1325 + HELP_STR);
1326 }
1327 if (options.getHBaseTable() != null && options.isDirect()) {
1328 throw new InvalidOptionsException("Direct import is incompatible with "
1329 + "HBase. Please remove parameter --direct");
1330 }
1331 }
1332
1333 /**
1334 * Given an array of extra arguments (usually populated via
1335 * this.extraArguments), determine the offset of the first '--'
1336 * argument in the list. Return 'extra.length' if there is none.
1337 */
1338 protected int getDashPosition(String [] extra) {
1339 int dashPos = extra.length;
1340 for (int i = 0; i < extra.length; i++) {
1341 if (extra[i].equals("--")) {
1342 dashPos = i;
1343 break;
1344 }
1345 }
1346
1347 return dashPos;
1348 }
1349 }
1350