b02e4fe7fda25c7f8171c7db17d15a7987459687
[sqoop.git] / src / java / org / apache / sqoop / tool / BaseSqoopTool.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.tool;
20
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.sql.SQLException;
26 import java.util.Arrays;
27 import java.util.Properties;
28
29 import org.apache.commons.cli.CommandLine;
30 import org.apache.commons.cli.Option;
31 import org.apache.commons.cli.OptionBuilder;
32 import org.apache.commons.cli.OptionGroup;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.util.StringUtils;
36 import org.apache.sqoop.manager.SupportedManagers;
37 import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
38 import org.apache.sqoop.util.CredentialsUtil;
39 import org.apache.sqoop.util.LoggingUtils;
40 import org.apache.sqoop.util.password.CredentialProviderHelper;
41
42 import org.apache.sqoop.ConnFactory;
43 import org.apache.sqoop.SqoopOptions;
44 import org.apache.sqoop.SqoopOptions.IncrementalMode;
45 import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
46 import org.apache.sqoop.cli.RelatedOptions;
47 import org.apache.sqoop.cli.ToolOptions;
48 import org.apache.sqoop.lib.DelimiterSet;
49 import org.apache.sqoop.manager.ConnManager;
50 import org.apache.sqoop.metastore.JobData;
51
52 /**
53 * Layer on top of SqoopTool that provides some basic common code
54 * that most SqoopTool implementations will use.
55 *
56 * Subclasses should call init() at the top of their run() method,
57 * and call destroy() at the end in a finally block.
58 */
59 public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
60
61 public static final String METADATA_TRANSACTION_ISOLATION_LEVEL = "metadata-transaction-isolation-level";
62
63 public static final Log LOG = LogFactory.getLog(
64 BaseSqoopTool.class.getName());
65
66 public static final String HELP_STR = "\nTry --help for usage instructions.";
67
68 // Here are all the arguments that are used by the standard sqoop tools.
69 // Their names are recorded here so that tools can share them and their
70 // use consistently. The argument parser applies the leading '--' to each
71 // string.
72 public static final String CONNECT_STRING_ARG = "connect";
73 public static final String CONN_MANAGER_CLASS_NAME =
74 "connection-manager";
75 public static final String CONNECT_PARAM_FILE = "connection-param-file";
76 public static final String DRIVER_ARG = "driver";
77 public static final String USERNAME_ARG = "username";
78 public static final String PASSWORD_ARG = "password";
79 public static final String PASSWORD_PROMPT_ARG = "P";
80 public static final String PASSWORD_PATH_ARG = "password-file";
81 public static final String PASSWORD_ALIAS_ARG = "password-alias";
82 public static final String DIRECT_ARG = "direct";
83 public static final String BATCH_ARG = "batch";
84 public static final String TABLE_ARG = "table";
85 public static final String STAGING_TABLE_ARG = "staging-table";
86 public static final String CLEAR_STAGING_TABLE_ARG = "clear-staging-table";
87 public static final String COLUMNS_ARG = "columns";
88 public static final String SPLIT_BY_ARG = "split-by";
89 public static final String SPLIT_LIMIT_ARG = "split-limit";
90 public static final String WHERE_ARG = "where";
91 public static final String HADOOP_HOME_ARG = "hadoop-home";
92 public static final String HADOOP_MAPRED_HOME_ARG = "hadoop-mapred-home";
93 public static final String HIVE_HOME_ARG = "hive-home";
94 public static final String WAREHOUSE_DIR_ARG = "warehouse-dir";
95 public static final String TARGET_DIR_ARG = "target-dir";
96 public static final String APPEND_ARG = "append";
97 public static final String DELETE_ARG = "delete-target-dir";
98 public static final String NULL_STRING = "null-string";
99 public static final String INPUT_NULL_STRING = "input-null-string";
100 public static final String NULL_NON_STRING = "null-non-string";
101 public static final String INPUT_NULL_NON_STRING = "input-null-non-string";
102 public static final String MAP_COLUMN_JAVA = "map-column-java";
103 public static final String MAP_COLUMN_HIVE = "map-column-hive";
104
105 public static final String FMT_SEQUENCEFILE_ARG = "as-sequencefile";
106 public static final String FMT_TEXTFILE_ARG = "as-textfile";
107 public static final String FMT_AVRODATAFILE_ARG = "as-avrodatafile";
108 public static final String FMT_PARQUETFILE_ARG = "as-parquetfile";
109 public static final String HIVE_IMPORT_ARG = "hive-import";
110 public static final String HIVE_TABLE_ARG = "hive-table";
111 public static final String HIVE_DATABASE_ARG = "hive-database";
112 public static final String HIVE_OVERWRITE_ARG = "hive-overwrite";
113 public static final String HIVE_DROP_DELIMS_ARG = "hive-drop-import-delims";
114 public static final String HIVE_DELIMS_REPLACEMENT_ARG =
115 "hive-delims-replacement";
116 public static final String HIVE_PARTITION_KEY_ARG = "hive-partition-key";
117 public static final String HIVE_PARTITION_VALUE_ARG = "hive-partition-value";
118 public static final String HIVE_EXTERNAL_TABLE_LOCATION_ARG = "external-table-dir";
119 public static final String HCATCALOG_PARTITION_KEYS_ARG =
120 "hcatalog-partition-keys";
121 public static final String HCATALOG_PARTITION_VALUES_ARG =
122 "hcatalog-partition-values";
123 public static final String CREATE_HIVE_TABLE_ARG =
124 "create-hive-table";
125 public static final String HCATALOG_TABLE_ARG = "hcatalog-table";
126 public static final String HCATALOG_DATABASE_ARG = "hcatalog-database";
127 public static final String CREATE_HCATALOG_TABLE_ARG =
128 "create-hcatalog-table";
129 public static final String DROP_AND_CREATE_HCATALOG_TABLE =
130 "drop-and-create-hcatalog-table";
131 public static final String HCATALOG_STORAGE_STANZA_ARG =
132 "hcatalog-storage-stanza";
133 public static final String HCATALOG_HOME_ARG = "hcatalog-home";
134 public static final String MAPREDUCE_JOB_NAME = "mapreduce-job-name";
135 public static final String NUM_MAPPERS_ARG = "num-mappers";
136 public static final String NUM_MAPPERS_SHORT_ARG = "m";
137 public static final String COMPRESS_ARG = "compress";
138 public static final String COMPRESSION_CODEC_ARG = "compression-codec";
139 public static final String COMPRESS_SHORT_ARG = "z";
140 public static final String DIRECT_SPLIT_SIZE_ARG = "direct-split-size";
141 public static final String INLINE_LOB_LIMIT_ARG = "inline-lob-limit";
142 public static final String FETCH_SIZE_ARG = "fetch-size";
143 public static final String EXPORT_PATH_ARG = "export-dir";
144 public static final String FIELDS_TERMINATED_BY_ARG = "fields-terminated-by";
145 public static final String LINES_TERMINATED_BY_ARG = "lines-terminated-by";
146 public static final String OPTIONALLY_ENCLOSED_BY_ARG =
147 "optionally-enclosed-by";
148 public static final String ENCLOSED_BY_ARG = "enclosed-by";
149 public static final String ESCAPED_BY_ARG = "escaped-by";
150 public static final String MYSQL_DELIMITERS_ARG = "mysql-delimiters";
151 public static final String INPUT_FIELDS_TERMINATED_BY_ARG =
152 "input-fields-terminated-by";
153 public static final String INPUT_LINES_TERMINATED_BY_ARG =
154 "input-lines-terminated-by";
155 public static final String INPUT_OPTIONALLY_ENCLOSED_BY_ARG =
156 "input-optionally-enclosed-by";
157 public static final String INPUT_ENCLOSED_BY_ARG = "input-enclosed-by";
158 public static final String INPUT_ESCAPED_BY_ARG = "input-escaped-by";
159 public static final String CODE_OUT_DIR_ARG = "outdir";
160 public static final String BIN_OUT_DIR_ARG = "bindir";
161 public static final String PACKAGE_NAME_ARG = "package-name";
162 public static final String CLASS_NAME_ARG = "class-name";
163 public static final String JAR_FILE_NAME_ARG = "jar-file";
164 public static final String SQL_QUERY_ARG = "query";
165 public static final String SQL_QUERY_BOUNDARY = "boundary-query";
166 public static final String SQL_QUERY_SHORT_ARG = "e";
167 public static final String VERBOSE_ARG = "verbose";
168 public static final String HELP_ARG = "help";
169 public static final String TEMP_ROOTDIR_ARG = "temporary-rootdir";
170 public static final String UPDATE_KEY_ARG = "update-key";
171 public static final String UPDATE_MODE_ARG = "update-mode";
172 public static final String CALL_ARG = "call";
173 public static final String SKIP_DISTCACHE_ARG = "skip-dist-cache";
174 public static final String RELAXED_ISOLATION = "relaxed-isolation";
175 public static final String THROW_ON_ERROR_ARG = "throw-on-error";
176 public static final String ORACLE_ESCAPING_DISABLED = "oracle-escaping-disabled";
177 public static final String ESCAPE_MAPPING_COLUMN_NAMES_ENABLED = "escape-mapping-column-names";
178
179 // Arguments for validation.
180 public static final String VALIDATE_ARG = "validate";
181 public static final String VALIDATOR_CLASS_ARG = "validator";
182 public static final String VALIDATION_THRESHOLD_CLASS_ARG =
183 "validation-threshold";
184 public static final String VALIDATION_FAILURE_HANDLER_CLASS_ARG =
185 "validation-failurehandler";
186
187 // Arguments for incremental imports.
188 public static final String INCREMENT_TYPE_ARG = "incremental";
189 public static final String INCREMENT_COL_ARG = "check-column";
190 public static final String INCREMENT_LAST_VAL_ARG = "last-value";
191
192 // Arguments for all table imports.
193 public static final String ALL_TABLE_EXCLUDES_ARG = "exclude-tables";
194
195 // HBase arguments.
196 public static final String HBASE_TABLE_ARG = "hbase-table";
197 public static final String HBASE_COL_FAM_ARG = "column-family";
198 public static final String HBASE_ROW_KEY_ARG = "hbase-row-key";
199 public static final String HBASE_BULK_LOAD_ENABLED_ARG =
200 "hbase-bulkload";
201 public static final String HBASE_CREATE_TABLE_ARG = "hbase-create-table";
202 public static final String HBASE_NULL_INCREMENTAL_MODE_ARG = "hbase-null-incremental-mode";
203
204 //Accumulo arguments.
205 public static final String ACCUMULO_TABLE_ARG = "accumulo-table";
206 public static final String ACCUMULO_COL_FAM_ARG = "accumulo-column-family";
207 public static final String ACCUMULO_ROW_KEY_ARG = "accumulo-row-key";
208 public static final String ACCUMULO_VISIBILITY_ARG = "accumulo-visibility";
209 public static final String ACCUMULO_CREATE_TABLE_ARG
210 = "accumulo-create-table";
211 public static final String ACCUMULO_BATCH_SIZE_ARG = "accumulo-batch-size";
212 public static final String ACCUMULO_MAX_LATENCY_ARG = "accumulo-max-latency";
213 public static final String ACCUMULO_ZOOKEEPERS_ARG = "accumulo-zookeepers";
214 public static final String ACCUMULO_INSTANCE_ARG = "accumulo-instance";
215 public static final String ACCUMULO_USER_ARG = "accumulo-user";
216 public static final String ACCUMULO_PASSWORD_ARG = "accumulo-password";
217
218
219 // Arguments for the saved job management system.
220 public static final String STORAGE_METASTORE_ARG = "meta-connect";
221 public static final String METASTORE_USER_ARG = "meta-username";
222 public static final String METASTORE_PASS_ARG = "meta-password";
223 public static final String JOB_CMD_CREATE_ARG = "create";
224 public static final String JOB_CMD_DELETE_ARG = "delete";
225 public static final String JOB_CMD_EXEC_ARG = "exec";
226 public static final String JOB_CMD_LIST_ARG = "list";
227 public static final String JOB_CMD_SHOW_ARG = "show";
228
229 // Arguments for the metastore.
230 public static final String METASTORE_SHUTDOWN_ARG = "shutdown";
231
232
233 // Arguments for merging datasets.
234 public static final String NEW_DATASET_ARG = "new-data";
235 public static final String OLD_DATASET_ARG = "onto";
236 public static final String MERGE_KEY_ARG = "merge-key";
237
238 // Reset number of mappers to one if there is no primary key avaliable and
239 // split by column is explicitly not provided
240
241 public static final String AUTORESET_TO_ONE_MAPPER = "autoreset-to-one-mapper";
242
243 static final String HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED = "--incremental lastmodified option for hive imports is not "
244 + "supported. Please remove the parameter --incremental lastmodified.";
245
246
247 public BaseSqoopTool() {
248 }
249
250 public BaseSqoopTool(String toolName) {
251 super(toolName);
252 }
253
254 protected ConnManager manager;
255
256 public ConnManager getManager() {
257 return manager;
258 }
259
260 public void setManager(ConnManager mgr) {
261 this.manager = mgr;
262 }
263
264 /**
265 * Should be called at the beginning of the run() method to initialize
266 * the connection manager, etc. If this succeeds (returns true), it should
267 * be paired with a call to destroy().
268 * @return true on success, false on failure.
269 */
270 protected boolean init(SqoopOptions sqoopOpts) {
271 // Get the connection to the database.
272 // Set the tool name in sqoop options
273 sqoopOpts.setToolName(getToolName());
274 try {
275 JobData data = new JobData(sqoopOpts, this);
276 this.manager = new ConnFactory(sqoopOpts.getConf()).getManager(data);
277 return true;
278 } catch (Exception e) {
279 LOG.error("Got error creating database manager: "
280 + StringUtils.stringifyException(e));
281 rethrowIfRequired(sqoopOpts, e);
282 }
283
284 return false;
285 }
286
287 protected void rethrowIfRequired(SqoopOptions options, Exception ex) {
288 if (!options.isThrowOnError()) {
289 return;
290 }
291
292 final RuntimeException exceptionToThrow;
293 if (ex instanceof RuntimeException) {
294 exceptionToThrow = (RuntimeException) ex;
295 } else {
296 exceptionToThrow = new RuntimeException(ex);
297 }
298
299 throw exceptionToThrow;
300 }
301
302
303 /**
304 * Should be called in a 'finally' block at the end of the run() method.
305 */
306 protected void destroy(SqoopOptions sqoopOpts) {
307 if (null != manager) {
308 try {
309 manager.close();
310 } catch (SQLException sqlE) {
311 LOG.warn("Error while closing connection: " + sqlE);
312 }
313 }
314 }
315
316 /**
317 * Examines a subset of the array presented, and determines if it
318 * contains any non-empty arguments. If so, logs the arguments
319 * and returns true.
320 *
321 * @param argv an array of strings to check.
322 * @param offset the first element of the array to check
323 * @param len the number of elements to check
324 * @return true if there are any non-null, non-empty argument strings
325 * present.
326 */
327 protected boolean hasUnrecognizedArgs(String [] argv, int offset, int len) {
328 if (argv == null) {
329 return false;
330 }
331
332 boolean unrecognized = false;
333 boolean printedBanner = false;
334 for (int i = offset; i < Math.min(argv.length, offset + len); i++) {
335 if (argv[i] != null && argv[i].length() > 0) {
336 if (!printedBanner) {
337 LOG.error("Error parsing arguments for " + getToolName() + ":");
338 printedBanner = true;
339 }
340 LOG.error("Unrecognized argument: " + argv[i]);
341 unrecognized = true;
342 }
343 }
344
345 return unrecognized;
346 }
347
348 protected boolean hasUnrecognizedArgs(String [] argv) {
349 if (null == argv) {
350 return false;
351 }
352 return hasUnrecognizedArgs(argv, 0, argv.length);
353 }
354
355
356 /**
357 * If argv contains an entry "--", return an array containing all elements
358 * after the "--" separator. Otherwise, return null.
359 * @param argv a set of arguments to scan for the subcommand arguments.
360 */
361 protected String [] getSubcommandArgs(String [] argv) {
362 if (null == argv) {
363 return null;
364 }
365
366 for (int i = 0; i < argv.length; i++) {
367 if (argv[i].equals("--")) {
368 return Arrays.copyOfRange(argv, i + 1, argv.length);
369 }
370 }
371
372 return null;
373 }
374
375 /**
376 * @return RelatedOptions used by job management tools.
377 */
378 protected RelatedOptions getJobOptions() {
379 RelatedOptions relatedOpts = new RelatedOptions(
380 "Job management arguments");
381 relatedOpts.addOption(OptionBuilder.withArgName("jdbc-uri")
382 .hasArg()
383 .withDescription("Specify JDBC connect string for the metastore")
384 .withLongOpt(STORAGE_METASTORE_ARG)
385 .create());
386 relatedOpts.addOption(OptionBuilder.withArgName("metastore-db-username")
387 .hasArg()
388 .withDescription("Specify the username string for the metastore")
389 .withLongOpt(METASTORE_USER_ARG)
390 .create());
391 relatedOpts.addOption(OptionBuilder.withArgName("metastore-db-password")
392 .hasArg()
393 .withDescription("Specify the password string for the metastore")
394 .withLongOpt(METASTORE_PASS_ARG)
395 .create());
396
397 // Create an option-group surrounding the operations a user
398 // can perform on jobs.
399 OptionGroup group = new OptionGroup();
400 group.addOption(OptionBuilder.withArgName("job-id")
401 .hasArg()
402 .withDescription("Create a new saved job")
403 .withLongOpt(JOB_CMD_CREATE_ARG)
404 .create());
405 group.addOption(OptionBuilder.withArgName("job-id")
406 .hasArg()
407 .withDescription("Delete a saved job")
408 .withLongOpt(JOB_CMD_DELETE_ARG)
409 .create());
410 group.addOption(OptionBuilder.withArgName("job-id")
411 .hasArg()
412 .withDescription("Show the parameters for a saved job")
413 .withLongOpt(JOB_CMD_SHOW_ARG)
414 .create());
415
416 Option execOption = OptionBuilder.withArgName("job-id")
417 .hasArg()
418 .withDescription("Run a saved job")
419 .withLongOpt(JOB_CMD_EXEC_ARG)
420 .create();
421 group.addOption(execOption);
422
423 group.addOption(OptionBuilder
424 .withDescription("List saved jobs")
425 .withLongOpt(JOB_CMD_LIST_ARG)
426 .create());
427
428 relatedOpts.addOptionGroup(group);
429
430 // Since the "common" options aren't used in the job tool,
431 // add these settings here.
432 relatedOpts.addOption(OptionBuilder
433 .withDescription("Print more information while working")
434 .withLongOpt(VERBOSE_ARG)
435 .create());
436 relatedOpts.addOption(OptionBuilder
437 .withDescription("Print usage instructions")
438 .withLongOpt(HELP_ARG)
439 .create());
440
441 return relatedOpts;
442 }
443
444 /**
445 * @return RelatedOptions used by most/all Sqoop tools.
446 */
447 protected RelatedOptions getCommonOptions() {
448 // Connection args (common)
449 RelatedOptions commonOpts = new RelatedOptions("Common arguments");
450 commonOpts.addOption(OptionBuilder.withArgName("jdbc-uri")
451 .hasArg().withDescription("Specify JDBC connect string")
452 .withLongOpt(CONNECT_STRING_ARG)
453 .create());
454 commonOpts.addOption(OptionBuilder.withArgName("class-name")
455 .hasArg().withDescription("Specify connection manager class name")
456 .withLongOpt(CONN_MANAGER_CLASS_NAME)
457 .create());
458 commonOpts.addOption(OptionBuilder.withArgName("properties-file")
459 .hasArg().withDescription("Specify connection parameters file")
460 .withLongOpt(CONNECT_PARAM_FILE)
461 .create());
462 commonOpts.addOption(OptionBuilder.withArgName("class-name")
463 .hasArg().withDescription("Manually specify JDBC driver class to use")
464 .withLongOpt(DRIVER_ARG)
465 .create());
466 commonOpts.addOption(OptionBuilder.withArgName("username")
467 .hasArg().withDescription("Set authentication username")
468 .withLongOpt(USERNAME_ARG)
469 .create());
470 commonOpts.addOption(OptionBuilder.withArgName("password")
471 .hasArg().withDescription("Set authentication password")
472 .withLongOpt(PASSWORD_ARG)
473 .create());
474 commonOpts.addOption(OptionBuilder.withArgName(PASSWORD_PATH_ARG)
475 .hasArg().withDescription("Set authentication password file path")
476 .withLongOpt(PASSWORD_PATH_ARG)
477 .create());
478 commonOpts.addOption(OptionBuilder
479 .withDescription("Read password from console")
480 .create(PASSWORD_PROMPT_ARG));
481 commonOpts.addOption(OptionBuilder.withArgName(PASSWORD_ALIAS_ARG)
482 .hasArg().withDescription("Credential provider password alias")
483 .withLongOpt(PASSWORD_ALIAS_ARG)
484 .create());
485 commonOpts.addOption(OptionBuilder.withArgName("dir")
486 .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG")
487 .withLongOpt(HADOOP_MAPRED_HOME_ARG)
488 .create());
489
490 commonOpts.addOption(OptionBuilder.withArgName("hdir")
491 .hasArg().withDescription("Override $HADOOP_MAPRED_HOME_ARG")
492 .withLongOpt(HADOOP_HOME_ARG)
493 .create());
494 commonOpts.addOption(OptionBuilder
495 .withDescription("Skip copying jars to distributed cache")
496 .withLongOpt(SKIP_DISTCACHE_ARG)
497 .create());
498
499 // misc (common)
500 commonOpts.addOption(OptionBuilder
501 .withDescription("Print more information while working")
502 .withLongOpt(VERBOSE_ARG)
503 .create());
504 commonOpts.addOption(OptionBuilder
505 .withDescription("Print usage instructions")
506 .withLongOpt(HELP_ARG)
507 .create());
508 commonOpts.addOption(OptionBuilder
509 .withDescription("Defines the temporary root directory for the import")
510 .withLongOpt(TEMP_ROOTDIR_ARG)
511 .hasArg()
512 .withArgName("rootdir")
513 .create());
514 commonOpts.addOption(OptionBuilder
515 .withDescription("Defines the transaction isolation level for metadata queries. "
516 + "For more details check java.sql.Connection javadoc or the JDBC specificaiton")
517 .withLongOpt(METADATA_TRANSACTION_ISOLATION_LEVEL)
518 .hasArg()
519 .withArgName("isolationlevel")
520 .create());
521 commonOpts.addOption(OptionBuilder
522 .withDescription("Rethrow a RuntimeException on error occurred during the job")
523 .withLongOpt(THROW_ON_ERROR_ARG)
524 .create());
525 // relax isolation requirements
526 commonOpts.addOption(OptionBuilder
527 .withDescription("Use read-uncommitted isolation for imports")
528 .withLongOpt(RELAXED_ISOLATION)
529 .create());
530
531 commonOpts.addOption(OptionBuilder
532 .withDescription("Disable the escaping mechanism of the Oracle/OraOop connection managers")
533 .withLongOpt(ORACLE_ESCAPING_DISABLED)
534 .hasArg()
535 .withArgName("boolean")
536 .create());
537
538 return commonOpts;
539 }
540
541 /**
542 * @param explicitHiveImport true if the user has an explicit --hive-import
543 * available, or false if this is implied by the tool.
544 * @return options governing interaction with Hive
545 */
546 protected RelatedOptions getHiveOptions(boolean explicitHiveImport) {
547 RelatedOptions hiveOpts = new RelatedOptions("Hive arguments");
548 if (explicitHiveImport) {
549 hiveOpts.addOption(OptionBuilder
550 .withDescription("Import tables into Hive "
551 + "(Uses Hive's default delimiters if none are set.)")
552 .withLongOpt(HIVE_IMPORT_ARG)
553 .create());
554 }
555
556 hiveOpts.addOption(OptionBuilder.withArgName("dir")
557 .hasArg().withDescription("Override $HIVE_HOME")
558 .withLongOpt(HIVE_HOME_ARG)
559 .create());
560 hiveOpts.addOption(OptionBuilder
561 .withDescription("Overwrite existing data in the Hive table")
562 .withLongOpt(HIVE_OVERWRITE_ARG)
563 .create());
564 hiveOpts.addOption(OptionBuilder
565 .withDescription("Fail if the target hive table exists")
566 .withLongOpt(CREATE_HIVE_TABLE_ARG)
567 .create());
568 hiveOpts.addOption(OptionBuilder.withArgName("table-name")
569 .hasArg()
570 .withDescription("Sets the table name to use when importing to hive")
571 .withLongOpt(HIVE_TABLE_ARG)
572 .create());
573 hiveOpts.addOption(OptionBuilder.withArgName("database-name")
574 .hasArg()
575 .withDescription("Sets the database name to use when importing to hive")
576 .withLongOpt(HIVE_DATABASE_ARG)
577 .create());
578 hiveOpts.addOption(OptionBuilder
579 .withDescription("Drop Hive record \\0x01 and row delimiters "
580 + "(\\n\\r) from imported string fields")
581 .withLongOpt(HIVE_DROP_DELIMS_ARG)
582 .create());
583 hiveOpts.addOption(OptionBuilder
584 .hasArg()
585 .withDescription("Replace Hive record \\0x01 and row delimiters "
586 + "(\\n\\r) from imported string fields with user-defined string")
587 .withLongOpt(HIVE_DELIMS_REPLACEMENT_ARG)
588 .create());
589 hiveOpts.addOption(OptionBuilder.withArgName("partition-key")
590 .hasArg()
591 .withDescription("Sets the partition key to use when importing to hive")
592 .withLongOpt(HIVE_PARTITION_KEY_ARG)
593 .create());
594 hiveOpts.addOption(OptionBuilder.withArgName("partition-value")
595 .hasArg()
596 .withDescription("Sets the partition value to use when importing "
597 + "to hive")
598 .withLongOpt(HIVE_PARTITION_VALUE_ARG)
599 .create());
600 hiveOpts.addOption(OptionBuilder.withArgName("hdfs path")
601 .hasArg()
602 .withDescription("Sets where the external table is in HDFS")
603 .withLongOpt(HIVE_EXTERNAL_TABLE_LOCATION_ARG)
604 .create());
605
606 hiveOpts.addOption(OptionBuilder
607 .hasArg()
608 .withDescription("Override mapping for specific column to hive"
609 + " types.")
610 .withLongOpt(MAP_COLUMN_HIVE)
611 .create());
612
613 return hiveOpts;
614 }
615
616 /**
617 * @return options governing interaction with HCatalog.
618 */
619 protected RelatedOptions getHCatalogOptions() {
620 RelatedOptions hCatOptions = new RelatedOptions("HCatalog arguments");
621 hCatOptions.addOption(OptionBuilder
622 .hasArg()
623 .withDescription("HCatalog table name")
624 .withLongOpt(HCATALOG_TABLE_ARG)
625 .create());
626 hCatOptions.addOption(OptionBuilder
627 .hasArg()
628 .withDescription("HCatalog database name")
629 .withLongOpt(HCATALOG_DATABASE_ARG)
630 .create());
631
632 hCatOptions.addOption(OptionBuilder.withArgName("dir")
633 .hasArg().withDescription("Override $HIVE_HOME")
634 .withLongOpt(HIVE_HOME_ARG)
635 .create());
636 hCatOptions.addOption(OptionBuilder.withArgName("hdir")
637 .hasArg().withDescription("Override $HCAT_HOME")
638 .withLongOpt(HCATALOG_HOME_ARG)
639 .create());
640 hCatOptions.addOption(OptionBuilder.withArgName("partition-key")
641 .hasArg()
642 .withDescription("Sets the partition key to use when importing to hive")
643 .withLongOpt(HIVE_PARTITION_KEY_ARG)
644 .create());
645 hCatOptions.addOption(OptionBuilder.withArgName("partition-value")
646 .hasArg()
647 .withDescription("Sets the partition value to use when importing "
648 + "to hive")
649 .withLongOpt(HIVE_PARTITION_VALUE_ARG)
650 .create());
651 hCatOptions.addOption(OptionBuilder
652 .hasArg()
653 .withDescription("Override mapping for specific column to hive"
654 + " types.")
655 .withLongOpt(MAP_COLUMN_HIVE)
656 .create());
657 hCatOptions.addOption(OptionBuilder.withArgName("partition-key")
658 .hasArg()
659 .withDescription("Sets the partition keys to use when importing to hive")
660 .withLongOpt(HCATCALOG_PARTITION_KEYS_ARG)
661 .create());
662 hCatOptions.addOption(OptionBuilder.withArgName("partition-value")
663 .hasArg()
664 .withDescription("Sets the partition values to use when importing "
665 + "to hive")
666 .withLongOpt(HCATALOG_PARTITION_VALUES_ARG)
667 .create());
668 return hCatOptions;
669 }
670
671 protected RelatedOptions getHCatImportOnlyOptions() {
672 RelatedOptions hCatOptions = new RelatedOptions(
673 "HCatalog import specific options");
674 hCatOptions.addOption(OptionBuilder
675 .withDescription("Create HCatalog before import")
676 .withLongOpt(CREATE_HCATALOG_TABLE_ARG)
677 .create());
678 hCatOptions.addOption(OptionBuilder
679 .withDescription("Drop and Create HCatalog before import")
680 .withLongOpt(DROP_AND_CREATE_HCATALOG_TABLE)
681 .create());
682 hCatOptions.addOption(OptionBuilder
683 .hasArg()
684 .withDescription("HCatalog storage stanza for table creation")
685 .withLongOpt(HCATALOG_STORAGE_STANZA_ARG)
686 .create());
687 return hCatOptions;
688 }
689
690 /**
691 * @return options governing output format delimiters
692 */
693 protected RelatedOptions getOutputFormatOptions() {
694 RelatedOptions formatOpts = new RelatedOptions(
695 "Output line formatting arguments");
696 formatOpts.addOption(OptionBuilder.withArgName("char")
697 .hasArg()
698 .withDescription("Sets the field separator character")
699 .withLongOpt(FIELDS_TERMINATED_BY_ARG)
700 .create());
701 formatOpts.addOption(OptionBuilder.withArgName("char")
702 .hasArg()
703 .withDescription("Sets the end-of-line character")
704 .withLongOpt(LINES_TERMINATED_BY_ARG)
705 .create());
706 formatOpts.addOption(OptionBuilder.withArgName("char")
707 .hasArg()
708 .withDescription("Sets a field enclosing character")
709 .withLongOpt(OPTIONALLY_ENCLOSED_BY_ARG)
710 .create());
711 formatOpts.addOption(OptionBuilder.withArgName("char")
712 .hasArg()
713 .withDescription("Sets a required field enclosing character")
714 .withLongOpt(ENCLOSED_BY_ARG)
715 .create());
716 formatOpts.addOption(OptionBuilder.withArgName("char")
717 .hasArg()
718 .withDescription("Sets the escape character")
719 .withLongOpt(ESCAPED_BY_ARG)
720 .create());
721 formatOpts.addOption(OptionBuilder
722 .withDescription("Uses MySQL's default delimiter set: "
723 + "fields: , lines: \\n escaped-by: \\ optionally-enclosed-by: '")
724 .withLongOpt(MYSQL_DELIMITERS_ARG)
725 .create());
726
727 return formatOpts;
728 }
729
730 /**
731 * @return options governing input format delimiters.
732 */
733 protected RelatedOptions getInputFormatOptions() {
734 RelatedOptions inputFormatOpts =
735 new RelatedOptions("Input parsing arguments");
736 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
737 .hasArg()
738 .withDescription("Sets the input field separator")
739 .withLongOpt(INPUT_FIELDS_TERMINATED_BY_ARG)
740 .create());
741 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
742 .hasArg()
743 .withDescription("Sets the input end-of-line char")
744 .withLongOpt(INPUT_LINES_TERMINATED_BY_ARG)
745 .create());
746 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
747 .hasArg()
748 .withDescription("Sets a field enclosing character")
749 .withLongOpt(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)
750 .create());
751 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
752 .hasArg()
753 .withDescription("Sets a required field encloser")
754 .withLongOpt(INPUT_ENCLOSED_BY_ARG)
755 .create());
756 inputFormatOpts.addOption(OptionBuilder.withArgName("char")
757 .hasArg()
758 .withDescription("Sets the input escape character")
759 .withLongOpt(INPUT_ESCAPED_BY_ARG)
760 .create());
761
762 return inputFormatOpts;
763 }
764
765 /**
766 * @param multiTable true if these options will be used for bulk code-gen.
767 * @return options related to code generation.
768 */
769 protected RelatedOptions getCodeGenOpts(boolean multiTable) {
770 RelatedOptions codeGenOpts =
771 new RelatedOptions("Code generation arguments");
772 codeGenOpts.addOption(OptionBuilder.withArgName("dir")
773 .hasArg()
774 .withDescription("Output directory for generated code")
775 .withLongOpt(CODE_OUT_DIR_ARG)
776 .create());
777 codeGenOpts.addOption(OptionBuilder.withArgName("dir")
778 .hasArg()
779 .withDescription("Output directory for compiled objects")
780 .withLongOpt(BIN_OUT_DIR_ARG)
781 .create());
782 codeGenOpts.addOption(OptionBuilder.withArgName("name")
783 .hasArg()
784 .withDescription("Put auto-generated classes in this package")
785 .withLongOpt(PACKAGE_NAME_ARG)
786 .create());
787 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
788 .hasArg()
789 .withDescription("Null string representation")
790 .withLongOpt(NULL_STRING)
791 .create());
792 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
793 .hasArg()
794 .withDescription("Input null string representation")
795 .withLongOpt(INPUT_NULL_STRING)
796 .create());
797 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
798 .hasArg()
799 .withDescription("Null non-string representation")
800 .withLongOpt(NULL_NON_STRING)
801 .create());
802 codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
803 .hasArg()
804 .withDescription("Input null non-string representation")
805 .withLongOpt(INPUT_NULL_NON_STRING)
806 .create());
807 codeGenOpts.addOption(OptionBuilder
808 .hasArg()
809 .withDescription("Override mapping for specific columns to java types")
810 .withLongOpt(MAP_COLUMN_JAVA)
811 .create());
812 codeGenOpts.addOption(OptionBuilder
813 .hasArg()
814 .withDescription("Disable special characters escaping in column names")
815 .withLongOpt(ESCAPE_MAPPING_COLUMN_NAMES_ENABLED)
816 .withArgName("boolean")
817 .create());
818
819 if (!multiTable) {
820 codeGenOpts.addOption(OptionBuilder.withArgName("name")
821 .hasArg()
822 .withDescription("Sets the generated class name. "
823 + "This overrides --" + PACKAGE_NAME_ARG + ". When combined "
824 + "with --" + JAR_FILE_NAME_ARG + ", sets the input class.")
825 .withLongOpt(CLASS_NAME_ARG)
826 .create());
827 }
828 return codeGenOpts;
829 }
830
831 protected RelatedOptions getHBaseOptions() {
832 RelatedOptions hbaseOpts =
833 new RelatedOptions("HBase arguments");
834 hbaseOpts.addOption(OptionBuilder.withArgName("table")
835 .hasArg()
836 .withDescription("Import to <table> in HBase")
837 .withLongOpt(HBASE_TABLE_ARG)
838 .create());
839 hbaseOpts.addOption(OptionBuilder.withArgName("family")
840 .hasArg()
841 .withDescription("Sets the target column family for the import")
842 .withLongOpt(HBASE_COL_FAM_ARG)
843 .create());
844 hbaseOpts.addOption(OptionBuilder.withArgName("col")
845 .hasArg()
846 .withDescription("Specifies which input column to use as the row key")
847 .withLongOpt(HBASE_ROW_KEY_ARG)
848 .create());
849 hbaseOpts.addOption(OptionBuilder
850 .withDescription("Enables HBase bulk loading")
851 .withLongOpt(HBASE_BULK_LOAD_ENABLED_ARG)
852 .create());
853 hbaseOpts.addOption(OptionBuilder
854 .withDescription("If specified, create missing HBase tables")
855 .withLongOpt(HBASE_CREATE_TABLE_ARG)
856 .create());
857 hbaseOpts.addOption(OptionBuilder.withArgName("nullmode")
858 .hasArg()
859 .withDescription("How to handle null values during incremental import into HBase.")
860 .withLongOpt(HBASE_NULL_INCREMENTAL_MODE_ARG)
861 .create());
862
863 return hbaseOpts;
864 }
865
866 protected RelatedOptions getAccumuloOptions() {
867 RelatedOptions accumuloOpts =
868 new RelatedOptions("Accumulo arguments");
869 accumuloOpts.addOption(OptionBuilder.withArgName("table")
870 .hasArg()
871 .withDescription("Import to <table> in Accumulo")
872 .withLongOpt(ACCUMULO_TABLE_ARG)
873 .create());
874 accumuloOpts.addOption(OptionBuilder.withArgName("family")
875 .hasArg()
876 .withDescription("Sets the target column family for the import")
877 .withLongOpt(ACCUMULO_COL_FAM_ARG)
878 .create());
879 accumuloOpts.addOption(OptionBuilder.withArgName("col")
880 .hasArg()
881 .withDescription("Specifies which input column to use as the row key")
882 .withLongOpt(ACCUMULO_ROW_KEY_ARG)
883 .create());
884 accumuloOpts.addOption(OptionBuilder.withArgName("vis")
885 .hasArg()
886 .withDescription("Visibility token to be applied to all rows imported")
887 .withLongOpt(ACCUMULO_VISIBILITY_ARG)
888 .create());
889 accumuloOpts.addOption(OptionBuilder
890 .withDescription("If specified, create missing Accumulo tables")
891 .withLongOpt(ACCUMULO_CREATE_TABLE_ARG)
892 .create());
893 accumuloOpts.addOption(OptionBuilder.withArgName("size")
894 .hasArg()
895 .withDescription("Batch size in bytes")
896 .withLongOpt(ACCUMULO_BATCH_SIZE_ARG)
897 .create());
898 accumuloOpts.addOption(OptionBuilder.withArgName("latency")
899 .hasArg()
900 .withDescription("Max write latency in milliseconds")
901 .withLongOpt(ACCUMULO_MAX_LATENCY_ARG)
902 .create());
903 accumuloOpts.addOption(OptionBuilder.withArgName("zookeepers")
904 .hasArg()
905 .withDescription("Comma-separated list of zookeepers (host:port)")
906 .withLongOpt(ACCUMULO_ZOOKEEPERS_ARG)
907 .create());
908 accumuloOpts.addOption(OptionBuilder.withArgName("instance")
909 .hasArg()
910 .withDescription("Accumulo instance name.")
911 .withLongOpt(ACCUMULO_INSTANCE_ARG)
912 .create());
913 accumuloOpts.addOption(OptionBuilder.withArgName("user")
914 .hasArg()
915 .withDescription("Accumulo user name.")
916 .withLongOpt(ACCUMULO_USER_ARG)
917 .create());
918 accumuloOpts.addOption(OptionBuilder.withArgName("password")
919 .hasArg()
920 .withDescription("Accumulo password.")
921 .withLongOpt(ACCUMULO_PASSWORD_ARG)
922 .create());
923
924 return accumuloOpts;
925 }
926
927 protected void applyAccumuloOptions(CommandLine in, SqoopOptions out) {
928 if (in.hasOption(ACCUMULO_TABLE_ARG)) {
929 out.setAccumuloTable(in.getOptionValue(ACCUMULO_TABLE_ARG));
930 }
931
932 if (in.hasOption(ACCUMULO_COL_FAM_ARG)) {
933 out.setAccumuloColFamily(in.getOptionValue(ACCUMULO_COL_FAM_ARG));
934 }
935
936 if (in.hasOption(ACCUMULO_ROW_KEY_ARG)) {
937 out.setAccumuloRowKeyColumn(in.getOptionValue(ACCUMULO_ROW_KEY_ARG));
938 }
939
940 if (in.hasOption(ACCUMULO_VISIBILITY_ARG)) {
941 out.setAccumuloVisibility(in.getOptionValue(ACCUMULO_VISIBILITY_ARG));
942 }
943
944 if (in.hasOption(ACCUMULO_CREATE_TABLE_ARG)) {
945 out.setCreateAccumuloTable(true);
946 }
947
948 if (in.hasOption(ACCUMULO_BATCH_SIZE_ARG)) {
949 out.setAccumuloBatchSize(Long.parseLong(
950 in.getOptionValue(ACCUMULO_BATCH_SIZE_ARG)));
951 }
952
953 if (in.hasOption(ACCUMULO_MAX_LATENCY_ARG)) {
954 out.setAccumuloMaxLatency(Long.parseLong(
955 in.getOptionValue(ACCUMULO_MAX_LATENCY_ARG)));
956 }
957
958 if (in.hasOption(ACCUMULO_ZOOKEEPERS_ARG)) {
959 out.setAccumuloZookeepers(in.getOptionValue(ACCUMULO_ZOOKEEPERS_ARG));
960 }
961
962 if (in.hasOption(ACCUMULO_INSTANCE_ARG)) {
963 out.setAccumuloInstance(in.getOptionValue(ACCUMULO_INSTANCE_ARG));
964 }
965
966 if (in.hasOption(ACCUMULO_USER_ARG)) {
967 out.setAccumuloUser(in.getOptionValue(ACCUMULO_USER_ARG));
968 }
969
970 if (in.hasOption(ACCUMULO_PASSWORD_ARG)) {
971 out.setAccumuloPassword(in.getOptionValue(ACCUMULO_PASSWORD_ARG));
972 }
973 }
974
975
976 @SuppressWarnings("static-access")
977 protected void addValidationOpts(RelatedOptions validationOptions) {
978 validationOptions.addOption(OptionBuilder
979 .withDescription("Validate the copy using the configured validator")
980 .withLongOpt(VALIDATE_ARG)
981 .create());
982 validationOptions.addOption(OptionBuilder
983 .withArgName(VALIDATOR_CLASS_ARG).hasArg()
984 .withDescription("Fully qualified class name for the Validator")
985 .withLongOpt(VALIDATOR_CLASS_ARG)
986 .create());
987 validationOptions.addOption(OptionBuilder
988 .withArgName(VALIDATION_THRESHOLD_CLASS_ARG).hasArg()
989 .withDescription("Fully qualified class name for ValidationThreshold")
990 .withLongOpt(VALIDATION_THRESHOLD_CLASS_ARG)
991 .create());
992 validationOptions.addOption(OptionBuilder
993 .withArgName(VALIDATION_FAILURE_HANDLER_CLASS_ARG).hasArg()
994 .withDescription("Fully qualified class name for "
995 + "ValidationFailureHandler")
996 .withLongOpt(VALIDATION_FAILURE_HANDLER_CLASS_ARG)
997 .create());
998 }
999
1000 /**
1001 * Apply common command-line to the state.
1002 */
1003 protected void applyCommonOptions(CommandLine in, SqoopOptions out)
1004 throws InvalidOptionsException {
1005
1006 // common options.
1007 if (in.hasOption(VERBOSE_ARG)) {
1008 // Immediately switch into DEBUG logging.
1009 out.setVerbose(true);
1010 LoggingUtils.setDebugLevel();
1011 LOG.debug("Enabled debug logging.");
1012 }
1013
1014 if (in.hasOption(HELP_ARG)) {
1015 ToolOptions toolOpts = new ToolOptions();
1016 configureOptions(toolOpts);
1017 printHelp(toolOpts);
1018 throw new InvalidOptionsException("");
1019 }
1020
1021 if (in.hasOption(TEMP_ROOTDIR_ARG)) {
1022 out.setTempRootDir(in.getOptionValue(TEMP_ROOTDIR_ARG));
1023 }
1024
1025 if (in.hasOption(THROW_ON_ERROR_ARG)) {
1026 LOG.debug("Throw exception on error during job is enabled.");
1027 out.setThrowOnError(true);
1028 }
1029
1030 if (in.hasOption(CONNECT_STRING_ARG)) {
1031 out.setConnectString(in.getOptionValue(CONNECT_STRING_ARG));
1032 }
1033
1034 if (in.hasOption(CONN_MANAGER_CLASS_NAME)) {
1035 out.setConnManagerClassName(in.getOptionValue(CONN_MANAGER_CLASS_NAME));
1036 }
1037
1038 if (in.hasOption(CONNECT_PARAM_FILE)) {
1039 File paramFile = new File(in.getOptionValue(CONNECT_PARAM_FILE));
1040 if (!paramFile.exists()) {
1041 throw new InvalidOptionsException(
1042 "Specified connection parameter file not found: " + paramFile);
1043 }
1044 InputStream inStream = null;
1045 Properties connectionParams = new Properties();
1046 try {
1047 inStream = new FileInputStream(
1048 new File(in.getOptionValue(CONNECT_PARAM_FILE)));
1049 connectionParams.load(inStream);
1050 } catch (IOException ex) {
1051 LOG.warn("Failed to load connection parameter file", ex);
1052 throw new InvalidOptionsException(
1053 "Error while loading connection parameter file: "
1054 + ex.getMessage());
1055 } finally {
1056 if (inStream != null) {
1057 try {
1058 inStream.close();
1059 } catch (IOException ex) {
1060 LOG.warn("Failed to close input stream", ex);
1061 }
1062 }
1063 }
1064 LOG.debug("Loaded connection parameters: " + connectionParams);
1065 out.setConnectionParams(connectionParams);
1066 }
1067
1068 if (in.hasOption(NULL_STRING)) {
1069 out.setNullStringValue(in.getOptionValue(NULL_STRING));
1070 }
1071
1072 if (in.hasOption(INPUT_NULL_STRING)) {
1073 out.setInNullStringValue(in.getOptionValue(INPUT_NULL_STRING));
1074 }
1075
1076 if (in.hasOption(NULL_NON_STRING)) {
1077 out.setNullNonStringValue(in.getOptionValue(NULL_NON_STRING));
1078 }
1079
1080 if (in.hasOption(INPUT_NULL_NON_STRING)) {
1081 out.setInNullNonStringValue(in.getOptionValue(INPUT_NULL_NON_STRING));
1082 }
1083
1084 if (in.hasOption(DRIVER_ARG)) {
1085 out.setDriverClassName(in.getOptionValue(DRIVER_ARG));
1086 }
1087
1088 if (in.hasOption(SKIP_DISTCACHE_ARG)) {
1089 LOG.debug("Disabling dist cache");
1090 out.setSkipDistCache(true);
1091 }
1092
1093 applyCredentialsOptions(in, out);
1094
1095
1096 if (in.hasOption(HADOOP_MAPRED_HOME_ARG)) {
1097 out.setHadoopMapRedHome(in.getOptionValue(HADOOP_MAPRED_HOME_ARG));
1098 // Only consider HADOOP_HOME if HADOOP_MAPRED_HOME is not set
1099 } else if (in.hasOption(HADOOP_HOME_ARG)) {
1100 out.setHadoopMapRedHome(in.getOptionValue(HADOOP_HOME_ARG));
1101 }
1102 if (in.hasOption(RELAXED_ISOLATION)) {
1103 out.setRelaxedIsolation(true);
1104 }
1105
1106 if (in.hasOption(METADATA_TRANSACTION_ISOLATION_LEVEL)) {
1107 String transactionLevel = in.getOptionValue(METADATA_TRANSACTION_ISOLATION_LEVEL);
1108 try {
1109 out.setMetadataTransactionIsolationLevel(JDBCTransactionLevels.valueOf(transactionLevel).getTransactionIsolationLevelValue());
1110 } catch (IllegalArgumentException e) {
1111 throw new RuntimeException("Only transaction isolation levels defined by "
1112 + "java.sql.Connection class are supported. Check the "
1113 + "java.sql.Connection javadocs for more details", e);
1114 }
1115 }
1116
1117 if (in.hasOption(ORACLE_ESCAPING_DISABLED)) {
1118 out.setOracleEscapingDisabled(Boolean.parseBoolean(in.getOptionValue(ORACLE_ESCAPING_DISABLED)));
1119 }
1120
1121 if (in.hasOption(ESCAPE_MAPPING_COLUMN_NAMES_ENABLED)) {
1122 out.setEscapeMappingColumnNamesEnabled(Boolean.parseBoolean(in.getOptionValue(
1123 ESCAPE_MAPPING_COLUMN_NAMES_ENABLED)));
1124 }
1125 }
1126
1127 private void applyCredentialsOptions(CommandLine in, SqoopOptions out)
1128 throws InvalidOptionsException {
1129 if (in.hasOption(USERNAME_ARG)) {
1130 out.setUsername(in.getOptionValue(USERNAME_ARG));
1131 if (null == out.getPassword()) {
1132 // Set password to empty if the username is set first,
1133 // to ensure that they're either both null or neither is.
1134 out.setPassword("");
1135 }
1136 }
1137
1138 if (in.hasOption(PASSWORD_ARG)) {
1139 LOG.warn("Setting your password on the command-line is insecure. "
1140 + "Consider using -" + PASSWORD_PROMPT_ARG + " instead.");
1141 out.setPassword(in.getOptionValue(PASSWORD_ARG));
1142 }
1143
1144 if (in.hasOption(PASSWORD_PROMPT_ARG)) {
1145 out.setPasswordFromConsole();
1146 }
1147
1148 if (in.hasOption(PASSWORD_PATH_ARG)) {
1149 if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)
1150 || in.hasOption(PASSWORD_ALIAS_ARG)) {
1151 throw new InvalidOptionsException("Only one of password, password "
1152 + "alias or path to a password file must be specified.");
1153 }
1154
1155 try {
1156 out.setPasswordFilePath(in.getOptionValue(PASSWORD_PATH_ARG));
1157 // apply password from file into password in options
1158 out.setPassword(CredentialsUtil.fetchPassword(out));
1159 // And allow the PasswordLoader to clean up any sensitive properties
1160 CredentialsUtil.cleanUpSensitiveProperties(out.getConf());
1161 } catch (IOException ex) {
1162 LOG.warn("Failed to load password file", ex);
1163 throw (InvalidOptionsException)
1164 new InvalidOptionsException("Error while loading password file: "
1165 + ex.getMessage()).initCause(ex);
1166 }
1167 }
1168 if (in.hasOption(PASSWORD_ALIAS_ARG)) {
1169 if (in.hasOption(PASSWORD_ARG) || in.hasOption(PASSWORD_PROMPT_ARG)
1170 || in.hasOption(PASSWORD_PATH_ARG)) {
1171 throw new InvalidOptionsException("Only one of password, password "
1172 + "alias or path to a password file must be specified.");
1173 }
1174 out.setPasswordAlias(in.getOptionValue(PASSWORD_ALIAS_ARG));
1175 if (!CredentialProviderHelper.isProviderAvailable()) {
1176 throw new InvalidOptionsException(
1177 "CredentialProvider facility not available in the hadoop "
1178 + " environment used");
1179 }
1180 try {
1181 out.setPassword(CredentialProviderHelper
1182 .resolveAlias(out.getConf(), in.getOptionValue(PASSWORD_ALIAS_ARG)));
1183 } catch (IOException ioe) {
1184 throw (InvalidOptionsException)
1185 new InvalidOptionsException("Unable to process alias")
1186 .initCause(ioe);
1187 }
1188 }
1189 }
1190
1191 protected void applyHiveOptions(CommandLine in, SqoopOptions out)
1192 throws InvalidOptionsException {
1193
1194 if (in.hasOption(HIVE_HOME_ARG)) {
1195 out.setHiveHome(in.getOptionValue(HIVE_HOME_ARG));
1196 }
1197
1198 if (in.hasOption(HIVE_IMPORT_ARG)) {
1199 out.setHiveImport(true);
1200 }
1201
1202 if (in.hasOption(HIVE_OVERWRITE_ARG)) {
1203 out.setOverwriteHiveTable(true);
1204 }
1205
1206 if (in.hasOption(CREATE_HIVE_TABLE_ARG)) {
1207 out.setFailIfHiveTableExists(true);
1208 }
1209
1210 if (in.hasOption(HIVE_TABLE_ARG)) {
1211 out.setHiveTableName(in.getOptionValue(HIVE_TABLE_ARG));
1212 }
1213
1214 if (in.hasOption(HIVE_DATABASE_ARG)) {
1215 out.setHiveDatabaseName(in.getOptionValue(HIVE_DATABASE_ARG));
1216 }
1217
1218 if (in.hasOption(HIVE_DROP_DELIMS_ARG)) {
1219 out.setHiveDropDelims(true);
1220 }
1221
1222 if (in.hasOption(HIVE_DELIMS_REPLACEMENT_ARG)) {
1223 out.setHiveDelimsReplacement(
1224 in.getOptionValue(HIVE_DELIMS_REPLACEMENT_ARG));
1225 }
1226
1227 if (in.hasOption(HIVE_PARTITION_KEY_ARG)) {
1228 out.setHivePartitionKey(in.getOptionValue(HIVE_PARTITION_KEY_ARG));
1229 }
1230
1231 if (in.hasOption(HIVE_PARTITION_VALUE_ARG)) {
1232 out.setHivePartitionValue(in.getOptionValue(HIVE_PARTITION_VALUE_ARG));
1233 }
1234
1235 if (in.hasOption(MAP_COLUMN_HIVE)) {
1236 out.setMapColumnHive(in.getOptionValue(MAP_COLUMN_HIVE));
1237 }
1238 if (in.hasOption(HIVE_EXTERNAL_TABLE_LOCATION_ARG)) {
1239 out.setHiveExternalTableDir(in.getOptionValue(HIVE_EXTERNAL_TABLE_LOCATION_ARG));
1240 }
1241
1242 }
1243
1244 protected void applyHCatalogOptions(CommandLine in, SqoopOptions out) {
1245 if (in.hasOption(HCATALOG_TABLE_ARG)) {
1246 out.setHCatTableName(in.getOptionValue(HCATALOG_TABLE_ARG));
1247 }
1248
1249 if (in.hasOption(HCATALOG_DATABASE_ARG)) {
1250 out.setHCatDatabaseName(in.getOptionValue(HCATALOG_DATABASE_ARG));
1251 }
1252
1253 if (in.hasOption(HCATALOG_STORAGE_STANZA_ARG)) {
1254 out.setHCatStorageStanza(in.getOptionValue(HCATALOG_STORAGE_STANZA_ARG));
1255 }
1256
1257 if (in.hasOption(CREATE_HCATALOG_TABLE_ARG)) {
1258 out.setCreateHCatalogTable(true);
1259 }
1260
1261 if (in.hasOption(DROP_AND_CREATE_HCATALOG_TABLE)) {
1262 out.setDropAndCreateHCatalogTable(true);
1263 }
1264
1265 if (in.hasOption(HCATALOG_HOME_ARG)) {
1266 out.setHCatHome(in.getOptionValue(HCATALOG_HOME_ARG));
1267 }
1268
1269 // Allow some of the hive options also
1270
1271 if (in.hasOption(HIVE_HOME_ARG)) {
1272 out.setHiveHome(in.getOptionValue(HIVE_HOME_ARG));
1273 }
1274
1275 if (in.hasOption(HCATCALOG_PARTITION_KEYS_ARG)) {
1276 out.setHCatalogPartitionKeys(
1277 in.getOptionValue(HCATCALOG_PARTITION_KEYS_ARG));
1278 }
1279
1280 if (in.hasOption(HCATALOG_PARTITION_VALUES_ARG)) {
1281 out.setHCatalogPartitionValues(
1282 in.getOptionValue(HCATALOG_PARTITION_VALUES_ARG));
1283 }
1284
1285 if (in.hasOption(HIVE_PARTITION_KEY_ARG)) {
1286 out.setHivePartitionKey(in.getOptionValue(HIVE_PARTITION_KEY_ARG));
1287 }
1288
1289 if (in.hasOption(HIVE_PARTITION_VALUE_ARG)) {
1290 out.setHivePartitionValue(in.getOptionValue(HIVE_PARTITION_VALUE_ARG));
1291 }
1292
1293 if (in.hasOption(MAP_COLUMN_HIVE)) {
1294 out.setMapColumnHive(in.getOptionValue(MAP_COLUMN_HIVE));
1295 }
1296
1297 }
1298
1299
1300 protected void applyOutputFormatOptions(CommandLine in, SqoopOptions out)
1301 throws InvalidOptionsException {
1302 if (in.hasOption(FIELDS_TERMINATED_BY_ARG)) {
1303 out.setFieldsTerminatedBy(SqoopOptions.toChar(
1304 in.getOptionValue(FIELDS_TERMINATED_BY_ARG)));
1305 out.setExplicitOutputDelims(true);
1306 }
1307
1308 if (in.hasOption(LINES_TERMINATED_BY_ARG)) {
1309 out.setLinesTerminatedBy(SqoopOptions.toChar(
1310 in.getOptionValue(LINES_TERMINATED_BY_ARG)));
1311 out.setExplicitOutputDelims(true);
1312 }
1313
1314 if (in.hasOption(OPTIONALLY_ENCLOSED_BY_ARG)) {
1315 out.setEnclosedBy(SqoopOptions.toChar(
1316 in.getOptionValue(OPTIONALLY_ENCLOSED_BY_ARG)));
1317 out.setOutputEncloseRequired(false);
1318 out.setExplicitOutputDelims(true);
1319 }
1320
1321 if (in.hasOption(ENCLOSED_BY_ARG)) {
1322 out.setEnclosedBy(SqoopOptions.toChar(
1323 in.getOptionValue(ENCLOSED_BY_ARG)));
1324 out.setOutputEncloseRequired(true);
1325 out.setExplicitOutputDelims(true);
1326 }
1327
1328 if (in.hasOption(ESCAPED_BY_ARG)) {
1329 out.setEscapedBy(SqoopOptions.toChar(
1330 in.getOptionValue(ESCAPED_BY_ARG)));
1331 out.setExplicitOutputDelims(true);
1332 }
1333
1334 if (in.hasOption(MYSQL_DELIMITERS_ARG)) {
1335 out.setOutputEncloseRequired(false);
1336 out.setFieldsTerminatedBy(',');
1337 out.setLinesTerminatedBy('\n');
1338 out.setEscapedBy('\\');
1339 out.setEnclosedBy('\'');
1340 out.setExplicitOutputDelims(true);
1341 }
1342 }
1343
1344 protected void applyInputFormatOptions(CommandLine in, SqoopOptions out)
1345 throws InvalidOptionsException {
1346 if (in.hasOption(INPUT_FIELDS_TERMINATED_BY_ARG)) {
1347 out.setInputFieldsTerminatedBy(SqoopOptions.toChar(
1348 in.getOptionValue(INPUT_FIELDS_TERMINATED_BY_ARG)));
1349 out.setExplicitInputDelims(true);
1350 }
1351
1352 if (in.hasOption(INPUT_LINES_TERMINATED_BY_ARG)) {
1353 out.setInputLinesTerminatedBy(SqoopOptions.toChar(
1354 in.getOptionValue(INPUT_LINES_TERMINATED_BY_ARG)));
1355 out.setExplicitInputDelims(true);
1356 }
1357
1358 if (in.hasOption(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)) {
1359 out.setInputEnclosedBy(SqoopOptions.toChar(
1360 in.getOptionValue(INPUT_OPTIONALLY_ENCLOSED_BY_ARG)));
1361 out.setInputEncloseRequired(false);
1362 out.setExplicitInputDelims(true);
1363 }
1364
1365 if (in.hasOption(INPUT_ENCLOSED_BY_ARG)) {
1366 out.setInputEnclosedBy(SqoopOptions.toChar(
1367 in.getOptionValue(INPUT_ENCLOSED_BY_ARG)));
1368 out.setInputEncloseRequired(true);
1369 out.setExplicitInputDelims(true);
1370 }
1371
1372 if (in.hasOption(INPUT_ESCAPED_BY_ARG)) {
1373 out.setInputEscapedBy(SqoopOptions.toChar(
1374 in.getOptionValue(INPUT_ESCAPED_BY_ARG)));
1375 out.setExplicitInputDelims(true);
1376 }
1377 }
1378
1379 protected void applyCodeGenOptions(CommandLine in, SqoopOptions out,
1380 boolean multiTable) throws InvalidOptionsException {
1381 if (in.hasOption(CODE_OUT_DIR_ARG)) {
1382 out.setCodeOutputDir(in.getOptionValue(CODE_OUT_DIR_ARG));
1383 }
1384
1385 if (in.hasOption(BIN_OUT_DIR_ARG)) {
1386 out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG));
1387 }
1388
1389 if (in.hasOption(PACKAGE_NAME_ARG)) {
1390 out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
1391 }
1392
1393 if (in.hasOption(MAP_COLUMN_JAVA)) {
1394 out.setMapColumnJava(in.getOptionValue(MAP_COLUMN_JAVA));
1395 }
1396
1397 if (!multiTable && in.hasOption(CLASS_NAME_ARG)) {
1398 out.setClassName(in.getOptionValue(CLASS_NAME_ARG));
1399 }
1400
1401 if (in.hasOption(ESCAPE_MAPPING_COLUMN_NAMES_ENABLED)) {
1402 out.setEscapeMappingColumnNamesEnabled(Boolean.parseBoolean(in.getOptionValue(
1403 ESCAPE_MAPPING_COLUMN_NAMES_ENABLED)));
1404 }
1405 }
1406
1407 protected void applyHBaseOptions(CommandLine in, SqoopOptions out) throws InvalidOptionsException {
1408 if (in.hasOption(HBASE_TABLE_ARG)) {
1409 out.setHBaseTable(in.getOptionValue(HBASE_TABLE_ARG));
1410 }
1411
1412 if (in.hasOption(HBASE_COL_FAM_ARG)) {
1413 out.setHBaseColFamily(in.getOptionValue(HBASE_COL_FAM_ARG));
1414 }
1415
1416 if (in.hasOption(HBASE_ROW_KEY_ARG)) {
1417 out.setHBaseRowKeyColumn(in.getOptionValue(HBASE_ROW_KEY_ARG));
1418 }
1419
1420 out.setHBaseBulkLoadEnabled(in.hasOption(HBASE_BULK_LOAD_ENABLED_ARG));
1421
1422 if (in.hasOption(HBASE_CREATE_TABLE_ARG)) {
1423 out.setCreateHBaseTable(true);
1424 }
1425
1426 if (in.hasOption(HBASE_NULL_INCREMENTAL_MODE_ARG)) {
1427 String nullMode = in.getOptionValue(HBASE_NULL_INCREMENTAL_MODE_ARG);
1428 if ("ignore".equals(nullMode)) {
1429 out.setHbaseNullIncrementalMode(SqoopOptions.HBaseNullIncrementalMode.Ignore);
1430 } else if ("delete".equals(nullMode)) {
1431 out.setHbaseNullIncrementalMode(SqoopOptions.HBaseNullIncrementalMode.Delete);
1432 } else {
1433 throw new InvalidOptionsException("Unknown HBase null incremental mode: "
1434 + nullMode + ". Use 'ignore' or 'delete'."
1435 + HELP_STR);
1436 }
1437 }
1438 }
1439
1440 protected void applyValidationOptions(CommandLine in, SqoopOptions out)
1441 throws InvalidOptionsException {
1442 if (in.hasOption(VALIDATE_ARG)) {
1443 out.setValidationEnabled(true);
1444 }
1445
1446 // Class Names are converted to Class in light of failing early
1447 if (in.hasOption(VALIDATOR_CLASS_ARG)) {
1448 out.setValidatorClass(
1449 getClassByName(in.getOptionValue(VALIDATOR_CLASS_ARG)));
1450 }
1451
1452 if (in.hasOption(VALIDATION_THRESHOLD_CLASS_ARG)) {
1453 out.setValidationThresholdClass(
1454 getClassByName(in.getOptionValue(VALIDATION_THRESHOLD_CLASS_ARG)));
1455 }
1456
1457 if (in.hasOption(VALIDATION_FAILURE_HANDLER_CLASS_ARG)) {
1458 out.setValidationFailureHandlerClass(getClassByName(
1459 in.getOptionValue(VALIDATION_FAILURE_HANDLER_CLASS_ARG)));
1460 }
1461 }
1462
1463 protected Class<?> getClassByName(String className)
1464 throws InvalidOptionsException {
1465 try {
1466 return Class.forName(className, true,
1467 Thread.currentThread().getContextClassLoader());
1468 } catch (ClassNotFoundException e) {
1469 throw new InvalidOptionsException(e.getMessage());
1470 }
1471 }
1472
1473 protected void validateCommonOptions(SqoopOptions options)
1474 throws InvalidOptionsException {
1475 if (options.getConnectString() == null) {
1476 throw new InvalidOptionsException(
1477 "Error: Required argument --connect is missing."
1478 + HELP_STR);
1479 }
1480 }
1481
1482 protected void validateCodeGenOptions(SqoopOptions options)
1483 throws InvalidOptionsException {
1484 if (options.getClassName() != null && options.getPackageName() != null) {
1485 throw new InvalidOptionsException(
1486 "--class-name overrides --package-name. You cannot use both."
1487 + HELP_STR);
1488 }
1489 }
1490
1491 protected void validateOutputFormatOptions(SqoopOptions options)
1492 throws InvalidOptionsException {
1493 if (options.doHiveImport()) {
1494 if (!options.explicitOutputDelims()) {
1495 // user hasn't manually specified delimiters, and wants to import
1496 // straight to Hive. Use Hive-style delimiters.
1497 LOG.info("Using Hive-specific delimiters for output. You can override");
1498 LOG.info("delimiters with --fields-terminated-by, etc.");
1499 options.setOutputDelimiters(DelimiterSet.HIVE_DELIMITERS);
1500 }
1501
1502 if (options.getOutputEscapedBy() != DelimiterSet.NULL_CHAR) {
1503 LOG.warn("Hive does not support escape characters in fields;");
1504 LOG.warn("parse errors in Hive may result from using --escaped-by.");
1505 }
1506
1507 if (options.getOutputEnclosedBy() != DelimiterSet.NULL_CHAR) {
1508 LOG.warn("Hive does not support quoted strings; parse errors");
1509 LOG.warn("in Hive may result from using --enclosed-by.");
1510 }
1511 }
1512 }
1513
1514 protected void validateHiveOptions(SqoopOptions options)
1515 throws InvalidOptionsException {
1516 if (options.getHiveDelimsReplacement() != null
1517 && options.doHiveDropDelims()) {
1518 throw new InvalidOptionsException("The " + HIVE_DROP_DELIMS_ARG
1519 + " option conflicts with the " + HIVE_DELIMS_REPLACEMENT_ARG
1520 + " option." + HELP_STR);
1521 }
1522
1523 // Make sure that one of hCatalog or hive jobs are used
1524 String hCatTable = options.getHCatTableName();
1525 if (hCatTable != null && options.doHiveImport()) {
1526 throw new InvalidOptionsException("The " + HCATALOG_TABLE_ARG
1527 + " option conflicts with the " + HIVE_IMPORT_ARG
1528 + " option." + HELP_STR);
1529 }
1530
1531 if (options.doHiveImport()
1532 && options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
1533 throw new InvalidOptionsException("Hive import is not compatible with "
1534 + "importing into AVRO format.");
1535 }
1536
1537 if (options.doHiveImport()
1538 && options.getFileLayout() == SqoopOptions.FileLayout.SequenceFile) {
1539 throw new InvalidOptionsException("Hive import is not compatible with "
1540 + "importing into SequenceFile format.");
1541 }
1542
1543 // Hive import and create hive table not compatible for ParquetFile format
1544 if (options.doHiveImport()
1545 && options.doFailIfHiveTableExists()
1546 && options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
1547 throw new InvalidOptionsException("Hive import and create hive table is not compatible with "
1548 + "importing into ParquetFile format.");
1549 }
1550
1551 if (options.doHiveImport()
1552 && options.getIncrementalMode().equals(IncrementalMode.DateLastModified)) {
1553 throw new InvalidOptionsException(HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED);
1554 }
1555
1556 if (options.doHiveImport()
1557 && options.isAppendMode()
1558 && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) {
1559 throw new InvalidOptionsException("Append mode for hive imports is not "
1560 + " yet supported. Please remove the parameter --append-mode");
1561 }
1562
1563 // Many users are reporting issues when they are trying to import data
1564 // directly into hive warehouse. This should prevent users from doing
1565 // so in case of a default location.
1566 String defaultHiveWarehouse = "/user/hive/warehouse";
1567 if (options.doHiveImport()
1568 && ((
1569 options.getWarehouseDir() != null
1570 && options.getWarehouseDir().startsWith(defaultHiveWarehouse)
1571 ) || (
1572 options.getTargetDir() != null
1573 && options.getTargetDir().startsWith(defaultHiveWarehouse)
1574 ))) {
1575 LOG.warn("It seems that you're doing hive import directly into default");
1576 LOG.warn("hive warehouse directory which is not supported. Sqoop is");
1577 LOG.warn("firstly importing data into separate directory and then");
1578 LOG.warn("inserting data into hive. Please consider removing");
1579 LOG.warn("--target-dir or --warehouse-dir into /user/hive/warehouse in");
1580 LOG.warn("case that you will detect any issues.");
1581 }
1582
1583 // Warn about using hive specific arguments without hive import itself
1584 // In HCatalog support some of the Hive options are reused
1585 if (!options.doHiveImport()
1586 && (((options.getHiveHome() != null
1587 && !options.getHiveHome().
1588 equals(SqoopOptions.getHiveHomeDefault())
1589 && hCatTable == null))
1590 || options.doOverwriteHiveTable()
1591 || options.doFailIfHiveTableExists()
1592 || (options.getHiveTableName() != null
1593 && !options.getHiveTableName().equals(options.getTableName()))
1594 || (options.getHivePartitionKey() != null && hCatTable == null)
1595 || (options.getHivePartitionValue() != null && hCatTable == null)
1596 || (options.getMapColumnHive().size() > 0 && hCatTable == null))) {
1597 LOG.warn("It seems that you've specified at least one of following:");
1598 LOG.warn("\t--hive-home");
1599 LOG.warn("\t--hive-overwrite");
1600 LOG.warn("\t--create-hive-table");
1601 LOG.warn("\t--hive-table");
1602 LOG.warn("\t--hive-partition-key");
1603 LOG.warn("\t--hive-partition-value");
1604 LOG.warn("\t--map-column-hive");
1605 LOG.warn("Without specifying parameter --hive-import. Please note that");
1606 LOG.warn("those arguments will not be used in this session. Either");
1607 LOG.warn("specify --hive-import to apply them correctly or remove them");
1608 LOG.warn("from command line to remove this warning.");
1609 LOG.info("Please note that --hive-home, --hive-partition-key, ");
1610 LOG.info("\t hive-partition-value and --map-column-hive options are ");
1611 LOG.info("\t are also valid for HCatalog imports and exports");
1612 }
1613 // importing to Hive external tables requires target directory to be set
1614 // for external table's location
1615 Boolean isNotHiveImportButExternalTableDirIsSet = !options.doHiveImport() && !org.apache.commons.lang.StringUtils.isBlank(options.getHiveExternalTableDir());
1616 if (isNotHiveImportButExternalTableDirIsSet) {
1617 LOG.warn("Importing to external Hive table requires --hive-import parameter to be set");
1618 throw new InvalidOptionsException("Importing to external Hive table requires --hive-import parameter to be set."
1619 + HELP_STR);
1620 }
1621 }
1622
1623 protected void validateAccumuloOptions(SqoopOptions options)
1624 throws InvalidOptionsException {
1625 if ((options.getAccumuloColFamily() != null
1626 && options.getAccumuloTable() == null)
1627 || (options.getAccumuloColFamily() == null
1628 && options.getAccumuloTable() != null)) {
1629 throw new InvalidOptionsException(
1630 "Both --accumulo-table and --accumulo-column-family must be set."
1631 + HELP_STR);
1632 }
1633
1634 if (options.getAccumuloTable() != null
1635 && options.getHBaseTable() != null) {
1636 throw new InvalidOptionsException("HBase import is incompatible with "
1637 + "Accumulo import.");
1638 }
1639 if (options.getAccumuloTable() != null
1640 && options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
1641 throw new InvalidOptionsException("Accumulo import is not compatible "
1642 + "with importing into file format.");
1643 }
1644 if (options.getAccumuloTable() != null
1645 && options.getHBaseColFamily() != null) {
1646 throw new InvalidOptionsException("Use --accumulo-column-family with "
1647 + "Accumulo import.");
1648 }
1649 if (options.getAccumuloTable() != null
1650 && options.getAccumuloUser() == null) {
1651 throw
1652 new InvalidOptionsException("Must specify Accumulo user.");
1653 }
1654 if (options.getAccumuloTable() != null
1655 && options.getAccumuloInstance() == null) {
1656 throw new
1657 InvalidOptionsException("Must specify Accumulo instance.");
1658 }
1659 if (options.getAccumuloTable() != null
1660 && options.getAccumuloZookeepers() == null) {
1661 throw new
1662 InvalidOptionsException("Must specify Zookeeper server(s).");
1663 }
1664 }
1665
1666 protected void validateHCatalogOptions(SqoopOptions options)
1667 throws InvalidOptionsException {
1668 // Make sure that one of hCatalog or hive jobs are used
1669 String hCatTable = options.getHCatTableName();
1670 if (hCatTable == null) {
1671 if (options.getHCatHome() != null && !options.getHCatHome().
1672 equals(SqoopOptions.getHCatHomeDefault())) {
1673 LOG.warn("--hcatalog-home option will be ignored in "
1674 + "non-HCatalog jobs");
1675 }
1676 if (options.getHCatDatabaseName() != null) {
1677 LOG.warn("--hcatalog-database option will be ignored "
1678 + "without --hcatalog-table");
1679 }
1680
1681 if (options.getHCatStorageStanza() != null) {
1682 LOG.warn("--hcatalog-storage-stanza option will be ignored "
1683 + "without --hatalog-table");
1684 }
1685 return;
1686 }
1687 if(isSet(options.getHCatTableName()) && SqoopHCatUtilities.isHCatView(options)){
1688 throw new InvalidOptionsException("Reads/Writes from and to Views are not supported by HCatalog");
1689 }
1690
1691 if (options.explicitInputDelims()) {
1692 LOG.warn("Input field/record delimiter options are not "
1693 + "used in HCatalog jobs unless the format is text. It is better "
1694 + "to use --hive-import in those cases. For text formats");
1695 }
1696 if (options.explicitOutputDelims()
1697 || options.getHiveDelimsReplacement() != null
1698 || options.doHiveDropDelims()) {
1699 LOG.warn("Output field/record delimiter options are not useful"
1700 + " in HCatalog jobs for most of the output types except text based "
1701 + " formats is text. It is better "
1702 + "to use --hive-import in those cases. For non text formats, ");
1703 }
1704 if (options.doHiveImport()) {
1705 throw new InvalidOptionsException("The " + HCATALOG_TABLE_ARG
1706 + " option conflicts with the " + HIVE_IMPORT_ARG
1707 + " option." + HELP_STR);
1708 }
1709 if (options.getTargetDir() != null) {
1710 throw new InvalidOptionsException("The " + TARGET_DIR_ARG
1711 + " option conflicts with the " + HCATALOG_TABLE_ARG
1712 + " option." + HELP_STR);
1713 }
1714 if (options.getWarehouseDir() != null) {
1715 throw new InvalidOptionsException("The " + WAREHOUSE_DIR_ARG
1716 + " option conflicts with the " + HCATALOG_TABLE_ARG
1717 + " option." + HELP_STR);
1718 }
1719
1720 if (options.isAppendMode()) {
1721 throw new InvalidOptionsException("Append mode for imports is not "
1722 + " compatible with HCatalog. Please remove the parameter"
1723 + "--append-mode");
1724 }
1725 if (options.getExportDir() != null) {
1726 throw new InvalidOptionsException("The " + EXPORT_PATH_ARG
1727 + " option conflicts with the " + HCATALOG_TABLE_ARG
1728 + " option." + HELP_STR);
1729 }
1730
1731 if (options.getFileLayout() == SqoopOptions.FileLayout.AvroDataFile) {
1732 throw new InvalidOptionsException("HCatalog job is not compatible with "
1733 + " AVRO format option " + FMT_AVRODATAFILE_ARG
1734 + " option." + HELP_STR);
1735
1736 }
1737
1738 if (options.getFileLayout() == SqoopOptions.FileLayout.SequenceFile) {
1739 throw new InvalidOptionsException("HCatalog job is not compatible with "
1740 + "SequenceFile format option " + FMT_SEQUENCEFILE_ARG
1741 + " option." + HELP_STR);
1742 }
1743
1744 if (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile) {
1745 throw new InvalidOptionsException("HCatalog job is not compatible with "
1746 + "SequenceFile format option " + FMT_PARQUETFILE_ARG
1747 + " option." + HELP_STR);
1748 }
1749
1750 if (options.getHCatalogPartitionKeys() != null
1751 && options.getHCatalogPartitionValues() == null) {
1752 throw new InvalidOptionsException("Either both --hcatalog-partition-keys"
1753 + " and --hcatalog-partition-values should be provided or both of these"
1754 + " options should be omitted.");
1755 }
1756
1757 if (options.getHCatalogPartitionKeys() != null) {
1758 if (options.getHivePartitionKey() != null) {
1759 LOG.warn("Both --hcatalog-partition-keys and --hive-partition-key"
1760 + "options are provided. --hive-partition-key option will be"
1761 + "ignored");
1762 }
1763
1764 String[] keys = options.getHCatalogPartitionKeys().split(",");
1765 String[] vals = options.getHCatalogPartitionValues().split(",");
1766
1767 if (keys.length != vals.length) {
1768 throw new InvalidOptionsException("Number of static partition keys "
1769 + "provided dpes match the number of partition values");
1770 }
1771
1772 for (int i = 0; i < keys.length; ++i) {
1773 String k = keys[i].trim();
1774 if (k.isEmpty()) {
1775 throw new InvalidOptionsException(
1776 "Invalid HCatalog static partition key at position " + i);
1777 }
1778 }
1779 for (int i = 0; i < vals.length; ++i) {
1780 String v = vals[i].trim();
1781 if (v.isEmpty()) {
1782 throw new InvalidOptionsException(
1783 "Invalid HCatalog static partition key at position " + v);
1784 }
1785 }
1786 } else {
1787 if (options.getHivePartitionKey() != null
1788 && options.getHivePartitionValue() == null) {
1789 throw new InvalidOptionsException("Either both --hive-partition-key and"
1790 + " --hive-partition-value options should be provided or both of "
1791 + "these options should be omitted");
1792 }
1793 }
1794 if (options.doCreateHCatalogTable() &&
1795 options.doDropAndCreateHCatalogTable()) {
1796 throw new InvalidOptionsException("Options --create-hcatalog-table" +
1797 " and --drop-and-create-hcatalog-table are mutually exclusive." +
1798 " Use any one of them");
1799 }
1800 }
1801
1802 private boolean isSet(String option) {
1803 return org.apache.commons.lang.StringUtils.isNotBlank(option);
1804 }
1805
1806 protected void validateHBaseOptions(SqoopOptions options)
1807 throws InvalidOptionsException {
1808 if ((options.getHBaseColFamily() != null && options.getHBaseTable() == null)
1809 || (options.getHBaseColFamily() == null
1810 && options.getHBaseTable() != null)) {
1811 throw new InvalidOptionsException(
1812 "Both --hbase-table and --column-family must be set together."
1813 + HELP_STR);
1814 }
1815
1816 if (options.isBulkLoadEnabled() && options.getHBaseTable() == null) {
1817 String validationMessage = String.format("Can't run import with %s "
1818 + "without %s",
1819 BaseSqoopTool.HBASE_BULK_LOAD_ENABLED_ARG,
1820 BaseSqoopTool.HBASE_TABLE_ARG);
1821 throw new InvalidOptionsException(validationMessage);
1822 }
1823
1824 if (options.getHBaseTable() != null && options.getFileLayout() != SqoopOptions.FileLayout.TextFile) {
1825 String validationMessage = String.format("Can't run HBase import with file layout: %s", options.getFileLayout());
1826 throw new InvalidOptionsException(validationMessage);
1827 }
1828 }
1829
1830 /**
1831 * Given an array of extra arguments (usually populated via
1832 * this.extraArguments), determine the offset of the first '--'
1833 * argument in the list. Return 'extra.length' if there is none.
1834 */
1835 protected int getDashPosition(String [] extra) {
1836 int dashPos = extra.length;
1837 for (int i = 0; i < extra.length; i++) {
1838 if (extra[i].equals("--")) {
1839 dashPos = i;
1840 break;
1841 }
1842 }
1843
1844 return dashPos;
1845 }
1846
1847 protected void validateHasDirectConnectorOption(SqoopOptions options) throws SqoopOptions.InvalidOptionsException {
1848 SupportedManagers m = SupportedManagers.createFrom(options);
1849 if (m != null && options.isDirect() && !m.hasDirectConnector()) {
1850 throw new SqoopOptions.InvalidOptionsException(
1851 "Was called with the --direct option, but no direct connector available.");
1852 }
1853 }
1854 }
1855