SQOOP-931: Integrate HCatalog with Sqoop
[sqoop.git] / src / test / org / apache / sqoop / hcat / HCatalogImportTest.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.hcat;
20
21 import java.io.IOException;
22 import java.io.PrintWriter;
23 import java.io.StringWriter;
24 import java.math.BigDecimal;
25 import java.nio.ByteBuffer;
26 import java.sql.Date;
27 import java.sql.Time;
28 import java.sql.Timestamp;
29 import java.sql.Types;
30 import java.util.ArrayList;
31 import java.util.Arrays;
32 import java.util.Collections;
33 import java.util.Comparator;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.TimeZone;
37
38 import org.apache.commons.logging.Log;
39 import org.apache.commons.logging.LogFactory;
40 import org.apache.hadoop.conf.Configuration;
41 import org.apache.hcatalog.data.HCatRecord;
42 import org.apache.hcatalog.data.schema.HCatFieldSchema;
43 import org.apache.hcatalog.data.schema.HCatSchema;
44 import org.apache.sqoop.hcat.HCatalogTestUtils.ColumnGenerator;
45 import org.apache.sqoop.hcat.HCatalogTestUtils.CreateMode;
46 import org.apache.sqoop.hcat.HCatalogTestUtils.KeyType;
47 import org.apache.sqoop.mapreduce.hcat.SqoopHCatUtilities;
48 import org.junit.Before;
49
50 import com.cloudera.sqoop.Sqoop;
51 import com.cloudera.sqoop.SqoopOptions;
52 import com.cloudera.sqoop.testutil.CommonArgs;
53 import com.cloudera.sqoop.testutil.ImportJobTestCase;
54 import com.cloudera.sqoop.tool.ImportTool;
55 import com.cloudera.sqoop.tool.SqoopTool;
56
57 /**
58 * Test that we can export HCatalog tables into databases.
59 */
60 public class HCatalogImportTest extends ImportJobTestCase {
61 private static final Log LOG =
62 LogFactory.getLog(HCatalogImportTest.class);
63 private final HCatalogTestUtils utils = HCatalogTestUtils.instance();
64 private List<String> extraTestArgs = null;
65 private List<String> configParams = null;
66
67 @Override
68 @Before
69 public void setUp() {
70 super.setUp();
71 try {
72 utils.initUtils();
73 extraTestArgs = new ArrayList<String>();
74 configParams = new ArrayList<String>();
75 } catch (Exception e) {
76 throw new RuntimeException("Error initializing HCatTestUtilis", e);
77 }
78 }
79
80 /**
81 * @return an argv for the CodeGenTool to use when creating tables to export.
82 */
83 protected String[] getCodeGenArgv(String... extraArgs) {
84 List<String> codeGenArgv = new ArrayList<String>();
85
86 if (null != extraArgs) {
87 for (String arg : extraArgs) {
88 codeGenArgv.add(arg);
89 }
90 }
91
92 codeGenArgv.add("--table");
93 codeGenArgv.add(getTableName());
94 codeGenArgv.add("--connect");
95 codeGenArgv.add(getConnectString());
96 codeGenArgv.add("--hcatalog-table");
97 codeGenArgv.add(getTableName());
98
99 return codeGenArgv.toArray(new String[0]);
100 }
101
102 private void setExtraArgs(List<String> args) {
103 extraTestArgs.clear();
104 if (args != null && args.size() > 0) {
105 extraTestArgs.addAll(args);
106 }
107 }
108
109 private List<String> getConfigParams() {
110 return configParams;
111 }
112
113 private void setConfigParams(List<String> params) {
114 configParams.clear();
115 if (params != null && params.size() > 0) {
116 configParams.addAll(params);
117 }
118 }
119 @Override
120 protected List<String> getExtraArgs(Configuration conf) {
121 List<String> addlArgsArray = new ArrayList<String>();
122 if (extraTestArgs != null && extraTestArgs.size() > 0) {
123 addlArgsArray.addAll(extraTestArgs);
124 }
125 Map<String, String> addlArgsMap = utils.getAddlTestArgs();
126 String[] argv = {};
127
128 if (addlArgsMap.containsKey("-libjars")) {
129 argv = new String[2];
130 argv[0] = "-libjars";
131 argv[1] = addlArgsMap.get("-libjars");
132 }
133 addlArgsArray.add("-m");
134 addlArgsArray.add("1");
135 addlArgsArray.add("--hcatalog-table");
136 addlArgsArray.add(getTableName());
137 for (String k : addlArgsMap.keySet()) {
138 if (!k.equals("-libjars")) {
139 addlArgsArray.add(k);
140 addlArgsArray.add(addlArgsMap.get(k));
141 }
142 }
143 return addlArgsArray;
144 }
145
146 @Override
147 protected String[] getArgv(boolean includeHadoopFlags, String[] colNames,
148 Configuration conf) {
149 if (null == colNames) {
150 colNames = getColNames();
151 }
152 String columnsString = "";
153 String splitByCol = null;
154 if (colNames != null) {
155 splitByCol = colNames[0];
156 for (String col : colNames) {
157 columnsString += col + ",";
158 }
159 }
160 ArrayList<String> args = new ArrayList<String>();
161
162 if (includeHadoopFlags) {
163 CommonArgs.addHadoopFlags(args);
164 }
165 args.addAll(getConfigParams());
166 args.add("--table");
167 args.add(getTableName());
168 if (colNames != null) {
169 args.add("--columns");
170 args.add(columnsString);
171 args.add("--split-by");
172 args.add(splitByCol);
173 }
174 args.add("--hcatalog-table");
175 args.add(getTableName());
176 args.add("--connect");
177 args.add(getConnectString());
178 args.addAll(getExtraArgs(conf));
179
180 return args.toArray(new String[0]);
181 }
182
183 private void validateHCatRecords(final List<HCatRecord> recs,
184 final HCatSchema schema, int expectedCount,
185 ColumnGenerator... cols) throws IOException {
186 if (recs.size() != expectedCount) {
187 fail("Expected records = " + expectedCount
188 + ", actual = " + recs.size());
189 return;
190 }
191 schema.getFieldNames();
192 Collections.sort(recs, new Comparator<HCatRecord>()
193 {
194 @Override
195 public int compare(HCatRecord hr1, HCatRecord hr2) {
196 try {
197 return hr1.getInteger("id", schema)
198 - hr2.getInteger("id", schema);
199 } catch (Exception e) {
200 LOG.warn("Exception caught while sorting hcat records " + e);
201 }
202 return 0;
203 }
204 });
205
206 Object expectedVal = null;
207 Object actualVal = null;
208 for (int i = 0; i < recs.size(); ++i) {
209 HCatRecord rec = recs.get(i);
210 expectedVal = i;
211 actualVal = rec.get("id", schema);
212 LOG.info("Validating field: id (expected = "
213 + expectedVal + ", actual = " + actualVal + ")");
214 HCatalogTestUtils.assertEquals(expectedVal, actualVal);
215 expectedVal = "textfield" + i;
216 actualVal = rec.get("msg", schema);
217 LOG.info("Validating field: msg (expected = "
218 + expectedVal + ", actual = " + actualVal + ")");
219 HCatalogTestUtils.assertEquals(rec.get("msg", schema), "textfield" + i);
220 for (ColumnGenerator col : cols) {
221 String name = col.getName().toLowerCase();
222 expectedVal = col.getHCatValue(i);
223 actualVal = rec.get(name, schema);
224 LOG.info("Validating field: " + name + " (expected = "
225 + expectedVal + ", actual = " + actualVal + ")");
226 HCatalogTestUtils.assertEquals(expectedVal, actualVal);
227 }
228 }
229 }
230
231 protected void runImport(SqoopTool tool, String[] argv) throws IOException {
232 // run the tool through the normal entry-point.
233 int ret;
234 try {
235 Configuration conf = getConf();
236 SqoopOptions opts = getSqoopOptions(conf);
237 Sqoop sqoop = new Sqoop(tool, conf, opts);
238 ret = Sqoop.runSqoop(sqoop, argv);
239 } catch (Exception e) {
240 LOG.error("Got exception running import: " + e.toString());
241 e.printStackTrace();
242 ret = 1;
243 }
244 if (0 != ret) {
245 throw new IOException("Import failure; return status " + ret);
246 }
247 }
248
249 private void runHCatImport(List<String> addlArgsArray,
250 int totalRecords, String table, ColumnGenerator[] cols,
251 String[] cNames) throws Exception {
252 runHCatImport(addlArgsArray, totalRecords, table, cols, cNames, false);
253 }
254
255 private void runHCatImport(List<String> addlArgsArray,
256 int totalRecords, String table, ColumnGenerator[] cols,
257 String[] cNames, boolean dontCreate) throws Exception {
258 CreateMode mode = CreateMode.CREATE;
259 if (dontCreate) {
260 mode = CreateMode.NO_CREATION;
261 }
262 HCatSchema tblSchema =
263 utils.createHCatTable(mode, totalRecords, table, cols);
264 utils.createSqlTable(getConnection(), false, totalRecords, table, cols);
265 Map<String, String> addlArgsMap = utils.getAddlTestArgs();
266 String[] argv = {};
267 addlArgsArray.add("-m");
268 addlArgsArray.add("1");
269 addlArgsArray.add("--hcatalog-table");
270 addlArgsArray.add(table);
271 if (addlArgsMap.containsKey("-libjars")) {
272 argv = new String[2];
273 argv[0] = "-libjars";
274 argv[1] = addlArgsMap.get("-libjars");
275 }
276 for (String k : addlArgsMap.keySet()) {
277 if (!k.equals("-libjars")) {
278 addlArgsArray.add(k);
279 addlArgsArray.add(addlArgsMap.get(k));
280 }
281 }
282 String[] colNames = null;
283 if (cNames != null) {
284 colNames = cNames;
285 } else {
286 colNames = new String[2 + cols.length];
287 colNames[0] = "ID";
288 colNames[1] = "MSG";
289 for (int i = 0; i < cols.length; ++i) {
290 colNames[2 + i] = cols[i].getName().toUpperCase();
291 }
292 }
293 String[] importArgs = getArgv(true, colNames, new Configuration());
294 LOG.debug("Import args = " + Arrays.toString(importArgs));
295 SqoopHCatUtilities.instance().setConfigured(false);
296 runImport(new ImportTool(), importArgs);
297 List<HCatRecord> recs = utils.readHCatRecords(null, table, null);
298 LOG.debug("HCat records ");
299 LOG.debug(utils.hCatRecordDump(recs, tblSchema));
300 validateHCatRecords(recs, tblSchema, 10, cols);
301 }
302
303 public void testIntTypes() throws Exception {
304 final int TOTAL_RECORDS = 1 * 10;
305 String table = getTableName().toUpperCase();
306 ColumnGenerator[] cols = new ColumnGenerator[] {
307 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
308 "boolean", Types.BOOLEAN, HCatFieldSchema.Type.BOOLEAN,
309 Boolean.TRUE, Boolean.TRUE, KeyType.NOT_A_KEY),
310 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
311 "tinyint", Types.INTEGER, HCatFieldSchema.Type.INT, 10,
312 10, KeyType.NOT_A_KEY),
313 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
314 "smallint", Types.INTEGER, HCatFieldSchema.Type.INT, 100,
315 100, KeyType.NOT_A_KEY),
316 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
317 "int", Types.INTEGER, HCatFieldSchema.Type.INT, 1000,
318 1000, KeyType.NOT_A_KEY),
319 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(4),
320 "bigint", Types.BIGINT, HCatFieldSchema.Type.BIGINT, 10000L,
321 10000L, KeyType.NOT_A_KEY),
322 };
323 List<String> addlArgsArray = new ArrayList<String>();
324 setExtraArgs(addlArgsArray);
325 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
326 }
327
328 public void testFloatTypes() throws Exception {
329 final int TOTAL_RECORDS = 1 * 10;
330 String table = getTableName().toUpperCase();
331 ColumnGenerator[] cols = new ColumnGenerator[] {
332 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
333 "float", Types.FLOAT, HCatFieldSchema.Type.FLOAT, 10.0F,
334 10.F, KeyType.NOT_A_KEY),
335 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
336 "real", Types.FLOAT, HCatFieldSchema.Type.FLOAT, 20.0F,
337 20.0F, KeyType.NOT_A_KEY),
338 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
339 "double", Types.DOUBLE, HCatFieldSchema.Type.DOUBLE, 30.0D,
340 30.0D, KeyType.NOT_A_KEY),
341 };
342 List<String> addlArgsArray = new ArrayList<String>();
343 setExtraArgs(addlArgsArray);
344 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
345 }
346
347 public void testNumberTypes() throws Exception {
348 final int TOTAL_RECORDS = 1 * 10;
349 String table = getTableName().toUpperCase();
350 ColumnGenerator[] cols = new ColumnGenerator[] {
351 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
352 "numeric(18,2)", Types.NUMERIC, HCatFieldSchema.Type.STRING, "1000",
353 new BigDecimal("1000"), KeyType.NOT_A_KEY),
354 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
355 "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.STRING, "2000",
356 new BigDecimal("2000"), KeyType.NOT_A_KEY),
357 };
358 List<String> addlArgsArray = new ArrayList<String>();
359 setExtraArgs(addlArgsArray);
360 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
361 }
362
363 public void testDateTypes() throws Exception {
364 final int TOTAL_RECORDS = 1 * 10;
365 String table = getTableName().toUpperCase();
366 ColumnGenerator[] cols = new ColumnGenerator[] {
367 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
368 "date", Types.DATE, HCatFieldSchema.Type.STRING, "2013-12-31",
369 new Date(113, 11, 31), KeyType.NOT_A_KEY),
370 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
371 "time", Types.TIME, HCatFieldSchema.Type.STRING, "10:11:12",
372 new Time(10, 11, 12), KeyType.NOT_A_KEY),
373 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
374 "timestamp", Types.TIMESTAMP, HCatFieldSchema.Type.STRING,
375 "2013-12-31 10:11:12.0", new Timestamp(113, 11, 31, 10, 11, 12, 0),
376 KeyType.NOT_A_KEY),
377 };
378 List<String> addlArgsArray = new ArrayList<String>();
379 setExtraArgs(addlArgsArray);
380 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
381 }
382
383 public void testDateTypesToBigInt() throws Exception {
384 final int TOTAL_RECORDS = 1 * 10;
385 long offset = TimeZone.getDefault().getRawOffset();
386 String table = getTableName().toUpperCase();
387 ColumnGenerator[] cols = new ColumnGenerator[] {
388 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
389 "date", Types.DATE, HCatFieldSchema.Type.BIGINT, 0 - offset,
390 new Date(70, 0, 1), KeyType.NOT_A_KEY),
391 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
392 "time", Types.TIME, HCatFieldSchema.Type.BIGINT, 36672000L - offset,
393 new Time(10, 11, 12), KeyType.NOT_A_KEY),
394 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(2),
395 "timestamp", Types.TIMESTAMP, HCatFieldSchema.Type.BIGINT,
396 36672000L - offset, new Timestamp(70, 0, 1, 10, 11, 12, 0),
397 KeyType.NOT_A_KEY),
398 };
399 List<String> addlArgsArray = new ArrayList<String>();
400 addlArgsArray.add("--map-column-hive");
401 addlArgsArray.add("COL0=bigint,COL1=bigint,COL2=bigint");
402 setExtraArgs(addlArgsArray);
403 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
404 }
405
406 public void testStringTypes() throws Exception {
407 final int TOTAL_RECORDS = 1 * 10;
408 String table = getTableName().toUpperCase();
409 ColumnGenerator[] cols = new ColumnGenerator[] {
410 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
411 "char(10)", Types.CHAR, HCatFieldSchema.Type.STRING, "string to test",
412 "string to test", KeyType.NOT_A_KEY),
413 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
414 "longvarchar", Types.LONGVARCHAR, HCatFieldSchema.Type.STRING,
415 "string to test", "string to test", KeyType.NOT_A_KEY),
416 };
417 List<String> addlArgsArray = new ArrayList<String>();
418 setExtraArgs(addlArgsArray);
419 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
420 }
421
422 public void testBinaryTypes() throws Exception {
423 ByteBuffer bb = ByteBuffer.wrap(new byte[] { 0, 1, 2 });
424 final int TOTAL_RECORDS = 1 * 10;
425 String table = getTableName().toUpperCase();
426 ColumnGenerator[] cols = new ColumnGenerator[] {
427 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
428 "binary(10)", Types.BINARY, HCatFieldSchema.Type.BINARY,
429 bb.array(), bb.array(), KeyType.NOT_A_KEY),
430 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
431 "longvarbinary", Types.BINARY, HCatFieldSchema.Type.BINARY,
432 bb.array(), bb.array(), KeyType.NOT_A_KEY),
433 };
434 List<String> addlArgsArray = new ArrayList<String>();
435 setExtraArgs(addlArgsArray);
436 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
437 }
438
439 public void testColumnProjection() throws Exception {
440 final int TOTAL_RECORDS = 1 * 10;
441 String table = getTableName().toUpperCase();
442 ColumnGenerator[] cols = new ColumnGenerator[] {
443 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
444 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
445 null, null, KeyType.NOT_A_KEY),
446 };
447 List<String> addlArgsArray = new ArrayList<String>();
448 List<String> cfgParams = new ArrayList<String>();
449 cfgParams.add("-D");
450 cfgParams.add(SqoopHCatUtilities.DEBUG_HCAT_IMPORT_MAPPER_PROP
451 + "=true");
452 setConfigParams(cfgParams);
453 String[] colNames = new String[] { "ID", "MSG" };
454 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, colNames);
455 }
456
457 public void testColumnProjectionMissingPartKeys() throws Exception {
458 final int TOTAL_RECORDS = 1 * 10;
459 String table = getTableName().toUpperCase();
460 ColumnGenerator[] cols = new ColumnGenerator[] {
461 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
462 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
463 null, null, KeyType.DYNAMIC_KEY),
464 };
465 List<String> addlArgsArray = new ArrayList<String>();
466 List<String> cfgParams = new ArrayList<String>();
467 cfgParams.add("-D");
468 cfgParams.add(SqoopHCatUtilities.DEBUG_HCAT_IMPORT_MAPPER_PROP
469 + "=true");
470 setConfigParams(cfgParams);
471 String[] colNames = new String[] { "ID", "MSG" };
472 try {
473 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, colNames);
474 fail("Column projection with missing dynamic partition keys must fail");
475 } catch (Throwable t) {
476 LOG.info("Job fails as expected : " + t);
477 StringWriter sw = new StringWriter();
478 t.printStackTrace(new PrintWriter(sw));
479 LOG.info("Exception stack trace = " + sw);
480 }
481 }
482 public void testStaticPartitioning() throws Exception {
483 final int TOTAL_RECORDS = 1 * 10;
484 String table = getTableName().toUpperCase();
485 ColumnGenerator[] cols = new ColumnGenerator[] {
486 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
487 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
488 "1", "1", KeyType.STATIC_KEY),
489 };
490 List<String> addlArgsArray = new ArrayList<String>();
491 addlArgsArray.add("--hive-partition-key");
492 addlArgsArray.add("col0");
493 addlArgsArray.add("--hive-partition-value");
494 addlArgsArray.add("1");
495 setExtraArgs(addlArgsArray);
496 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
497 }
498
499 public void testDynamicPartitioning() throws Exception {
500 final int TOTAL_RECORDS = 1 * 10;
501 String table = getTableName().toUpperCase();
502 ColumnGenerator[] cols = new ColumnGenerator[] {
503 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
504 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
505 "1", "1", KeyType.DYNAMIC_KEY),
506 };
507 List<String> addlArgsArray = new ArrayList<String>();
508 setExtraArgs(addlArgsArray);
509 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
510 }
511
512 public void testStaicAndDynamicPartitioning() throws Exception {
513 final int TOTAL_RECORDS = 1 * 10;
514 String table = getTableName().toUpperCase();
515 ColumnGenerator[] cols = new ColumnGenerator[] {
516 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
517 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
518 "1", "1", KeyType.STATIC_KEY),
519 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
520 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
521 "2", "2", KeyType.DYNAMIC_KEY),
522 };
523 List<String> addlArgsArray = new ArrayList<String>();
524 addlArgsArray.add("--hive-partition-key");
525 addlArgsArray.add("col0");
526 addlArgsArray.add("--hive-partition-value");
527 addlArgsArray.add("1");
528 setExtraArgs(addlArgsArray);
529 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
530 }
531
532 /**
533 * Test other file formats.
534 */
535 public void testSequenceFile() throws Exception {
536 final int TOTAL_RECORDS = 1 * 10;
537 String table = getTableName().toUpperCase();
538 ColumnGenerator[] cols = new ColumnGenerator[] {
539 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
540 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "1",
541 "1", KeyType.STATIC_KEY),
542 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
543 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "2",
544 "2", KeyType.DYNAMIC_KEY), };
545 List<String> addlArgsArray = new ArrayList<String>();
546 addlArgsArray.add("--hive-partition-key");
547 addlArgsArray.add("col0");
548 addlArgsArray.add("--hive-partition-value");
549 addlArgsArray.add("1");
550 setExtraArgs(addlArgsArray);
551 utils.setStorageInfo(HCatalogTestUtils.STORED_AS_SEQFILE);
552 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
553 }
554
555 public void testTextFile() throws Exception {
556 final int TOTAL_RECORDS = 1 * 10;
557 String table = getTableName().toUpperCase();
558 ColumnGenerator[] cols = new ColumnGenerator[] {
559 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
560 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
561 "1", "1", KeyType.STATIC_KEY),
562 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
563 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
564 "2", "2", KeyType.DYNAMIC_KEY),
565 };
566 List<String> addlArgsArray = new ArrayList<String>();
567 addlArgsArray.add("--hive-partition-key");
568 addlArgsArray.add("col0");
569 addlArgsArray.add("--hive-partition-value");
570 addlArgsArray.add("1");
571 setExtraArgs(addlArgsArray);
572 utils.setStorageInfo(HCatalogTestUtils.STORED_AS_TEXT);
573 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
574 }
575
576 public void testTableCreation() throws Exception {
577 final int TOTAL_RECORDS = 1 * 10;
578 String table = getTableName().toUpperCase();
579 ColumnGenerator[] cols = new ColumnGenerator[] {
580 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
581 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
582 "1", "1", KeyType.STATIC_KEY),
583 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
584 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
585 "2", "2", KeyType.DYNAMIC_KEY),
586 };
587 List<String> addlArgsArray = new ArrayList<String>();
588 addlArgsArray.add("--create-hcatalog-table");
589 setExtraArgs(addlArgsArray);
590 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true);
591 }
592
593 public void testTableCreationWithPartition() throws Exception {
594 final int TOTAL_RECORDS = 1 * 10;
595 String table = getTableName().toUpperCase();
596 ColumnGenerator[] cols = new ColumnGenerator[] {
597 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
598 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
599 "1", "1", KeyType.NOT_A_KEY),
600 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
601 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
602 "2", "2", KeyType.STATIC_KEY),
603 };
604 List<String> addlArgsArray = new ArrayList<String>();
605 addlArgsArray.add("--hive-partition-key");
606 addlArgsArray.add("col1");
607 addlArgsArray.add("--hive-partition-value");
608 addlArgsArray.add("2");
609 addlArgsArray.add("--create-hcatalog-table");
610 setExtraArgs(addlArgsArray);
611 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true);
612 }
613
614 public void testTableCreationWithStorageStanza() throws Exception {
615 final int TOTAL_RECORDS = 1 * 10;
616 String table = getTableName().toUpperCase();
617 ColumnGenerator[] cols = new ColumnGenerator[] {
618 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
619 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
620 "1", "1", KeyType.NOT_A_KEY),
621 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
622 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
623 "2", "2", KeyType.STATIC_KEY),
624 };
625 List<String> addlArgsArray = new ArrayList<String>();
626 addlArgsArray.add("--hive-partition-key");
627 addlArgsArray.add("col1");
628 addlArgsArray.add("--hive-partition-value");
629 addlArgsArray.add("2");
630 addlArgsArray.add("--create-hcatalog-table");
631 addlArgsArray.add("--hcatalog-storage-stanza");
632 addlArgsArray.add(HCatalogTestUtils.STORED_AS_TEXT);
633 setExtraArgs(addlArgsArray);
634 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true);
635 }
636
637 public void testHiveDropDelims() throws Exception {
638 final int TOTAL_RECORDS = 1 * 10;
639 String table = getTableName().toUpperCase();
640 ColumnGenerator[] cols = new ColumnGenerator[] {
641 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
642 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
643 "Test", "\u0001\n\rTest", KeyType.NOT_A_KEY),
644 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
645 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
646 "Test2", "\u0001\r\nTest2", KeyType.NOT_A_KEY),
647 };
648 List<String> addlArgsArray = new ArrayList<String>();
649 addlArgsArray.add("--hive-drop-import-delims");
650 setExtraArgs(addlArgsArray);
651 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
652 }
653
654 public void testHiveDelimsReplacement() throws Exception {
655 final int TOTAL_RECORDS = 1 * 10;
656 String table = getTableName().toUpperCase();
657 ColumnGenerator[] cols = new ColumnGenerator[] {
658 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
659 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
660 "^^^Test", "\u0001\n\rTest", KeyType.NOT_A_KEY),
661 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
662 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING,
663 "^^^Test2", "\u0001\r\nTest2", KeyType.NOT_A_KEY),
664 };
665 List<String> addlArgsArray = new ArrayList<String>();
666 addlArgsArray.add("--hive-delims-replacement");
667 addlArgsArray.add("^");
668 setExtraArgs(addlArgsArray);
669 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
670 }
671
672 public void testDynamicKeyInMiddle() throws Exception {
673 final int TOTAL_RECORDS = 1 * 10;
674 String table = getTableName().toUpperCase();
675 ColumnGenerator[] cols = new ColumnGenerator[] {
676 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
677 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "1",
678 "1", KeyType.NOT_A_KEY),
679 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
680 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "2",
681 "2", KeyType.DYNAMIC_KEY), };
682 List<String> addlArgsArray = new ArrayList<String>();
683 setExtraArgs(addlArgsArray);
684 utils.setStorageInfo(HCatalogTestUtils.STORED_AS_SEQFILE);
685 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null);
686 }
687
688 public void testCreateTableWithPreExistingTable() throws Exception {
689 final int TOTAL_RECORDS = 1 * 10;
690 String table = getTableName().toUpperCase();
691 ColumnGenerator[] cols = new ColumnGenerator[] {
692 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
693 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "1",
694 "1", KeyType.NOT_A_KEY),
695 HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
696 "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, "2",
697 "2", KeyType.DYNAMIC_KEY), };
698 List<String> addlArgsArray = new ArrayList<String>();
699 addlArgsArray.add("--create-hcatalog-table");
700 setExtraArgs(addlArgsArray);
701 try {
702 // Precreate table
703 utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols);
704 runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols, null, true);
705 fail("HCatalog job with --create-hcatalog-table and pre-existing"
706 + " table should fail");
707 } catch (Exception e) {
708 LOG.debug("Caught expected exception while running "
709 + " create-hcatalog-table with pre-existing table test", e);
710 }
711 }
712 }