8bdc3beb3677312ec0ee2e612616358bca4ca838
[sqoop.git] / src / test / org / apache / sqoop / hive / TestTableDefWriter.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.hive;
20
21 import java.util.Map;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.sqoop.manager.ConnManager;
27 import org.apache.sqoop.util.SqlTypeMap;
28
29 import org.apache.sqoop.SqoopOptions;
30 import org.apache.sqoop.testutil.HsqldbTestServer;
31
32 import org.junit.Before;
33 import org.junit.Rule;
34 import org.junit.Test;
35 import org.junit.rules.ExpectedException;
36
37 import java.sql.Types;
38
39 import static org.junit.Assert.assertEquals;
40 import static org.junit.Assert.assertFalse;
41 import static org.junit.Assert.assertNotNull;
42 import static org.junit.Assert.assertTrue;
43 import static org.mockito.Matchers.anyString;
44 import static org.mockito.Mockito.mock;
45 import static org.mockito.Mockito.when;
46
47
48 /**
49 * Test Hive DDL statement generation.
50 */
51 public class TestTableDefWriter {
52
53 public static final Log LOG = LogFactory.getLog(
54 TestTableDefWriter.class.getName());
55
56 private ConnManager connManager;
57
58 private Configuration conf;
59
60 private SqoopOptions options;
61
62 private TableDefWriter writer;
63
64 private String inputTable;
65
66 private String outputTable;
67
68 @Rule
69 public ExpectedException thrown = ExpectedException.none();
70
71 @Before
72 public void before() {
73 inputTable = HsqldbTestServer.getTableName();
74 outputTable = "outputTable";
75 connManager = mock(ConnManager.class);
76 conf = new Configuration();
77 options = new SqoopOptions();
78 when(connManager.getColumnTypes(anyString())).thenReturn(new SqlTypeMap<String, Integer>());
79 when(connManager.getColumnNames(anyString())).thenReturn(new String[]{});
80
81 writer = new TableDefWriter(options, connManager, inputTable, outputTable, conf, false);
82 }
83
84 // Test getHiveOctalCharCode and expect an IllegalArgumentException.
85 private void expectExceptionInCharCode(int charCode) {
86 thrown.expect(IllegalArgumentException.class);
87 thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException with out-of-range Hive delimiter");
88 TableDefWriter.getHiveOctalCharCode(charCode);
89 }
90
91 @Test
92 public void testHiveOctalCharCode() {
93 assertEquals("\\000", TableDefWriter.getHiveOctalCharCode(0));
94 assertEquals("\\001", TableDefWriter.getHiveOctalCharCode(1));
95 assertEquals("\\012", TableDefWriter.getHiveOctalCharCode((int) '\n'));
96 assertEquals("\\177", TableDefWriter.getHiveOctalCharCode(0177));
97
98 expectExceptionInCharCode(4096);
99 expectExceptionInCharCode(0200);
100 expectExceptionInCharCode(254);
101 }
102
103 @Test
104 public void testDifferentTableNames() throws Exception {
105 String createTable = writer.getCreateTableStmt();
106 String loadData = writer.getLoadDataStmt();
107
108 LOG.debug("Create table stmt: " + createTable);
109 LOG.debug("Load data stmt: " + loadData);
110
111 // Assert that the statements generated have the form we expect.
112 assertTrue(createTable.indexOf(
113 "CREATE TABLE IF NOT EXISTS `outputTable`") != -1);
114 assertTrue(loadData.indexOf("INTO TABLE `outputTable`") != -1);
115 assertTrue(loadData.indexOf("/" + inputTable + "'") != -1);
116 }
117
118 @Test
119 public void testDifferentTargetDirs() throws Exception {
120 String targetDir = "targetDir";
121
122 // Specify a different target dir from input table name
123 options.setTargetDir(targetDir);
124
125 String createTable = writer.getCreateTableStmt();
126 String loadData = writer.getLoadDataStmt();
127
128 LOG.debug("Create table stmt: " + createTable);
129 LOG.debug("Load data stmt: " + loadData);
130
131 // Assert that the statements generated have the form we expect.
132 assertTrue(createTable.indexOf(
133 "CREATE TABLE IF NOT EXISTS `" + outputTable + "`") != -1);
134 assertTrue(loadData.indexOf("INTO TABLE `" + outputTable + "`") != -1);
135 assertTrue(loadData.indexOf("/" + targetDir + "'") != -1);
136 }
137
138 @Test
139 public void testPartitions() throws Exception {
140 options.setHivePartitionKey("ds");
141 options.setHivePartitionValue("20110413");
142
143 String createTable = writer.getCreateTableStmt();
144 String loadData = writer.getLoadDataStmt();
145
146 assertNotNull(createTable);
147 assertNotNull(loadData);
148 assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) "
149 + "PARTITIONED BY (ds STRING) "
150 + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' "
151 + "LINES TERMINATED BY '\\012' STORED AS TEXTFILE", createTable);
152 assertTrue(loadData.endsWith(" PARTITION (ds='20110413')"));
153 }
154
155 @Test
156 public void testLzoSplitting() throws Exception {
157 options.setUseCompression(true);
158 options.setCompressionCodec("lzop");
159
160 String createTable = writer.getCreateTableStmt();
161 String loadData = writer.getLoadDataStmt();
162
163 assertNotNull(createTable);
164 assertNotNull(loadData);
165 assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) "
166 + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' "
167 + "LINES TERMINATED BY '\\012' STORED AS "
168 + "INPUTFORMAT 'com.hadoop.mapred.DeprecatedLzoTextInputFormat' "
169 + "OUTPUTFORMAT "
170 + "'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'",
171 createTable);
172 }
173
174 @Test
175 public void testUserMappingNoDecimal() throws Exception {
176 options.setMapColumnHive("id=STRING,value=INTEGER");
177
178 Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
179 colTypes.put("id", Types.INTEGER);
180 colTypes.put("value", Types.VARCHAR);
181
182 setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
183
184 String createTable = writer.getCreateTableStmt();
185
186 assertNotNull(createTable);
187
188 assertTrue(createTable.contains("`id` STRING"));
189 assertTrue(createTable.contains("`value` INTEGER"));
190
191 assertFalse(createTable.contains("`id` INTEGER"));
192 assertFalse(createTable.contains("`value` STRING"));
193 }
194
195 @Test
196 public void testUserMappingWithDecimal() throws Exception {
197 options.setMapColumnHive("id=STRING,value2=DECIMAL(13,5),value1=INTEGER,value3=DECIMAL(4,5),value4=VARCHAR(255)");
198
199 Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
200 colTypes.put("id", Types.INTEGER);
201 colTypes.put("value1", Types.VARCHAR);
202 colTypes.put("value2", Types.DOUBLE);
203 colTypes.put("value3", Types.FLOAT);
204 colTypes.put("value4", Types.CHAR);
205
206 setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
207
208 String createTable = writer.getCreateTableStmt();
209
210 assertNotNull(createTable);
211
212 assertTrue(createTable.contains("`id` STRING"));
213 assertTrue(createTable.contains("`value1` INTEGER"));
214 assertTrue(createTable.contains("`value2` DECIMAL(13,5)"));
215 assertTrue(createTable.contains("`value3` DECIMAL(4,5)"));
216 assertTrue(createTable.contains("`value4` VARCHAR(255)"));
217
218 assertFalse(createTable.contains("`id` INTEGER"));
219 assertFalse(createTable.contains("`value1` STRING"));
220 assertFalse(createTable.contains("`value2` DOUBLE"));
221 assertFalse(createTable.contains("`value3` FLOAT"));
222 assertFalse(createTable.contains("`value4` CHAR"));
223 }
224
225 @Test
226 public void testUserMappingFailWhenCantBeApplied() throws Exception {
227 options.setMapColumnHive("id=STRING,value=INTEGER");
228
229 Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
230 colTypes.put("id", Types.INTEGER);
231 setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
232
233 thrown.expect(IllegalArgumentException.class);
234 thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException on non applied Hive type mapping");
235 writer.getCreateTableStmt();
236 }
237
238 @Test
239 public void testHiveDatabase() throws Exception {
240 options.setHiveDatabaseName("db");
241
242 String createTable = writer.getCreateTableStmt();
243 assertNotNull(createTable);
244 assertTrue(createTable.contains("`db`.`outputTable`"));
245
246 String loadStmt = writer.getLoadDataStmt();
247 assertNotNull(loadStmt);
248 assertTrue(createTable.contains("`db`.`outputTable`"));
249 }
250
251 private void setUpMockConnManager(String tableName, Map<String, Integer> typeMap) {
252 when(connManager.getColumnTypes(tableName)).thenReturn(typeMap);
253 when(connManager.getColumnNames(tableName)).thenReturn(typeMap.keySet().toArray(new String[]{}));
254 }
255
256 }