SQOOP-3309: Implement HiveServer2 client
[sqoop.git] / src / test / org / apache / sqoop / hive / TestTableDefWriter.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.hive;
20
21 import java.util.Map;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.sqoop.manager.ConnManager;
27 import org.apache.sqoop.util.SqlTypeMap;
28
29 import org.apache.sqoop.SqoopOptions;
30 import org.apache.sqoop.testutil.HsqldbTestServer;
31
32 import org.junit.Before;
33 import org.junit.Rule;
34 import org.junit.Test;
35 import org.junit.rules.ExpectedException;
36
37 import java.sql.Types;
38
39 import static org.junit.Assert.assertEquals;
40 import static org.junit.Assert.assertFalse;
41 import static org.junit.Assert.assertNotNull;
42 import static org.junit.Assert.assertTrue;
43 import static org.mockito.Matchers.anyString;
44 import static org.mockito.Mockito.mock;
45 import static org.mockito.Mockito.verify;
46 import static org.mockito.Mockito.when;
47
48
49 /**
50 * Test Hive DDL statement generation.
51 */
52 public class TestTableDefWriter {
53
54 public static final Log LOG = LogFactory.getLog(
55 TestTableDefWriter.class.getName());
56
57 private ConnManager connManager;
58
59 private Configuration conf;
60
61 private SqoopOptions options;
62
63 private TableDefWriter writer;
64
65 private String inputTable;
66
67 private String outputTable;
68
69 @Rule
70 public ExpectedException thrown = ExpectedException.none();
71
72 @Before
73 public void before() {
74 inputTable = HsqldbTestServer.getTableName();
75 outputTable = "outputTable";
76 connManager = mock(ConnManager.class);
77 conf = new Configuration();
78 options = new SqoopOptions();
79 when(connManager.getColumnTypes(anyString())).thenReturn(new SqlTypeMap<String, Integer>());
80 when(connManager.getColumnNames(anyString())).thenReturn(new String[]{});
81
82 writer = new TableDefWriter(options, connManager, inputTable, outputTable, conf, false);
83 }
84
85 // Test getHiveOctalCharCode and expect an IllegalArgumentException.
86 private void expectExceptionInCharCode(int charCode) {
87 thrown.expect(IllegalArgumentException.class);
88 thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException with out-of-range Hive delimiter");
89 TableDefWriter.getHiveOctalCharCode(charCode);
90 }
91
92 @Test
93 public void testHiveOctalCharCode() {
94 assertEquals("\\000", TableDefWriter.getHiveOctalCharCode(0));
95 assertEquals("\\001", TableDefWriter.getHiveOctalCharCode(1));
96 assertEquals("\\012", TableDefWriter.getHiveOctalCharCode((int) '\n'));
97 assertEquals("\\177", TableDefWriter.getHiveOctalCharCode(0177));
98
99 expectExceptionInCharCode(4096);
100 expectExceptionInCharCode(0200);
101 expectExceptionInCharCode(254);
102 }
103
104 @Test
105 public void testDifferentTableNames() throws Exception {
106 String createTable = writer.getCreateTableStmt();
107 String loadData = writer.getLoadDataStmt();
108
109 LOG.debug("Create table stmt: " + createTable);
110 LOG.debug("Load data stmt: " + loadData);
111
112 // Assert that the statements generated have the form we expect.
113 assertTrue(createTable.indexOf(
114 "CREATE TABLE IF NOT EXISTS `outputTable`") != -1);
115 assertTrue(loadData.indexOf("INTO TABLE `outputTable`") != -1);
116 assertTrue(loadData.indexOf("/" + inputTable + "'") != -1);
117 }
118
119 @Test
120 public void testDifferentTargetDirs() throws Exception {
121 String targetDir = "targetDir";
122
123 // Specify a different target dir from input table name
124 options.setTargetDir(targetDir);
125
126 String createTable = writer.getCreateTableStmt();
127 String loadData = writer.getLoadDataStmt();
128
129 LOG.debug("Create table stmt: " + createTable);
130 LOG.debug("Load data stmt: " + loadData);
131
132 // Assert that the statements generated have the form we expect.
133 assertTrue(createTable.indexOf(
134 "CREATE TABLE IF NOT EXISTS `" + outputTable + "`") != -1);
135 assertTrue(loadData.indexOf("INTO TABLE `" + outputTable + "`") != -1);
136 assertTrue(loadData.indexOf("/" + targetDir + "'") != -1);
137 }
138
139 @Test
140 public void testPartitions() throws Exception {
141 options.setHivePartitionKey("ds");
142 options.setHivePartitionValue("20110413");
143
144 String createTable = writer.getCreateTableStmt();
145 String loadData = writer.getLoadDataStmt();
146
147 assertNotNull(createTable);
148 assertNotNull(loadData);
149 assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) "
150 + "PARTITIONED BY (ds STRING) "
151 + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' "
152 + "LINES TERMINATED BY '\\012' STORED AS TEXTFILE", createTable);
153 assertTrue(loadData.endsWith(" PARTITION (ds='20110413')"));
154 }
155
156 @Test
157 public void testLzoSplitting() throws Exception {
158 options.setUseCompression(true);
159 options.setCompressionCodec("lzop");
160
161 String createTable = writer.getCreateTableStmt();
162 String loadData = writer.getLoadDataStmt();
163
164 assertNotNull(createTable);
165 assertNotNull(loadData);
166 assertEquals("CREATE TABLE IF NOT EXISTS `outputTable` ( ) "
167 + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\054' "
168 + "LINES TERMINATED BY '\\012' STORED AS "
169 + "INPUTFORMAT 'com.hadoop.mapred.DeprecatedLzoTextInputFormat' "
170 + "OUTPUTFORMAT "
171 + "'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'",
172 createTable);
173 }
174
175 @Test
176 public void testUserMappingNoDecimal() throws Exception {
177 options.setMapColumnHive("id=STRING,value=INTEGER");
178
179 Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
180 colTypes.put("id", Types.INTEGER);
181 colTypes.put("value", Types.VARCHAR);
182
183 setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
184
185 String createTable = writer.getCreateTableStmt();
186
187 assertNotNull(createTable);
188
189 assertTrue(createTable.contains("`id` STRING"));
190 assertTrue(createTable.contains("`value` INTEGER"));
191
192 assertFalse(createTable.contains("`id` INTEGER"));
193 assertFalse(createTable.contains("`value` STRING"));
194 }
195
196 @Test
197 public void testUserMappingWithDecimal() throws Exception {
198 options.setMapColumnHive("id=STRING,value2=DECIMAL(13,5),value1=INTEGER,value3=DECIMAL(4,5),value4=VARCHAR(255)");
199
200 Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
201 colTypes.put("id", Types.INTEGER);
202 colTypes.put("value1", Types.VARCHAR);
203 colTypes.put("value2", Types.DOUBLE);
204 colTypes.put("value3", Types.FLOAT);
205 colTypes.put("value4", Types.CHAR);
206
207 setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
208
209 String createTable = writer.getCreateTableStmt();
210
211 assertNotNull(createTable);
212
213 assertTrue(createTable.contains("`id` STRING"));
214 assertTrue(createTable.contains("`value1` INTEGER"));
215 assertTrue(createTable.contains("`value2` DECIMAL(13,5)"));
216 assertTrue(createTable.contains("`value3` DECIMAL(4,5)"));
217 assertTrue(createTable.contains("`value4` VARCHAR(255)"));
218
219 assertFalse(createTable.contains("`id` INTEGER"));
220 assertFalse(createTable.contains("`value1` STRING"));
221 assertFalse(createTable.contains("`value2` DOUBLE"));
222 assertFalse(createTable.contains("`value3` FLOAT"));
223 assertFalse(createTable.contains("`value4` CHAR"));
224 }
225
226 @Test
227 public void testUserMappingFailWhenCantBeApplied() throws Exception {
228 options.setMapColumnHive("id=STRING,value=INTEGER");
229
230 Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
231 colTypes.put("id", Types.INTEGER);
232 setUpMockConnManager(HsqldbTestServer.getTableName(), colTypes);
233
234 thrown.expect(IllegalArgumentException.class);
235 thrown.reportMissingExceptionWithMessage("Expected IllegalArgumentException on non applied Hive type mapping");
236 writer.getCreateTableStmt();
237 }
238
239 @Test
240 public void testHiveDatabase() throws Exception {
241 options.setHiveDatabaseName("db");
242
243 String createTable = writer.getCreateTableStmt();
244 assertNotNull(createTable);
245 assertTrue(createTable.contains("`db`.`outputTable`"));
246
247 String loadStmt = writer.getLoadDataStmt();
248 assertNotNull(loadStmt);
249 assertTrue(createTable.contains("`db`.`outputTable`"));
250 }
251
252 @Test
253 public void testGetCreateTableStmtDiscardsConnection() throws Exception {
254 writer.getCreateTableStmt();
255
256 verify(connManager).discardConnection(true);
257 }
258
259 private void setUpMockConnManager(String tableName, Map<String, Integer> typeMap) {
260 when(connManager.getColumnTypes(tableName)).thenReturn(typeMap);
261 when(connManager.getColumnNames(tableName)).thenReturn(typeMap.keySet().toArray(new String[]{}));
262 }
263
264 }