SQOOP-931: Integrate HCatalog with Sqoop
[sqoop.git] / src / perftest / ExportStressTest.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 import java.io.*;
20 import java.sql.*;
21 import java.util.*;
22 import org.apache.hadoop.fs.*;
23 import org.apache.hadoop.conf.*;
24 import org.apache.hadoop.util.*;
25
26 import com.cloudera.sqoop.Sqoop;
27 import com.cloudera.sqoop.SqoopOptions;
28 import com.cloudera.sqoop.tool.ExportTool;
29 import com.cloudera.sqoop.tool.SqoopTool;
30
31 /**
32 * Stress test export procedure by running a large-scale export to MySQL.
33 * This requires MySQL be configured with a database that can be accessed by
34 * the specified username without a password. The user must be able to create
35 * and drop tables in the database.
36 *
37 * Run with: src/scripts/run-perftest.sh ExportStressTest \
38 * (connect-str) (username)
39 */
40 public class ExportStressTest extends Configured implements Tool {
41
42 // Export 10 GB of data. Each record is ~100 bytes.
43 public static final int NUM_FILES = 10;
44 public static final int RECORDS_PER_FILE = 10 * 1024 * 1024;
45
46 public static final String ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
47
48 public ExportStressTest() {
49 }
50
51 public void createFile(int fileId) throws IOException {
52 Configuration conf = getConf();
53 FileSystem fs = FileSystem.get(conf);
54 Path dirPath = new Path("ExportStressTest");
55 fs.mkdirs(dirPath);
56 Path filePath = new Path(dirPath, "input-" + fileId);
57
58 OutputStream os = fs.create(filePath);
59 Writer w = new BufferedWriter(new OutputStreamWriter(os));
60 for (int i = 0; i < RECORDS_PER_FILE; i++) {
61 long v = (long) i + ((long) RECORDS_PER_FILE * (long) fileId);
62 w.write("" + v + "," + ALPHABET + ALPHABET + ALPHABET + ALPHABET + "\n");
63
64 }
65 w.close();
66 os.close();
67 }
68
69 /** Create a set of data files to export. */
70 public void createData() throws IOException {
71 Configuration conf = getConf();
72 FileSystem fs = FileSystem.get(conf);
73 Path dirPath = new Path("ExportStressTest");
74 if (fs.exists(dirPath)) {
75 System.out.println(
76 "Export directory appears to already exist. Skipping data-gen.");
77 return;
78 }
79
80 for (int i = 0; i < NUM_FILES; i++) {
81 createFile(i);
82 }
83 }
84
85 /** Create a table to hold our results. Drop any existing definition. */
86 public void createTable(String connectStr, String username) throws Exception {
87 Class.forName("com.mysql.jdbc.Driver"); // Load mysql driver.
88
89 Connection conn = DriverManager.getConnection(connectStr, username, null);
90 conn.setAutoCommit(false);
91 PreparedStatement stmt = conn.prepareStatement(
92 "DROP TABLE IF EXISTS ExportStressTestTable",
93 ResultSet.TYPE_FORWARD_ONLY,
94 ResultSet.CONCUR_READ_ONLY);
95 stmt.executeUpdate();
96 stmt.close();
97
98 stmt = conn.prepareStatement(
99 "CREATE TABLE ExportStressTestTable(id INT NOT NULL PRIMARY KEY, "
100 + "msg VARCHAR(110)) Engine=InnoDB", ResultSet.TYPE_FORWARD_ONLY,
101 ResultSet.CONCUR_READ_ONLY);
102 stmt.executeUpdate();
103 stmt.close();
104 conn.commit();
105 conn.close();
106 }
107
108 /**
109 * Actually run the export of the generated data to the user-created table.
110 */
111 public void runExport(String connectStr, String username) throws Exception {
112 SqoopOptions options = new SqoopOptions(getConf());
113 options.setConnectString(connectStr);
114 options.setTableName("ExportStressTestTable");
115 options.setUsername(username);
116 options.setExportDir("ExportStressTest");
117 options.setNumMappers(4);
118 options.setLinesTerminatedBy('\n');
119 options.setFieldsTerminatedBy(',');
120 options.setExplicitOutputDelims(true);
121
122 SqoopTool exportTool = new ExportTool();
123 Sqoop sqoop = new Sqoop(exportTool, getConf(), options);
124 int ret = Sqoop.runSqoop(sqoop, new String[0]);
125 if (0 != ret) {
126 throw new Exception("Error doing export; ret=" + ret);
127 }
128 }
129
130 @Override
131 public int run(String [] args) {
132 String connectStr = args[0];
133 String username = args[1];
134
135 try {
136 createData();
137 createTable(connectStr, username);
138 runExport(connectStr, username);
139 } catch (Exception e) {
140 System.err.println("Error: " + StringUtils.stringifyException(e));
141 return 1;
142 }
143
144 return 0;
145 }
146
147 public static void main(String [] args) throws Exception {
148 ExportStressTest test = new ExportStressTest();
149 int ret = ToolRunner.run(test, args);
150 System.exit(ret);
151 }
152 }