SQOOP-2843: Sqoop2: Enable SSL/TLS for API calls
authorJarek Jarcec Cecho <jarcec@apache.org>
Fri, 26 Feb 2016 17:37:04 +0000 (09:37 -0800)
committerJarek Jarcec Cecho <jarcec@apache.org>
Fri, 26 Feb 2016 17:37:04 +0000 (09:37 -0800)
(Abraham Fine via Jarek Jarcec Cecho)

13 files changed:
core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
dist/src/main/conf/sqoop.properties
server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
server/src/main/java/org/apache/sqoop/server/common/ServerError.java
test/pom.xml
test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java
test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java [new file with mode: 0644]
test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java
test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java [moved from test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java with 98% similarity]
test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java [moved from test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java with 99% similarity]
test/src/test/java/org/apache/sqoop/integration/serverproperties/ConnectorClasspathIsolationTest.java [moved from test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java with 99% similarity]
test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java [new file with mode: 0644]
test/src/test/resources/server-properties-tests-suite.xml [moved from test/src/test/resources/connector-loading-tests-suite.xml with 86% similarity]

index 6f32e04..91a1b8b 100644 (file)
@@ -37,6 +37,13 @@ public final class SecurityConstants {
     PREFIX_SECURITY_CONFIG + "authentication.";
 
   /**
+   * All tls related configuration is prefixed with this:
+   * <tt>org.apache.sqoop.security.tls.</tt>
+   */
+  public static final String PREFIX_TLS_CONFIG =
+    PREFIX_SECURITY_CONFIG + "tls.";
+
+  /**
    * The config specifies the sqoop authentication type (SIMPLE, KERBEROS).
    * The default type is SIMPLE
    * <tt>org.apache.sqoop.security.authentication.type</tt>.
@@ -158,6 +165,42 @@ public final class SecurityConstants {
           PREFIX_AUTHORIZATION_CONFIG + "server_name";
 
   /**
+   * The config specifies the whether the http server should use TLS.
+   * <tt>org.apache.sqoop.security.tls.enabled</tt>.
+   */
+  public static final String TLS_ENABLED =
+          PREFIX_TLS_CONFIG + "enabled";
+
+  /**
+   * The config specifies the tls protocol version
+   * <tt>org.apache.sqoop.security.tls.protocol</tt>.
+   */
+  public static final String TLS_PROTOCOL =
+    PREFIX_TLS_CONFIG + "protocol";
+
+  /**
+   * The config specifies the location of the JKS formatted keystore
+   * <tt>org.apache.sqoop.security.tls.keystore</tt>.
+   */
+  public static final String KEYSTORE_LOCATION =
+          PREFIX_TLS_CONFIG + "keystore";
+
+  /**
+   * The config specifies the password of the JKS formatted keystore
+   * <tt>org.apache.sqoop.security.tls.keystorepassword</tt>.
+   */
+  public static final String KEYSTORE_PASSWORD =
+          PREFIX_TLS_CONFIG + "keystore_password";
+
+  /**
+   * The config specifies the password for the private key in the JKS formatted
+   * keystore
+   * <tt>org.apache.sqoop.security.tls.keymanagerpassword</tt>.
+   */
+  public static final String KEYMANAGER_PASSWORD =
+    PREFIX_TLS_CONFIG + "keymanager_password";
+
+  /**
    * The config specifies the token kind in delegation token.
    */
   public static final String TOKEN_KIND = "sqoop_token_kind";
index 620146d..767d3f2 100755 (executable)
@@ -180,6 +180,15 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
 #org.apache.sqoop.security.authorization.authentication_provider=org.apache.sqoop.security.authorization.DefaultAuthenticationProvider
 #org.apache.sqoop.security.authorization.server_name=SqoopServer1
 
+#
+# SSL/TLS configuration
+#
+#org.apache.sqoop.security.tls.enabled=false
+#org.apache.sqoop.security.tls.protocol="TLSv1.2"
+#org.apache.sqoop.security.tls.keystore=
+#org.apache.sqoop.security.tls.keystore_password=
+
+
 # External connectors load path
 # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
 #
index 2c4cb7a..60368af 100644 (file)
 package org.apache.sqoop.server;
 
 import org.apache.log4j.Logger;
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.core.SqoopConfiguration;
 import org.apache.sqoop.core.SqoopServer;
 import org.apache.sqoop.filter.SqoopAuthenticationFilter;
+import org.apache.sqoop.security.SecurityConstants;
+import org.apache.sqoop.server.common.ServerError;
 import org.apache.sqoop.server.v1.*;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.ExecutorThreadPool;
 import org.eclipse.jetty.server.ServerConnector;
 
@@ -58,7 +67,45 @@ public class SqoopJettyServer {
     webServer = new Server(threadPool);
 
     // Connector configs
-    ServerConnector connector = new ServerConnector(webServer);
+    ServerConnector connector;
+
+    MapContext configurationContext = SqoopConfiguration.getInstance().getContext();
+
+    if (configurationContext.getBoolean(SecurityConstants.TLS_ENABLED, false)) {
+      String keyStorePath = configurationContext.getString(SecurityConstants.KEYSTORE_LOCATION);
+      if (keyStorePath == null) {
+        throw new SqoopException(ServerError.SERVER_0007);
+      }
+
+      SslContextFactory sslContextFactory = new SslContextFactory();
+      sslContextFactory.setKeyStorePath(keyStorePath);
+
+      String protocol = configurationContext.getString(SecurityConstants.TLS_PROTOCOL);
+      if (protocol != null && protocol.length() > 0) {
+        sslContextFactory.setProtocol(protocol.trim());
+      }
+
+      String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD);
+      if (keyStorePassword != null && keyStorePassword.length() > 0) {
+        sslContextFactory.setKeyStorePassword(keyStorePassword);
+      }
+
+      String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD);
+      if (keyManagerPassword != null && keyManagerPassword.length() > 0) {
+        sslContextFactory.setKeyManagerPassword(keyManagerPassword);
+      }
+
+      HttpConfiguration https = new HttpConfiguration();
+      https.addCustomizer(new SecureRequestCustomizer());
+
+      connector = new ServerConnector(webServer,
+        new SslConnectionFactory(sslContextFactory, "http/1.1"),
+        new HttpConnectionFactory(https));
+    } else {
+      connector = new ServerConnector(webServer);
+    }
+
+
     connector.setPort(sqoopJettyContext.getPort());
     webServer.addConnector(connector);
     webServer.setHandler(createServletContextHandler());
index 1b021cf..a644e9f 100644 (file)
@@ -38,6 +38,9 @@ public enum ServerError implements ErrorCode {
 
   /** Entity requested doesn't exist*/
   SERVER_0006("Entity requested doesn't exist"),
+
+  /** TLS enabled but keystore location not set*/
+  SERVER_0007("TLS enabled but keystore location not set"),
   ;
 
   private final String message;
index 1e88b34..4bac683 100644 (file)
@@ -274,17 +274,17 @@ limitations under the License.
             </configuration>
           </execution>
           <execution>
-            <id>connector-loading-test</id>
+            <id>server-properties-test</id>
             <goals>
               <goal>test</goal>
             </goals>
             <phase>integration-test</phase>
             <configuration>
               <suiteXmlFiles>
-                <suiteXmlFile>src/test/resources/connector-loading-tests-suite.xml</suiteXmlFile>
+                <suiteXmlFile>src/test/resources/server-properties-tests-suite.xml</suiteXmlFile>
               </suiteXmlFiles>
               <properties>
-                <suitename>connector-loading-tests</suitename>
+                <suitename>server-properties-tests</suitename>
               </properties>
             </configuration>
           </execution>
index 83f960b..bca4412 100644 (file)
 package org.apache.sqoop.test.kdc;
 
 import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
 import java.lang.reflect.Constructor;
-import java.math.BigInteger;
 import java.net.URL;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.Key;
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.KeyStore;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
 import java.security.Principal;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
-import java.security.SecureRandom;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
 import java.util.Collection;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -53,19 +34,16 @@ import java.util.concurrent.Callable;
 import javax.security.auth.Subject;
 import javax.security.auth.login.AppConfigurationEntry;
 import javax.security.auth.login.LoginContext;
-import javax.security.auth.x500.X500Principal;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.minikdc.MiniKdc;
-import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
-import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.sqoop.client.SqoopClient;
 import org.apache.sqoop.model.MConnector;
 import org.apache.sqoop.test.utils.HdfsUtils;
+import org.apache.sqoop.test.utils.SecurityUtils;
 import org.apache.sqoop.test.utils.SqoopUtils;
-import org.bouncycastle.x509.X509V1CertificateGenerator;
 
 /**
  * Represents a Minikdc setup. Minikdc should be only used together with
@@ -107,17 +85,17 @@ public class MiniKdcRunner extends KdcRunner {
     config.set("dfs.datanode.kerberos.principal", hadoopPrincipal);
     config.set("dfs.datanode.keytab.file", hadoopKeytabFile);
     String sslKeystoresDir = getTemporaryPath() + "/ssl-keystore";
-    String sslConfDir = getClasspathDir(MiniKdcRunner.class);
+    String sslConfDir = SqoopUtils.getClasspathDir(MiniKdcRunner.class);
     FileUtils.deleteDirectory(new File(sslKeystoresDir));
     FileUtils.forceMkdir(new File(sslKeystoresDir));
-    setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true);
-    config.set("dfs.https.server.keystore.resource", getSSLConfigFileName("ssl-server"));
+    SecurityUtils.setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true);
+    config.set("dfs.https.server.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-server"));
     // Configurations used by both NameNode and DataNode
     config.set("dfs.block.access.token.enable", "true");
     config.set("dfs.http.policy", "HTTPS_ONLY");
     // Configurations used by DFSClient
     config.set("dfs.data.transfer.protection", "privacy");
-    config.set("dfs.client.https.keystore.resource", getSSLConfigFileName("ssl-client"));
+    config.set("dfs.client.https.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-client"));
 
     // YARN related configurations
     config.set("yarn.resourcemanager.principal", hadoopPrincipal);
@@ -331,219 +309,4 @@ public class MiniKdcRunner extends KdcRunner {
       }
     }
   }
-
-  @SuppressWarnings("rawtypes")
-  private static String getClasspathDir(Class klass) throws Exception {
-    String file = klass.getName();
-    file = file.replace('.', '/') + ".class";
-    URL url = Thread.currentThread().getContextClassLoader().getResource(file);
-    String baseDir = url.toURI().getPath();
-    baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1);
-    return baseDir;
-  }
-
-  /**
-   * Performs complete setup of SSL configuration. This includes keys, certs,
-   * keystores, truststores, the server SSL configuration file,
-   * the client SSL configuration file.
-   *
-   * @param keystoresDir String directory to save keystores
-   * @param sslConfDir String directory to save SSL configuration files
-   * @param conf Configuration
-   * @param useClientCert boolean true to make the client present a cert in the
-   * SSL handshake
-   * @param trustStore boolean true to create truststore, false not to create it
-   */
-  private void setupSSLConfig(String keystoresDir, String sslConfDir,
-      Configuration conf, boolean useClientCert, boolean trustStore)
-          throws Exception {
-    String clientKS = keystoresDir + "/clientKS.jks";
-    String clientPassword = "clientP";
-    String serverKS = keystoresDir + "/serverKS.jks";
-    String serverPassword = "serverP";
-    String trustKS = null;
-    String trustPassword = "trustP";
-
-    File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client"));
-    File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server"));
-
-    Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>();
-
-    if (useClientCert) {
-      KeyPair cKP = generateKeyPair("RSA");
-      X509Certificate cCert = generateCertificate("CN=localhost, O=client", cKP, 30, "SHA1withRSA");
-      createKeyStore(clientKS, clientPassword, "client", cKP.getPrivate(), cCert);
-      certs.put("client", cCert);
-    }
-
-    KeyPair sKP = generateKeyPair("RSA");
-    X509Certificate sCert = generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA");
-    createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert);
-    certs.put("server", sCert);
-
-    if (trustStore) {
-      trustKS = keystoresDir + "/trustKS.jks";
-      createTrustStore(trustKS, trustPassword, certs);
-    }
-
-    Configuration clientSSLConf = createSSLConfig(
-        SSLFactory.Mode.CLIENT, clientKS, clientPassword, clientPassword, trustKS);
-    Configuration serverSSLConf = createSSLConfig(
-        SSLFactory.Mode.SERVER, serverKS, serverPassword, serverPassword, trustKS);
-
-    saveConfig(sslClientConfFile, clientSSLConf);
-    saveConfig(sslServerConfFile, serverSSLConf);
-
-    conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL");
-    conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName());
-    conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName());
-    conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert);
-  }
-
-  /**
-   * Returns an SSL configuration file name.  Under parallel test
-   * execution, this file name is parameterized by a unique ID to ensure that
-   * concurrent tests don't collide on an SSL configuration file.
-   *
-   * @param base the base of the file name
-   * @return SSL configuration file name for base
-   */
-  private static String getSSLConfigFileName(String base) {
-    String testUniqueForkId = System.getProperty("test.unique.fork.id");
-    String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : "";
-    return base + fileSuffix + ".xml";
-  }
-
-  /**
-   * Creates SSL configuration.
-   *
-   * @param mode SSLFactory.Mode mode to configure
-   * @param keystore String keystore file
-   * @param password String store password, or null to avoid setting store
-   *   password
-   * @param keyPassword String key password, or null to avoid setting key
-   *   password
-   * @param trustKS String truststore file
-   * @return Configuration for SSL
-   */
-  private static Configuration createSSLConfig(SSLFactory.Mode mode,
-      String keystore, String password, String keyPassword, String trustKS) {
-    String trustPassword = "trustP";
-
-    Configuration sslConf = new Configuration(false);
-    if (keystore != null) {
-      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
-          FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore);
-    }
-    if (password != null) {
-      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
-          FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), password);
-    }
-    if (keyPassword != null) {
-      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
-          FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY),
-          keyPassword);
-    }
-    if (trustKS != null) {
-      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
-          FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
-    }
-    sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
-        FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
-        trustPassword);
-    sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
-        FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
-
-    return sslConf;
-  }
-
-  private static KeyPair generateKeyPair(String algorithm)
-      throws NoSuchAlgorithmException {
-    KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm);
-    keyGen.initialize(1024);
-    return keyGen.genKeyPair();
-  }
-
-  /**
-   * Create a self-signed X.509 Certificate.
-   *
-   * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB"
-   * @param pair the KeyPair
-   * @param days how many days from now the Certificate is valid for
-   * @param algorithm the signing algorithm, eg "SHA1withRSA"
-   * @return the self-signed certificate
-   */
-  private static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm)
-      throws CertificateEncodingException,
-             InvalidKeyException,
-             IllegalStateException,
-             NoSuchProviderException, NoSuchAlgorithmException, SignatureException{
-    Date from = new Date();
-    Date to = new Date(from.getTime() + days * 86400000l);
-    BigInteger sn = new BigInteger(64, new SecureRandom());
-    KeyPair keyPair = pair;
-    X509V1CertificateGenerator certGen = new X509V1CertificateGenerator();
-    X500Principal dnName = new X500Principal(dn);
-
-    certGen.setSerialNumber(sn);
-    certGen.setIssuerDN(dnName);
-    certGen.setNotBefore(from);
-    certGen.setNotAfter(to);
-    certGen.setSubjectDN(dnName);
-    certGen.setPublicKey(keyPair.getPublic());
-    certGen.setSignatureAlgorithm(algorithm);
-
-    X509Certificate cert = certGen.generate(pair.getPrivate());
-    return cert;
-  }
-
-  private static void createKeyStore(String filename,
-      String password, String alias,
-      Key privateKey, Certificate cert)
-      throws GeneralSecurityException, IOException {
-    KeyStore ks = KeyStore.getInstance("JKS");
-    ks.load(null, null); // initialize
-    ks.setKeyEntry(alias, privateKey, password.toCharArray(),
-        new Certificate[]{cert});
-    saveKeyStore(ks, filename, password);
-  }
-
-  private static <T extends Certificate> void createTrustStore(
-      String filename, String password, Map<String, T> certs)
-      throws GeneralSecurityException, IOException {
-    KeyStore ks = KeyStore.getInstance("JKS");
-    ks.load(null, null); // initialize
-    for (Map.Entry<String, T> cert : certs.entrySet()) {
-      ks.setCertificateEntry(cert.getKey(), cert.getValue());
-    }
-    saveKeyStore(ks, filename, password);
-  }
-
-  private static void saveKeyStore(KeyStore ks, String filename,
-      String password)
-      throws GeneralSecurityException, IOException {
-    FileOutputStream out = new FileOutputStream(filename);
-    try {
-      ks.store(out, password.toCharArray());
-    } finally {
-      out.close();
-    }
-  }
-
-  /**
-   * Saves configuration to a file.
-   *
-   * @param file File to save
-   * @param conf Configuration contents to write to file
-   * @throws IOException if there is an I/O error saving the file
-   */
-  private static void saveConfig(File file, Configuration conf)
-      throws IOException {
-    Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
-    try {
-      conf.writeXml(writer);
-    } finally {
-      writer.close();
-    }
-  }
 }
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java
new file mode 100644 (file)
index 0000000..35b9b7d
--- /dev/null
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
+import org.apache.hadoop.security.ssl.SSLFactory;
+import org.bouncycastle.x509.X509V1CertificateGenerator;
+
+import javax.security.auth.x500.X500Principal;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.math.BigInteger;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.Key;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+public class SecurityUtils {
+
+  public static final String CLIENT_KEYSTORE = "clientKS.jks";
+  public static final String SERVER_KEYSTORE = "serverKS.jks";
+  public static final String TRUSTSTORE = "trustKS.jks";
+
+  public static final String CLIENT_KEY_PASSWORD = "client_key";
+  public static final String CLIENT_KEY_STORE_PASSWORD = "client_keystore";
+  public static final String SERVER_KEY_PASSWORD = "server_key";
+  public static final String SERVER_KEY_STORE_PASSWORD = "server_keystore";
+
+  /**
+   * Performs complete setup of SSL configuration. This includes keys, certs,
+   * keystores, truststores, the server SSL configuration file,
+   * the client SSL configuration file.
+   *
+   * @param keystoresDir String directory to save keystores
+   * @param sslConfDir String directory to save SSL configuration files
+   * @param conf Configuration
+   * @param useClientCert boolean true to make the client present a cert in the
+   * SSL handshake
+   * @param trustStore boolean true to create truststore, false not to create it
+   */
+  public static X509Certificate setupSSLConfig(String keystoresDir, String sslConfDir,
+                                    Configuration conf, boolean useClientCert, boolean trustStore)
+    throws Exception {
+    String clientKeyStorePath = keystoresDir + "/" + CLIENT_KEYSTORE;
+    String serverKeyStorePath = keystoresDir + "/" + SERVER_KEYSTORE;
+    String serverPassword = "serverP";
+    String trustKS = null;
+    String trustPassword = "trustP";
+
+    File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client"));
+    File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server"));
+
+    Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>();
+
+    String hostname = SqoopUtils.getLocalHostName();
+
+    if (useClientCert) {
+      KeyPair cKP = generateKeyPair("RSA");
+      X509Certificate cCert = generateCertificate("CN=" + hostname + ", O=client", cKP, 30, "SHA1withRSA");
+      createKeyStore(clientKeyStorePath, CLIENT_KEY_PASSWORD, CLIENT_KEY_STORE_PASSWORD, "client", cKP.getPrivate(), cCert);
+      certs.put("client", cCert);
+    }
+
+    KeyPair sKP = generateKeyPair("RSA");
+    X509Certificate sCert = generateCertificate("CN=" + hostname + ", O=server", sKP, 30, "SHA1withRSA");
+    createKeyStore(serverKeyStorePath, SERVER_KEY_PASSWORD, SERVER_KEY_STORE_PASSWORD, "server", sKP.getPrivate(), sCert);
+    certs.put("server", sCert);
+
+    if (trustStore) {
+      trustKS = keystoresDir + TRUSTSTORE;
+      createTrustStore(trustKS, trustPassword, certs);
+    }
+
+    Configuration clientSSLConf = createSSLConfig(
+      SSLFactory.Mode.CLIENT, clientKeyStorePath, CLIENT_KEY_STORE_PASSWORD, CLIENT_KEY_PASSWORD, trustKS);
+    Configuration serverSSLConf = createSSLConfig(
+      SSLFactory.Mode.SERVER, serverKeyStorePath, SERVER_KEY_STORE_PASSWORD, SERVER_KEY_PASSWORD, trustKS);
+
+    saveConfig(sslClientConfFile, clientSSLConf);
+    saveConfig(sslServerConfFile, serverSSLConf);
+
+    conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL");
+    conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName());
+    conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName());
+    conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert);
+
+    return sCert;
+  }
+
+  /**
+   * Returns an SSL configuration file name.  Under parallel test
+   * execution, this file name is parameterized by a unique ID to ensure that
+   * concurrent tests don't collide on an SSL configuration file.
+   *
+   * @param base the base of the file name
+   * @return SSL configuration file name for base
+   */
+  public static String getSSLConfigFileName(String base) {
+    String testUniqueForkId = System.getProperty("test.unique.fork.id");
+    String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : "";
+    return base + fileSuffix + ".xml";
+  }
+
+  /**
+   * Creates SSL configuration.
+   *
+   * @param mode SSLFactory.Mode mode to configure
+   * @param keystore String keystore file
+   * @param keyStorePassword String store password, or null to avoid setting store
+   *   password
+   * @param keyPassword String key password, or null to avoid setting key
+   *   password
+   * @param trustKS String truststore file
+   * @return Configuration for SSL
+   */
+  public static Configuration createSSLConfig(SSLFactory.Mode mode,
+                                              String keystore, String keyStorePassword,
+                                              String keyPassword, String trustKS) {
+    String trustPassword = "trustP";
+
+    Configuration sslConf = new Configuration(false);
+    if (keystore != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore);
+    }
+
+    if (keyStorePassword != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), keyStorePassword);
+    }
+    if (keyPassword != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY),
+        keyPassword);
+    }
+    if (trustKS != null) {
+      sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+        FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
+    }
+    sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
+      trustPassword);
+    sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+      FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
+
+    return sslConf;
+  }
+
+  public static KeyPair generateKeyPair(String algorithm)
+    throws NoSuchAlgorithmException {
+    KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm);
+    keyGen.initialize(1024);
+    return keyGen.genKeyPair();
+  }
+
+  /**
+   * Create a self-signed X.509 Certificate.
+   *
+   * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB"
+   * @param pair the KeyPair
+   * @param days how many days from now the Certificate is valid for
+   * @param algorithm the signing algorithm, eg "SHA1withRSA"
+   * @return the self-signed certificate
+   */
+  public static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm)
+    throws CertificateEncodingException,
+    InvalidKeyException,
+    IllegalStateException,
+    NoSuchProviderException, NoSuchAlgorithmException, SignatureException {
+    Date from = new Date();
+    Date to = new Date(from.getTime() + days * 86400000l);
+    BigInteger sn = new BigInteger(64, new SecureRandom());
+    KeyPair keyPair = pair;
+    X509V1CertificateGenerator certGen = new X509V1CertificateGenerator();
+    X500Principal dnName = new X500Principal(dn);
+
+    certGen.setSerialNumber(sn);
+    certGen.setIssuerDN(dnName);
+    certGen.setNotBefore(from);
+    certGen.setNotAfter(to);
+    certGen.setSubjectDN(dnName);
+    certGen.setPublicKey(keyPair.getPublic());
+    certGen.setSignatureAlgorithm(algorithm);
+
+    X509Certificate cert = certGen.generate(pair.getPrivate());
+    return cert;
+  }
+
+  public static void createKeyStore(String filename,
+                                    String keyPassword, String keyStorePassword,
+                                    String alias, Key privateKey, Certificate cert)
+    throws GeneralSecurityException, IOException {
+    KeyStore ks = KeyStore.getInstance("JKS");
+    ks.load(null, null); // initialize
+    ks.setKeyEntry(alias, privateKey, keyPassword.toCharArray(),
+      new Certificate[]{cert});
+    saveKeyStore(ks, filename, keyStorePassword);
+  }
+
+  public static <T extends Certificate> void createTrustStore(
+    String filename, String password, Map<String, T> certs)
+    throws GeneralSecurityException, IOException {
+    KeyStore ks = KeyStore.getInstance("JKS");
+    ks.load(null, null); // initialize
+    for (Map.Entry<String, T> cert : certs.entrySet()) {
+      ks.setCertificateEntry(cert.getKey(), cert.getValue());
+    }
+    saveKeyStore(ks, filename, password);
+  }
+
+  public static void saveKeyStore(KeyStore ks, String filename,
+                                   String password)
+    throws GeneralSecurityException, IOException {
+    FileOutputStream out = new FileOutputStream(filename);
+    try {
+      ks.store(out, password.toCharArray());
+    } finally {
+      out.close();
+    }
+  }
+
+  /**
+   * Saves configuration to a file.
+   *
+   * @param file File to save
+   * @param conf Configuration contents to write to file
+   * @throws IOException if there is an I/O error saving the file
+   */
+  public static void saveConfig(File file, Configuration conf)
+    throws IOException {
+    Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+    try {
+      conf.writeXml(writer);
+    } finally {
+      writer.close();
+    }
+  }
+}
index 6614b19..3e0566f 100644 (file)
@@ -18,6 +18,7 @@
 package org.apache.sqoop.test.utils;
 
 import java.net.InetAddress;
+import java.net.URL;
 import java.net.UnknownHostException;
 import java.util.Locale;
 import java.util.Random;
@@ -44,7 +45,7 @@ public class SqoopUtils {
     object.setName(prefix + rand.nextLong());
   }
 
-  //Retrieve the FQDN of the current host
+  // Retrieve the FQDN of the current host
   public static String getLocalHostName() {
     String fqdn;
     try {
@@ -54,4 +55,26 @@ public class SqoopUtils {
     }
     return fqdn;
   }
+
+  // Retrieve the IP address of the current host
+  public static String getLocalIpAddress() {
+    String address;
+    try {
+      address = InetAddress.getLocalHost().getHostAddress();
+    } catch (UnknownHostException e1) {
+      address = "127.0.0.1";
+    }
+    return address;
+  }
+
+
+  @SuppressWarnings("rawtypes")
+  public static String getClasspathDir(Class klass) throws Exception {
+    String file = klass.getName();
+    file = file.replace('.', '/') + ".class";
+    URL url = Thread.currentThread().getContextClassLoader().getResource(file);
+    String baseDir = url.toURI().getPath();
+    baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1);
+    return baseDir;
+  }
 }
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.sqoop.integration.connectorloading;
+package org.apache.sqoop.integration.serverproperties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.sqoop.common.SqoopException;
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.sqoop.integration.connectorloading;
+package org.apache.sqoop.integration.serverproperties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.sqoop.core.ConfigurationConstants;
diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java
new file mode 100644 (file)
index 0000000..17503f3
--- /dev/null
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.serverproperties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
+import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
+import org.apache.sqoop.security.SecurityConstants;
+import org.apache.sqoop.test.infrastructure.Infrastructure;
+import org.apache.sqoop.test.infrastructure.SqoopTestCase;
+import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.apache.sqoop.test.utils.SecurityUtils;
+import org.apache.sqoop.test.utils.SqoopUtils;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.core.MediaType;
+import java.io.File;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.cert.Certificate;
+import java.security.cert.X509Certificate;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+
+@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class, DatabaseInfrastructureProvider.class})
+@Test(groups = {"no-real-cluster"})
+public class SslTest extends SqoopTestCase {
+
+  private SqoopMiniCluster sqoopMiniCluster;
+  private SSLContext defaultSslContext;
+  private HostnameVerifier defaultHostNameVerifier;
+
+  public static class SslSqoopMiniCluster extends JettySqoopMiniCluster {
+
+    private String keyStoreFilePath;
+    private String keyStorePassword;
+    private String keyManagerPassword;
+
+    public SslSqoopMiniCluster(String temporaryPath, Configuration configuration, String keyStoreFilePath, String keyStorePassword, String keyManagerPassword) throws Exception {
+      super(temporaryPath, configuration);
+      this.keyStoreFilePath = keyStoreFilePath;
+      this.keyStorePassword = keyStorePassword;
+      this.keyManagerPassword = keyManagerPassword;
+    }
+
+    @Override
+    protected Map<String, String> getSecurityConfiguration() {
+      Map<String, String> properties = super.getSecurityConfiguration();
+
+      properties.put(SecurityConstants.TLS_ENABLED, String.valueOf(true));
+      properties.put(SecurityConstants.TLS_PROTOCOL, "TLSv1.2");
+      properties.put(SecurityConstants.KEYSTORE_LOCATION, keyStoreFilePath);
+      properties.put(SecurityConstants.KEYSTORE_PASSWORD, keyStorePassword);
+      properties.put(SecurityConstants.KEYMANAGER_PASSWORD, keyManagerPassword);
+
+      return properties;
+    }
+  }
+
+  @BeforeMethod
+  public void backupSslContext() throws Exception {
+    defaultSslContext = SSLContext.getDefault();
+    defaultHostNameVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
+  }
+
+  @AfterMethod
+  public void restoreSslContext() {
+    SSLContext.setDefault(defaultSslContext);
+    HttpsURLConnection.setDefaultHostnameVerifier(defaultHostNameVerifier);
+  }
+
+  @AfterMethod
+  public void stopCluster() throws Exception {
+    sqoopMiniCluster.stop();
+  }
+
+  @Test
+  public void testSslInUse() throws Exception {
+    String sslKeystoresDir = getTemporaryPath() + "ssl-keystore/";
+    String sslConfDir = SqoopUtils.getClasspathDir(SslTest.class);
+    FileUtils.deleteDirectory(new File(sslKeystoresDir));
+    FileUtils.forceMkdir(new File(sslKeystoresDir));
+    X509Certificate serverCertificate = SecurityUtils.setupSSLConfig(
+      sslKeystoresDir, sslConfDir, new Configuration(), false, true);
+
+    sqoopMiniCluster =
+      new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()), getHadoopConf(), sslKeystoresDir + SecurityUtils.SERVER_KEYSTORE, SecurityUtils.SERVER_KEY_STORE_PASSWORD, SecurityUtils.SERVER_KEY_PASSWORD);
+
+    KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
+    if (kdcProvider != null) {
+      sqoopMiniCluster.setKdc(kdcProvider.getInstance());
+    }
+
+    sqoopMiniCluster.start();
+
+    // Bypass hostname verification
+    HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
+      public boolean verify(String hostname, SSLSession session) {
+        try {
+          if (hostname.equals((new URL(sqoopMiniCluster.getServerUrl())).getHost())) {
+            return true;
+          }
+        } catch (MalformedURLException e) {
+          return false;
+        }
+        return false;
+      }
+    });
+
+    SslContextFactory sslContextFactory = new SslContextFactory();
+    sslContextFactory.setKeyStorePath(sslKeystoresDir + SecurityUtils.TRUSTSTORE);
+
+    sslContextFactory.start();
+
+    SSLContext.setDefault(sslContextFactory.getSslContext());
+
+    initSqoopClient(sqoopMiniCluster.getServerUrl());
+
+    // Make a request and check the cert
+    URL url = new URL(sqoopMiniCluster.getServerUrl() + "version?" +
+      PseudoAuthenticator.USER_NAME + "=" + System.getProperty("user.name"));
+    HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, getAuthToken());
+    conn.setRequestMethod(HttpMethod.GET);
+    conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON);
+
+    assertEquals(conn.getResponseCode(), 200);
+
+    HttpsURLConnection secured = (HttpsURLConnection) conn;
+    Certificate actualCertificate = secured.getServerCertificates()[0];
+    assertEquals(actualCertificate, serverCertificate);
+  }
+
+}
@@ -18,16 +18,16 @@ limitations under the License.
 
 <!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
 
-<suite name="ConnectorLoadingTests" verbose="2" parallel="false">
+<suite name="ServerPropertiesTests" verbose="2" parallel="false">
 
     <listeners>
         <listener class-name="org.apache.sqoop.test.testng.SqoopTestListener" />
         <listener class-name="org.apache.sqoop.test.testng.ReconfigureLogListener" />
     </listeners>
 
-    <test name="ConnectorLoadingTests">
+    <test name="ServerPropertiesTests">
         <packages>
-            <package name="org.apache.sqoop.integration.connectorloading"/>
+            <package name="org.apache.sqoop.integration.serverproperties"/>
         </packages>
     </test>