SQOOP-3391: Test storing AWS credentials in Hadoop CredentialProvider during import
authorBoglarka Egyed <bogi@apache.org>
Thu, 18 Oct 2018 14:08:13 +0000 (16:08 +0200)
committerBoglarka Egyed <bogi@apache.org>
Thu, 18 Oct 2018 14:08:13 +0000 (16:08 +0200)
(Boglarka Egyed)

build.gradle
build.xml
conf/password-file.txt [new file with mode: 0644]
conf/wrong-password-file.txt [new file with mode: 0644]
gradle.properties
ivy.xml
ivy/libraries.properties
src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java [new file with mode: 0644]
src/test/org/apache/sqoop/testutil/S3TestUtils.java

index 7a0712e..2340bce 100644 (file)
@@ -141,6 +141,7 @@ dependencies {
     testCompile group: 'junit', name: 'junit', version: junitVersion
     testCompile group: 'org.assertj', name: 'assertj-core', version: assertjVersion
     testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoallVersion
+    testCompile group: 'com.github.stefanbirkner', name: 'system-rules', version: systemRulesVersion
     testCompile group: 'org.apache.zookeeper', name: 'zookeeper', version: zookeeperVersion, ext: 'jar'
 }
 
index f397531..995a513 100644 (file)
--- a/build.xml
+++ b/build.xml
     <copy todir="${test.build.extraconf}/oraoop">
       <fileset dir="${test.dir}/oraoop"/>
     </copy>
+    <copy file="${basedir}/conf/password-file.txt"
+          todir="${test.build.extraconf}" />
+    <copy file="${basedir}/conf/wrong-password-file.txt"
+          todir="${test.build.extraconf}" />
     <junit
       printsummary="yes" showoutput="${test.output}"
       haltonfailure="no" fork="yes" maxmemory="5120m"
diff --git a/conf/password-file.txt b/conf/password-file.txt
new file mode 100644 (file)
index 0000000..c09b861
--- /dev/null
@@ -0,0 +1 @@
+credProviderPwd
\ No newline at end of file
diff --git a/conf/wrong-password-file.txt b/conf/wrong-password-file.txt
new file mode 100644 (file)
index 0000000..5fe145b
--- /dev/null
@@ -0,0 +1 @@
+wrongCredProviderPwd
\ No newline at end of file
index 4808ec7..722bc8b 100644 (file)
@@ -48,6 +48,7 @@ commonsnetVersion=3.1
 log4jVersion=1.2.16
 junitVersion=4.12
 mockitoallVersion=1.9.5
+systemRulesVersion=1.17.0
 assertjVersion=2.8.0
 
 checkstyleVersion=5.5
diff --git a/ivy.xml b/ivy.xml
index 91157ca..6805fc3 100644 (file)
--- a/ivy.xml
+++ b/ivy.xml
@@ -105,6 +105,8 @@ under the License.
       conf="test->default"/>
     <dependency org="org.mockito" name="mockito-all"
       rev="${mockito-all.version}" conf="test->default"/>
+    <dependency org="com.github.stefanbirkner" name="system-rules"
+                rev="${system-rules.version}" conf="test->default"/>
     <!-- We're only using H2 for tests as it supports stored
          procedures; once we move to HSQLDB 2.x we can drop
          this -->
index 2ca95ee..3511a6f 100644 (file)
@@ -41,6 +41,7 @@ ivy.version=2.3.0
 junit.version=4.12
 assertj.version=2.8.0
 mockito-all.version=1.9.5
+system-rules.version=1.17.0
 
 h2.version=1.3.170
 
index 1d6481a..4e79f0a 100644 (file)
@@ -85,6 +85,8 @@ public class CredentialProviderHelper {
   // Should track what is in CredentialProvider class.
   public static final String CREDENTIAL_PROVIDER_PATH =
     "hadoop.security.credential.provider.path";
+  public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
+          "hadoop.security.credstore.java-keystore-provider.password-file";
 
   public static boolean isProviderAvailable() {
 
diff --git a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
new file mode 100644 (file)
index 0000000..e03eb64
--- /dev/null
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.s3;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.security.alias.CredentialShell;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.sqoop.testutil.ArgumentArrayBuilder;
+import org.apache.sqoop.testutil.DefaultS3CredentialGenerator;
+import org.apache.sqoop.testutil.ImportJobTestCase;
+import org.apache.sqoop.testutil.S3CredentialGenerator;
+import org.apache.sqoop.testutil.S3TestUtils;
+import org.apache.sqoop.testutil.TextFileTestUtils;
+import org.apache.sqoop.util.password.CredentialProviderHelper;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.EnvironmentVariables;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import static junit.framework.TestCase.fail;
+
+public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
+    public static final Log LOG = LogFactory.getLog(
+            TestS3ImportWithHadoopCredProvider.class.getName());
+
+    private static S3CredentialGenerator s3CredentialGenerator;
+
+    private static String providerPathDefault;
+    private static String providerPathEnv;
+    private static String providerPathPwdFile;
+
+    @ClassRule
+    public static final EnvironmentVariables environmentVariables
+            = new EnvironmentVariables();
+    private static File providerFileDefault;
+    private static File providerFileEnvPwd;
+    private static File providerFilePwdFile;
+
+    private FileSystem s3Client;
+
+    private static final String PASSWORD_FILE_NAME = "password-file.txt";
+    private static final String HADOOP_CREDSTORE_PASSWORD_ENV_NAME = "HADOOP_CREDSTORE_PASSWORD";
+
+    @Rule
+    public ExpectedException thrown = ExpectedException.none();
+
+    @BeforeClass
+    public static void setupS3Credentials() throws Exception {
+        String generatorCommand = S3TestUtils.getGeneratorCommand();
+        if (generatorCommand != null) {
+            s3CredentialGenerator = new DefaultS3CredentialGenerator(generatorCommand);
+        }
+        generateTempProviderFileNames();
+        fillCredentialProviderDefault();
+        fillCredentialProviderPwdFile();
+        fillCredentialProviderEnv();
+    }
+
+    @Before
+    public void setup() throws IOException {
+        S3TestUtils.runTestCaseOnlyIfS3CredentialsAreSet(s3CredentialGenerator);
+        super.setUp();
+        S3TestUtils.createTestTableFromInputData(this);
+        s3Client = S3TestUtils.setupS3ImportTestCase(s3CredentialGenerator);
+        environmentVariables.clear(HADOOP_CREDSTORE_PASSWORD_ENV_NAME);
+    }
+
+    @After
+    public void cleanUpTargetDir() {
+        S3TestUtils.tearDownS3ImportTestCase(s3Client);
+        super.tearDown();
+    }
+
+    @AfterClass
+    public static void deleteTemporaryCredFiles() {
+        providerFileDefault.deleteOnExit();
+        providerFileEnvPwd.deleteOnExit();
+        providerFilePwdFile.deleteOnExit();
+    }
+
+    @Test
+    public void testCredentialProviderDefaultSucceeds() throws Exception {
+        runImport(getArgs(providerPathDefault,false, null));
+        TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
+    }
+
+    @Test
+    public void testCredentialProviderEnvSucceeds() throws Exception {
+        setHadoopCredStorePwdEnvVar();
+        runImport(getArgs(providerPathEnv,false, null));
+        TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
+    }
+
+    @Test
+    public void testCredentialProviderPwdFileSucceeds() throws Exception {
+        runImport(getArgs(providerPathPwdFile,true, PASSWORD_FILE_NAME));
+        TextFileTestUtils.verify(S3TestUtils.getExpectedTextOutput(), s3Client, S3TestUtils.getTargetDirPath());
+    }
+
+    @Test
+    public void testCredentialProviderWithNoProviderPathFails() throws Exception {
+        thrown.expect(IOException.class);
+        runImport(getArgs(null,false, null));
+    }
+
+    @Test
+    public void testCredentialProviderWithNoEnvFails() throws Exception {
+        thrown.expect(IOException.class);
+        runImport(getArgs(providerPathEnv,false, null));
+    }
+
+    @Test
+    public void testCredentialProviderWithWrongPwdFileFails() throws Exception {
+        thrown.expect(IOException.class);
+        runImport(getArgs(providerPathPwdFile,true, "wrong-password-file.txt"));
+    }
+
+    @Test
+    public void testCredentialProviderWithNoPwdFileFails() throws Exception {
+        thrown.expect(IOException.class);
+        runImport(getArgs(providerPathPwdFile,true, null));
+    }
+
+    private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
+        ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
+
+        builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
+        if (withPwdFile) {
+            builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, pwdFile);
+        }
+        return builder.build();
+    }
+
+    private static void fillCredentialProviderDefault() throws Exception {
+        fillCredentialProvider(new Configuration(), providerPathDefault);
+    }
+
+    private static void fillCredentialProviderEnv() throws Exception {
+        setHadoopCredStorePwdEnvVar();
+        fillCredentialProvider(new Configuration(), providerPathEnv);
+    }
+
+    private static void fillCredentialProviderPwdFile() throws Exception {
+        Configuration conf = new Configuration();
+        conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE, PASSWORD_FILE_NAME);
+        fillCredentialProvider(conf, providerPathPwdFile);
+    }
+
+    private static void generateTempProviderFileNames() throws IOException {
+        providerFileDefault = Files.createTempFile("test-default-pwd-", ".jceks").toFile();
+        boolean deleted = providerFileDefault.delete();
+        providerFileEnvPwd = Files.createTempFile("test-env-pwd-", ".jceks").toFile();
+        deleted &= providerFileEnvPwd.delete();
+        providerFilePwdFile = Files.createTempFile("test-file-pwd-", ".jceks").toFile();
+        deleted &= providerFilePwdFile.delete();
+        if (!deleted) {
+            fail("Could not delete temporary provider files");
+        }
+        providerPathDefault = "jceks://file/" + providerFileDefault.getAbsolutePath();
+        providerPathEnv = "jceks://file/" + providerFileEnvPwd.getAbsolutePath();
+        providerPathPwdFile = "jceks://file/" + providerFilePwdFile.getAbsolutePath();
+    }
+
+    private static void runCredentialProviderCreateCommand(String command, Configuration conf) throws Exception {
+        ToolRunner.run(conf, new CredentialShell(), command.split(" "));
+    }
+
+    private static String getCreateCommand(String credentialKey, String credentialValue, String providerPath) {
+        return "create " + credentialKey + " -value " + credentialValue + " -provider " + providerPath;
+    }
+
+    private static void fillCredentialProvider(Configuration conf, String providerPath) throws Exception {
+        runCredentialProviderCreateCommand(getCreateCommand(Constants.ACCESS_KEY, s3CredentialGenerator.getS3AccessKey(), providerPath), conf);
+        runCredentialProviderCreateCommand(getCreateCommand(Constants.SECRET_KEY, s3CredentialGenerator.getS3SecretKey(), providerPath), conf);
+
+        if (s3CredentialGenerator.getS3SessionToken() != null) {
+            runCredentialProviderCreateCommand(getCreateCommand(Constants.SESSION_TOKEN, s3CredentialGenerator.getS3SessionToken(), providerPath), conf);
+        }
+    }
+
+    private static void setHadoopCredStorePwdEnvVar() {
+        environmentVariables.set(HADOOP_CREDSTORE_PASSWORD_ENV_NAME, "credProviderPwd");
+    }
+}
index c9d17bc..97d53bb 100644 (file)
@@ -178,6 +178,18 @@ public class S3TestUtils {
                 .withOption("target-dir", getTargetDirPath().toString());
     }
 
+    public static ArgumentArrayBuilder getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(BaseSqoopTestCase testCase) {
+
+        ArgumentArrayBuilder builder = new ArgumentArrayBuilder();
+        return builder.withCommonHadoopFlags()
+                .withProperty("fs.s3a.impl.disable.cache", "true")
+                .withProperty(Constants.AWS_CREDENTIALS_PROVIDER, getTemporaryCredentialsProviderClass())
+                .withOption("connect", testCase.getConnectString())
+                .withOption("num-mappers", "1")
+                .withOption("table", testCase.getTableName())
+                .withOption("target-dir", getTargetDirPath().toString());
+    }
+
     public static ArgumentArrayBuilder getArgumentArrayBuilderForS3UnitTestsWithFileFormatOption(BaseSqoopTestCase testCase,
                                                                                                  S3CredentialGenerator s3CredentialGenerator,
                                                                                                  String fileFormat) {