Repository: sqoop
Updated Branches:
refs/heads/trunk 2bf6f3ccd -> 08eb5bdc4
SQOOP-3410: Test S3 import with fs.s3a.security.credential.provider.path
(Boglarka Egyed via Szabolcs Vasas)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/08eb5bdc
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/08eb5bdc
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/08eb5bdc
Branch: refs/heads/trunk
Commit: 08eb5bdc405c22a25cf680390bbd438f6513199c
Parents: 2bf6f3c
Author: Szabolcs Vasas <vasas@apache.org>
Authored: Fri Nov 23 15:15:02 2018 +0100
Committer: Szabolcs Vasas <vasas@apache.org>
Committed: Fri Nov 23 15:15:02 2018 +0100
----------------------------------------------------------------------
src/docs/user/s3.txt | 6 ++++--
.../util/password/CredentialProviderHelper.java | 6 ++++--
.../credentials/TestPassingSecurePassword.java | 4 ++--
.../s3/TestS3ImportWithHadoopCredProvider.java | 21 +++++++++++++++++++-
4 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/docs/user/s3.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/s3.txt b/src/docs/user/s3.txt
index 6ff828c..e3d9771 100644
--- a/src/docs/user/s3.txt
+++ b/src/docs/user/s3.txt
@@ -173,8 +173,10 @@ https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Pr
For a guide to the Hadoop Credential Provider API please see the Hadoop documentation at
https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html.
-After creating a credential file with the credential entries the URL to the provider can
be set via the
-+hadoop.security.credential.provider.path+ property.
+After creating a credential file with the credential entries the URL to the provider can
be set via either the
++hadoop.security.credential.provider.path+ or the +fs.s3a.security.credential.provider.path+
property. For learning
+more about the precedence of these please see the Hadoop AWS documentation at
+https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Configure_the_hadoop.security.credential.provider.path_property.
Hadoop Credential Provider is often protected by password supporting three options:
http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
index 4e79f0a..e659def 100644
--- a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
+++ b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
@@ -83,8 +83,10 @@ public class CredentialProviderHelper {
// Should track what is specified in JavaKeyStoreProvider class.
public static final String SCHEME_NAME = "jceks";
// Should track what is in CredentialProvider class.
- public static final String CREDENTIAL_PROVIDER_PATH =
+ public static final String HADOOP_CREDENTIAL_PROVIDER_PATH =
"hadoop.security.credential.provider.path";
+ public static final String S3A_CREDENTIAL_PROVIDER_PATH =
+ "fs.s3a.security.credential.provider.path";
public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
"hadoop.security.credstore.java-keystore-provider.password-file";
@@ -103,7 +105,7 @@ public class CredentialProviderHelper {
public static String resolveAlias(Configuration conf, String alias)
throws IOException {
LOG.debug("Resolving alias with credential provider path set to "
- + conf.get(CREDENTIAL_PROVIDER_PATH));
+ + conf.get(HADOOP_CREDENTIAL_PROVIDER_PATH));
try {
char[] cred = (char[])
methGetPassword.invoke(conf, new Object[] { alias });
http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
index dca3195..4a8e7dc 100644
--- a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
+++ b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
@@ -397,7 +397,7 @@ public class TestPassingSecurePassword extends BaseSqoopTestCase {
"://file/" + credDir.getAbsolutePath() + "/" + jksFile;
File file = new File(credDir, jksFile);
file.delete();
- conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+ conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
ourUrl);
CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
@@ -439,7 +439,7 @@ public class TestPassingSecurePassword extends BaseSqoopTestCase {
"://file/" + credDir.getAbsolutePath() + "/" + jksFile;
File file = new File(credDir, jksFile);
file.delete();
- conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+ conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
ourUrl);
CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
index e1d7cbd..534d5a9 100644
--- a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
+++ b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
@@ -32,6 +32,7 @@ import org.apache.sqoop.testutil.ImportJobTestCase;
import org.apache.sqoop.testutil.S3CredentialGenerator;
import org.apache.sqoop.testutil.S3TestUtils;
import org.apache.sqoop.testutil.TextFileTestUtils;
+import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
import org.apache.sqoop.util.password.CredentialProviderHelper;
import org.junit.After;
import org.junit.AfterClass;
@@ -43,18 +44,36 @@ import org.junit.Test;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.util.Arrays;
import static junit.framework.TestCase.fail;
@Category(S3Test.class)
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class)
public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
+
+ @Parameterized.Parameters(name = "credentialProviderPathProperty = {0}")
+ public static Iterable<? extends Object> parameters() {
+ return Arrays.asList(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
+ CredentialProviderHelper.S3A_CREDENTIAL_PROVIDER_PATH);
+ }
+
public static final Log LOG = LogFactory.getLog(
TestS3ImportWithHadoopCredProvider.class.getName());
+ private String credentialProviderPathProperty;
+
+ public TestS3ImportWithHadoopCredProvider(String credentialProviderPathProperty) {
+ this.credentialProviderPathProperty = credentialProviderPathProperty;
+ }
+
private static S3CredentialGenerator s3CredentialGenerator;
private static String providerPathDefault;
@@ -156,7 +175,7 @@ public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase
{
private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
- builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
+ builder.withProperty(credentialProviderPathProperty, providerPath);
if (withPwdFile) {
builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE,
pwdFile);
}
|